uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Cdrc to accept static fields in case of .csv files

Browse Source
This commit is contained in:
DanB
2014-07-03 19:50:14 +02:00
committed by Radu Ioan Fericean
parent bb102e65c2
commit 392071da85
5 changed files with 138 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cdrc/cdrc.go
View File

@@ -28,6 +28,7 @@ import (
"os"
"path"
"strconv"
"strings"
"time"
"unicode/utf8"
@@ -85,16 +86,20 @@ func (self *Cdrc) Run() error {
}
// Takes the record out of csv and turns it into http form which can be posted
func (self *Cdrc) recordForkCdr(record []string) (*utils.StoredCdr, error) {
func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) {
storedCdr := &utils.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceId, ExtraFields: make(map[string]string), Cost: -1}
var err error
for cfgFieldName, cfgFieldRSR := range self.cdrFields {
var fieldVal string
if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cdrType) {
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName)
} else {
fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx])
if strings.HasPrefix(cfgFieldRSR.Id, utils.STATIC_VALUE_PREFIX) {
fieldVal = cfgFieldRSR.ParseValue("PLACEHOLDER")
} else { // Dynamic value extracted using index
if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName)
} else {
fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx])
}
}
} else { // Modify here when we add more supported cdr formats
fieldVal = "UNKNOWN"
@@ -199,7 +204,7 @@ func (self *Cdrc) processFile(filePath string) error {
engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
continue // Other csv related errors, ignore
}
storedCdr, err := self.recordForkCdr(record)
storedCdr, err := self.recordToStoredCdr(record)
if err != nil {
engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
continue

125
cdrc/cdrc_test.go
View File

@@ -19,9 +19,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package cdrc
import (
//"bytes"
//"encoding/csv"
//"fmt"
"github.com/cgrates/cgrates/cdrs"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
//"io"
"reflect"
"testing"
"time"
@@ -35,13 +39,13 @@ func TestRecordForkCdr(t *testing.T) {
cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, csvSepRune,
cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil}
cdrRow := []string{"firstField", "secondField"}
_, err := cdrc.recordForkCdr(cdrRow)
_, err := cdrc.recordToStoredCdr(cdrRow)
if err == nil {
t.Error("Failed to corectly detect missing fields from record")
}
cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", "prepaid", "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963",
"2013-02-03 19:50:00", "2013-02-03 19:54:00", "62000000000", "supplier1", "172.16.1.1"}
rtCdr, err := cdrc.recordForkCdr(cdrRow)
rtCdr, err := cdrc.recordToStoredCdr(cdrRow)
if err != nil {
t.Error("Failed to parse CDR in rated cdr", err)
}
@@ -68,3 +72,120 @@ func TestRecordForkCdr(t *testing.T) {
t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
}
}
/*
func TestDnTdmCdrs(t *testing.T) {
tdmCdrs := `
49773280254,0049LN130676000285,N_IP_0676_00-Internet 0676 WRAP 13,02.07.2014 15:24:40,02.07.2014 15:24:40,1,25,Peak,0.000000,49DE13
49893252121,0049651515477,N_MO_MRAP_00-WRAP Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,8,Peak,0.003920,49651
49497361022,0049LM0409005226,N_MO_MTMB_00-RW-Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,43,Peak,0.021050,49MTMB
`
cgrConfig, _ := config.NewDefaultCGRConfig()
eCdrs := []*utils.StoredCdr{
&utils.StoredCdr{
CgrId: utils.Sha1("49773280254", time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC).String()),
TOR: utils.VOICE,
AccId: "49773280254",
CdrHost: "0.0.0.0",
CdrSource: cgrConfig.CdrcSourceId,
ReqType: "rated",
Direction: "*out",
Tenant: "sip.test.deanconnect.nl",
Category: "call",
Account: "+49773280254",
Subject: "+49773280254",
Destination: "+49676000285",
SetupTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC),
AnswerTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC),
Usage: time.Duration(25) * time.Second,
Cost: -1,
},
&utils.StoredCdr{
CgrId: utils.Sha1("49893252121", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()),
TOR: utils.VOICE,
AccId: "49893252121",
CdrHost: "0.0.0.0",
CdrSource: cgrConfig.CdrcSourceId,
ReqType: "rated",
Direction: "*out",
Tenant: "sip.test.deanconnect.nl",
Category: "call",
Account: "+49893252121",
Subject: "+49893252121",
Destination: "+49651515477",
SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
Usage: time.Duration(8) * time.Second,
Cost: -1,
},
&utils.StoredCdr{
CgrId: utils.Sha1("49497361022", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()),
TOR: utils.VOICE,
AccId: "49497361022",
CdrHost: "0.0.0.0",
CdrSource: cgrConfig.CdrcSourceId,
ReqType: "rated",
Direction: "*out",
Tenant: "sip.test.deanconnect.nl",
Category: "call",
Account: "+49497361022",
Subject: "+49497361022",
Destination: "+499005226",
SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
Usage: time.Duration(43) * time.Second,
Cost: -1,
},
}
torFld, _ := utils.NewRSRField("^*voice")
acntFld, _ := utils.NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`)
reqTypeFld, _ := utils.NewRSRField("^rated")
dirFld, _ := utils.NewRSRField("^*out")
tenantFld, _ := utils.NewRSRField("^sip.test.deanconnect.nl")
categFld, _ := utils.NewRSRField("^call")
dstFld, _ := utils.NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/`)
usageFld, _ := utils.NewRSRField(`~6:s/^(\d+)$/${1}s/`)
cgrConfig.CdrcCdrFields = map[string]*utils.RSRField{
utils.TOR: torFld,
utils.ACCID: &utils.RSRField{Id: "0"},
utils.REQTYPE: reqTypeFld,
utils.DIRECTION: dirFld,
utils.TENANT: tenantFld,
utils.CATEGORY: categFld,
utils.ACCOUNT: acntFld,
utils.SUBJECT: acntFld,
utils.DESTINATION: dstFld,
utils.SETUP_TIME: &utils.RSRField{Id: "4"},
utils.ANSWER_TIME: &utils.RSRField{Id: "4"},
utils.USAGE: usageFld,
}
cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, ',',
cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil}
cdrsContent := bytes.NewReader([]byte(tdmCdrs))
csvReader := csv.NewReader(cdrsContent)
cdrs := make([]*utils.StoredCdr, 0)
for {
cdrCsv, err := csvReader.Read()
if err != nil && err == io.EOF {
break // End of file
} else if err != nil {
t.Error("Unexpected error:", err)
}
if cdr, err := cdrc.recordToStoredCdr(cdrCsv); err != nil {
t.Error("Unexpected error: ", err)
} else {
cdrs = append(cdrs, cdr)
}
}
if !reflect.DeepEqual(eCdrs, cdrs) {
for _, ecdr := range eCdrs {
fmt.Printf("Cdr expected: %+v\n", ecdr)
}
for _, cdr := range cdrs {
fmt.Printf("Cdr processed: %+v\n", cdr)
}
t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
}
}
*/

4
utils/coreutils.go
View File

@@ -118,7 +118,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) {
fsTimestamp := regexp.MustCompile(`^\d{16}$`)
unixTimestampRule := regexp.MustCompile(`^\d{10}$`)
oneLineTimestampRule := regexp.MustCompile(`^\d{14}$`)
twoSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{2}\d{2}:\d{2}:\d{2}$`)
oneSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{1}\d{2}:\d{2}:\d{2}$`)
switch {
case rfc3339Rule.MatchString(tmStr):
return time.Parse(time.RFC3339, tmStr)
@@ -142,7 +142,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) {
return nilTime, nil
case oneLineTimestampRule.MatchString(tmStr):
return time.Parse("20060102150405", tmStr)
case twoSpaceTimestampRule.MatchString(tmStr):
case oneSpaceTimestampRule.MatchString(tmStr):
return time.Parse("02.01.2006 15:04:05", tmStr)
}
return nilTime, errors.New("Unsupported time format")

8
utils/storedcdr.go
View File

@@ -163,14 +163,6 @@ func (storedCdr *StoredCdr) AsHttpForm() url.Values {
// Used in mediation, primaryMandatory marks whether missing field out of request represents error or can be ignored
func (storedCdr *StoredCdr) ForkCdr(runId string, reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld *RSRField,
extraFlds []*RSRField, primaryMandatory bool) (*StoredCdr, error) {
// A more elegant solution for the future to fix
/*for _, fld := range []*RSRField{reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld} {
if fld == nil {
tmp, _ := NewRSRField(META_DEFAULT)
*fld = *tmp
}
}
*/
if reqTypeFld == nil {
reqTypeFld, _ = NewRSRField(META_DEFAULT)
}

4
utils/utils_test.go
View File

@@ -211,8 +211,8 @@ func TestParseTimeDetectLayout(t *testing.T) {
} else if !olTm.Equal(expectedTime) {
t.Errorf("Unexpected time parsed: %v, expecting: %v", olTm, expectedTime)
}
twoSpaceTmStr := "08.04.2014 22:14:29"
tsTm, err := ParseTimeDetectLayout(twoSpaceTmStr)
oneSpaceTmStr := "08.04.2014 22:14:29"
tsTm, err := ParseTimeDetectLayout(oneSpaceTmStr)
expectedTime = time.Date(2014, 4, 8, 22, 14, 29, 0, time.UTC)
if err != nil {
t.Error(err)