From a014ef840181aa0092f03f464901fa45635ef6a0 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 3 Jul 2014 19:50:14 +0200 Subject: [PATCH] Cdrc to accept static fields in case of .csv files --- cdrc/cdrc.go | 17 +++--- cdrc/cdrc_test.go | 125 +++++++++++++++++++++++++++++++++++++++++++- utils/coreutils.go | 4 +- utils/storedcdr.go | 8 --- utils/utils_test.go | 4 +- 5 files changed, 138 insertions(+), 20 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index 6facf7271..f485dc61f 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -28,6 +28,7 @@ import ( "os" "path" "strconv" + "strings" "time" "unicode/utf8" @@ -85,16 +86,20 @@ func (self *Cdrc) Run() error { } // Takes the record out of csv and turns it into http form which can be posted -func (self *Cdrc) recordForkCdr(record []string) (*utils.StoredCdr, error) { +func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) { storedCdr := &utils.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var err error for cfgFieldName, cfgFieldRSR := range self.cdrFields { var fieldVal string if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cdrType) { - if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) - } else { - fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + if strings.HasPrefix(cfgFieldRSR.Id, utils.STATIC_VALUE_PREFIX) { + fieldVal = cfgFieldRSR.ParseValue("PLACEHOLDER") + } else { // Dynamic value extracted using index + if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { + return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) + } else { + fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + } } } else { // Modify here when we add more supported cdr formats fieldVal = "UNKNOWN" @@ -199,7 +204,7 @@ func (self *Cdrc) processFile(filePath string) error { engine.Logger.Err(fmt.Sprintf(" Error in csv file: %s", err.Error())) continue // Other csv related errors, ignore } - storedCdr, err := self.recordForkCdr(record) + storedCdr, err := self.recordToStoredCdr(record) if err != nil { engine.Logger.Err(fmt.Sprintf(" Error in csv file: %s", err.Error())) continue diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index c1c59a77b..7874c1e59 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -19,9 +19,13 @@ along with this program. If not, see package cdrc import ( + //"bytes" + //"encoding/csv" + //"fmt" "github.com/cgrates/cgrates/cdrs" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + //"io" "reflect" "testing" "time" @@ -35,13 +39,13 @@ func TestRecordForkCdr(t *testing.T) { cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, csvSepRune, cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil} cdrRow := []string{"firstField", "secondField"} - _, err := cdrc.recordForkCdr(cdrRow) + _, err := cdrc.recordToStoredCdr(cdrRow) if err == nil { t.Error("Failed to corectly detect missing fields from record") } cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", "prepaid", "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963", "2013-02-03 19:50:00", "2013-02-03 19:54:00", "62000000000", "supplier1", "172.16.1.1"} - rtCdr, err := cdrc.recordForkCdr(cdrRow) + rtCdr, err := cdrc.recordToStoredCdr(cdrRow) if err != nil { t.Error("Failed to parse CDR in rated cdr", err) } @@ -68,3 +72,120 @@ func TestRecordForkCdr(t *testing.T) { t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) } } + +/* +func TestDnTdmCdrs(t *testing.T) { + tdmCdrs := ` +49773280254,0049LN130676000285,N_IP_0676_00-Internet 0676 WRAP 13,02.07.2014 15:24:40,02.07.2014 15:24:40,1,25,Peak,0.000000,49DE13 +49893252121,0049651515477,N_MO_MRAP_00-WRAP Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,8,Peak,0.003920,49651 +49497361022,0049LM0409005226,N_MO_MTMB_00-RW-Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,43,Peak,0.021050,49MTMB +` + cgrConfig, _ := config.NewDefaultCGRConfig() + eCdrs := []*utils.StoredCdr{ + &utils.StoredCdr{ + CgrId: utils.Sha1("49773280254", time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49773280254", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49773280254", + Subject: "+49773280254", + Destination: "+49676000285", + SetupTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC), + Usage: time.Duration(25) * time.Second, + Cost: -1, + }, + &utils.StoredCdr{ + CgrId: utils.Sha1("49893252121", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49893252121", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49893252121", + Subject: "+49893252121", + Destination: "+49651515477", + SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + Usage: time.Duration(8) * time.Second, + Cost: -1, + }, + &utils.StoredCdr{ + CgrId: utils.Sha1("49497361022", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49497361022", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49497361022", + Subject: "+49497361022", + Destination: "+499005226", + SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + Usage: time.Duration(43) * time.Second, + Cost: -1, + }, + } + torFld, _ := utils.NewRSRField("^*voice") + acntFld, _ := utils.NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`) + reqTypeFld, _ := utils.NewRSRField("^rated") + dirFld, _ := utils.NewRSRField("^*out") + tenantFld, _ := utils.NewRSRField("^sip.test.deanconnect.nl") + categFld, _ := utils.NewRSRField("^call") + dstFld, _ := utils.NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/`) + usageFld, _ := utils.NewRSRField(`~6:s/^(\d+)$/${1}s/`) + cgrConfig.CdrcCdrFields = map[string]*utils.RSRField{ + utils.TOR: torFld, + utils.ACCID: &utils.RSRField{Id: "0"}, + utils.REQTYPE: reqTypeFld, + utils.DIRECTION: dirFld, + utils.TENANT: tenantFld, + utils.CATEGORY: categFld, + utils.ACCOUNT: acntFld, + utils.SUBJECT: acntFld, + utils.DESTINATION: dstFld, + utils.SETUP_TIME: &utils.RSRField{Id: "4"}, + utils.ANSWER_TIME: &utils.RSRField{Id: "4"}, + utils.USAGE: usageFld, + } + cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, ',', + cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil} + cdrsContent := bytes.NewReader([]byte(tdmCdrs)) + csvReader := csv.NewReader(cdrsContent) + cdrs := make([]*utils.StoredCdr, 0) + for { + cdrCsv, err := csvReader.Read() + if err != nil && err == io.EOF { + break // End of file + } else if err != nil { + t.Error("Unexpected error:", err) + } + if cdr, err := cdrc.recordToStoredCdr(cdrCsv); err != nil { + t.Error("Unexpected error: ", err) + } else { + cdrs = append(cdrs, cdr) + } + } + if !reflect.DeepEqual(eCdrs, cdrs) { + for _, ecdr := range eCdrs { + fmt.Printf("Cdr expected: %+v\n", ecdr) + } + for _, cdr := range cdrs { + fmt.Printf("Cdr processed: %+v\n", cdr) + } + t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs) + } + +} +*/ diff --git a/utils/coreutils.go b/utils/coreutils.go index dcffef59e..46a6632ae 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -118,7 +118,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) { fsTimestamp := regexp.MustCompile(`^\d{16}$`) unixTimestampRule := regexp.MustCompile(`^\d{10}$`) oneLineTimestampRule := regexp.MustCompile(`^\d{14}$`) - twoSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{2}\d{2}:\d{2}:\d{2}$`) + oneSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{1}\d{2}:\d{2}:\d{2}$`) switch { case rfc3339Rule.MatchString(tmStr): return time.Parse(time.RFC3339, tmStr) @@ -142,7 +142,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) { return nilTime, nil case oneLineTimestampRule.MatchString(tmStr): return time.Parse("20060102150405", tmStr) - case twoSpaceTimestampRule.MatchString(tmStr): + case oneSpaceTimestampRule.MatchString(tmStr): return time.Parse("02.01.2006 15:04:05", tmStr) } return nilTime, errors.New("Unsupported time format") diff --git a/utils/storedcdr.go b/utils/storedcdr.go index ca821bda7..2c2d03a5a 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -163,14 +163,6 @@ func (storedCdr *StoredCdr) AsHttpForm() url.Values { // Used in mediation, primaryMandatory marks whether missing field out of request represents error or can be ignored func (storedCdr *StoredCdr) ForkCdr(runId string, reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld *RSRField, extraFlds []*RSRField, primaryMandatory bool) (*StoredCdr, error) { - // A more elegant solution for the future to fix - /*for _, fld := range []*RSRField{reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld} { - if fld == nil { - tmp, _ := NewRSRField(META_DEFAULT) - *fld = *tmp - } - } - */ if reqTypeFld == nil { reqTypeFld, _ = NewRSRField(META_DEFAULT) } diff --git a/utils/utils_test.go b/utils/utils_test.go index e71f1a6e5..e904b9633 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -211,8 +211,8 @@ func TestParseTimeDetectLayout(t *testing.T) { } else if !olTm.Equal(expectedTime) { t.Errorf("Unexpected time parsed: %v, expecting: %v", olTm, expectedTime) } - twoSpaceTmStr := "08.04.2014 22:14:29" - tsTm, err := ParseTimeDetectLayout(twoSpaceTmStr) + oneSpaceTmStr := "08.04.2014 22:14:29" + tsTm, err := ParseTimeDetectLayout(oneSpaceTmStr) expectedTime = time.Date(2014, 4, 8, 22, 14, 29, 0, time.UTC) if err != nil { t.Error(err)