diff --git a/apier/cdrs.go b/apier/cdrs.go index ce9ceaf9f..d6483e0c7 100644 --- a/apier/cdrs.go +++ b/apier/cdrs.go @@ -58,7 +58,7 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E } else { defer fileOut.Close() } - csvWriter := cdrexporter.NewCsvCdrWriter(fileOut, self.Config.RoundingDecimals, self.Config.CdreExtraFields) + csvWriter := cdrexporter.NewCsvCdrWriter(fileOut, self.Config.RoundingDecimals, self.Config.CdreExportedFields) for _, cdr := range cdrs { if err := csvWriter.Write(cdr); err != nil { os.Remove(fileName) diff --git a/cdrexporter/csv.go b/cdrexporter/csv.go index a85cfcb10..6080ba4c5 100644 --- a/cdrexporter/csv.go +++ b/cdrexporter/csv.go @@ -22,38 +22,32 @@ import ( "encoding/csv" "github.com/cgrates/cgrates/utils" "io" - "sort" - "strconv" ) type CsvCdrWriter struct { - writer *csv.Writer - roundDecimals int // Round floats like Cost using this number of decimals - extraFields []string // Extra fields to append after primary ones, order important + writer *csv.Writer + roundDecimals int // Round floats like Cost using this number of decimals + exportedFields []*utils.RSRField // The fields exported, order important } -func NewCsvCdrWriter(writer io.Writer, roundDecimals int, extraFields []string) *CsvCdrWriter { - return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, extraFields} +func NewCsvCdrWriter(writer io.Writer, roundDecimals int, exportedFields []*utils.RSRField) *CsvCdrWriter { + return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, exportedFields} } -func (dcw *CsvCdrWriter) Write(cdr *utils.StoredCdr) error { - primaryFields := []string{cdr.CgrId, cdr.MediationRunId, cdr.AccId, cdr.CdrHost, cdr.ReqType, cdr.Direction, cdr.Tenant, cdr.TOR, cdr.Account, cdr.Subject, - cdr.Destination, cdr.SetupTime.String(), cdr.AnswerTime.String(), strconv.Itoa(int(cdr.Duration)), strconv.FormatFloat(cdr.Cost, 'f', dcw.roundDecimals, 64)} - if len(dcw.extraFields) == 0 { - dcw.extraFields = utils.MapKeys(cdr.ExtraFields) - sort.Strings(dcw.extraFields) // Controlled order in case of dynamic extra fields +func (csvwr *CsvCdrWriter) Write(cdr *utils.StoredCdr) error { + row := make([]string, len(csvwr.exportedFields)) + for idx, fld := range csvwr.exportedFields { // Add primary fields + var fldVal string + if fld.Id == utils.COST { + fldVal = cdr.FormatCost(csvwr.roundDecimals) + } else { + fldVal = cdr.ExportFieldValue(fld.Id) + } + row[idx] = fld.ParseValue(fldVal) } - lenPrimary := len(primaryFields) - row := make([]string, lenPrimary+len(dcw.extraFields)) - for idx, fld := range primaryFields { // Add primary fields - row[idx] = fld - } - for idx, fldKey := range dcw.extraFields { // Add extra fields - row[lenPrimary+idx] = cdr.ExtraFields[fldKey] - } - return dcw.writer.Write(row) + return csvwr.writer.Write(row) } -func (dcw *CsvCdrWriter) Close() { - dcw.writer.Flush() +func (csvwr *CsvCdrWriter) Close() { + csvwr.writer.Flush() } diff --git a/cdrexporter/csv_test.go b/cdrexporter/csv_test.go index a3941f94c..0e5c7a31d 100644 --- a/cdrexporter/csv_test.go +++ b/cdrexporter/csv_test.go @@ -20,6 +20,7 @@ package cdrexporter import ( "bytes" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "strings" "testing" @@ -28,17 +29,19 @@ import ( func TestCsvCdrWriter(t *testing.T) { writer := &bytes.Buffer{} - csvCdrWriter := NewCsvCdrWriter(writer, 4, []string{"extra3", "extra1"}) + cfg, _ := config.NewDefaultCGRConfig() + exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"}) + csvCdrWriter := NewCsvCdrWriter(writer, 4, exportedFields) ratedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), - Duration: 10, MediationRunId: utils.DEFAULT_RUNID, + Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01, } csvCdrWriter.Write(ratedCdr) csvCdrWriter.Close() - expected := "b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,val_extra1" + expected := `b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,"",val_extra1` result := strings.TrimSpace(writer.String()) if result != expected { - t.Errorf("Expected %s received %s.", expected, result) + t.Errorf("Expected: \n%s received: \n%s.", expected, result) } } diff --git a/cdrs/fscdr.go b/cdrs/fscdr.go index 811626251..65f09f9dd 100644 --- a/cdrs/fscdr.go +++ b/cdrs/fscdr.go @@ -120,9 +120,7 @@ func (fsCdr FSCdr) GetExtraFields() map[string]string { if !foundInVars { origFieldVal = fsCdr.searchExtraField(field.Id, fsCdr.body) } - if len(origFieldVal) != 0 { // Found a value, parse it - extraFields[field.Id] = field.ParseValue(origFieldVal) - } + extraFields[field.Id] = field.ParseValue(origFieldVal) } return extraFields } diff --git a/config/config.go b/config/config.go index 066a22f2b..932d70d66 100644 --- a/config/config.go +++ b/config/config.go @@ -88,7 +88,7 @@ type CGRConfig struct { CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> CdreCdrFormat string // Format of the exported CDRs. - CdreExtraFields []string // Extra fields list to add in exported CDRs + CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs CdreDir string // Path towards exported cdrs directory CdrcEnabled bool // Enable CDR client functionality CdrcCdrs string // Address where to reach CDR server @@ -192,7 +192,6 @@ func (self *CGRConfig) setDefaults() error { self.CDRSExtraFields = []*utils.RSRField{} self.CDRSMediator = "" self.CdreCdrFormat = "csv" - self.CdreExtraFields = []string{} self.CdreDir = "/var/log/cgrates/cdr/cdrexport/csv" self.CdrcEnabled = false self.CdrcCdrs = utils.INTERNAL @@ -257,6 +256,23 @@ func (self *CGRConfig) setDefaults() error { self.MailerAuthUser = "cgrates" self.MailerAuthPass = "CGRateS.org" self.MailerFromAddr = "cgr-mailer@localhost.localdomain" + self.CdreExportedFields = []*utils.RSRField{ + &utils.RSRField{Id: utils.CGRID}, + &utils.RSRField{Id: utils.MEDI_RUNID}, + &utils.RSRField{Id: utils.ACCID}, + &utils.RSRField{Id: utils.CDRHOST}, + &utils.RSRField{Id: utils.REQTYPE}, + &utils.RSRField{Id: utils.DIRECTION}, + &utils.RSRField{Id: utils.TENANT}, + &utils.RSRField{Id: utils.TOR}, + &utils.RSRField{Id: utils.ACCOUNT}, + &utils.RSRField{Id: utils.SUBJECT}, + &utils.RSRField{Id: utils.DESTINATION}, + &utils.RSRField{Id: utils.SETUP_TIME}, + &utils.RSRField{Id: utils.ANSWER_TIME}, + &utils.RSRField{Id: utils.DURATION}, + &utils.RSRField{Id: utils.COST}, + } return nil } @@ -448,9 +464,12 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("cdre", "cdr_format"); hasOpt { cfg.CdreCdrFormat, _ = c.GetString("cdre", "cdr_format") } - if hasOpt = c.HasOption("cdre", "extra_fields"); hasOpt { - if cfg.CdreExtraFields, errParse = ConfigSlice(c, "cdre", "extra_fields"); errParse != nil { + if hasOpt = c.HasOption("cdre", "exported_fields"); hasOpt { + extraFieldsStr, _ := c.GetString("cdre", "exported_fields") + if extraFields, err := ParseRSRFields(extraFieldsStr); err != nil { return nil, errParse + } else { + cfg.CdreExportedFields = extraFields } } if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt { diff --git a/config/config_test.go b/config/config_test.go index 897eea1a8..b1ace7da3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -81,7 +81,6 @@ func TestDefaults(t *testing.T) { eCfg.CDRSExtraFields = []*utils.RSRField{} eCfg.CDRSMediator = "" eCfg.CdreCdrFormat = "csv" - eCfg.CdreExtraFields = []string{} eCfg.CdreDir = "/var/log/cgrates/cdr/cdrexport/csv" eCfg.CdrcEnabled = false eCfg.CdrcCdrs = utils.INTERNAL @@ -146,6 +145,23 @@ func TestDefaults(t *testing.T) { eCfg.MailerAuthUser = "cgrates" eCfg.MailerAuthPass = "CGRateS.org" eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain" + eCfg.CdreExportedFields = []*utils.RSRField{ + &utils.RSRField{Id: utils.CGRID}, + &utils.RSRField{Id: utils.MEDI_RUNID}, + &utils.RSRField{Id: utils.ACCID}, + &utils.RSRField{Id: utils.CDRHOST}, + &utils.RSRField{Id: utils.REQTYPE}, + &utils.RSRField{Id: utils.DIRECTION}, + &utils.RSRField{Id: utils.TENANT}, + &utils.RSRField{Id: utils.TOR}, + &utils.RSRField{Id: utils.ACCOUNT}, + &utils.RSRField{Id: utils.SUBJECT}, + &utils.RSRField{Id: utils.DESTINATION}, + &utils.RSRField{Id: utils.SETUP_TIME}, + &utils.RSRField{Id: utils.ANSWER_TIME}, + &utils.RSRField{Id: utils.DURATION}, + &utils.RSRField{Id: utils.COST}, + } if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) @@ -214,7 +230,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}} eCfg.CDRSMediator = "test" eCfg.CdreCdrFormat = "test" - eCfg.CdreExtraFields = []string{"test"} + eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}} eCfg.CdreDir = "test" eCfg.CdrcEnabled = true eCfg.CdrcCdrs = "test" diff --git a/config/test_data.txt b/config/test_data.txt index 222449d48..6079dc4c6 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -49,8 +49,8 @@ mediator = test # Address where to reach the Mediator. Empty for disabling me [cdre] cdr_format = test # Exported CDRs format -extra_fields = test # List of extra fields to be exported out in CDRs export_dir = test # Path where the exported CDRs will be placed +exported_fields = test # List of fields in the exported CDRs [cdrc] enabled = true # Enable CDR client functionality diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 2ef48accc..cf66d26e9 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -51,8 +51,9 @@ [cdre] # cdr_format = csv # Exported CDRs format -# extra_fields = # List of extra fields to be exported out in CDRs # export_dir = /var/log/cgrates/cdr/cdrexport/csv # Path where the exported CDRs will be placed +# exported_fields = cgrid,mediation_runid,accid,cdrhost,reqtype,direction,tenant,tor,account,subject,destination,setup_time,answer_time,duration,cost + # List of fields in the exported CDRs [cdrc] # enabled = false # Enable CDR client functionality diff --git a/utils/consts.go b/utils/consts.go index 6acb85c3b..cdf62b3d1 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -82,6 +82,8 @@ const ( SETUP_TIME = "setup_time" ANSWER_TIME = "answer_time" DURATION = "duration" + MEDI_RUNID = "mediation_runid" + COST = "cost" DEFAULT_RUNID = "default" STATIC_VALUE_PREFIX = "^" CDRE_CSV = "csv" diff --git a/utils/researchreplace.go b/utils/researchreplace.go index 51126c7aa..668fa2795 100644 --- a/utils/researchreplace.go +++ b/utils/researchreplace.go @@ -28,9 +28,12 @@ type ReSearchReplace struct { ReplaceTemplate string } -func (self *ReSearchReplace) Process(source string) string { +func (rsr *ReSearchReplace) Process(source string) string { + if rsr.SearchRegexp == nil { + return "" + } res := []byte{} - match := self.SearchRegexp.FindStringSubmatchIndex(source) - res = self.SearchRegexp.ExpandString(res, self.ReplaceTemplate, source, match) + match := rsr.SearchRegexp.FindStringSubmatchIndex(source) + res = rsr.SearchRegexp.ExpandString(res, rsr.ReplaceTemplate, source, match) return string(res) } diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 540fced07..4c5c1db22 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -25,6 +25,9 @@ type RSRField struct { // Parse the field value from a string func (rsrf *RSRField) ParseValue(value string) string { + if len(value) == 0 { + return value + } if rsrf.RSRule != nil { value = rsrf.RSRule.Process(value) } diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 7c79fe442..60dd46a09 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -134,6 +134,11 @@ func (storedCdr *StoredCdr) GetExtraFields() map[string]string { return storedCdr.ExtraFields } +// Return cost as string, formated with number of decimals configured +func (storedCdr *StoredCdr) FormatCost(roundDecimals int) string { + return strconv.FormatFloat(storedCdr.Cost, 'f', roundDecimals, 64) +} + func (storedCdr *StoredCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld string, extraFlds []string, fieldsMandatory bool) (*StoredCdr, error) { return storedCdr, nil } @@ -159,3 +164,43 @@ func (storedCdr *StoredCdr) AsRawCdrHttpForm() url.Values { } return v } + +// Used to export fields as string, primary fields are const labeled +func (storedCdr *StoredCdr) ExportFieldValue(fldName string) string { + switch fldName { + case CGRID: + return storedCdr.CgrId + case ACCID: + return storedCdr.AccId + case CDRHOST: + return storedCdr.CdrHost + case CDRSOURCE: + return storedCdr.CdrSource + case REQTYPE: + return storedCdr.ReqType + case DIRECTION: + return storedCdr.Direction + case TENANT: + return storedCdr.Tenant + case TOR: + return storedCdr.TOR + case ACCOUNT: + return storedCdr.Account + case SUBJECT: + return storedCdr.Subject + case DESTINATION: + return storedCdr.Destination + case SETUP_TIME: + return storedCdr.SetupTime.String() + case ANSWER_TIME: + return storedCdr.AnswerTime.String() + case DURATION: + return strconv.FormatFloat(storedCdr.Duration.Seconds(), 'f', -1, 64) + case MEDI_RUNID: + return storedCdr.MediationRunId + case COST: + return strconv.FormatFloat(storedCdr.Cost, 'f', -1, 64) // Recommended to use FormatCost + default: + return storedCdr.ExtraFields[fldName] + } +} diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index c3fe07cf0..6fad3e290 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -148,3 +148,42 @@ func TestAsRawCdrHttpForm(t *testing.T) { t.Errorf("Expected: %s, received: %s", ratedCdr.ExtraFields["fieldextr2"], cdrForm.Get("fieldextr2")) } } + +func TestExportFieldValue(t *testing.T) { + cdr := StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if cdr.ExportFieldValue(CGRID) != cdr.CgrId || + cdr.ExportFieldValue(ACCID) != cdr.AccId || + cdr.ExportFieldValue(CDRHOST) != cdr.CdrHost || + cdr.ExportFieldValue(CDRSOURCE) != cdr.CdrSource || + cdr.ExportFieldValue(REQTYPE) != cdr.ReqType || + cdr.ExportFieldValue(DIRECTION) != cdr.Direction || + cdr.ExportFieldValue(TENANT) != cdr.Tenant || + cdr.ExportFieldValue(TOR) != cdr.TOR || + cdr.ExportFieldValue(ACCOUNT) != cdr.Account || + cdr.ExportFieldValue(SUBJECT) != cdr.Subject || + cdr.ExportFieldValue(DESTINATION) != cdr.Destination || + cdr.ExportFieldValue(SETUP_TIME) != "0001-01-01 00:00:00 +0000 UTC" || + cdr.ExportFieldValue(ANSWER_TIME) != cdr.AnswerTime.String() || + cdr.ExportFieldValue(DURATION) != "10" || + cdr.ExportFieldValue(MEDI_RUNID) != cdr.MediationRunId || + cdr.ExportFieldValue(COST) != "1.01" || + cdr.ExportFieldValue("field_extr1") != cdr.ExtraFields["field_extr1"] || + cdr.ExportFieldValue("fieldextr2") != cdr.ExtraFields["fieldextr2"] || + cdr.ExportFieldValue("dummy_field") != "" { + t.Error("Unexpected filed value received") + } +} + +func TestFormatCost(t *testing.T) { + cdr := StoredCdr{Cost: 1.01} + if cdr.FormatCost(4) != "1.0100" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(4)) + } + cdr = StoredCdr{Cost: 1.01001} + if cdr.FormatCost(4) != "1.0100" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(4)) + } +}