From e4c9cf561a7aca2dfaa0bdf2da44968d3336a380 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 2 Oct 2014 19:02:23 +0200 Subject: [PATCH] Refactoring CDRE and CDRC configurations and functionality --- apier/v1/apier_local_test.go | 2 +- apier/v1/cdre.go | 8 +- cdrc/cdrc.go | 67 ++- cdrc/cdrc_local_test.go | 52 ++- cdrc/cdrc_test.go | 20 +- cdre/cdrexporter.go | 195 ++++---- cdre/cdrexporter_test.go | 37 +- cdre/fixedwidth_test.go | 94 ++-- cmd/cgr-engine/cgr-engine.go | 28 +- config/cdreconfig.go | 248 +++------- config/cdreconfig_test.go | 553 ++++++----------------- config/config.go | 136 ++---- config/config_test.go | 117 ++--- config/helpers_test.go | 23 +- config/xmlcdrc.go | 87 ---- config/xmlcdrc_test.go | 118 ++--- config/xmlcdre.go | 154 ------- config/xmlcdre_test.go | 286 ++++++------ config/xmlconfig.go | 102 +++-- engine/fscdr_test.go | 2 +- general_tests/multiplecdrc_local_test.go | 12 +- general_tests/tutorial_local_test.go | 1 + utils/consts.go | 6 + utils/coreutils.go | 9 + utils/rsrfield.go | 16 +- utils/rsrfield_test.go | 6 +- utils/storedcdr_test.go | 2 +- 27 files changed, 832 insertions(+), 1549 deletions(-) delete mode 100644 config/xmlcdrc.go delete mode 100644 config/xmlcdre.go diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 90f1af766..1acaf4ef8 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -73,7 +73,7 @@ func TestCreateDirs(t *testing.T) { if !*testLocal { return } - for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} { + for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcInstances[0].CdrInDir, cfg.CdrcInstances[0].CdrOutDir, cfg.HistoryDir} { if err := os.RemoveAll(pathDir); err != nil { t.Fatal("Error removing folder: ", pathDir, err) } diff --git a/apier/v1/cdre.go b/apier/v1/cdre.go index de92ebfe5..59ea72d92 100644 --- a/apier/v1/cdre.go +++ b/apier/v1/cdre.go @@ -114,11 +114,13 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E if xmlTemplates := self.Config.XmlCfgDocument.GetCdreCfgs(expTplStr[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates == nil { return fmt.Errorf("%s:ExportTemplate", utils.ERR_NOT_FOUND) } else { - exportTemplate = xmlTemplates[expTplStr[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig() + if exportTemplate, err = config.NewCdreConfigFromXmlCdreCfg(xmlTemplates[expTplStr[len(utils.XML_PROFILE_PREFIX):]]); err != nil { + return fmt.Errorf("%s:ExportTemplate:%s", utils.ERR_SERVER_ERROR, err.Error()) + } } } else { - exportTemplate, _ = config.NewDefaultCdreConfig() - if contentFlds, err := config.NewCdreCdrFieldsFromIds(exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH, + exportTemplate = config.NewDefaultCdreConfig() + if contentFlds, err := config.NewCfgCdrFieldsFromIds(exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH, strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } else { diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index fa997822e..8dd5e228e 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -28,10 +28,10 @@ import ( "os" "path" "strconv" - "strings" "time" "unicode/utf8" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "github.com/howeyc/fsnotify" @@ -42,13 +42,13 @@ const ( FS_CSV = "freeswitch_csv" ) -func NewCdrc(cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId string, runDelay time.Duration, csvSep string, cdrFields map[string][]*utils.RSRField, cdrServer *engine.CDRS) (*Cdrc, error) { - if len(csvSep) != 1 { - return nil, fmt.Errorf("Unsupported csv separator: %s", csvSep) +func NewCdrc(cdrcCfg *config.CdrcConfig, httpSkipTlsCheck bool, cdrServer *engine.CDRS) (*Cdrc, error) { + if len(cdrcCfg.FieldSeparator) != 1 { + return nil, fmt.Errorf("Unsupported csv separator: %s", cdrcCfg.FieldSeparator) } - csvSepRune, _ := utf8.DecodeRune([]byte(csvSep)) - cdrc := &Cdrc{cdrsAddress: cdrsAddress, cdrType: cdrType, cdrInDir: cdrInDir, cdrOutDir: cdrOutDir, - cdrSourceId: cdrSourceId, runDelay: runDelay, csvSep: csvSepRune, cdrFields: cdrFields, cdrServer: cdrServer} + csvSepRune, _ := utf8.DecodeRune([]byte(cdrcCfg.FieldSeparator)) + cdrc := &Cdrc{cdrsAddress: cdrcCfg.CdrsAddress, cdrType: cdrcCfg.CdrType, cdrInDir: cdrcCfg.CdrInDir, cdrOutDir: cdrcCfg.CdrOutDir, + cdrSourceId: cdrcCfg.CdrSourceId, runDelay: cdrcCfg.RunDelay, csvSep: csvSepRune, cdrFields: cdrcCfg.CdrFields, httpSkipTlsCheck: httpSkipTlsCheck, cdrServer: cdrServer} // Before processing, make sure in and out folders exist for _, dir := range []string{cdrc.cdrInDir, cdrc.cdrOutDir} { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { @@ -65,11 +65,12 @@ type Cdrc struct { cdrInDir, cdrOutDir, cdrSourceId string - runDelay time.Duration - csvSep rune - cdrFields map[string][]*utils.RSRField - cdrServer *engine.CDRS // Reference towards internal cdrServer if that is the case - httpClient *http.Client + runDelay time.Duration + csvSep rune + cdrFields []*config.CfgCdrField + httpSkipTlsCheck bool + cdrServer *engine.CDRS // Reference towards internal cdrServer if that is the case + httpClient *http.Client } // When called fires up folder monitoring, either automated via inotify or manual by sleeping between processing @@ -77,7 +78,7 @@ func (self *Cdrc) Run() error { if self.runDelay == time.Duration(0) { // Automated via inotify return self.trackCDRFiles() } - // No automated, process and sleep approach + // Not automated, process and sleep approach for { self.processCdrDir() time.Sleep(self.runDelay) @@ -88,24 +89,40 @@ func (self *Cdrc) Run() error { func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) { storedCdr := &utils.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var err error - for cfgFieldName, cfgFieldRSRs := range self.cdrFields { + for _, cdrFldCfg := range self.cdrFields { var fieldVal string if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cdrType) { - for _, cfgFieldRSR := range cfgFieldRSRs { - if strings.HasPrefix(cfgFieldRSR.Id, utils.STATIC_VALUE_PREFIX) { - fieldVal += cfgFieldRSR.ParseValue("PLACEHOLDER") - } else { // Dynamic value extracted using index - if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) - } else { - fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + if cdrFldCfg.Type == utils.CDRFIELD { + for _, cfgFieldRSR := range cdrFldCfg.Value { + if cfgFieldRSR.IsStatic() { + fieldVal += cfgFieldRSR.ParseValue("") + } else { // Dynamic value extracted using index + if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { + return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag) + } else { + fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + } } } + } else if cdrFldCfg.Type == utils.HTTP_POST { + var outValByte []byte + var httpAddr string + for _, rsrFld := range cdrFldCfg.Value { + httpAddr += rsrFld.ParseValue("") + } + if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, record); err == nil { + fieldVal = string(outValByte) + if len(fieldVal) == 0 && cdrFldCfg.Mandatory { + return nil, fmt.Errorf("MandatoryIeMissing: thEmpty result for http_post field: %s", cdrFldCfg.Tag) + } + } + } else { + return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type) } } else { // Modify here when we add more supported cdr formats - fieldVal = "UNKNOWN" + return nil, fmt.Errorf("Unsupported CDR file format: %s", self.cdrType) } - switch cfgFieldName { + switch cdrFldCfg.CdrFieldId { case utils.TOR: storedCdr.TOR = fieldVal case utils.ACCID: @@ -137,7 +154,7 @@ func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) { return nil, fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error()) } default: // Extra fields will not match predefined so they all show up here - storedCdr.ExtraFields[cfgFieldName] = fieldVal + storedCdr.ExtraFields[cdrFldCfg.CdrFieldId] = fieldVal } } diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index 041598edc..7a774520b 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -48,6 +48,7 @@ README: var cfgPath string var cfg *config.CGRConfig +var cdrcCfg *config.CdrcConfig var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") @@ -57,6 +58,9 @@ var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for r func init() { cfgPath = path.Join(*dataDir, "conf", "samples", "apier_local_test.cfg") cfg, _ = config.NewCGRConfigFromFile(&cfgPath) + if len(cfg.CdrcInstances) > 0 { + cdrcCfg = cfg.CdrcInstances[0] + } } var fileContent1 = `accid11,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 @@ -125,38 +129,41 @@ func TestCreateCdrFiles(t *testing.T) { if !*testLocal { return } - if err := os.RemoveAll(cfg.CdrcCdrInDir); err != nil { - t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err) + if cdrcCfg == nil { + t.Fatal("Empty default cdrc configuration") } - if err := os.MkdirAll(cfg.CdrcCdrInDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cfg.CdrcCdrInDir, err) + if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil { + t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err) } - if err := os.RemoveAll(cfg.CdrcCdrOutDir); err != nil { - t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err) + if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil { + t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err) } - if err := os.MkdirAll(cfg.CdrcCdrOutDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cfg.CdrcCdrOutDir, err) + if err := os.RemoveAll(cdrcCfg.CdrOutDir); err != nil { + t.Fatal("Error removing folder: ", cdrcCfg.CdrOutDir, err) } - if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil { + if err := os.MkdirAll(cdrcCfg.CdrOutDir, 0755); err != nil { + t.Fatal("Error creating folder: ", cdrcCfg.CdrOutDir, err) + } + if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil { t.Fatal(err.Error) } - if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil { + if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil { t.Fatal(err.Error) } + } func TestProcessCdrDir(t *testing.T) { if !*testLocal { return } - if cfg.CdrcCdrs == utils.INTERNAL { // For now we only test over network - return + if cdrcCfg.CdrsAddress == utils.INTERNAL { // For now we only test over network + cdrcCfg.CdrsAddress = "127.0.0.1:2013" } if err := startEngine(); err != nil { t.Fatal(err.Error()) } - cdrc, err := NewCdrc(cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, cfg.CdrcCsvSep, - cfg.CdrcCdrFields, nil) + cdrc, err := NewCdrc(cdrcCfg, true, nil) if err != nil { t.Fatal(err.Error()) } @@ -171,13 +178,13 @@ func TestCreateCdr3File(t *testing.T) { if !*testLocal { return } - if err := os.RemoveAll(cfg.CdrcCdrInDir); err != nil { - t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err) + if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil { + t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err) } - if err := os.MkdirAll(cfg.CdrcCdrInDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cfg.CdrcCdrInDir, err) + if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil { + t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err) } - if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil { + if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil { t.Fatal(err.Error) } } @@ -186,14 +193,13 @@ func TestProcessCdr3Dir(t *testing.T) { if !*testLocal { return } - if cfg.CdrcCdrs == utils.INTERNAL { // For now we only test over network - return + if cdrcCfg.CdrsAddress == utils.INTERNAL { // For now we only test over network + cdrcCfg.CdrsAddress = "127.0.0.1:2013" } if err := startEngine(); err != nil { t.Fatal(err.Error()) } - cdrc, err := NewCdrc(cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, ";", - cfg.CdrcCdrFields, nil) + cdrc, err := NewCdrc(cdrcCfg, true, nil) if err != nil { t.Fatal(err.Error()) } diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index faae9cd25..86e86565c 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -19,27 +19,23 @@ along with this program. If not, see package cdrc import ( - //"bytes" - //"encoding/csv" - //"fmt" "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" - //"io" "reflect" "testing" "time" - "unicode/utf8" ) func TestRecordForkCdr(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() - cgrConfig.CdrcCdrFields["supplier"] = []*utils.RSRField{&utils.RSRField{Id: "14"}} - csvSepRune, _ := utf8.DecodeRune([]byte(cgrConfig.CdrcCsvSep)) - cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, csvSepRune, - cgrConfig.CdrcCdrFields, new(engine.CDRS), nil} + cdrcConfig := cgrConfig.CdrcInstances[0] + cdrcConfig.CdrFields = append(cdrcConfig.CdrFields, &config.CfgCdrField{Tag: "SupplierTest", Type: utils.CDRFIELD, CdrFieldId: "supplier", Value: []*utils.RSRField{&utils.RSRField{Id: "14"}}}) + cdrc, err := NewCdrc(cdrcConfig, true, nil) + if err != nil { + t.Error(err) + } cdrRow := []string{"firstField", "secondField"} - _, err := cdrc.recordToStoredCdr(cdrRow) + _, err = cdrc.recordToStoredCdr(cdrRow) if err == nil { t.Error("Failed to corectly detect missing fields from record") } @@ -54,7 +50,7 @@ func TestRecordForkCdr(t *testing.T) { TOR: cdrRow[2], AccId: cdrRow[3], CdrHost: "0.0.0.0", // Got it over internal interface - CdrSource: cgrConfig.CdrcSourceId, + CdrSource: cdrcConfig.CdrSourceId, ReqType: cdrRow[4], Direction: cdrRow[5], Tenant: cdrRow[6], diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index 4a45008c2..069f96b3e 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -28,19 +28,13 @@ import ( "io" "os" "strconv" - "strings" "time" ) const ( COST_DETAILS = "cost_details" - FILLER = "filler" - CONSTANT = "constant" - METATAG = "metatag" CONCATENATED_CDRFIELD = "concatenated_cdrfield" - COMBIMED = "combimed" DATETIME = "datetime" - HTTP_POST = "http_post" META_EXPORTID = "export_id" META_TIMENOW = "time_now" META_FIRSTCDRATIME = "first_cdr_atime" @@ -121,20 +115,25 @@ func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) return string(ccJson), nil } -func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *utils.StoredCdr, filterRule, fieldRule *utils.RSRField) (string, error) { - fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule) - if !fltrPass { - return "", nil - } - for _, cdr := range cdre.cdrs { - if cdr.CgrId != processedCdr.CgrId { - continue // We only care about cdrs with same primary cdr behind +func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *utils.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) { + var combinedVal string // Will result as combination of the field values, filters must match + for _, filterRule := range cfgCdrFld.Filter { + fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule) + if !fltrPass { + return "", nil } - if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { - return cdr.FieldAsString(fieldRule), nil + for _, cdr := range cdre.cdrs { + if cdr.CgrId != processedCdr.CgrId { + continue // We only care about cdrs with same primary cdr behind + } + if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte + for _, rsrRule := range cfgCdrFld.Value { + combinedVal += cdr.FieldAsString(rsrRule) + } + } } } - return "", nil + return combinedVal, nil } // Check if the destination should be masked in output @@ -145,17 +144,20 @@ func (cdre *CdrExporter) maskedDestination(destination string) bool { return false } -func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, fltrRl, fieldRl *utils.RSRField, layout string) (string, error) { - if fieldRl == nil { +func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) { + if len(cfgCdrFld.Value) == 0 { return "", nil } - if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { - return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) + for _, fltrRl := range cfgCdrFld.Filter { + if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { + return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) + } } + layout := cfgCdrFld.Layout if len(layout) == 0 { layout = time.RFC3339 } - if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(fieldRl)); err != nil { + if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(cfgCdrFld.Value[0])); err != nil { // Only one rule makes sense here return "", err } else { return dtFld.Format(layout), nil @@ -163,39 +165,43 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, fltrRl, field } // Extracts the value specified by cfgHdr out of cdr -func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, fltrRl, rsrFld *utils.RSRField, layout string) (string, error) { - if rsrFld == nil { - return "", nil - } - if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { - return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) +func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) { + for _, fltrRl := range cfgCdrFld.Filter { + if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { + return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) + } } + layout := cfgCdrFld.Layout if len(layout) == 0 { layout = time.RFC3339 } - var cdrVal string - switch rsrFld.Id { - case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb - if cdrVal, err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil { - return "", err + var retVal string // Concatenate the resulting values + for _, rsrFld := range cfgCdrFld.Value { + var cdrVal string + switch rsrFld.Id { + case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb + if cdrVal, err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil { + return "", err + } + case utils.COST: + cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals) + case utils.USAGE: + cdrVal = cdr.FormatUsage(layout) + case utils.SETUP_TIME: + cdrVal = cdr.SetupTime.Format(layout) + case utils.ANSWER_TIME: // Format time based on layout + cdrVal = cdr.AnswerTime.Format(layout) + case utils.DESTINATION: + cdrVal = cdr.FieldAsString(rsrFld) + if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) { + cdrVal = MaskDestination(cdrVal, cdre.maskLen) + } + default: + cdrVal = cdr.FieldAsString(rsrFld) } - case utils.COST: - cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals) - case utils.USAGE: - cdrVal = cdr.FormatUsage(layout) - case utils.SETUP_TIME: - cdrVal = cdr.SetupTime.Format(layout) - case utils.ANSWER_TIME: // Format time based on layout - cdrVal = cdr.AnswerTime.Format(layout) - case utils.DESTINATION: - cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}) - if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) { - cdrVal = MaskDestination(cdrVal, cdre.maskLen) - } - default: - cdrVal = cdr.FieldAsString(rsrFld) + retVal += cdrVal } - return rsrFld.ParseValue(cdrVal), nil + return retVal, nil } // Handle various meta functions used in header/trailer @@ -212,15 +218,12 @@ func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) { case META_NRCDRS: return strconv.Itoa(cdre.numberOfRecords), nil case META_DURCDRS: - //return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil emulatedCdr := &utils.StoredCdr{TOR: utils.VOICE, Usage: cdre.totalDuration} return emulatedCdr.FormatUsage(arg), nil case META_SMSUSAGE: - //return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil emulatedCdr := &utils.StoredCdr{TOR: utils.SMS, Usage: cdre.totalSmsUsage} return emulatedCdr.FormatUsage(arg), nil case META_DATAUSAGE: - //return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil emulatedCdr := &utils.StoredCdr{TOR: utils.DATA, Usage: cdre.totalDataUsage} return emulatedCdr.FormatUsage(arg), nil case META_COSTCDRS: @@ -241,23 +244,27 @@ func (cdre *CdrExporter) composeHeader() error { for _, cfgFld := range cdre.exportTemplate.HeaderFields { var outVal string switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value + case utils.FILLER: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value - case METATAG: - outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + case utils.CONSTANT: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } + case utils.METATAG: + outVal, err = cdre.metaHandler(cfgFld.CdrFieldId, cfgFld.Layout) default: return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) } if err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error())) + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error())) return err } fmtOut := outVal if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error())) + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error())) return err } cdre.header = append(cdre.header, fmtOut) @@ -270,23 +277,27 @@ func (cdre *CdrExporter) composeTrailer() error { for _, cfgFld := range cdre.exportTemplate.TrailerFields { var outVal string switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value + case utils.FILLER: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value - case METATAG: - outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + case utils.CONSTANT: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } + case utils.METATAG: + outVal, err = cdre.metaHandler(cfgFld.CdrFieldId, cfgFld.Layout) default: return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) } if err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error())) + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error())) return err } fmtOut := outVal if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error())) + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error())) return err } cdre.trailer = append(cdre.trailer, fmtOut) @@ -310,38 +321,38 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error { for idx, cfgFld := range cdre.exportTemplate.ContentFields { var outVal string switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value + case utils.FILLER: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value + case utils.CONSTANT: + for _, rsrFld := range cfgFld.Value { + outVal += rsrFld.ParseValue("") + } case utils.CDRFIELD: - outVal, err = cdre.cdrFieldValue(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout) + outVal, err = cdre.cdrFieldValue(cdr, cfgFld) case DATETIME: - outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout) - case HTTP_POST: + outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld) + case utils.HTTP_POST: var outValByte []byte - if outValByte, err = utils.HttpJsonPost(cfgFld.Value, cdre.httpSkipTlsCheck, cdr); err == nil { + var httpAddr string + for _, rsrFld := range cfgFld.Value { + httpAddr += rsrFld.ParseValue("") + } + if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil { outVal = string(outValByte) if len(outVal) == 0 && cfgFld.Mandatory { - err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Name) + err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag) } } - case COMBIMED: - outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField()) - case CONCATENATED_CDRFIELD: - for _, fld := range strings.Split(cfgFld.Value, ",") { - if fldOut, err := cdre.cdrFieldValue(cdr, cfgFld.Filter, &utils.RSRField{Id: fld}, cfgFld.Layout); err != nil { - break // The error will be reported bellow - } else { - outVal += fldOut - } - } - case METATAG: - if cfgFld.Value == META_MASKDESTINATION { - outVal, err = cdre.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})) + case utils.COMBIMED: + outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld) + case utils.METATAG: + if cfgFld.CdrFieldId == META_MASKDESTINATION { + outVal, err = cdre.metaHandler(cfgFld.CdrFieldId, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})) } else { - outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + outVal, err = cdre.metaHandler(cfgFld.CdrFieldId, cfgFld.Layout) } } if err != nil { @@ -350,7 +361,7 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error { } fmtOut := outVal if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error())) + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Tag, outVal, err.Error())) return err } cdrRow[idx] += fmtOut diff --git a/cdre/cdrexporter_test.go b/cdre/cdrexporter_test.go index ed2c1afef..b147f312c 100644 --- a/cdre/cdrexporter_test.go +++ b/cdre/cdrexporter_test.go @@ -51,20 +51,23 @@ func TestCdreGetCombimedCdrFieldVal(t *testing.T) { Category: "call", Account: "1000", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Usage: time.Duration(10) * time.Second, MediationRunId: "RETAIL1", Cost: 5.01}, } - cdre, err := NewCdrExporter(cdrs, logDb, cfg.CdreDefaultInstance, cfg.CdreDefaultInstance.CdrFormat, cfg.CdreDefaultInstance.FieldSeparator, "firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify) if err != nil { t.Error("Unexpected error received: ", err) } - fltrRule, _ := utils.NewRSRField("~mediation_runid:s/default/RUN_RTL/") - if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], fltrRule, &utils.RSRField{Id: "cost"}); err != nil { + fltrRule, _ := utils.ParseRSRFields("~mediation_runid:s/default/RUN_RTL/", utils.INFIELD_SEP) + val, _ := utils.ParseRSRFields("cost", utils.INFIELD_SEP) + cfgCdrFld, _ := config.NewCfgCdrFieldWithDefaults(false, val, fltrRule, nil, nil, nil, nil, nil, nil, nil, nil) + if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil { t.Error(err) } else if costVal != "1.01" { t.Error("Expecting: 1.01, received: ", costVal) } - fltrRule, _ = utils.NewRSRField("~mediation_runid:s/default/RETAIL1/") - if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], fltrRule, &utils.RSRField{Id: "account"}); err != nil { + fltrRule, _ = utils.ParseRSRFields("~mediation_runid:s/default/RETAIL1/", utils.INFIELD_SEP) + val, _ = utils.ParseRSRFields("account", utils.INFIELD_SEP) + cfgCdrFld, _ = config.NewCfgCdrFieldWithDefaults(false, val, fltrRule, nil, nil, nil, nil, nil, nil, nil, nil) + if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil { t.Error(err) } else if acntVal != "1000" { t.Error("Expecting: 1000, received: ", acntVal) @@ -78,18 +81,24 @@ func TestGetDateTimeFieldVal(t *testing.T) { Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01, ExtraFields: map[string]string{"stop_time": "2014-06-11 19:19:00 +0000 UTC", "fieldextr2": "valextr2"}} - if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, nil, &utils.RSRField{Id: "stop_time"}, "2006-01-02 15:04:05"); err != nil { + val, _ := utils.ParseRSRFields("stop_time", utils.INFIELD_SEP) + layout := "2006-01-02 15:04:05" + cfgCdrFld, _ := config.NewCfgCdrFieldWithDefaults(false, val, nil, nil, nil, nil, nil, &layout, nil, nil, nil) + if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err != nil { t.Error(err) } else if cdrVal != "2014-06-11 19:19:00" { t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal) } // Test filter - fltrRule, _ := utils.NewRSRField("~tenant:s/(.+)/itsyscom.com/") - if _, err := cdreTst.getDateTimeFieldVal(cdrTst, fltrRule, &utils.RSRField{Id: "stop_time"}, "2006-01-02 15:04:05"); err == nil { + fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP) + cfgCdrFld, _ = config.NewCfgCdrFieldWithDefaults(false, val, fltr, nil, nil, nil, nil, &layout, nil, nil, nil) + if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil { t.Error(err) } + val, _ = utils.ParseRSRFields("fieldextr2", utils.INFIELD_SEP) + cfgCdrFld, _ = config.NewCfgCdrFieldWithDefaults(false, val, nil, nil, nil, nil, nil, &layout, nil, nil, nil) // Test time parse error - if _, err := cdreTst.getDateTimeFieldVal(cdrTst, nil, &utils.RSRField{Id: "fieldextr2"}, "2006-01-02 15:04:05"); err == nil { + if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil { t.Error("Should give error here, got none.") } } @@ -100,14 +109,16 @@ func TestCdreCdrFieldValue(t *testing.T) { ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01} - fltrRule, _ := utils.NewRSRField("~tenant:s/(.+)/cgrates.org/") - if val, err := cdre.cdrFieldValue(cdr, fltrRule, &utils.RSRField{Id: "destination"}, ""); err != nil { + val, _ := utils.ParseRSRFields("destination", utils.INFIELD_SEP) + cfgCdrFld, _ := config.NewCfgCdrFieldWithDefaults(false, val, nil, nil, nil, nil, nil, nil, nil, nil, nil) + if val, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err != nil { t.Error(err) } else if val != cdr.Destination { t.Errorf("Expecting: %s, received: %s", cdr.Destination, val) } - fltrRule, _ = utils.NewRSRField("~tenant:s/(.+)/itsyscom.com/") - if _, err := cdre.cdrFieldValue(cdr, fltrRule, &utils.RSRField{Id: "destination"}, ""); err == nil { + fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP) + cfgCdrFld, _ = config.NewCfgCdrFieldWithDefaults(false, val, fltr, nil, nil, nil, nil, nil, nil, nil, nil) + if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil { t.Error("Failed to use filter") } } diff --git a/cdre/fixedwidth_test.go b/cdre/fixedwidth_test.go index 44613217f..028ddc8d2 100644 --- a/cdre/fixedwidth_test.go +++ b/cdre/fixedwidth_test.go @@ -28,50 +28,52 @@ import ( "time" ) -var hdrCfgFlds = []*config.CgrXmlCfgCdrField{ - &config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "10", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3}, - &config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3}, - &config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: "export_id", Width: 5, Strip: "right", Padding: "zeroleft"}, - &config.CgrXmlCfgCdrField{Name: "LastCdr", Type: METATAG, Value: META_LASTCDRATIME, Width: 12, Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "FileCreationfTime", Type: METATAG, Value: "time_now", Width: 12, Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "FileVersion", Type: CONSTANT, Value: "01", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 105}, +var hdrCfgFlds = []*config.XmlCfgCdrField{ + &config.XmlCfgCdrField{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("10"), Width: utils.IntPointer(2)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer("export_id"), Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("LastCdr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME), Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("FileCreationfTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer("time_now"), Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("FileVersion"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("01"), Width: utils.IntPointer(2)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(105)}, } -var contentCfgFlds = []*config.CgrXmlCfgCdrField{ - &config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "20", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "Account", Type: utils.CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Subject", Type: utils.CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "CLI", Type: utils.CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Destination", Type: utils.CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "TOR", Type: CONSTANT, Value: "02", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "SubtypeTOR", Type: CONSTANT, Value: "11", Width: 4, Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "SetupTime", Type: utils.CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "Duration", Type: utils.CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS}, - &config.CgrXmlCfgCdrField{Name: "DataVolume", Type: FILLER, Width: 6}, - &config.CgrXmlCfgCdrField{Name: "TaxCode", Type: CONSTANT, Value: "1", Width: 1}, - &config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: utils.CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "ProductId", Type: utils.CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "NetworkId", Type: CONSTANT, Value: "3", Width: 1}, - &config.CgrXmlCfgCdrField{Name: "CallId", Type: utils.CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8}, - &config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8}, - &config.CgrXmlCfgCdrField{Name: "TerminationCode", Type: CONCATENATED_CDRFIELD, Value: "operator,product", Width: 5, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Cost", Type: utils.CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"}, - &config.CgrXmlCfgCdrField{Name: "DestinationPrivacy", Type: METATAG, Value: META_MASKDESTINATION, Width: 1}, +var contentCfgFlds = []*config.XmlCfgCdrField{ + &config.XmlCfgCdrField{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("20"), Width: utils.IntPointer(2)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Account"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCOUNT), Width: utils.IntPointer(12), Strip: utils.StringPointer("left"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Subject"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SUBJECT), Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("CLI"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("cli"), Width: utils.IntPointer(15), Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Destination"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.DESTINATION), Width: utils.IntPointer(24), Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("TOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("02"), Width: utils.IntPointer(2)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("SubtypeTOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("11"), Width: utils.IntPointer(4), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("SetupTime"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SETUP_TIME), Width: utils.IntPointer(12), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), + Layout: utils.StringPointer("020106150400")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Duration"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.USAGE), Width: utils.IntPointer(6), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), + Layout: utils.StringPointer(utils.SECONDS)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("DataVolume"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(6)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("TaxCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("1"), Width: utils.IntPointer(1)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("OperatorCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("opercode"), Width: utils.IntPointer(2), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("ProductId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("productid"), Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("NetworkId"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("3"), Width: utils.IntPointer(1)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("CallId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCID), Width: utils.IntPointer(16), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("TerminationCode"), Type: utils.StringPointer(CONCATENATED_CDRFIELD), Value: utils.StringPointer("operator,product"), Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Cost"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.COST), Width: utils.IntPointer(9), Padding: utils.StringPointer("zeroleft")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("DestinationPrivacy"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_MASKDESTINATION), Width: utils.IntPointer(1)}, } -var trailerCfgFlds = []*config.CgrXmlCfgCdrField{ - &config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "90", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3}, - &config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3}, - &config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: META_EXPORTID, Width: 5, Strip: "right", Padding: "zeroleft"}, - &config.CgrXmlCfgCdrField{Name: "NumberOfRecords", Type: METATAG, Value: META_NRCDRS, Width: 6, Padding: "zeroleft"}, - &config.CgrXmlCfgCdrField{Name: "CdrsDuration", Type: METATAG, Value: META_DURCDRS, Width: 8, Padding: "zeroleft", Layout: "seconds"}, - &config.CgrXmlCfgCdrField{Name: "FirstCdrTime", Type: METATAG, Value: META_FIRSTCDRATIME, Width: 12, Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "LastCdrTime", Type: METATAG, Value: META_LASTCDRATIME, Width: 12, Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 93}, +var trailerCfgFlds = []*config.XmlCfgCdrField{ + &config.XmlCfgCdrField{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("90"), Width: utils.IntPointer(2)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID), Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("NumberOfRecords"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_NRCDRS), Width: utils.IntPointer(6), Padding: utils.StringPointer("zeroleft")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("CdrsDuration"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_DURCDRS), Width: utils.IntPointer(8), Padding: utils.StringPointer("zeroleft"), Layout: utils.StringPointer(utils.SECONDS)}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("FirstCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_FIRSTCDRATIME), Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("LastCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME), Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")}, + &config.XmlCfgCdrField{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(93)}, } // Write one CDR and test it's results only for content buffer @@ -95,7 +97,11 @@ func TestWriteCdr(t *testing.T) { Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } - cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr}, logDb, exportTpl.AsCdreConfig(), utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify) + cdreCfg, err := config.NewCdreConfigFromXmlCdreCfg(exportTpl) + if err != nil { + t.Error(err) + } + cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr}, logDb, cdreCfg, utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify) if err != nil { t.Error(err) } @@ -171,7 +177,11 @@ func TestWriteCdrs(t *testing.T) { ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"}, } cfg, _ := config.NewDefaultCGRConfig() - cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4}, logDb, exportTpl.AsCdreConfig(), utils.CDRE_FIXED_WIDTH, ',', + cdreCfg, err := config.NewCdreConfigFromXmlCdreCfg(exportTpl) + if err != nil { + t.Error(err) + } + cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4}, logDb, cdreCfg, utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify) if err != nil { t.Error(err) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 0579481e4..9e7627983 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -59,7 +59,6 @@ var ( raterEnabled = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config") schedEnabled = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config") cdrsEnabled = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config") - cdrcEnabled = flag.Bool("cdrc", false, "Enforce starting of the cdrc service overwriting config") mediatorEnabled = flag.Bool("mediator", false, "Enforce starting of the mediator service overwriting config") pidFile = flag.String("pid", "", "Write pid file") bal = balancer2go.NewBalancer() @@ -125,11 +124,11 @@ func startMediator(responder *engine.Responder, loggerDb engine.LogStorage, cdrD } // Fires up a cdrc instance -func startCdrc(cdrsChan chan struct{}, cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId string, runDelay time.Duration, csvSep string, cdrFields map[string][]*utils.RSRField) { - if cdrsAddress == utils.INTERNAL { +func startCdrc(cdrsChan chan struct{}, cdrcCfg *config.CdrcConfig, httpSkipTlsCheck bool, cdrServer *engine.CDRS) { + if cdrcCfg.CdrsAddress == utils.INTERNAL { <-cdrsChan // Wait for CDRServer to come up before start processing } - cdrc, err := cdrc.NewCdrc(cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId, runDelay, csvSep, cdrFields, cdrServer) + cdrc, err := cdrc.NewCdrc(cdrcCfg, httpSkipTlsCheck, cdrServer) if err != nil { engine.Logger.Crit(fmt.Sprintf("Cdrc config parsing error: %s", err.Error())) exitChan <- true @@ -329,9 +328,6 @@ func main() { if *cdrsEnabled { cfg.CDRSEnabled = *cdrsEnabled } - if *cdrcEnabled { - cfg.CdrcEnabled = *cdrcEnabled - } if *mediatorEnabled { cfg.MediatorEnabled = *mediatorEnabled } @@ -491,19 +487,13 @@ func main() { go shutdownSessionmanagerSingnalHandler() } var cdrcEnabled bool - if cfg.CdrcEnabled { // Start default cdrc configured in csv here - cdrcEnabled = true - go startCdrc(cdrsChan, cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, cfg.CdrcCsvSep, cfg.CdrcCdrFields) - } - if cfg.XmlCfgDocument != nil { - for _, xmlCdrc := range cfg.XmlCfgDocument.GetCdrcCfgs("") { - if !xmlCdrc.Enabled { - continue - } - cdrcEnabled = true - go startCdrc(cdrsChan, xmlCdrc.CdrsAddress, xmlCdrc.CdrType, xmlCdrc.CdrInDir, xmlCdrc.CdrOutDir, - xmlCdrc.CdrSourceId, time.Duration(xmlCdrc.RunDelay), xmlCdrc.CsvSeparator, xmlCdrc.CdrRSRFields()) + for _, cdrcConfig := range cfg.CdrcInstances { + if cdrcConfig.Enabled == false { + continue // Ignore not enabled + } else if !cdrcEnabled { + cdrcEnabled = true // Mark that at least one cdrc service is active } + go startCdrc(cdrsChan, cdrcConfig, cfg.HttpSkipTlsVerify, cdrServer) } if cdrcEnabled { engine.Logger.Info("Starting CGRateS CDR client.") diff --git a/config/cdreconfig.go b/config/cdreconfig.go index 2ad092353..e83336aca 100644 --- a/config/cdreconfig.go +++ b/config/cdreconfig.go @@ -19,32 +19,72 @@ along with this program. If not, see package config import ( - "errors" "github.com/cgrates/cgrates/utils" ) -// Converts a list of field identifiers into proper CDR field content -func NewCdreCdrFieldsFromIds(withFixedWith bool, fldsIds ...string) ([]*CdreCdrField, error) { - cdrFields := make([]*CdreCdrField, len(fldsIds)) - for idx, fldId := range fldsIds { - if parsedRsr, err := utils.NewRSRField(fldId); err != nil { - return nil, err - } else { - cdrFld := &CdreCdrField{Name: fldId, Type: utils.CDRFIELD, Value: fldId, valueAsRsrField: parsedRsr} - if err := cdrFld.setDefaultFieldProperties(withFixedWith); err != nil { // Set default fixed width properties to be used later if needed - return nil, err - - } - cdrFields[idx] = cdrFld - } - } - return cdrFields, nil +func NewDefaultCdreConfig() *CdreConfig { + cdreCfg := new(CdreConfig) + cdreCfg.setDefaults() + return cdreCfg } -func NewDefaultCdreConfig() (*CdreConfig, error) { - cdreCfg := new(CdreConfig) - if err := cdreCfg.setDefaults(); err != nil { - return nil, err +func NewCdreConfigFromXmlCdreCfg(xmlCdreCfg *CgrXmlCdreCfg) (*CdreConfig, error) { + var err error + cdreCfg := NewDefaultCdreConfig() + if xmlCdreCfg.CdrFormat != nil { + cdreCfg.CdrFormat = *xmlCdreCfg.CdrFormat + } + if xmlCdreCfg.FieldSeparator != nil && len(*xmlCdreCfg.FieldSeparator) == 1 { + sepStr := *xmlCdreCfg.FieldSeparator + cdreCfg.FieldSeparator = rune(sepStr[0]) + } + if xmlCdreCfg.DataUsageMultiplyFactor != nil { + cdreCfg.DataUsageMultiplyFactor = *xmlCdreCfg.DataUsageMultiplyFactor + } + if xmlCdreCfg.CostMultiplyFactor != nil { + cdreCfg.CostMultiplyFactor = *xmlCdreCfg.CostMultiplyFactor + } + if xmlCdreCfg.CostRoundingDecimals != nil { + cdreCfg.CostRoundingDecimals = *xmlCdreCfg.CostRoundingDecimals + } + if xmlCdreCfg.CostShiftDigits != nil { + cdreCfg.CostShiftDigits = *xmlCdreCfg.CostShiftDigits + } + if xmlCdreCfg.MaskDestId != nil { + cdreCfg.MaskDestId = *xmlCdreCfg.MaskDestId + } + if xmlCdreCfg.MaskLength != nil { + cdreCfg.MaskLength = *xmlCdreCfg.MaskLength + } + if xmlCdreCfg.ExportDir != nil { + cdreCfg.ExportDir = *xmlCdreCfg.ExportDir + } + if xmlCdreCfg.Header != nil { + cdreCfg.HeaderFields = make([]*CfgCdrField, len(xmlCdreCfg.Header.Fields)) + for idx, xmlFld := range xmlCdreCfg.Header.Fields { + cdreCfg.HeaderFields[idx], err = NewCfgCdrFieldFromCgrXmlCfgCdrField(xmlFld, cdreCfg.CdrFormat == utils.CDRE_FIXED_WIDTH) + if err != nil { + return nil, err + } + } + } + if xmlCdreCfg.Content != nil { + cdreCfg.ContentFields = make([]*CfgCdrField, len(xmlCdreCfg.Content.Fields)) + for idx, xmlFld := range xmlCdreCfg.Content.Fields { + cdreCfg.ContentFields[idx], err = NewCfgCdrFieldFromCgrXmlCfgCdrField(xmlFld, cdreCfg.CdrFormat == utils.CDRE_FIXED_WIDTH) + if err != nil { + return nil, err + } + } + } + if xmlCdreCfg.Trailer != nil { + cdreCfg.TrailerFields = make([]*CfgCdrField, len(xmlCdreCfg.Trailer.Fields)) + for idx, xmlFld := range xmlCdreCfg.Trailer.Fields { + cdreCfg.TrailerFields[idx], err = NewCfgCdrFieldFromCgrXmlCfgCdrField(xmlFld, cdreCfg.CdrFormat == utils.CDRE_FIXED_WIDTH) + if err != nil { + return nil, err + } + } } return cdreCfg, nil } @@ -60,9 +100,9 @@ type CdreConfig struct { MaskDestId string MaskLength int ExportDir string - HeaderFields []*CdreCdrField - ContentFields []*CdreCdrField - TrailerFields []*CdreCdrField + HeaderFields []*CfgCdrField + ContentFields []*CfgCdrField + TrailerFields []*CfgCdrField } // Set here defaults @@ -76,7 +116,7 @@ func (cdreCfg *CdreConfig) setDefaults() error { cdreCfg.MaskDestId = "" cdreCfg.MaskLength = 0 cdreCfg.ExportDir = "/var/log/cgrates/cdre" - if flds, err := NewCdreCdrFieldsFromIds(false, utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, + if flds, err := NewCfgCdrFieldsFromIds(false, utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, utils.COST); err != nil { return err } else { @@ -84,161 +124,3 @@ func (cdreCfg *CdreConfig) setDefaults() error { } return nil } - -type CdreCdrField struct { - Name string - Type string - Value string - Width int - Strip string - Padding string - Layout string - Filter *utils.RSRField - Mandatory bool - valueAsRsrField *utils.RSRField // Cached if the need arrises -} - -func (cdrField *CdreCdrField) ValueAsRSRField() *utils.RSRField { - return cdrField.valueAsRsrField -} - -// Should be called on .fwv configuration without providing default values for fixed with parameters -func (cdrField *CdreCdrField) setDefaultFieldProperties(fixedWidth bool) error { - if cdrField.valueAsRsrField == nil { - return errors.New("Missing valueAsRsrField") - } - switch cdrField.valueAsRsrField.Id { - case utils.CGRID: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 40 - } - case utils.ORDERID: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 11 - cdrField.Padding = "left" - } - case utils.TOR: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 6 - cdrField.Padding = "left" - } - case utils.ACCID: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 36 - cdrField.Strip = "left" - cdrField.Padding = "left" - } - case utils.CDRHOST: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 15 - cdrField.Strip = "left" - cdrField.Padding = "left" - } - case utils.CDRSOURCE: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 15 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.REQTYPE: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 13 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.DIRECTION: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 4 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.TENANT: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.CATEGORY: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 10 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.ACCOUNT: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.SUBJECT: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.DESTINATION: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.SETUP_TIME: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "2006-01-02T15:04:05Z07:00" - } - case utils.ANSWER_TIME: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "2006-01-02T15:04:05Z07:00" - } - case utils.USAGE: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.MEDI_RUNID: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 20 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - case utils.COST: - cdrField.Mandatory = true - if fixedWidth { - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - default: - cdrField.Mandatory = false - if fixedWidth { - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - } - } - return nil -} diff --git a/config/cdreconfig_test.go b/config/cdreconfig_test.go index c7edcf7aa..455bdf296 100644 --- a/config/cdreconfig_test.go +++ b/config/cdreconfig_test.go @@ -19,58 +19,42 @@ along with this program. If not, see package config import ( + "fmt" "github.com/cgrates/cgrates/utils" "reflect" "testing" ) -func TestCdreCfgNewCdreCdrFieldsFromIds(t *testing.T) { - expectedFlds := []*CdreCdrField{ - &CdreCdrField{ - Name: utils.CGRID, - Type: utils.CDRFIELD, - Value: utils.CGRID, - Width: 40, - Strip: "", - Padding: "", - Layout: "", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, +func TestNewCfgCdrFieldsFromIds(t *testing.T) { + expectedFlds := []*CfgCdrField{ + &CfgCdrField{ + Tag: utils.CGRID, + Type: utils.CDRFIELD, + CdrFieldId: utils.CGRID, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.CGRID}}, + Width: 40, + Mandatory: true, }, - &CdreCdrField{ - Name: "extra1", - Type: utils.CDRFIELD, - Value: "extra1", - Width: 30, - Strip: "xright", - Padding: "left", - Layout: "", - Mandatory: false, - valueAsRsrField: &utils.RSRField{Id: "extra1"}, + &CfgCdrField{ + Tag: "extra1", + Type: utils.CDRFIELD, + CdrFieldId: "extra1", + Value: []*utils.RSRField{ + &utils.RSRField{Id: "extra1"}}, + Width: 30, + Strip: "xright", + Padding: "left", + Mandatory: false, }, } - if cdreFlds, err := NewCdreCdrFieldsFromIds(true, utils.CGRID, "extra1"); err != nil { + if cdreFlds, err := NewCfgCdrFieldsFromIds(true, utils.CGRID, "extra1"); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedFlds, cdreFlds) { t.Errorf("Expected: %v, received: %v", expectedFlds, cdreFlds) } } -func TestCdreCfgValueAsRSRField(t *testing.T) { - cdreCdrFld := &CdreCdrField{ - Name: utils.CGRID, - Type: utils.CDRFIELD, - Value: utils.CGRID, - Width: 10, - Strip: "xright", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, - } - if rsrVal := cdreCdrFld.ValueAsRSRField(); rsrVal != cdreCdrFld.valueAsRsrField { - t.Error("Unexpected value received: ", rsrVal) - } -} - func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { eCdreCfg := new(CdreConfig) eCdreCfg.CdrFormat = utils.CSV @@ -82,402 +66,133 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { eCdreCfg.MaskDestId = "" eCdreCfg.MaskLength = 0 eCdreCfg.ExportDir = "/var/log/cgrates/cdre" - eCdreCfg.ContentFields = []*CdreCdrField{ - &CdreCdrField{ - Name: utils.CGRID, - Type: utils.CDRFIELD, - Value: utils.CGRID, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + eCdreCfg.ContentFields = []*CfgCdrField{ + &CfgCdrField{ + Tag: utils.CGRID, + Type: utils.CDRFIELD, + CdrFieldId: utils.CGRID, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.CGRID}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.MEDI_RUNID, - Type: utils.CDRFIELD, - Value: utils.MEDI_RUNID, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, + &CfgCdrField{ + Tag: utils.MEDI_RUNID, + Type: utils.CDRFIELD, + CdrFieldId: utils.MEDI_RUNID, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.MEDI_RUNID}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.TOR, - Type: utils.CDRFIELD, - Value: utils.TOR, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.TOR}, + &CfgCdrField{ + Tag: utils.TOR, + Type: utils.CDRFIELD, + CdrFieldId: utils.TOR, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.TOR}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.ACCID, - Type: utils.CDRFIELD, - Value: utils.ACCID, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, + &CfgCdrField{ + Tag: utils.ACCID, + Type: utils.CDRFIELD, + CdrFieldId: utils.ACCID, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.ACCID}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.REQTYPE, - Type: utils.CDRFIELD, - Value: utils.REQTYPE, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, + &CfgCdrField{ + Tag: utils.REQTYPE, + Type: utils.CDRFIELD, + CdrFieldId: utils.REQTYPE, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.REQTYPE}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.DIRECTION, - Type: utils.CDRFIELD, - Value: utils.DIRECTION, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, + &CfgCdrField{ + Tag: utils.DIRECTION, + Type: utils.CDRFIELD, + CdrFieldId: utils.DIRECTION, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.DIRECTION}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.TENANT, - Type: utils.CDRFIELD, - Value: utils.TENANT, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, + &CfgCdrField{ + Tag: utils.TENANT, + Type: utils.CDRFIELD, + CdrFieldId: utils.TENANT, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.TENANT}}, + Mandatory: true}, + &CfgCdrField{ + Tag: utils.CATEGORY, + Type: utils.CDRFIELD, + CdrFieldId: utils.CATEGORY, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.CATEGORY}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.CATEGORY, - Type: utils.CDRFIELD, - Value: utils.CATEGORY, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, + &CfgCdrField{ + Tag: utils.ACCOUNT, + Type: utils.CDRFIELD, + CdrFieldId: utils.ACCOUNT, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.ACCOUNT}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.ACCOUNT, - Type: utils.CDRFIELD, - Value: utils.ACCOUNT, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, + &CfgCdrField{ + Tag: utils.SUBJECT, + Type: utils.CDRFIELD, + CdrFieldId: utils.SUBJECT, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.SUBJECT}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.SUBJECT, - Type: utils.CDRFIELD, - Value: utils.SUBJECT, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, + &CfgCdrField{ + Tag: utils.DESTINATION, + Type: utils.CDRFIELD, + CdrFieldId: utils.DESTINATION, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.DESTINATION}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.DESTINATION, - Type: utils.CDRFIELD, - Value: utils.DESTINATION, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, + &CfgCdrField{ + Tag: utils.SETUP_TIME, + Type: utils.CDRFIELD, + CdrFieldId: utils.SETUP_TIME, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.SETUP_TIME}}, + Layout: "2006-01-02T15:04:05Z07:00", + Mandatory: true, }, - &CdreCdrField{ - Name: utils.SETUP_TIME, - Type: utils.CDRFIELD, - Value: utils.SETUP_TIME, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, + &CfgCdrField{ + Tag: utils.ANSWER_TIME, + Type: utils.CDRFIELD, + CdrFieldId: utils.ANSWER_TIME, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.ANSWER_TIME}}, + Layout: "2006-01-02T15:04:05Z07:00", + Mandatory: true, }, - &CdreCdrField{ - Name: utils.ANSWER_TIME, - Type: utils.CDRFIELD, - Value: utils.ANSWER_TIME, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, + &CfgCdrField{ + Tag: utils.USAGE, + Type: utils.CDRFIELD, + CdrFieldId: utils.USAGE, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.USAGE}}, + Mandatory: true, }, - &CdreCdrField{ - Name: utils.USAGE, - Type: utils.CDRFIELD, - Value: utils.USAGE, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, - }, - &CdreCdrField{ - Name: utils.COST, - Type: utils.CDRFIELD, - Value: utils.COST, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.COST}, + &CfgCdrField{ + Tag: utils.COST, + Type: utils.CDRFIELD, + CdrFieldId: utils.COST, + Value: []*utils.RSRField{ + &utils.RSRField{Id: utils.COST}}, + Mandatory: true, }, } - if cdreCfg, err := NewDefaultCdreConfig(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCfg, cdreCfg) { + if cdreCfg := NewDefaultCdreConfig(); !reflect.DeepEqual(eCdreCfg, cdreCfg) { + for _, fld := range cdreCfg.ContentFields { + fmt.Printf("Have field: %+v\n", fld) + } t.Errorf("Expecting: %v, received: %v", eCdreCfg, cdreCfg) } } - -func TestCdreCfgSetDefaultFieldProperties(t *testing.T) { - cdreCdrFld := &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, - } - eCdreCdrFld := &CdreCdrField{ - Width: 40, - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.ORDERID}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 11, - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ORDERID}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.TOR}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 6, - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.TOR}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 36, - Strip: "left", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.CDRHOST}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 15, - Strip: "left", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CDRHOST}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.CDRSOURCE}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 15, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CDRSOURCE}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 13, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 4, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 24, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 10, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 24, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 24, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 24, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 30, - Strip: "xright", - Padding: "left", - Layout: "2006-01-02T15:04:05Z07:00", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 30, - Strip: "xright", - Padding: "left", - Layout: "2006-01-02T15:04:05Z07:00", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 30, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 20, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: utils.COST}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 24, - Strip: "xright", - Padding: "left", - Mandatory: true, - valueAsRsrField: &utils.RSRField{Id: utils.COST}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } - cdreCdrFld = &CdreCdrField{ - valueAsRsrField: &utils.RSRField{Id: "extra_1"}, - } - eCdreCdrFld = &CdreCdrField{ - Width: 30, - Strip: "xright", - Padding: "left", - Mandatory: false, - valueAsRsrField: &utils.RSRField{Id: "extra_1"}, - } - if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { - t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) - } -} diff --git a/config/config.go b/config/config.go index 197644567..ae7cc9196 100644 --- a/config/config.go +++ b/config/config.go @@ -88,23 +88,15 @@ type CGRConfig struct { RaterBalancer string // balancer address host:port BalancerEnabled bool SchedulerEnabled bool - CDRSEnabled bool // Enable CDR Server service - CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs - CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> - CDRSStats string // Address where to reach the Mediator. <""|intenal> - CDRSStoreDisable bool // When true, CDRs will not longer be saved in stordb, useful for cdrstats only scenario - CDRStatsEnabled bool // Enable CDR Stats service - CDRStatConfig *CdrStatsConfig // Active cdr stats configuration instances - CdreDefaultInstance *CdreConfig // Will be used in the case no specific one selected by API - CdrcEnabled bool // Enable CDR client functionality - CdrcCdrs string // Address where to reach CDR server - CdrcRunDelay time.Duration // Sleep interval between consecutive runs, 0 to use automation via inotify - CdrcCdrType string // CDR file format . - CdrcCsvSep string // Separator used in case of csv files. One character only supported. - CdrcCdrInDir string // Absolute path towards the directory where the CDRs are stored. - CdrcCdrOutDir string // Absolute path towards the directory where processed CDRs will be moved. - CdrcSourceId string // Tag identifying the source of the CDRs within CGRS database. - CdrcCdrFields map[string][]*utils.RSRField // FieldName/RSRField format. Index number in case of .csv cdrs. + CDRSEnabled bool // Enable CDR Server service + CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs + CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> + CDRSStats string // Address where to reach the Mediator. <""|intenal> + CDRSStoreDisable bool // When true, CDRs will not longer be saved in stordb, useful for cdrstats only scenario + CDRStatsEnabled bool // Enable CDR Stats service + CDRStatConfig *CdrStatsConfig // Active cdr stats configuration instances + CdreDefaultInstance *CdreConfig // Will be used in the case no specific one selected by API + CdrcInstances []*CdrcConfig // Number of CDRC instances running imports SMEnabled bool SMSwitchType string SMRater string // address where to access rater. Can be internal, direct rater address or the address of a balancer @@ -185,29 +177,8 @@ func (self *CGRConfig) setDefaults() error { self.CDRSStoreDisable = false self.CDRStatsEnabled = false self.CDRStatConfig = NewCdrStatsConfigWithDefaults() - self.CdreDefaultInstance, _ = NewDefaultCdreConfig() - self.CdrcEnabled = false - self.CdrcCdrs = utils.INTERNAL - self.CdrcRunDelay = time.Duration(0) - self.CdrcCdrType = utils.CSV - self.CdrcCsvSep = string(utils.CSV_SEP) - self.CdrcCdrInDir = "/var/log/cgrates/cdrc/in" - self.CdrcCdrOutDir = "/var/log/cgrates/cdrc/out" - self.CdrcSourceId = "csv" - self.CdrcCdrFields = map[string][]*utils.RSRField{ - utils.TOR: []*utils.RSRField{&utils.RSRField{Id: "2"}}, - utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "3"}}, - utils.REQTYPE: []*utils.RSRField{&utils.RSRField{Id: "4"}}, - utils.DIRECTION: []*utils.RSRField{&utils.RSRField{Id: "5"}}, - utils.TENANT: []*utils.RSRField{&utils.RSRField{Id: "6"}}, - utils.CATEGORY: []*utils.RSRField{&utils.RSRField{Id: "7"}}, - utils.ACCOUNT: []*utils.RSRField{&utils.RSRField{Id: "8"}}, - utils.SUBJECT: []*utils.RSRField{&utils.RSRField{Id: "9"}}, - utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "10"}}, - utils.SETUP_TIME: []*utils.RSRField{&utils.RSRField{Id: "11"}}, - utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "12"}}, - utils.USAGE: []*utils.RSRField{&utils.RSRField{Id: "13"}}, - } + self.CdreDefaultInstance = NewDefaultCdreConfig() + self.CdrcInstances = []*CdrcConfig{NewDefaultCdrcConfig()} // This instance is just for the sake of defaults, it will be replaced when the file is loaded with the one resulted from there self.MediatorEnabled = false self.MediatorRater = utils.INTERNAL self.MediatorReconnects = 3 @@ -249,15 +220,18 @@ func (self *CGRConfig) setDefaults() error { } func (self *CGRConfig) checkConfigSanity() error { - if self.CdrcEnabled { - if len(self.CdrcCdrFields) == 0 { - return errors.New("CdrC enabled but no fields to be processed defined!") - } - if self.CdrcCdrType == utils.CSV { - for _, rsrFldLst := range self.CdrcCdrFields { - for _, rsrFld := range rsrFldLst { - if _, errConv := strconv.Atoi(rsrFld.Id); errConv != nil { - return fmt.Errorf("CDR fields must be indices in case of .csv files, have instead: %s", rsrFld.Id) + // CDRC sanity checks + for _, cdrcInst := range self.CdrcInstances { + if cdrcInst.Enabled == true { + if len(cdrcInst.CdrFields) == 0 { + return errors.New("CdrC enabled but no fields to be processed defined!") + } + if cdrcInst.CdrType == utils.CSV { + for _, cdrFld := range cdrcInst.CdrFields { + for _, rsrFld := range cdrFld.Value { + if _, errConv := strconv.Atoi(rsrFld.Id); errConv != nil { + return fmt.Errorf("CDR fields must be indices in case of .csv files, have instead: %s", rsrFld.Id) + } } } } @@ -478,10 +452,12 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { exportTemplate, _ := c.GetString("cdre", "export_template") if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) { if xmlTemplates := cfg.XmlCfgDocument.GetCdreCfgs(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates != nil { - cfg.CdreDefaultInstance = xmlTemplates[exportTemplate[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig() + if cfg.CdreDefaultInstance, err = NewCdreConfigFromXmlCdreCfg(xmlTemplates[exportTemplate[len(utils.XML_PROFILE_PREFIX):]]); err != nil { + return nil, err + } } } else { // Not loading out of template - if flds, err := NewCdreCdrFieldsFromIds(cfg.CdreDefaultInstance.CdrFormat == utils.CDRE_FIXED_WIDTH, + if flds, err := NewCfgCdrFieldsFromIds(cfg.CdreDefaultInstance.CdrFormat == utils.CDRE_FIXED_WIDTH, strings.Split(exportTemplate, string(utils.CSV_SEP))...); err != nil { return nil, err } else { @@ -492,53 +468,19 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt { cfg.CdreDefaultInstance.ExportDir, _ = c.GetString("cdre", "export_dir") } - if hasOpt = c.HasOption("cdrc", "enabled"); hasOpt { - cfg.CdrcEnabled, _ = c.GetBool("cdrc", "enabled") + // CDRC Default instance parsing + if cdrcFileCfgInst, err := NewCdrcConfigFromFileParams(c); err != nil { + return nil, err + } else { + cfg.CdrcInstances = []*CdrcConfig{cdrcFileCfgInst} } - if hasOpt = c.HasOption("cdrc", "cdrs"); hasOpt { - cfg.CdrcCdrs, _ = c.GetString("cdrc", "cdrs") - } - if hasOpt = c.HasOption("cdrc", "run_delay"); hasOpt { - durStr, _ := c.GetString("cdrc", "run_delay") - if cfg.CdrcRunDelay, err = utils.ParseDurationWithSecs(durStr); err != nil { - return nil, err - } - } - if hasOpt = c.HasOption("cdrc", "cdr_type"); hasOpt { - cfg.CdrcCdrType, _ = c.GetString("cdrc", "cdr_type") - } - if hasOpt = c.HasOption("cdrc", "csv_separator"); hasOpt { - cfg.CdrcCsvSep, _ = c.GetString("cdrc", "csv_separator") - } - if hasOpt = c.HasOption("cdrc", "cdr_in_dir"); hasOpt { - cfg.CdrcCdrInDir, _ = c.GetString("cdrc", "cdr_in_dir") - } - if hasOpt = c.HasOption("cdrc", "cdr_out_dir"); hasOpt { - cfg.CdrcCdrOutDir, _ = c.GetString("cdrc", "cdr_out_dir") - } - if hasOpt = c.HasOption("cdrc", "cdr_source_id"); hasOpt { - cfg.CdrcSourceId, _ = c.GetString("cdrc", "cdr_source_id") - } - // ParseCdrcCdrFields - torIdFld, _ := c.GetString("cdrc", "tor_field") - accIdFld, _ := c.GetString("cdrc", "accid_field") - reqtypeFld, _ := c.GetString("cdrc", "reqtype_field") - directionFld, _ := c.GetString("cdrc", "direction_field") - tenantFld, _ := c.GetString("cdrc", "tenant_field") - categoryFld, _ := c.GetString("cdrc", "category_field") - acntFld, _ := c.GetString("cdrc", "account_field") - subjectFld, _ := c.GetString("cdrc", "subject_field") - destFld, _ := c.GetString("cdrc", "destination_field") - setupTimeFld, _ := c.GetString("cdrc", "setup_time_field") - answerTimeFld, _ := c.GetString("cdrc", "answer_time_field") - durFld, _ := c.GetString("cdrc", "usage_field") - extraFlds, _ := c.GetString("cdrc", "extra_fields") - if len(torIdFld) != 0 || len(accIdFld) != 0 || len(reqtypeFld) != 0 || len(directionFld) != 0 || len(tenantFld) != 0 || len(categoryFld) != 0 || len(acntFld) != 0 || - len(subjectFld) != 0 || len(destFld) != 0 || len(setupTimeFld) != 0 || len(answerTimeFld) != 0 || len(durFld) != 0 || len(extraFlds) != 0 { - // We overwrite the defaults only if at least one of the fields were defined - if cfg.CdrcCdrFields, err = ParseCdrcCdrFields(torIdFld, accIdFld, reqtypeFld, directionFld, tenantFld, categoryFld, acntFld, subjectFld, destFld, - setupTimeFld, answerTimeFld, durFld, extraFlds); err != nil { - return nil, err + if cfg.XmlCfgDocument != nil { // Add the possible configured instances inside xml doc + for id, xmlCdrcCfg := range cfg.XmlCfgDocument.GetCdrcCfgs("") { + if cdrcInst, err := NewCdrcConfigFromCgrXmlCdrcCfg(id, xmlCdrcCfg); err != nil { + return nil, err + } else { + cfg.CdrcInstances = append(cfg.CdrcInstances, cdrcInst) + } } } if hasOpt = c.HasOption("mediator", "enabled"); hasOpt { diff --git a/config/config_test.go b/config/config_test.go index 1d7c5dc30..e7c88484d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -79,7 +79,8 @@ func TestDefaults(t *testing.T) { eCfg.RaterBalancer = "" eCfg.BalancerEnabled = false eCfg.SchedulerEnabled = false - eCfg.CdreDefaultInstance, _ = NewDefaultCdreConfig() + eCfg.CdreDefaultInstance = NewDefaultCdreConfig() + eCfg.CdrcInstances = []*CdrcConfig{NewDefaultCdrcConfig()} eCfg.CDRSEnabled = false eCfg.CDRSExtraFields = []*utils.RSRField{} eCfg.CDRSMediator = "" @@ -87,28 +88,6 @@ func TestDefaults(t *testing.T) { eCfg.CDRSStoreDisable = false eCfg.CDRStatsEnabled = false eCfg.CDRStatConfig = &CdrStatsConfig{Id: utils.META_DEFAULT, QueueLength: 50, TimeWindow: time.Duration(1) * time.Hour, Metrics: []string{"ASR", "ACD", "ACC"}} - eCfg.CdrcEnabled = false - eCfg.CdrcCdrs = utils.INTERNAL - eCfg.CdrcRunDelay = time.Duration(0) - eCfg.CdrcCdrType = "csv" - eCfg.CdrcCsvSep = string(utils.CSV_SEP) - eCfg.CdrcCdrInDir = "/var/log/cgrates/cdrc/in" - eCfg.CdrcCdrOutDir = "/var/log/cgrates/cdrc/out" - eCfg.CdrcSourceId = "csv" - eCfg.CdrcCdrFields = map[string][]*utils.RSRField{ - utils.TOR: []*utils.RSRField{&utils.RSRField{Id: "2"}}, - utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "3"}}, - utils.REQTYPE: []*utils.RSRField{&utils.RSRField{Id: "4"}}, - utils.DIRECTION: []*utils.RSRField{&utils.RSRField{Id: "5"}}, - utils.TENANT: []*utils.RSRField{&utils.RSRField{Id: "6"}}, - utils.CATEGORY: []*utils.RSRField{&utils.RSRField{Id: "7"}}, - utils.ACCOUNT: []*utils.RSRField{&utils.RSRField{Id: "8"}}, - utils.SUBJECT: []*utils.RSRField{&utils.RSRField{Id: "9"}}, - utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "10"}}, - utils.SETUP_TIME: []*utils.RSRField{&utils.RSRField{Id: "11"}}, - utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "12"}}, - utils.USAGE: []*utils.RSRField{&utils.RSRField{Id: "13"}}, - } eCfg.MediatorEnabled = false eCfg.MediatorRater = utils.INTERNAL eCfg.MediatorReconnects = 3 @@ -164,20 +143,6 @@ func TestSanityCheck(t *testing.T) { t.Error("Invalid defaults: ", err) } cfg = &CGRConfig{} - cfg.CdrcEnabled = true - if err := cfg.checkConfigSanity(); err == nil { - t.Error("Failed to detect missing CDR fields definition") - } - cfg.CdrcCdrType = utils.CSV - cfg.CdrcCdrFields = map[string][]*utils.RSRField{utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "test"}}} - if err := cfg.checkConfigSanity(); err == nil { - t.Error("Failed to detect improper use of CDR field names") - } - cfg.CdrcCdrFields = map[string][]*utils.RSRField{"extra1": []*utils.RSRField{&utils.RSRField{Id: "test"}}} - if err := cfg.checkConfigSanity(); err == nil { - t.Error("Failed to detect improper use of CDR field names") - } - cfg = &CGRConfig{} cfg.CDRSStats = utils.INTERNAL if err := cfg.checkConfigSanity(); err == nil { t.Error("Failed detecting improper CDRStats configuration within CDRS") @@ -258,30 +223,32 @@ func TestConfigFromFile(t *testing.T) { MaskDestId: "test", MaskLength: 99, ExportDir: "test"} - eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds(false, "test") - eCfg.CdrcEnabled = true - eCfg.CdrcCdrs = "test" - eCfg.CdrcRunDelay = time.Duration(99) * time.Second - eCfg.CdrcCdrType = "test" - eCfg.CdrcCsvSep = ";" - eCfg.CdrcCdrInDir = "test" - eCfg.CdrcCdrOutDir = "test" - eCfg.CdrcSourceId = "test" - eCfg.CdrcCdrFields = map[string][]*utils.RSRField{ - utils.TOR: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.REQTYPE: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.DIRECTION: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.TENANT: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.CATEGORY: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.ACCOUNT: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.SUBJECT: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.SETUP_TIME: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - utils.USAGE: []*utils.RSRField{&utils.RSRField{Id: "test"}}, - "test": []*utils.RSRField{&utils.RSRField{Id: "test"}}, + eCfg.CdreDefaultInstance.ContentFields, _ = NewCfgCdrFieldsFromIds(false, "test") + cdrcCfg := NewDefaultCdrcConfig() + cdrcCfg.Enabled = true + cdrcCfg.CdrsAddress = "test" + cdrcCfg.RunDelay = time.Duration(99) * time.Second + cdrcCfg.CdrType = "test" + cdrcCfg.FieldSeparator = ";" + cdrcCfg.CdrInDir = "test" + cdrcCfg.CdrOutDir = "test" + cdrcCfg.CdrSourceId = "test" + cdrcCfg.CdrFields = []*CfgCdrField{ + &CfgCdrField{Tag: utils.TOR, Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.ACCID, Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.REQTYPE, Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.DIRECTION, Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.TENANT, Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.CATEGORY, Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.ACCOUNT, Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.SUBJECT, Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.DESTINATION, Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: utils.SETUP_TIME, Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true, Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: utils.ANSWER_TIME, Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true, Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: utils.USAGE, Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}, Mandatory: true}, + &CfgCdrField{Tag: "test", Type: utils.CDRFIELD, CdrFieldId: "test", Value: []*utils.RSRField{&utils.RSRField{Id: "test"}}}, } + eCfg.CdrcInstances = []*CdrcConfig{cdrcCfg} eCfg.MediatorEnabled = true eCfg.MediatorRater = "test" eCfg.MediatorReconnects = 99 @@ -325,6 +292,7 @@ func TestConfigFromFile(t *testing.T) { t.Log(cfg) t.Error("Loading of configuration from file failed!") } + } func TestCdrsExtraFields(t *testing.T) { @@ -359,12 +327,10 @@ func TestCdreExtraFields(t *testing.T) { cdr_format = csv export_template = cgrid,mediation_runid,accid `) - expectedFlds := []*CdreCdrField{ - &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true}, - &CdreCdrField{Name: "mediation_runid", Type: utils.CDRFIELD, Value: "mediation_runid", valueAsRsrField: &utils.RSRField{Id: "mediation_runid"}, - Mandatory: true}, - &CdreCdrField{Name: "accid", Type: utils.CDRFIELD, Value: "accid", valueAsRsrField: &utils.RSRField{Id: "accid"}, - Mandatory: true}, + expectedFlds := []*CfgCdrField{ + &CfgCdrField{Tag: "cgrid", Type: utils.CDRFIELD, CdrFieldId: "cgrid", Value: []*utils.RSRField{&utils.RSRField{Id: "cgrid"}}, Mandatory: true}, + &CfgCdrField{Tag: "mediation_runid", Type: utils.CDRFIELD, CdrFieldId: "mediation_runid", Value: []*utils.RSRField{&utils.RSRField{Id: "mediation_runid"}}, Mandatory: true}, + &CfgCdrField{Tag: "accid", Type: utils.CDRFIELD, CdrFieldId: "accid", Value: []*utils.RSRField{&utils.RSRField{Id: "accid"}}, Mandatory: true}, } expCdreCfg := &CdreConfig{CdrFormat: utils.CSV, FieldSeparator: utils.CSV_SEP, CostRoundingDecimals: -1, ExportDir: "/var/log/cgrates/cdre", ContentFields: expectedFlds} if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { @@ -377,10 +343,9 @@ cdr_format = csv export_template = cgrid,~effective_caller_id_number:s/(\d+)/+$1/ `) rsrField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`) - expectedFlds = []*CdreCdrField{ - &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true}, - &CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`, - valueAsRsrField: rsrField, Mandatory: false}} + expectedFlds = []*CfgCdrField{ + &CfgCdrField{Tag: "cgrid", Type: utils.CDRFIELD, CdrFieldId: "cgrid", Value: []*utils.RSRField{&utils.RSRField{Id: "cgrid"}}, Mandatory: true}, + &CfgCdrField{Tag: "effective_caller_id_number", Type: utils.CDRFIELD, CdrFieldId: "effective_caller_id_number", Value: []*utils.RSRField{rsrField}, Mandatory: false}} expCdreCfg.ContentFields = expectedFlds if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { t.Error("Could not parse the config", err.Error()) @@ -395,15 +360,3 @@ export_template = cgrid,~accid:s/(\d)/$1,runid t.Error("Failed to detect failed RSRParsing") } } - -func TestCdrcCdrDefaultFields(t *testing.T) { - cdrcCfg := []byte(`[cdrc] -enabled = true -`) - cfgDefault, _ := NewDefaultCGRConfig() - if cfg, err := NewCGRConfigFromBytes(cdrcCfg); err != nil { - t.Error("Could not parse the config", err.Error()) - } else if !reflect.DeepEqual(cfg.CdrcCdrFields, cfgDefault.CdrcCdrFields) { - t.Errorf("Unexpected value for CdrcCdrFields: %v", cfg.CdrcCdrFields) - } -} diff --git a/config/helpers_test.go b/config/helpers_test.go index 8f0d462ae..8238bafa0 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -100,22 +100,6 @@ usage_fields = test1, test2 } func TestParseCdrcCdrFields(t *testing.T) { - eFieldsCfg := []byte(`[cdrc] -cdr_type = test -tor_field = tor1 -accid_field = accid1 -reqtype_field = reqtype1 -direction_field = direction1 -tenant_field = tenant1 -category_field = category1 -account_field = account1 -subject_field = subject1 -destination_field = destination1 -setup_time_field = setuptime1 -answer_time_field = answertime1 -usage_field = duration1 -extra_fields = extra1:extraval1,extra2:extraval1 -`) eCdrcCdrFlds := map[string][]*utils.RSRField{ utils.TOR: []*utils.RSRField{&utils.RSRField{Id: "tor1"}}, utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "accid1"}}, @@ -132,9 +116,10 @@ extra_fields = extra1:extraval1,extra2:extraval1 "extra1": []*utils.RSRField{&utils.RSRField{Id: "extraval1"}}, "extra2": []*utils.RSRField{&utils.RSRField{Id: "extraval1"}}, } - if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { + if cdrFlds, err := ParseCdrcCdrFields("tor1", "accid1", "reqtype1", "direction1", "tenant1", "category1", "account1", "subject1", "destination1", + "setuptime1", "answertime1", "duration1", "extra1:extraval1,extra2:extraval1"); err != nil { t.Error("Could not parse the config", err.Error()) - } else if !reflect.DeepEqual(cfg.CdrcCdrFields, eCdrcCdrFlds) { - t.Errorf("Expecting: %v, received: %v, tor: %v", eCdrcCdrFlds, cfg.CdrcCdrFields, cfg.CdrcCdrFields[utils.TOR]) + } else if !reflect.DeepEqual(eCdrcCdrFlds, cdrFlds) { + t.Errorf("Expecting: %v, received: %v, tor: %v", eCdrcCdrFlds, cdrFlds, cdrFlds[utils.TOR]) } } diff --git a/config/xmlcdrc.go b/config/xmlcdrc.go deleted file mode 100644 index 01bdd0896..000000000 --- a/config/xmlcdrc.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Real-time Charging System for Telecom & ISP environments -Copyright (C) 2012-2014 ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package config - -import ( - "encoding/xml" - "github.com/cgrates/cgrates/utils" -) - -type CgrXmlCdrcCfg struct { - Enabled bool `xml:"enabled"` // Enable/Disable the - CdrsAddress string `xml:"cdrs_address"` // The address where CDRs can be reached - CdrType string `xml:"cdr_type"` // The type of CDR to process - CsvSeparator string `xml:"field_separator"` // The separator to use when reading csvs - RunDelay int64 `xml:"run_delay"` // Delay between runs - CdrInDir string `xml:"cdr_in_dir"` // Folder to process CDRs from - CdrOutDir string `xml:"cdr_out_dir"` // Folder to move processed CDRs to - CdrSourceId string `xml:"cdr_source_id"` // Source identifier for the processed CDRs - CdrFields []*CdrcField `xml:"fields>field"` -} - -// Set the defaults -func (cdrcCfg *CgrXmlCdrcCfg) setDefaults() error { - dfCfg, _ := NewDefaultCGRConfig() - if len(cdrcCfg.CdrsAddress) == 0 { - cdrcCfg.CdrsAddress = dfCfg.CdrcCdrs - } - if len(cdrcCfg.CdrType) == 0 { - cdrcCfg.CdrType = dfCfg.CdrcCdrType - } - if len(cdrcCfg.CsvSeparator) == 0 { - cdrcCfg.CsvSeparator = dfCfg.CdrcCsvSep - } - if len(cdrcCfg.CdrInDir) == 0 { - cdrcCfg.CdrInDir = dfCfg.CdrcCdrInDir - } - if len(cdrcCfg.CdrOutDir) == 0 { - cdrcCfg.CdrOutDir = dfCfg.CdrcCdrOutDir - } - if len(cdrcCfg.CdrSourceId) == 0 { - cdrcCfg.CdrSourceId = dfCfg.CdrcSourceId - } - if len(cdrcCfg.CdrFields) == 0 { - for key, cfgRsrFields := range dfCfg.CdrcCdrFields { - cdrcCfg.CdrFields = append(cdrcCfg.CdrFields, &CdrcField{Id: key, Value: "PLACEHOLDER", rsrFields: cfgRsrFields}) - } - } - return nil -} - -func (cdrcCfg *CgrXmlCdrcCfg) CdrRSRFields() map[string][]*utils.RSRField { - rsrFields := make(map[string][]*utils.RSRField) - for _, fld := range cdrcCfg.CdrFields { - rsrFields[fld.Id] = fld.rsrFields - } - return rsrFields -} - -type CdrcField struct { - XMLName xml.Name `xml:"field"` - Id string `xml:"id,attr"` - Value string `xml:"value,attr"` - rsrFields []*utils.RSRField -} - -func (cdrcFld *CdrcField) PopulateRSRFields() (err error) { - if cdrcFld.rsrFields, err = utils.ParseRSRFields(cdrcFld.Value, utils.INFIELD_SEP); err != nil { - return err - } - return nil -} diff --git a/config/xmlcdrc_test.go b/config/xmlcdrc_test.go index c7f3e8d53..901f049b9 100644 --- a/config/xmlcdrc_test.go +++ b/config/xmlcdrc_test.go @@ -28,21 +28,7 @@ import ( var cfgDocCdrc *CgrXmlCfgDocument // Will be populated by first test -func TestPopulateRSRFIeld(t *testing.T) { - cdrcField := CdrcField{Id: "TEST1", Value: `~effective_caller_id_number:s/(\d+)/+$1/`} - if err := cdrcField.PopulateRSRFields(); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if cdrcField.rsrFields == nil { - t.Error("Failed loading the RSRField") - } - cdrcField = CdrcField{Id: "TEST2", Value: `99`} - if err := cdrcField.PopulateRSRFields(); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if cdrcField.rsrFields == nil { - t.Error("Failed loading the RSRField") - } -} - +/* func TestSetDefaults(t *testing.T) { cfgXmlStr := ` @@ -73,33 +59,34 @@ func TestSetDefaults(t *testing.T) { t.Error("Failed loading default configuration") } } +*/ func TestParseXmlCdrcConfig(t *testing.T) { cfgXmlStr := ` - + true internal csv - , + , 0 /var/log/cgrates/cdrc/in /var/log/cgrates/cdrc/out freeswitch_csv - - - - - - - - - - - - - + + + + + + + + + + + + + ` @@ -120,52 +107,35 @@ func TestGetCdrcCfgs(t *testing.T) { if cdrcfgs == nil { t.Error("No config instance returned") } - expectCdrc := &CgrXmlCdrcCfg{Enabled: true, CdrsAddress: "internal", CdrType: "csv", CsvSeparator: ",", - RunDelay: 0, CdrInDir: "/var/log/cgrates/cdrc/in", CdrOutDir: "/var/log/cgrates/cdrc/out", CdrSourceId: "freeswitch_csv"} - cdrFlds := []*CdrcField{ - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.ACCID, Value: "0;13"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.REQTYPE, Value: "1"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.DIRECTION, Value: "2"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.TENANT, Value: "3"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.CATEGORY, Value: "4"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.ACCOUNT, Value: "5"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.SUBJECT, Value: "6"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.DESTINATION, Value: "7"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.SETUP_TIME, Value: "8"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.ANSWER_TIME, Value: "9"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: utils.USAGE, Value: "10"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: "extr1", Value: "11"}, - &CdrcField{XMLName: xml.Name{Local: "field"}, Id: "extr2", Value: "12"}} - for _, fld := range cdrFlds { - fld.PopulateRSRFields() - } + enabled := true + cdrsAddr := "internal" + cdrType := "csv" + fldSep := "," + runDelay := int64(0) + cdrInDir := "/var/log/cgrates/cdrc/in" + cdrOutDir := "/var/log/cgrates/cdrc/out" + cdrSrcId := "freeswitch_csv" + expectCdrc := &CgrXmlCdrcCfg{Enabled: &enabled, CdrsAddress: &cdrsAddr, CdrType: &cdrType, FieldSeparator: &fldSep, + RunDelay: &runDelay, CdrInDir: &cdrInDir, CdrOutDir: &cdrOutDir, CdrSourceId: &cdrSrcId} + accIdTag, reqTypeTag, dirTag, tntTag, categTag, acntTag, subjTag, dstTag, sTimeTag, aTimeTag, usageTag, extr1, extr2 := utils.ACCID, + utils.REQTYPE, utils.DIRECTION, utils.TENANT, utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, "extr1", "extr2" + accIdVal, reqVal, dirVal, tntVal, categVal, acntVal, subjVal, dstVal, sTimeVal, aTimeVal, usageVal, extr1Val, extr2Val := "0;13", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + cdrFlds := []*XmlCfgCdrField{ + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &accIdTag, Value: &accIdVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &reqTypeTag, Value: &reqVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &dirTag, Value: &dirVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &tntTag, Value: &tntVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &categTag, Value: &categVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &acntTag, Value: &acntVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &subjTag, Value: &subjVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &dstTag, Value: &dstVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &sTimeTag, Value: &sTimeVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &aTimeTag, Value: &aTimeVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &usageTag, Value: &usageVal}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &extr1, Value: &extr1Val}, + &XmlCfgCdrField{XMLName: xml.Name{Local: "field"}, Tag: &extr2, Value: &extr2Val}} expectCdrc.CdrFields = cdrFlds if !reflect.DeepEqual(expectCdrc, cdrcfgs["CDRC-CSV1"]) { t.Errorf("Expecting: %v, received: %v", expectCdrc, cdrcfgs["CDRC-CSV1"]) } } - -func TestCdrRSRFields(t *testing.T) { - cdrcfgs := cfgDocCdrc.GetCdrcCfgs("CDRC-CSV1") - if cdrcfgs == nil { - t.Error("No config instance returned") - } - eRSRFields := map[string][]*utils.RSRField{ - utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "0"}, &utils.RSRField{Id: "13"}}, - utils.REQTYPE: []*utils.RSRField{&utils.RSRField{Id: "1"}}, - utils.DIRECTION: []*utils.RSRField{&utils.RSRField{Id: "2"}}, - utils.TENANT: []*utils.RSRField{&utils.RSRField{Id: "3"}}, - utils.CATEGORY: []*utils.RSRField{&utils.RSRField{Id: "4"}}, - utils.ACCOUNT: []*utils.RSRField{&utils.RSRField{Id: "5"}}, - utils.SUBJECT: []*utils.RSRField{&utils.RSRField{Id: "6"}}, - utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "7"}}, - utils.SETUP_TIME: []*utils.RSRField{&utils.RSRField{Id: "8"}}, - utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "9"}}, - utils.USAGE: []*utils.RSRField{&utils.RSRField{Id: "10"}}, - "extr1": []*utils.RSRField{&utils.RSRField{Id: "11"}}, - "extr2": []*utils.RSRField{&utils.RSRField{Id: "12"}}, - } - if rsrFields := cdrcfgs["CDRC-CSV1"].CdrRSRFields(); !reflect.DeepEqual(rsrFields, eRSRFields) { - t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFields) - } -} diff --git a/config/xmlcdre.go b/config/xmlcdre.go deleted file mode 100644 index e775a9e00..000000000 --- a/config/xmlcdre.go +++ /dev/null @@ -1,154 +0,0 @@ -/* -Real-time Charging System for Telecom & ISP environments -Copyright (C) 2012-2014 ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package config - -import ( - "encoding/xml" - "github.com/cgrates/cgrates/utils" -) - -// The CdrExporter configuration instance -type CgrXmlCdreCfg struct { - CdrFormat *string `xml:"cdr_format"` - FieldSeparator *string `xml:"field_separator"` - DataUsageMultiplyFactor *float64 `xml:"data_usage_multiply_factor"` - CostMultiplyFactor *float64 `xml:"cost_multiply_factor"` - CostRoundingDecimals *int `xml:"cost_rounding_decimals"` - CostShiftDigits *int `xml:"cost_shift_digits"` - MaskDestId *string `xml:"mask_destination_id"` - MaskLength *int `xml:"mask_length"` - ExportDir *string `xml:"export_dir"` - Header *CgrXmlCfgCdrHeader `xml:"export_template>header"` - Content *CgrXmlCfgCdrContent `xml:"export_template>content"` - Trailer *CgrXmlCfgCdrTrailer `xml:"export_template>trailer"` -} - -func (xmlCdreCfg *CgrXmlCdreCfg) AsCdreConfig() *CdreConfig { - cdreCfg, _ := NewDefaultCdreConfig() - if xmlCdreCfg.CdrFormat != nil { - cdreCfg.CdrFormat = *xmlCdreCfg.CdrFormat - } - if xmlCdreCfg.FieldSeparator != nil && len(*xmlCdreCfg.FieldSeparator) == 1 { - sepStr := *xmlCdreCfg.FieldSeparator - cdreCfg.FieldSeparator = rune(sepStr[0]) - } - if xmlCdreCfg.DataUsageMultiplyFactor != nil { - cdreCfg.DataUsageMultiplyFactor = *xmlCdreCfg.DataUsageMultiplyFactor - } - if xmlCdreCfg.CostMultiplyFactor != nil { - cdreCfg.CostMultiplyFactor = *xmlCdreCfg.CostMultiplyFactor - } - if xmlCdreCfg.CostRoundingDecimals != nil { - cdreCfg.CostRoundingDecimals = *xmlCdreCfg.CostRoundingDecimals - } - if xmlCdreCfg.CostShiftDigits != nil { - cdreCfg.CostShiftDigits = *xmlCdreCfg.CostShiftDigits - } - if xmlCdreCfg.MaskDestId != nil { - cdreCfg.MaskDestId = *xmlCdreCfg.MaskDestId - } - if xmlCdreCfg.MaskLength != nil { - cdreCfg.MaskLength = *xmlCdreCfg.MaskLength - } - if xmlCdreCfg.ExportDir != nil { - cdreCfg.ExportDir = *xmlCdreCfg.ExportDir - } - if xmlCdreCfg.Header != nil { - cdreCfg.HeaderFields = make([]*CdreCdrField, len(xmlCdreCfg.Header.Fields)) - for idx, xmlFld := range xmlCdreCfg.Header.Fields { - cdreCfg.HeaderFields[idx] = xmlFld.AsCdreCdrField() - } - } - if xmlCdreCfg.Content != nil { - cdreCfg.ContentFields = make([]*CdreCdrField, len(xmlCdreCfg.Content.Fields)) - for idx, xmlFld := range xmlCdreCfg.Content.Fields { - cdreCfg.ContentFields[idx] = xmlFld.AsCdreCdrField() - } - } - if xmlCdreCfg.Trailer != nil { - cdreCfg.TrailerFields = make([]*CdreCdrField, len(xmlCdreCfg.Trailer.Fields)) - for idx, xmlFld := range xmlCdreCfg.Trailer.Fields { - cdreCfg.TrailerFields[idx] = xmlFld.AsCdreCdrField() - } - } - return cdreCfg -} - -// CDR header -type CgrXmlCfgCdrHeader struct { - XMLName xml.Name `xml:"header"` - Fields []*CgrXmlCfgCdrField `xml:"fields>field"` -} - -// CDR content -type CgrXmlCfgCdrContent struct { - XMLName xml.Name `xml:"content"` - Fields []*CgrXmlCfgCdrField `xml:"fields>field"` -} - -// CDR trailer -type CgrXmlCfgCdrTrailer struct { - XMLName xml.Name `xml:"trailer"` - Fields []*CgrXmlCfgCdrField `xml:"fields>field"` -} - -// CDR field -type CgrXmlCfgCdrField struct { - XMLName xml.Name `xml:"field"` - Name string `xml:"name,attr"` - Type string `xml:"type,attr"` - Value string `xml:"value,attr"` - Width int `xml:"width,attr"` // Field width - Strip string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright> - Padding string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right> - Layout string `xml:"layout,attr"` // Eg. time format layout - Filter string `xml:"filter,attr"` // Eg. combimed filters - Mandatory bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported - valueAsRsrField *utils.RSRField // Cached if the need arrises - filterAsRsrField *utils.RSRField -} - -func (cdrFld *CgrXmlCfgCdrField) populateRSRField() (err error) { - cdrFld.valueAsRsrField, err = utils.NewRSRField(cdrFld.Value) - return err -} - -func (cdrFld *CgrXmlCfgCdrField) populateFltrRSRField() (err error) { - cdrFld.filterAsRsrField, err = utils.NewRSRField(cdrFld.Filter) - return err -} - -func (cdrFld *CgrXmlCfgCdrField) ValueAsRSRField() *utils.RSRField { - return cdrFld.valueAsRsrField -} - -func (cdrFld *CgrXmlCfgCdrField) AsCdreCdrField() *CdreCdrField { - return &CdreCdrField{ - Name: cdrFld.Name, - Type: cdrFld.Type, - Value: cdrFld.Value, - Width: cdrFld.Width, - Strip: cdrFld.Strip, - Padding: cdrFld.Padding, - Layout: cdrFld.Layout, - Filter: cdrFld.filterAsRsrField, - Mandatory: cdrFld.Mandatory, - valueAsRsrField: cdrFld.valueAsRsrField, - } -} diff --git a/config/xmlcdre_test.go b/config/xmlcdre_test.go index 8b332be33..59f424155 100644 --- a/config/xmlcdre_test.go +++ b/config/xmlcdre_test.go @@ -19,7 +19,6 @@ along with this program. If not, see package config import ( - "fmt" "github.com/cgrates/cgrates/utils" "reflect" "strings" @@ -28,25 +27,6 @@ import ( var cfgDoc *CgrXmlCfgDocument // Will be populated by first test -func TestXmlCdreCfgPopulateCdreRSRFIeld(t *testing.T) { - cdreField := CgrXmlCfgCdrField{Name: "TEST1", Type: "cdrfield", Value: `~effective_caller_id_number:s/(\d+)/+$1/`} - if err := cdreField.populateRSRField(); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if cdreField.valueAsRsrField == nil { - t.Error("Failed loading the RSRField") - } - valRSRField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`) - if recv := cdreField.ValueAsRSRField(); !reflect.DeepEqual(valRSRField, recv) { - t.Errorf("Expecting %v, received %v", valRSRField, recv) - } - /*cdreField = CgrXmlCfgCdrField{Name: "TEST1", Type: "constant", Value: `someval`} - if err := cdreField.populateRSRField(); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if cdreField.valueAsRsrField != nil { - t.Error("Should not load the RSRField") - }*/ -} - func TestXmlCdreCfgParseXmlConfig(t *testing.T) { cfgXmlStr := ` @@ -62,51 +42,51 @@ func TestXmlCdreCfgParseXmlConfig(t *testing.T) {
- - - - - - - - + + + + + + + +
- - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + @@ -115,12 +95,12 @@ func TestXmlCdreCfgParseXmlConfig(t *testing.T) { - - - - - - + + + + + + @@ -162,7 +142,7 @@ func TestXmlCdreCfgGetCdreCfg(t *testing.T) { } } -func TestXmlCdreCfgAsCdreConfig(t *testing.T) { +func TestNewCdreConfigFromXmlCdreCfg(t *testing.T) { cfgXmlStr := ` @@ -178,23 +158,23 @@ func TestXmlCdreCfgAsCdreConfig(t *testing.T) {
- - + +
- - - - - + + + + + - - + + @@ -222,85 +202,99 @@ func TestXmlCdreCfgAsCdreConfig(t *testing.T) { MaskLength: 1, ExportDir: "/var/log/cgrates/cdre", } - fltrCombiMed, _ := utils.NewRSRField("~mediation_runid:s/DEFAULT/SECOND_RUN/") - eCdreCfg.HeaderFields = []*CdreCdrField{ - &CdreCdrField{ - Name: "TypeOfRecord", - Type: "constant", - Value: "10", - Width: 2, - valueAsRsrField: &utils.RSRField{Id: "10"}}, - &CdreCdrField{ - Name: "LastCdr", - Type: "metatag", - Value: "last_cdr_time", - Layout: "020106150400", - Width: 12, - valueAsRsrField: &utils.RSRField{Id: "last_cdr_time"}}, - } - eCdreCfg.ContentFields = []*CdreCdrField{ - &CdreCdrField{ - Name: "OperatorCode", - Type: "cdrfield", - Value: "operator", - Width: 2, - valueAsRsrField: &utils.RSRField{Id: "operator"}, + fltrCombiMed, _ := utils.ParseRSRFields("~mediation_runid:s/DEFAULT/SECOND_RUN/", utils.INFIELD_SEP) + torVal, _ := utils.ParseRSRFields("^10", utils.INFIELD_SEP) + lastCdrVal, _ := utils.ParseRSRFields("^last_cdr_time", utils.INFIELD_SEP) + eCdreCfg.HeaderFields = []*CfgCdrField{ + &CfgCdrField{ + Tag: "TypeOfRecord", + Type: "constant", + Value: torVal, + Width: 2, }, - &CdreCdrField{ - Name: "ProductId", - Type: "cdrfield", - Value: "productid", - Width: 5, - valueAsRsrField: &utils.RSRField{Id: "productid"}, - }, - &CdreCdrField{ - Name: "NetworkId", - Type: "constant", - Value: "3", - Width: 1, - valueAsRsrField: &utils.RSRField{Id: "3"}, - }, - &CdreCdrField{ - Name: "FromHttpPost1", - Type: "http_post", - Value: "https://localhost:8000", - Width: 10, - Strip: "xright", - Padding: "left", - valueAsRsrField: &utils.RSRField{Id: "https://localhost:8000"}, - }, - &CdreCdrField{ - Name: "CombiMed1", - Type: "combimed", - Value: "cost", - Width: 10, - Strip: "xright", - Padding: "left", - Filter: fltrCombiMed, - valueAsRsrField: &utils.RSRField{Id: "cost"}, + &CfgCdrField{ + Tag: "LastCdr", + Type: "metatag", + CdrFieldId: "last_cdr_time", + Value: lastCdrVal, + Layout: "020106150400", + Strip: "xright", + Padding: "left", + Width: 12, }, } - eCdreCfg.TrailerFields = []*CdreCdrField{ - &CdreCdrField{ - Name: "DistributorCode", - Type: "constant", - Value: "VOI", - Width: 3, - valueAsRsrField: &utils.RSRField{Id: "VOI"}, + networkIdVal, _ := utils.ParseRSRFields("^3", utils.INFIELD_SEP) + fromHttpPost1Val, _ := utils.ParseRSRFields("^https://localhost:8000", utils.INFIELD_SEP) + eCdreCfg.ContentFields = []*CfgCdrField{ + &CfgCdrField{ + Tag: "OperatorCode", + Type: "cdrfield", + CdrFieldId: "operator", + Value: []*utils.RSRField{ + &utils.RSRField{Id: "operator"}}, + Width: 2, + Strip: "xright", + Padding: "left", }, - &CdreCdrField{ - Name: "FileSeqNr", - Type: "metatag", - Value: "export_id", - Width: 5, - Padding: "zeroleft", - valueAsRsrField: &utils.RSRField{Id: "export_id"}, + &CfgCdrField{ + Tag: "ProductId", + Type: "cdrfield", + CdrFieldId: "productid", + Value: []*utils.RSRField{ + &utils.RSRField{Id: "productid"}}, + Width: 5, + Strip: "xright", + Padding: "left", + }, + &CfgCdrField{ + Tag: "NetworkId", + Type: "constant", + Value: networkIdVal, + Width: 1, + }, + &CfgCdrField{ + Tag: "FromHttpPost1", + Type: "http_post", + Value: fromHttpPost1Val, + Width: 10, + Strip: "xright", + Padding: "left", + }, + &CfgCdrField{ + Tag: "CombiMed1", + Type: "combimed", + CdrFieldId: "cost", + Value: []*utils.RSRField{ + &utils.RSRField{Id: "cost"}}, + Width: 10, + Strip: "xright", + Padding: "left", + Filter: fltrCombiMed, + Mandatory: true, }, } - if rcvCdreCfg := xmlCdreCfgs["CDRE-FW2"].AsCdreConfig(); !reflect.DeepEqual(rcvCdreCfg, eCdreCfg) { - for _, fld := range rcvCdreCfg.ContentFields { - fmt.Printf("Fld: %+v\n", fld) - } + distribCodeVal, _ := utils.ParseRSRFields("^VOI", utils.INFIELD_SEP) + fileSeqNrVal, _ := utils.ParseRSRFields("^export_id", utils.INFIELD_SEP) + eCdreCfg.TrailerFields = []*CfgCdrField{ + &CfgCdrField{ + Tag: "DistributorCode", + Type: "constant", + Value: distribCodeVal, + Width: 3, + }, + &CfgCdrField{ + Tag: "FileSeqNr", + Type: "metatag", + CdrFieldId: "export_id", + Value: fileSeqNrVal, + Width: 5, + Strip: "xright", + Padding: "zeroleft", + }, + } + if rcvCdreCfg, err := NewCdreConfigFromXmlCdreCfg(xmlCdreCfgs["CDRE-FW2"]); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvCdreCfg, eCdreCfg) { t.Errorf("Expecting: %v, received: %v", eCdreCfg, rcvCdreCfg) } } diff --git a/config/xmlconfig.go b/config/xmlconfig.go index 5c2c903cd..e80d65031 100644 --- a/config/xmlconfig.go +++ b/config/xmlconfig.go @@ -38,6 +38,68 @@ func ParseCgrXmlConfig(reader io.Reader) (*CgrXmlCfgDocument, error) { return xmlConfig, nil } +// XML CDR field, used for both cdrc and cdre +type XmlCfgCdrField struct { + XMLName xml.Name `xml:"field"` + Tag *string `xml:"tag,attr"` + Type *string `xml:"type,attr"` + CdrFieldId *string `xml:"cdr_field,attr"` + Value *string `xml:"value,attr"` + Width *int `xml:"width,attr"` // Field width + Strip *string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright> + Padding *string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right> + Layout *string `xml:"layout,attr"` // Eg. time format layout + Filter *string `xml:"filter,attr"` // Eg. combimed filters + Mandatory *bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported +} + +// One CDRC Configuration instance +type CgrXmlCdrcCfg struct { + Enabled *bool `xml:"enabled"` // Enable/Disable the + CdrsAddress *string `xml:"cdrs_address"` // The address where CDRs can be reached + CdrType *string `xml:"cdr_type"` // The type of CDR to process + FieldSeparator *string `xml:"field_separator"` // The separator to use when reading csvs + RunDelay *int64 `xml:"run_delay"` // Delay between runs + CdrInDir *string `xml:"cdr_in_dir"` // Folder to process CDRs from + CdrOutDir *string `xml:"cdr_out_dir"` // Folder to move processed CDRs to + CdrSourceId *string `xml:"cdr_source_id"` // Source identifier for the processed CDRs + CdrFields []*XmlCfgCdrField `xml:"fields>field"` +} + +// The CdrExporter configuration instance +type CgrXmlCdreCfg struct { + CdrFormat *string `xml:"cdr_format"` + FieldSeparator *string `xml:"field_separator"` + DataUsageMultiplyFactor *float64 `xml:"data_usage_multiply_factor"` + CostMultiplyFactor *float64 `xml:"cost_multiply_factor"` + CostRoundingDecimals *int `xml:"cost_rounding_decimals"` + CostShiftDigits *int `xml:"cost_shift_digits"` + MaskDestId *string `xml:"mask_destination_id"` + MaskLength *int `xml:"mask_length"` + ExportDir *string `xml:"export_dir"` + Header *CgrXmlCfgCdrHeader `xml:"export_template>header"` + Content *CgrXmlCfgCdrContent `xml:"export_template>content"` + Trailer *CgrXmlCfgCdrTrailer `xml:"export_template>trailer"` +} + +// CDR header +type CgrXmlCfgCdrHeader struct { + XMLName xml.Name `xml:"header"` + Fields []*XmlCfgCdrField `xml:"fields>field"` +} + +// CDR content +type CgrXmlCfgCdrContent struct { + XMLName xml.Name `xml:"content"` + Fields []*XmlCfgCdrField `xml:"fields>field"` +} + +// CDR trailer +type CgrXmlCfgCdrTrailer struct { + XMLName xml.Name `xml:"trailer"` + Fields []*XmlCfgCdrField `xml:"fields>field"` +} + // Define a format for configuration file, one doc contains more configuration instances, identified by section, type and id type CgrXmlCfgDocument struct { XMLName xml.Name `xml:"document"` @@ -83,13 +145,6 @@ func (xmlCfg *CgrXmlCfgDocument) cacheCdrcCfgs() error { } else if cdrcCfg == nil { return fmt.Errorf("Could not unmarshal config instance: %s", cfgInst.Id) } - // Cache rsr fields - for _, fld := range cdrcCfg.CdrFields { - if err := fld.PopulateRSRFields(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Id, err.Error()) - } - } - cdrcCfg.setDefaults() xmlCfg.cdrcs[cfgInst.Id] = cdrcCfg } return nil @@ -108,39 +163,6 @@ func (xmlCfg *CgrXmlCfgDocument) cacheCdreCfgs() error { } else if cdreCfg == nil { return fmt.Errorf("Could not unmarshal CgrXmlCdreCfg: %s", cfgInst.Id) } - if cdreCfg.Header != nil { - // Cache rsr fields - for _, fld := range cdreCfg.Header.Fields { - if err := fld.populateRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - if err := fld.populateFltrRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - } - } - if cdreCfg.Content != nil { - // Cache rsr fields - for _, fld := range cdreCfg.Content.Fields { - if err := fld.populateRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - if err := fld.populateFltrRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - } - } - if cdreCfg.Trailer != nil { - // Cache rsr fields - for _, fld := range cdreCfg.Trailer.Fields { - if err := fld.populateRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - if err := fld.populateFltrRSRField(); err != nil { - return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) - } - } - } xmlCfg.cdres[cfgInst.Id] = cdreCfg } return nil diff --git a/engine/fscdr_test.go b/engine/fscdr_test.go index b137ecf3f..8496ecb4b 100644 --- a/engine/fscdr_test.go +++ b/engine/fscdr_test.go @@ -74,7 +74,7 @@ func TestSearchExtraFieldLast(t *testing.T) { func TestSearchExtraField(t *testing.T) { fsCdr, _ := NewFSCdr(body) rsrSt1, _ := utils.NewRSRField("^injected_value") - rsrSt2, _ := utils.NewRSRField("^injected_hdr/injected_value/") + rsrSt2, _ := utils.NewRSRField("^injected_hdr::injected_value/") cfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "caller_id_name"}, rsrSt1, rsrSt2} extraFields := fsCdr.getExtraFields() if len(extraFields) != 3 || extraFields["caller_id_name"] != "dan" || diff --git a/general_tests/multiplecdrc_local_test.go b/general_tests/multiplecdrc_local_test.go index fe3a18ac3..b1466ea7f 100644 --- a/general_tests/multiplecdrc_local_test.go +++ b/general_tests/multiplecdrc_local_test.go @@ -96,9 +96,9 @@ func TestCreateCdrDirs(t *testing.T) { if !*testLocal { return } - for _, cdrcDir := range []string{cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, - cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrInDir, cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrOutDir, - cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrInDir, cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrOutDir} { + for _, cdrcDir := range []string{cfg.CdrcInstances[0].CdrInDir, cfg.CdrcInstances[0].CdrOutDir, + *cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrInDir, *cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrOutDir, + *cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrInDir, *cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrOutDir} { if err := os.RemoveAll(cdrcDir); err != nil { t.Fatal("Error removing folder: ", cdrcDir, err) } @@ -151,7 +151,7 @@ dbafe9c8614c785a65aabd116dd3959c3c56f7f7,default,*voice,dsafdsag,rated,*out,cgra if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1), 0644); err != nil { t.Fatal(err.Error) } - if err := os.Rename(tmpFilePath, path.Join(cfg.CdrcCdrInDir, fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join(cfg.CdrcInstances[0].CdrInDir, fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } @@ -169,7 +169,7 @@ func TestHandleCdr2File(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil { t.Fatal(err.Error) } - if err := os.Rename(tmpFilePath, path.Join(cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrInDir, fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join(*cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV2")["CDRC-CSV2"].CdrInDir, fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } @@ -186,7 +186,7 @@ func TestHandleCdr3File(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil { t.Fatal(err.Error) } - if err := os.Rename(tmpFilePath, path.Join(cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrInDir, fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join(*cfg.XmlCfgDocument.GetCdrcCfgs("CDRC-CSV3")["CDRC-CSV3"].CdrInDir, fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index b909b800f..1d71c849a 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -91,6 +91,7 @@ func TestTutLclStartEngine(t *testing.T) { t.Fatal("Cannot find cgr-engine executable") } exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it + time.Sleep(time.Duration(*waitRater) * time.Millisecond) engine := exec.Command(enginePath, "-config", tutCfgPath) if err := engine.Start(); err != nil { t.Fatal(err) diff --git a/utils/consts.go b/utils/consts.go index b36d486aa..6b76ad3df 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -76,6 +76,7 @@ const ( FALLBACK_SEP = ';' INFIELD_SEP = ";" FIELDS_SEP = "," + STATIC_HDRVAL_SEP = "::" REGEXP_PREFIX = "~" JSON = "json" GOB = "gob" @@ -105,6 +106,7 @@ const ( STATIC_VALUE_PREFIX = "^" CSV = "csv" CDRE_DRYRUN = "dry_run" + COMBIMED = "combimed" INTERNAL = "internal" ZERO_RATING_SUBJECT_PREFIX = "*zero" OK = "OK" @@ -159,6 +161,10 @@ const ( CREATE_TARIFFPLAN_TABLES_SQL = "create_tariffplan_tables.sql" TEST_SQL = "TEST_SQL" EMPTY = "_empty_" + CONSTANT = "constant" + FILLER = "filler" + METATAG = "metatag" + HTTP_POST = "http_post" ) var ( diff --git a/utils/coreutils.go b/utils/coreutils.go index 64ca6ad23..4b9ce11c6 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -315,3 +315,12 @@ func Unzip(src, dest string) error { return nil } + +// Utilities to provide pointers where we need to define ad-hoc +func StringPointer(str string) *string { + return &str +} + +func IntPointer(i int) *int { + return &i +} diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 4f4fa604b..b3efc19ad 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -30,12 +30,12 @@ func NewRSRField(fldStr string) (*RSRField, error) { } if strings.HasPrefix(fldStr, STATIC_VALUE_PREFIX) { // Special case when RSR is defined as static header/value var staticHdr, staticVal string - if splt := strings.Split(fldStr, "/"); len(splt) == 3 { // Using / as separator since ':' is often use in date/time fields - if len(splt[2]) != 0 { // Last split has created empty element - return nil, fmt.Errorf("Invalid static header/value combination: %s", fldStr) - } + if splt := strings.Split(fldStr, STATIC_HDRVAL_SEP); len(splt) == 2 { // Using / as separator since ':' is often use in date/time fields staticHdr, staticVal = splt[0][1:], splt[1] // Strip the / suffix - } else if len(splt) == 2 { + if strings.HasSuffix(staticVal, "/") { // If value ends with /, strip it since it is a part of the definition syntax + staticVal = staticVal[:len(staticVal)-1] + } + } else if len(splt) > 2 { return nil, fmt.Errorf("Invalid RSRField string: %s", fldStr) } else { staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix @@ -102,13 +102,13 @@ func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a reg } // Parses list of RSRFields, used for example as multiple filters in derived charging -func ParseRSRFields(fldsStr, sep string) ([]*RSRField, error) { +func ParseRSRFields(fldsStr, sep string) (RSRFields, error) { //rsrRlsPattern := regexp.MustCompile(`^(~\w+:s/.+/.*/)|(\^.+(/.+/)?)(;(~\w+:s/.+/.*/)|(\^.+(/.+/)?))*$`) //ToDo:Fix here rule able to confirm the content if len(fldsStr) == 0 { return nil, nil } rulesSplt := strings.Split(fldsStr, sep) - rsrFields := make([]*RSRField, len(rulesSplt)) + rsrFields := make(RSRFields, len(rulesSplt)) for idx, ruleStr := range rulesSplt { if rsrField, err := NewRSRField(ruleStr); err != nil { return nil, err @@ -118,3 +118,5 @@ func ParseRSRFields(fldsStr, sep string) ([]*RSRField, error) { } return rsrFields, nil } + +type RSRFields []*RSRField diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 6b9339aab..f90fb1461 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -89,7 +89,7 @@ func TestConvertPlusNationalAnd00(t *testing.T) { } func TestRSRParseStatic(t *testing.T) { - if rsrField, err := NewRSRField("^static_header/static_value/"); err != nil { + if rsrField, err := NewRSRField("^static_header::static_value/"); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, &RSRField{Id: "static_header", staticValue: "static_value"}) { t.Errorf("Unexpected RSRField received: %v", rsrField) @@ -150,14 +150,14 @@ func TestParseRSRFields(t *testing.T) { rsrFld2, _ := NewRSRField(`~subject:s/^0\d{9}$//`) rsrFld3, _ := NewRSRField(`^destination/+4912345/`) rsrFld4, _ := NewRSRField(`~mediation_runid:s/^default$/default/`) - eRSRFields := []*RSRField{rsrFld1, rsrFld2, rsrFld3, rsrFld4} + eRSRFields := RSRFields{rsrFld1, rsrFld2, rsrFld3, rsrFld4} if rsrFlds, err := ParseRSRFields(fieldsStr1, INFIELD_SEP); err != nil { t.Error("Unexpected error: ", err) } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) } fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` - expectParsedFields := []*RSRField{ + expectParsedFields := RSRFields{ &RSRField{Id: "host"}, &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index d8f022747..8bda93ea1 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -112,7 +112,7 @@ func TestPassesFieldFilter(t *testing.T) { if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Passing filter") } - torFltr, _ := NewRSRField(`^tor/*voice/`) + torFltr, _ := NewRSRField(`^tor::*voice/`) if pass, _ := cdr.PassesFieldFilter(torFltr); !pass { t.Error("Not passing filter") }