From 520451657dbaca170a467bceca5dfafee5aa6c45 Mon Sep 17 00:00:00 2001 From: TeoV Date: Mon, 27 Aug 2018 05:13:50 -0400 Subject: [PATCH] CacheFields as CfgCdrField --- cdrc/csv.go | 4 +- cdrc/fwv.go | 12 +- cdrc/fwv_it_test.go | 3 +- cdrc/partial_cdr.go | 10 +- cdrc/partialcsv_it_test.go | 6 +- config/cdrcconfig.go | 8 +- config/config_defaults.go | 28 +-- config/config_json_test.go | 30 +-- config/config_test.go | 58 ++--- config/configcdrc_test.go | 240 ++++++++++---------- config/libconfig_json.go | 2 +- data/conf/samples/cdrc_partcsv/cgrates.json | 18 +- 12 files changed, 206 insertions(+), 213 deletions(-) diff --git a/cdrc/csv.go b/cdrc/csv.go index 47e97ccf4..a73f550c8 100644 --- a/cdrc/csv.go +++ b/cdrc/csv.go @@ -34,7 +34,7 @@ import ( func NewCsvRecordsProcessor(csvReader *csv.Reader, timezone, fileName string, dfltCdrcCfg *config.CdrcConfig, cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, unpairedRecordsCache *UnpairedRecordsCache, partialRecordsCache *PartialRecordsCache, - cacheDumpFields []*config.FCTemplate, filterS *engine.FilterS) *CsvRecordsProcessor { + cacheDumpFields []*config.CfgCdrField, filterS *engine.FilterS) *CsvRecordsProcessor { return &CsvRecordsProcessor{csvReader: csvReader, timezone: timezone, fileName: fileName, dfltCdrcCfg: dfltCdrcCfg, cdrcCfgs: cdrcCfgs, httpSkipTlsCheck: httpSkipTlsCheck, unpairedRecordsCache: unpairedRecordsCache, partialRecordsCache: partialRecordsCache, partialCacheDumpFields: cacheDumpFields, filterS: filterS} @@ -51,7 +51,7 @@ type CsvRecordsProcessor struct { httpSkipTlsCheck bool unpairedRecordsCache *UnpairedRecordsCache // Shared by cdrc so we can cache for all files in a folder partialRecordsCache *PartialRecordsCache // Cache records which are of type "Partial" - partialCacheDumpFields []*config.FCTemplate + partialCacheDumpFields []*config.CfgCdrField filterS *engine.FilterS } diff --git a/cdrc/fwv.go b/cdrc/fwv.go index fa9e3b0e6..491adf1b3 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -34,9 +34,11 @@ import ( "github.com/cgrates/cgrates/utils" ) -func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, cdrcCfgs []*config.CdrcConfig, httpClient *http.Client, +func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, + cdrcCfgs []*config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool, timezone string, filterS *engine.FilterS) *FwvRecordsProcessor { - return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone, filterS: filterS} + return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, + httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone, filterS: filterS} } type FwvRecordsProcessor struct { @@ -290,15 +292,15 @@ func (fP *fwvProvider) FieldAsInterface(fldPath []string) (data interface{}, err if err != nil { return nil, err } - if startIndex < len(fP.req) { - return "", fmt.Errorf("Invalid start index : %+v", startIndex) + if startIndex > len(fP.req) { + return "", fmt.Errorf("StartIndex : %+v is greater than : %+v", startIndex, len(fP.req)) } finalIndex, err := strconv.Atoi(indexes[1]) if err != nil { return nil, err } if finalIndex > len(fP.req) { - return "", fmt.Errorf("Invalid final index : %+v", finalIndex) + return "", fmt.Errorf("FinalIndex : %+v is greater than : %+v", finalIndex, len(fP.req)) } data = fP.req[startIndex:finalIndex] fP.cache.Set(fldPath, data, false) diff --git a/cdrc/fwv_it_test.go b/cdrc/fwv_it_test.go index 717588101..427ffd195 100644 --- a/cdrc/fwv_it_test.go +++ b/cdrc/fwv_it_test.go @@ -132,7 +132,7 @@ func TestFwvitInitCdrDb(t *testing.T) { func TestFwvitProcessFiles(t *testing.T) { fileName := "test1.fwv" - if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil { + if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0755); err != nil { t.Fatal(err.Error()) } if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CdrInDir, fileName)); err != nil { @@ -147,7 +147,6 @@ func TestFwvitProcessFiles(t *testing.T) { if len(filesOutDir) != 1 { t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir)) } - time.Sleep(time.Duration(1) * time.Second) } func TestFwvitAnalyseCDRs(t *testing.T) { diff --git a/cdrc/partial_cdr.go b/cdrc/partial_cdr.go index 2b8141e6a..9a5aa5aba 100644 --- a/cdrc/partial_cdr.go +++ b/cdrc/partial_cdr.go @@ -70,9 +70,7 @@ func (prc *PartialRecordsCache) dumpPartialRecords(originID string) { csvWriter := csv.NewWriter(fileOut) csvWriter.Comma = prc.csvSep for _, cdr := range prc.partialRecords[originID].cdrs { - //FOR THE MOMENTN USED - // nil instead of prc.partialRecords[originID].cacheDumpFields - expRec, err := cdr.AsExportRecord(nil, prc.httpSkipTlsCheck, nil, prc.roundDecimals) + expRec, err := cdr.AsExportRecord(prc.partialRecords[originID].cacheDumpFields, prc.httpSkipTlsCheck, nil, prc.roundDecimals) if err != nil { return nil, err } @@ -180,15 +178,15 @@ func (prc *PartialRecordsCache) MergePartialCDRRecord(pCDR *PartialCDRRecord) (* return pCDRIf.(*engine.CDR), err } -func NewPartialCDRRecord(cdr *engine.CDR, cacheDumpFlds []*config.FCTemplate) *PartialCDRRecord { +func NewPartialCDRRecord(cdr *engine.CDR, cacheDumpFlds []*config.CfgCdrField) *PartialCDRRecord { return &PartialCDRRecord{cdrs: []*engine.CDR{cdr}, cacheDumpFields: cacheDumpFlds} } // PartialCDRRecord is a record which can be updated later // different from PartialFlatstoreRecordsCache which is incomplete (eg: need to calculate duration out of 2 records) type PartialCDRRecord struct { - cdrs []*engine.CDR // Number of CDRs - cacheDumpFields []*config.FCTemplate // Fields template to use when dumping from cache on disk + cdrs []*engine.CDR // Number of CDRs + cacheDumpFields []*config.CfgCdrField // Fields template to use when dumping from cache on disk } // Part of sort interface diff --git a/cdrc/partialcsv_it_test.go b/cdrc/partialcsv_it_test.go index dca7edbd7..587b37ed2 100644 --- a/cdrc/partialcsv_it_test.go +++ b/cdrc/partialcsv_it_test.go @@ -104,7 +104,7 @@ func TestPartcsvITRpcConn(t *testing.T) { func TestPartcsvITHandleCdr1File(t *testing.T) { fileName := "file1.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent1), 0644); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent1), 0755); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { @@ -116,7 +116,7 @@ func TestPartcsvITHandleCdr1File(t *testing.T) { func TestPartcsvITHandleCdr2File(t *testing.T) { fileName := "file2.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent2), 0644); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent2), 0755); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { @@ -128,7 +128,7 @@ func TestPartcsvITHandleCdr2File(t *testing.T) { func TestPartcsvITHandleCdr3File(t *testing.T) { fileName := "file3.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent3), 0644); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent3), 0755); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn2, fileName)); err != nil { diff --git a/config/cdrcconfig.go b/config/cdrcconfig.go index 59c14d61c..58d6dec2b 100644 --- a/config/cdrcconfig.go +++ b/config/cdrcconfig.go @@ -49,7 +49,7 @@ type CdrcConfig struct { HeaderFields []*FCTemplate ContentFields []*FCTemplate TrailerFields []*FCTemplate - CacheDumpFields []*FCTemplate + CacheDumpFields []*CfgCdrField } func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { @@ -142,7 +142,9 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { self.TrailerFields = FCTemplatesFromFCTemapltesJsonCfg(*jsnCfg.Trailer_fields) } if jsnCfg.Cache_dump_fields != nil { - self.CacheDumpFields = FCTemplatesFromFCTemapltesJsonCfg(*jsnCfg.Cache_dump_fields) + if self.CacheDumpFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Cache_dump_fields); err != nil { + return err + } } return nil } @@ -180,7 +182,7 @@ func (self *CdrcConfig) Clone() *CdrcConfig { clnCdrc.HeaderFields = make([]*FCTemplate, len(self.HeaderFields)) clnCdrc.ContentFields = make([]*FCTemplate, len(self.ContentFields)) clnCdrc.TrailerFields = make([]*FCTemplate, len(self.TrailerFields)) - clnCdrc.CacheDumpFields = make([]*FCTemplate, len(self.CacheDumpFields)) + clnCdrc.CacheDumpFields = make([]*CfgCdrField, len(self.CacheDumpFields)) for idx, fld := range self.HeaderFields { clonedVal := *fld clnCdrc.HeaderFields[idx] = &clonedVal diff --git a/config/config_defaults.go b/config/config_defaults.go index b9d6adab5..3cf32cb8e 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -271,20 +271,20 @@ const CGRATES_CFG_JSON = ` ], "trailer_fields": [], // template of the import trailer fields "cache_dump_fields": [ // template used when dumping cached CDR, eg: partial CDRs - {"id": "CGRID", "type": "*composed", "value": "CGRID"}, - {"id": "RunID", "type": "*composed", "value": "RunID"}, - {"id": "TOR", "type": "*composed", "value": "ToR"}, - {"id": "OriginID", "type": "*composed", "value": "OriginID"}, - {"id": "RequestType", "type": "*composed", "value": "RequestType"}, - {"id": "Tenant", "type": "*composed", "value": "Tenant"}, - {"id": "Category", "type": "*composed", "value": "Category"}, - {"id": "Account", "type": "*composed", "value": "Account"}, - {"id": "Subject", "type": "*composed", "value": "Subject"}, - {"id": "Destination", "type": "*composed", "value": "Destination"}, - {"id": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"id": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"id": "Usage", "type": "*composed", "value": "Usage"}, - {"id": "Cost", "type": "*composed", "value": "Cost"}, + {"tag": "CGRID", "type": "*composed", "value": "CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "RunID"}, + {"tag": "TOR", "type": "*composed", "value": "ToR"}, + {"tag": "OriginID", "type": "*composed", "value": "OriginID"}, + {"tag": "RequestType", "type": "*composed", "value": "RequestType"}, + {"tag": "Tenant", "type": "*composed", "value": "Tenant"}, + {"tag": "Category", "type": "*composed", "value": "Category"}, + {"tag": "Account", "type": "*composed", "value": "Account"}, + {"tag": "Subject", "type": "*composed", "value": "Subject"}, + {"tag": "Destination", "type": "*composed", "value": "Destination"}, + {"tag": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag": "Usage", "type": "*composed", "value": "Usage"}, + {"tag": "Cost", "type": "*composed", "value": "Cost"}, ], }, ], diff --git a/config/config_json_test.go b/config/config_json_test.go index 62858d1d4..ac1a4611f 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -398,49 +398,49 @@ func TestDfCdrcJsonCfg(t *testing.T) { &FcTemplateJsonCfg{Id: utils.StringPointer("Usage"), Field_id: utils.StringPointer(utils.Usage), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~13"), Mandatory: utils.BoolPointer(true)}, } - cacheDumpFields := []*FcTemplateJsonCfg{ - &FcTemplateJsonCfg{Id: utils.StringPointer("CGRID"), + cacheDumpFields := []*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer("CGRID"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.CGRID)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("RunID"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("RunID"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.RunID)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("TOR"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("TOR"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.ToR)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("OriginID"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("OriginID"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.OriginID)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("RequestType"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("RequestType"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.RequestType)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Tenant"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Tenant"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Tenant)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Category"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Category"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Category)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Account"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Account"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Account)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Subject"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Subject)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Destination"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Destination)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("SetupTime"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.SetupTime), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, - &FcTemplateJsonCfg{Id: utils.StringPointer("AnswerTime"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("AnswerTime"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.AnswerTime), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Usage"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Usage"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.Usage)}, - &FcTemplateJsonCfg{Id: utils.StringPointer("Cost"), + &CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer(utils.COST)}, } diff --git a/config/config_test.go b/config/config_test.go index 5276d95ee..936e447c6 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -202,37 +202,37 @@ func TestCgrCfgCDRC(t *testing.T) { Value: NewRSRParsersMustCompile("~9:s/^(\\d+)$/${1}s/", true)}, }, TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - &FCTemplate{ID: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.CGRID, true)}, - &FCTemplate{ID: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RunID, true)}, - &FCTemplate{ID: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.ToR, true)}, - &FCTemplate{ID: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.OriginID, true)}, - &FCTemplate{ID: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RequestType, true)}, - &FCTemplate{ID: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Tenant, true)}, - &FCTemplate{ID: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Category, true)}, - &FCTemplate{ID: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Account, true)}, - &FCTemplate{ID: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Subject, true)}, - &FCTemplate{ID: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Destination, true)}, - &FCTemplate{ID: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.SetupTime, true), + CacheDumpFields: []*CfgCdrField{ + &CfgCdrField{Tag: "CGRID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.CGRID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RunID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RunID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TOR", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.ToR, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "OriginID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.OriginID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RequestType", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RequestType, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Tenant", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Tenant, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Category", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Category, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Account", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Account, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Subject", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Subject, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Destination", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Destination, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SetupTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.SetupTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.AnswerTime, true), + &CfgCdrField{Tag: "AnswerTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.AnswerTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Usage, true)}, - &FCTemplate{ID: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.COST, true)}, + &CfgCdrField{Tag: "Usage", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Usage, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Cost", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.COST, utils.INFIELD_SEP)}, }, }, } diff --git a/config/configcdrc_test.go b/config/configcdrc_test.go index 63883a38d..22ee00822 100644 --- a/config/configcdrc_test.go +++ b/config/configcdrc_test.go @@ -79,37 +79,35 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - &FCTemplate{ID: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.CGRID, true)}, - &FCTemplate{ID: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RunID, true)}, - &FCTemplate{ID: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.ToR, true)}, - &FCTemplate{ID: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.OriginID, true)}, - &FCTemplate{ID: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RequestType, true)}, - &FCTemplate{ID: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Tenant, true)}, - &FCTemplate{ID: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Category, true)}, - &FCTemplate{ID: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Account, true)}, - &FCTemplate{ID: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Subject, true)}, - &FCTemplate{ID: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Destination, true)}, - &FCTemplate{ID: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.SetupTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.AnswerTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Usage, true)}, - &FCTemplate{ID: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.COST, true)}, + CacheDumpFields: []*CfgCdrField{ + &CfgCdrField{Tag: "CGRID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.CGRID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RunID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RunID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TOR", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.ToR, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "OriginID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.OriginID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RequestType", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RequestType, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Tenant", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Tenant, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Category", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Category, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Account", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Account, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Subject", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Subject, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Destination", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Destination, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SetupTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.SetupTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "AnswerTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.AnswerTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "Usage", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Usage, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Cost", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.COST, utils.INFIELD_SEP)}, }, }, } @@ -158,37 +156,35 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - &FCTemplate{ID: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.CGRID, true)}, - &FCTemplate{ID: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RunID, true)}, - &FCTemplate{ID: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.ToR, true)}, - &FCTemplate{ID: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.OriginID, true)}, - &FCTemplate{ID: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RequestType, true)}, - &FCTemplate{ID: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Tenant, true)}, - &FCTemplate{ID: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Category, true)}, - &FCTemplate{ID: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Account, true)}, - &FCTemplate{ID: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Subject, true)}, - &FCTemplate{ID: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Destination, true)}, - &FCTemplate{ID: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.SetupTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.AnswerTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Usage, true)}, - &FCTemplate{ID: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.COST, true)}, + CacheDumpFields: []*CfgCdrField{ + &CfgCdrField{Tag: "CGRID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.CGRID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RunID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RunID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TOR", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.ToR, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "OriginID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.OriginID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RequestType", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RequestType, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Tenant", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Tenant, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Category", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Category, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Account", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Account, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Subject", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Subject, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Destination", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Destination, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SetupTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.SetupTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "AnswerTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.AnswerTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "Usage", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Usage, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Cost", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.COST, utils.INFIELD_SEP)}, }, }, } @@ -221,37 +217,35 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { Value: NewRSRParsersMustCompile("~9:s/^(\\d+)$/${1}s/", true)}, }, TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - &FCTemplate{ID: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.CGRID, true)}, - &FCTemplate{ID: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RunID, true)}, - &FCTemplate{ID: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.ToR, true)}, - &FCTemplate{ID: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.OriginID, true)}, - &FCTemplate{ID: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RequestType, true)}, - &FCTemplate{ID: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Tenant, true)}, - &FCTemplate{ID: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Category, true)}, - &FCTemplate{ID: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Account, true)}, - &FCTemplate{ID: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Subject, true)}, - &FCTemplate{ID: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Destination, true)}, - &FCTemplate{ID: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.SetupTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.AnswerTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Usage, true)}, - &FCTemplate{ID: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.COST, true)}, + CacheDumpFields: []*CfgCdrField{ + &CfgCdrField{Tag: "CGRID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.CGRID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RunID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RunID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TOR", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.ToR, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "OriginID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.OriginID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RequestType", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RequestType, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Tenant", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Tenant, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Category", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Category, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Account", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Account, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Subject", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Subject, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Destination", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Destination, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SetupTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.SetupTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "AnswerTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.AnswerTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "Usage", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Usage, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Cost", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.COST, utils.INFIELD_SEP)}, }, }, } @@ -300,37 +294,35 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - &FCTemplate{ID: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.CGRID, true)}, - &FCTemplate{ID: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RunID, true)}, - &FCTemplate{ID: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.ToR, true)}, - &FCTemplate{ID: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.OriginID, true)}, - &FCTemplate{ID: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.RequestType, true)}, - &FCTemplate{ID: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Tenant, true)}, - &FCTemplate{ID: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Category, true)}, - &FCTemplate{ID: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Account, true)}, - &FCTemplate{ID: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Subject, true)}, - &FCTemplate{ID: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Destination, true)}, - &FCTemplate{ID: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.SetupTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.AnswerTime, true), - Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{ID: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.Usage, true)}, - &FCTemplate{ID: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.COST, true)}, + CacheDumpFields: []*CfgCdrField{ + &CfgCdrField{Tag: "CGRID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.CGRID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RunID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RunID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TOR", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.ToR, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "OriginID", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.OriginID, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "RequestType", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.RequestType, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Tenant", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Tenant, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Category", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Category, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Account", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Account, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Subject", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Subject, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Destination", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Destination, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SetupTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.SetupTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "AnswerTime", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.AnswerTime, utils.INFIELD_SEP), Layout: "2006-01-02T15:04:05Z07:00"}, + &CfgCdrField{Tag: "Usage", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.Usage, utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Cost", Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile(utils.COST, utils.INFIELD_SEP)}, }, }, } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index bc65f7890..648b8329f 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -210,7 +210,7 @@ type CdrcJsonCfg struct { Header_fields *[]*FcTemplateJsonCfg Content_fields *[]*FcTemplateJsonCfg Trailer_fields *[]*FcTemplateJsonCfg - Cache_dump_fields *[]*FcTemplateJsonCfg + Cache_dump_fields *[]*CdrFieldJsonCfg } // SM-Generic config section diff --git a/data/conf/samples/cdrc_partcsv/cgrates.json b/data/conf/samples/cdrc_partcsv/cgrates.json index 4b4335bcd..2f4f3c969 100644 --- a/data/conf/samples/cdrc_partcsv/cgrates.json +++ b/data/conf/samples/cdrc_partcsv/cgrates.json @@ -51,15 +51,15 @@ {"id": "Partial", "field_id": "Partial", "type": "*composed", "value": "true", "field_filter": "10(partial)"}, ], "cache_dump_fields": [ - {"id": "OriginID", "type": "*composed", "value": "OriginID"}, - {"id": "OrderID", "type": "*composed", "value": "OrderID"}, - {"id": "RequestType", "type": "*composed", "value": "RequestType"}, - {"id": "Account", "type": "*composed", "value": "Account"}, - {"id": "Destination", "type": "*composed", "value": "Destination"}, - {"id": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"id": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"id": "Usage", "type": "*composed", "value": "Usage"}, - {"id": "Cost", "type": "*composed", "value": "Cost"}, + {"tag": "OriginID", "type": "*composed", "value": "OriginID"}, + {"tag": "OrderID", "type": "*composed", "value": "OrderID"}, + {"tag": "RequestType", "type": "*composed", "value": "RequestType"}, + {"tag": "Account", "type": "*composed", "value": "Account"}, + {"tag": "Destination", "type": "*composed", "value": "Destination"}, + {"tag": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag": "Usage", "type": "*composed", "value": "Usage"}, + {"tag": "Cost", "type": "*composed", "value": "Cost"}, ], }, {