diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index b55f5897a..281fda50a 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -55,9 +55,9 @@ Common parameters within configs processed: Parameters specific per config instance: * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields */ -func NewCdrc(cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, cdrs rpcclient.RpcClientConnection, +func NewCdrc(cdrcCfgs []*config.CdrcCfg, httpSkipTlsCheck bool, cdrs rpcclient.RpcClientConnection, closeChan chan struct{}, dfltTimezone string, roundDecimals int, filterS *engine.FilterS) (*Cdrc, error) { - var cdrcCfg *config.CdrcConfig + var cdrcCfg *config.CdrcCfg for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break } @@ -91,8 +91,8 @@ func NewCdrc(cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, cdrs rpcclien type Cdrc struct { httpSkipTlsCheck bool - cdrcCfgs []*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name) - dfltCdrcCfg *config.CdrcConfig + cdrcCfgs []*config.CdrcCfg // All cdrc config profiles attached to this CDRC (key will be profile instance name) + dfltCdrcCfg *config.CdrcCfg timezone string cdrs rpcclient.RpcClientConnection httpClient *http.Client diff --git a/cdrc/csv.go b/cdrc/csv.go index 351d91c76..37ffd0067 100644 --- a/cdrc/csv.go +++ b/cdrc/csv.go @@ -32,7 +32,7 @@ import ( ) func NewCsvRecordsProcessor(csvReader *csv.Reader, timezone, fileName string, - dfltCdrcCfg *config.CdrcConfig, cdrcCfgs []*config.CdrcConfig, + dfltCdrcCfg *config.CdrcCfg, cdrcCfgs []*config.CdrcCfg, httpSkipTlsCheck bool, unpairedRecordsCache *UnpairedRecordsCache, partialRecordsCache *PartialRecordsCache, cacheDumpFields []*config.FCTemplate, filterS *engine.FilterS) *CsvRecordsProcessor { return &CsvRecordsProcessor{csvReader: csvReader, timezone: timezone, fileName: fileName, @@ -45,8 +45,8 @@ type CsvRecordsProcessor struct { csvReader *csv.Reader timezone string // Timezone for CDRs which are not clearly specifying it fileName string - dfltCdrcCfg *config.CdrcConfig - cdrcCfgs []*config.CdrcConfig + dfltCdrcCfg *config.CdrcCfg + cdrcCfgs []*config.CdrcCfg processedRecordsNr int64 // Number of content records in file httpSkipTlsCheck bool unpairedRecordsCache *UnpairedRecordsCache // Shared by cdrc so we can cache for all files in a folder @@ -135,7 +135,7 @@ func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.CDR, } // Takes the record out of csv and turns it into storedCdr which can be processed by CDRS -func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *config.CdrcConfig, tenant string) (*engine.CDR, error) { +func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *config.CdrcCfg, tenant string) (*engine.CDR, error) { storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var err error csvProvider := newCsvProvider(record) // used for filterS and for RSRParsers diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go index 0077fb05f..1508455bd 100644 --- a/cdrc/csv_it_test.go +++ b/cdrc/csv_it_test.go @@ -49,8 +49,8 @@ README: var csvCfgPath string var csvCfg *config.CGRConfig -var cdrcCfgs []*config.CdrcConfig -var cdrcCfg *config.CdrcConfig +var cdrcCfgs []*config.CdrcCfg +var cdrcCfg *config.CdrcCfg var cdrcRpc *rpc.Client var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") diff --git a/cdrc/csv_test.go b/cdrc/csv_test.go index b450273df..6a7f570e1 100644 --- a/cdrc/csv_test.go +++ b/cdrc/csv_test.go @@ -34,7 +34,7 @@ func TestCsvRecordToCDR(t *testing.T) { cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.FCTemplate{ Tag: utils.RunID, Type: utils.META_COMPOSED, FieldId: utils.RunID, Value: config.NewRSRParsersMustCompile("*default", true)}) - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcConfig{cdrcConfig}} + csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcCfg{cdrcConfig}} cdrRow := []string{"firstField", "secondField"} _, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig, "cgrates.org") if err == nil { @@ -76,12 +76,12 @@ func TestCsvDataMultiplyFactor(t *testing.T) { cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] cdrcConfig.CdrSourceId = "TEST_CDRC" cdrcConfig.ContentFields = []*config.FCTemplate{ - &config.FCTemplate{Tag: "TORField", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TORField", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("~0", true)}, - &config.FCTemplate{Tag: "UsageField", Type: utils.META_COMPOSED, FieldId: utils.Usage, + {Tag: "UsageField", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: config.NewRSRParsersMustCompile("~1", true)}, } - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcConfig{cdrcConfig}} + csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcCfg{cdrcConfig}} csvProcessor.cdrcCfgs[0].DataUsageMultiplyFactor = 0 cdrRow := []string{"*data", "1"} rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig, "cgrates.org") diff --git a/cdrc/flatstore_it_test.go b/cdrc/flatstore_it_test.go index 491bfe626..e6a670ecb 100644 --- a/cdrc/flatstore_it_test.go +++ b/cdrc/flatstore_it_test.go @@ -36,7 +36,7 @@ import ( var flatstoreCfgPath string var flatstoreCfg *config.CGRConfig var flatstoreRpc *rpc.Client -var flatstoreCdrcCfg *config.CdrcConfig +var flatstoreCdrcCfg *config.CdrcCfg var fullSuccessfull = `INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475 BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475 diff --git a/cdrc/fwv.go b/cdrc/fwv.go index a43f6c085..2b06404f5 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -34,8 +34,8 @@ import ( "github.com/cgrates/cgrates/utils" ) -func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, - cdrcCfgs []*config.CdrcConfig, httpClient *http.Client, +func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcCfg, + cdrcCfgs []*config.CdrcCfg, httpClient *http.Client, httpSkipTlsCheck bool, timezone string, filterS *engine.FilterS) *FwvRecordsProcessor { return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone, filterS: filterS} @@ -43,8 +43,8 @@ func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, type FwvRecordsProcessor struct { file *os.File - dfltCfg *config.CdrcConfig // General parameters - cdrcCfgs []*config.CdrcConfig + dfltCfg *config.CdrcCfg // General parameters + cdrcCfgs []*config.CdrcCfg httpClient *http.Client httpSkipTlsCheck bool timezone string @@ -139,7 +139,7 @@ func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { } // Converts a record (header or normal) to CDR -func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *config.CdrcConfig, cfgKey string) (*engine.CDR, error) { +func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *config.CdrcCfg, cfgKey string) (*engine.CDR, error) { var err error var lazyHttpFields []*config.FCTemplate var cfgFields []*config.FCTemplate diff --git a/cdrc/fwv_it_test.go b/cdrc/fwv_it_test.go index 4e7d71fd5..d774d7f53 100644 --- a/cdrc/fwv_it_test.go +++ b/cdrc/fwv_it_test.go @@ -37,7 +37,7 @@ import ( var fwvCfgPath string var fwvCfg *config.CGRConfig var fwvRpc *rpc.Client -var fwvCdrcCfg *config.CdrcConfig +var fwvCdrcCfg *config.CdrcCfg var FW_CDR_FILE1 = `HDR0001DDB ABC Some Connect A.B. DDB-Some-10022-20120711-309.CDR 00030920120711100255 CDR0000010 0 20120708181506000123451234 0040123123120 004 000018009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009 diff --git a/cdrc/xml.go b/cdrc/xml.go index 726ba076e..1569f84fa 100644 --- a/cdrc/xml.go +++ b/cdrc/xml.go @@ -80,7 +80,7 @@ func handlerSubstractUsage(xmlElement *xmlquery.Node, argsTpl config.RSRParsers, } func NewXMLRecordsProcessor(recordsReader io.Reader, cdrPath utils.HierarchyPath, timezone string, - httpSkipTlsCheck bool, cdrcCfgs []*config.CdrcConfig, filterS *engine.FilterS) (*XMLRecordsProcessor, error) { + httpSkipTlsCheck bool, cdrcCfgs []*config.CdrcCfg, filterS *engine.FilterS) (*XMLRecordsProcessor, error) { //create doc doc, err := xmlquery.Parse(recordsReader) if err != nil { @@ -99,7 +99,7 @@ type XMLRecordsProcessor struct { cdrPath utils.HierarchyPath // path towards one CDR element timezone string httpSkipTlsCheck bool - cdrcCfgs []*config.CdrcConfig // individual configs for the folder CDRC is monitoring + cdrcCfgs []*config.CdrcCfg // individual configs for the folder CDRC is monitoring filterS *engine.FilterS } @@ -138,7 +138,7 @@ func (xmlProc *XMLRecordsProcessor) ProcessNextRecord() (cdrs []*engine.CDR, err return cdrs, nil } -func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity *xmlquery.Node, cdrcCfg *config.CdrcConfig, tenant string) (*engine.CDR, error) { +func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity *xmlquery.Node, cdrcCfg *config.CdrcCfg, tenant string) (*engine.CDR, error) { cdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var lazyHttpFields []*config.FCTemplate var err error diff --git a/cdrc/xml_it_test.go b/cdrc/xml_it_test.go index 704dce606..7a7fa4cc3 100644 --- a/cdrc/xml_it_test.go +++ b/cdrc/xml_it_test.go @@ -35,8 +35,8 @@ import ( var xmlCfgPath string var xmlCfg *config.CGRConfig -var cdrcXmlCfgs []*config.CdrcConfig -var cdrcXmlCfg *config.CdrcConfig +var cdrcXmlCfgs []*config.CdrcCfg +var cdrcXmlCfg *config.CdrcCfg var cdrcXmlRPC *rpc.Client var xmlPathIn1, xmlPathOut1 string diff --git a/cdrc/xml_test.go b/cdrc/xml_test.go index 11a04a86a..8fd11f33f 100644 --- a/cdrc/xml_test.go +++ b/cdrc/xml_test.go @@ -207,8 +207,8 @@ func TestXMLHandlerSubstractUsage(t *testing.T) { } func TestXMLRPProcess(t *testing.T) { - cdrcCfgs := []*config.CdrcConfig{ - &config.CdrcConfig{ + cdrcCfgs := []*config.CdrcCfg{ + { ID: "TestXML", Enabled: true, CdrFormat: "xml", @@ -216,25 +216,25 @@ func TestXMLRPProcess(t *testing.T) { CDRPath: utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), CdrSourceId: "TestXML", ContentFields: []*config.FCTemplate{ - &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, - &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.localCallId", true), Mandatory: true}, - &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, - &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true), Mandatory: true}, - &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, - &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userNumber", true), Mandatory: true}, - &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.calledNumber", true), Mandatory: true}, - &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.startTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, + {Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, @@ -257,7 +257,7 @@ func TestXMLRPProcess(t *testing.T) { t.Error(err) } expectedCDRs := []*engine.CDR{ - &engine.CDR{CGRID: "1f045359a0784d15e051d7e41ae30132b139d714", + {CGRID: "1f045359a0784d15e051d7e41ae30132b139d714", OriginHost: "0.0.0.0", Source: "TestXML", OriginID: "25160047719:0", ToR: "*voice", RequestType: "*rated", Tenant: "cgrates.org", Category: "call", Account: "1001", Destination: "+4986517174963", @@ -272,8 +272,8 @@ func TestXMLRPProcess(t *testing.T) { } func TestXMLRPProcessWithNewFilters(t *testing.T) { - cdrcCfgs := []*config.CdrcConfig{ - &config.CdrcConfig{ + cdrcCfgs := []*config.CdrcCfg{ + { ID: "XMLWithFilters", Enabled: true, CdrFormat: "xml", @@ -282,25 +282,25 @@ func TestXMLRPProcessWithNewFilters(t *testing.T) { CdrSourceId: "XMLWithFilters", Filters: []string{"*string:broadWorksCDR.cdrData.headerModule.type:Normal"}, ContentFields: []*config.FCTemplate{ - &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, - &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.localCallId", true), Mandatory: true}, - &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, - &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true), Mandatory: true}, - &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, - &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userNumber", true), Mandatory: true}, - &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.calledNumber", true), Mandatory: true}, - &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.startTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, + {Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, @@ -329,7 +329,7 @@ func TestXMLRPProcessWithNewFilters(t *testing.T) { t.Error(err) } expectedCDRs := []*engine.CDR{ - &engine.CDR{CGRID: "1f045359a0784d15e051d7e41ae30132b139d714", + {CGRID: "1f045359a0784d15e051d7e41ae30132b139d714", OriginHost: "0.0.0.0", Source: "XMLWithFilters", OriginID: "25160047719:0", ToR: "*voice", RequestType: "*rated", Tenant: "cgrates.org", Category: "call", Account: "1001", Destination: "+4986517174963", @@ -531,8 +531,8 @@ func TestXMLElementText3(t *testing.T) { } func TestXMLRPNestingSeparator(t *testing.T) { - cdrcCfgs := []*config.CdrcConfig{ - &config.CdrcConfig{ + cdrcCfgs := []*config.CdrcCfg{ + { ID: "msw_xml", Enabled: true, CdrFormat: "xml", @@ -541,25 +541,25 @@ func TestXMLRPNestingSeparator(t *testing.T) { CdrSourceId: "zw_cfs1", Filters: []string{}, ContentFields: []*config.FCTemplate{ - &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, - &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.SignalingInfo.PChargingVector.icidvalue", true), Mandatory: true}, - &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, - &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: config.NewRSRParsersMustCompile("XX.liquid.tel", true), Mandatory: true}, - &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, - &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.OrigParty.SubscriberAddr", true), Mandatory: true}, - &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.RoutingInfo.DestAddr", true), Mandatory: true}, - &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.RingingTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.ConnectTime", true), Mandatory: true}, - &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, + {Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.ReleaseTime;|;~File.CDRs.Call.ConnectTime", true), Mandatory: true}, @@ -588,7 +588,7 @@ func TestXMLRPNestingSeparator(t *testing.T) { t.Error(err) } expectedCDRs := []*engine.CDR{ - &engine.CDR{CGRID: "0ad7f9554ff8fc5b3a7cebbe7431bbf809bc5144", + {CGRID: "0ad7f9554ff8fc5b3a7cebbe7431bbf809bc5144", OriginHost: "0.0.0.0", Source: "zw_cfs1", OriginID: "46d7974398c2671016afccc3f2c428c7", ToR: "*voice", RequestType: "*rated", Tenant: "XX.liquid.tel", Category: "call", Account: "+27110493421", Destination: "+270843073451", diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 695bdb089..53a63a230 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -92,7 +92,7 @@ func startCdrcs(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConn } // Start CDRCs for _, cdrcCfgs := range cfg.CdrcProfiles { - var enabledCfgs []*config.CdrcConfig + var enabledCfgs []*config.CdrcCfg for _, cdrcCfg := range cdrcCfgs { // Take a random config out since they should be the same if cdrcCfg.Enabled { enabledCfgs = append(enabledCfgs, cdrcCfg) @@ -111,11 +111,11 @@ func startCdrcs(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConn } // Fires up a cdrc instance -func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConnection, cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, +func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConnection, cdrcCfgs []*config.CdrcCfg, httpSkipTlsCheck bool, closeChan chan struct{}, exitChan chan bool, filterSChan chan *engine.FilterS) { filterS := <-filterSChan filterSChan <- filterS - var cdrcCfg *config.CdrcConfig + var cdrcCfg *config.CdrcCfg for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break } diff --git a/config/cdrcconfig.go b/config/cdrccfg.go similarity index 95% rename from config/cdrcconfig.go rename to config/cdrccfg.go index 6a2ddcced..0a43c6cf9 100644 --- a/config/cdrcconfig.go +++ b/config/cdrccfg.go @@ -24,7 +24,7 @@ import ( "github.com/cgrates/cgrates/utils" ) -type CdrcConfig struct { +type CdrcCfg struct { ID string // free-form text identifying this CDRC instance Enabled bool // Enable/Disable the profile DryRun bool // Do not post CDRs to the server @@ -51,7 +51,7 @@ type CdrcConfig struct { CacheDumpFields []*FCTemplate } -func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { +func (self *CdrcCfg) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { if jsnCfg == nil { return nil } @@ -151,11 +151,12 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { return nil } -// Clone itself into a new CdrcConfig -func (self *CdrcConfig) Clone() *CdrcConfig { - clnCdrc := new(CdrcConfig) +// Clone itself into a new CdrcCfg +func (self *CdrcCfg) Clone() *CdrcCfg { + clnCdrc := new(CdrcCfg) clnCdrc.ID = self.ID clnCdrc.Enabled = self.Enabled + clnCdrc.DryRun = self.DryRun clnCdrc.CdrsConns = make([]*HaPoolConfig, len(self.CdrsConns)) for idx, cdrConn := range self.CdrsConns { clonedVal := *cdrConn @@ -173,12 +174,14 @@ func (self *CdrcConfig) Clone() *CdrcConfig { for i, path := range self.CDRPath { clnCdrc.CDRPath[i] = path } + clnCdrc.FailedCallsPrefix = self.FailedCallsPrefix clnCdrc.Filters = make([]string, len(self.Filters)) for i, fltr := range self.Filters { clnCdrc.Filters[i] = fltr } clnCdrc.Tenant = self.Tenant clnCdrc.CdrSourceId = self.CdrSourceId + clnCdrc.ContinueOnSuccess = self.ContinueOnSuccess clnCdrc.PartialRecordCache = self.PartialRecordCache clnCdrc.PartialCacheExpiryAction = self.PartialCacheExpiryAction clnCdrc.HeaderFields = make([]*FCTemplate, len(self.HeaderFields)) diff --git a/config/cdrccfg_test.go b/config/cdrccfg_test.go new file mode 100644 index 000000000..492884733 --- /dev/null +++ b/config/cdrccfg_test.go @@ -0,0 +1,138 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package config + +import ( + "reflect" + "strings" + "testing" + "time" + + "github.com/cgrates/cgrates/utils" +) + +var cdrcCfg = CdrcCfg{ + ID: "*default", + CdrsConns: []*HaPoolConfig{{Address: utils.MetaInternal}}, + CdrFormat: "csv", + FieldSeparator: ',', + MaxOpenFiles: 1024, + DataUsageMultiplyFactor: 1024, + CdrInDir: "/var/spool/cgrates/cdrc/in", + CdrOutDir: "/var/spool/cgrates/cdrc/out", + FailedCallsPrefix: "missed_calls", + CDRPath: []string{""}, + CdrSourceId: "freeswitch_csv", + Filters: []string{}, + Tenant: NewRSRParsersMustCompile("cgrates.org", true), + PartialRecordCache: time.Duration(10 * time.Second), + PartialCacheExpiryAction: "*dump_to_file", + HeaderFields: []*FCTemplate{}, + ContentFields: []*FCTemplate{ + { + Tag: "TOR", + FieldId: "ToR", + Type: "*composed", + Value: NewRSRParsersMustCompile("~2", true), + Mandatory: true, + }, + }, + TrailerFields: []*FCTemplate{}, + CacheDumpFields: []*FCTemplate{ + { + Tag: "CGRID", + Type: "*composed", + Value: NewRSRParsersMustCompile("~CGRID", true), + }, + }, +} + +func TestCdrcCfgloadFromJsonCfg(t *testing.T) { + var cdrccfg, expected CdrcCfg + if err := cdrccfg.loadFromJsonCfg(nil); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(cdrccfg, expected) { + t.Errorf("Expected: %+v ,recived: %+v", expected, cdrccfg) + } + if err := cdrccfg.loadFromJsonCfg(new(CdrcJsonCfg)); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(cdrccfg, expected) { + t.Errorf("Expected: %+v ,recived: %+v", expected, cdrccfg) + } + cfgJSONStr := `{ +"cdrc": [ + { + "id": "*default", // identifier of the CDRC runner + "enabled": false, // enable CDR client functionality + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "cdrs_conns": [ + {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> + ], + "cdr_format": "csv", // CDR file format + "field_separator": ",", // separator used in case of csv files + "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited + "data_usage_multiply_factor": 1024, // conversion factor for data usage + "cdr_in_dir": "/var/spool/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored + "cdr_out_dir": "/var/spool/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved + "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records + "cdr_path": "", // path towards one CDR element in case of XML CDRs + "cdr_source_id": "freeswitch_csv", // free form field, tag identifying the source of the CDRs within CDRS database + "filters" :[], // new filters used in FilterS subsystem + "tenant": "cgrates.org", // default tenant + "continue_on_success": false, // continue to the next template if executed + "partial_record_cache": "10s", // duration to cache partial records when not pairing + "partial_cache_expiry_action": "*dump_to_file", // action taken when cache when records in cache are timed-out <*dump_to_file|*post_cdr> + "header_fields": [], // template of the import header fields + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "~2", "mandatory": true}, + ], + "trailer_fields": [], // template of the import trailer fields + "cache_dump_fields": [ // template used when dumping cached CDR, eg: partial CDRs + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + ], + }, +], +}` + expected = cdrcCfg + if jsnCfg, err := NewCgrJsonCfgFromReader(strings.NewReader(cfgJSONStr)); err != nil { + t.Error(err) + } else if jsnCacheCfg, err := jsnCfg.CdrcJsonCfg(); err != nil { + t.Error(err) + } else if err = cdrccfg.loadFromJsonCfg(jsnCacheCfg[0]); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expected, cdrccfg) { + t.Errorf("Expected: %+v , recived: %+v", utils.ToJSON(expected), utils.ToJSON(cdrccfg)) + } +} + +func TestCdrcCfgClone(t *testing.T) { + clnCdrcCfg := *cdrcCfg.Clone() + if !reflect.DeepEqual(cdrcCfg, clnCdrcCfg) { + t.Errorf("Expected: %+v , recived: %+v", cdrcCfg, clnCdrcCfg) + } + cdrcCfg.ContentFields[0].Tag = "CGRID" + if reflect.DeepEqual(cdrcCfg, clnCdrcCfg) { // MOdifying a field after clone should not affect cloned instance + t.Errorf("Cloned result: %+v", utils.ToJSON(clnCdrcCfg)) + } + clnCdrcCfg.ContentFields[0].FieldId = "destination" + if cdrcCfg.ContentFields[0].FieldId != "ToR" { + t.Error("Unexpected change of FieldId: ", cdrcCfg.ContentFields[0].FieldId) + } +} diff --git a/config/cdrcconfig_test.go b/config/cdrcconfig_test.go deleted file mode 100644 index 77948b3bb..000000000 --- a/config/cdrcconfig_test.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package config diff --git a/config/config.go b/config/config.go index 0bc365efe..22a051f34 100755 --- a/config/config.go +++ b/config/config.go @@ -143,6 +143,7 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { cfg.cdrsCfg = new(CdrsCfg) cfg.cdrStatsCfg = new(CdrStatsCfg) cfg.CdreProfiles = make(map[string]*CdreCfg) + cfg.CdrcProfiles = make(map[string][]*CdrcCfg) cfg.analyzerSCfg = new(AnalyzerSCfg) cfg.sessionSCfg = new(SessionSCfg) @@ -255,10 +256,10 @@ func NewCGRConfigFromFolder(cfgDir string) (*CGRConfig, error) { type CGRConfig struct { MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file - CdreProfiles map[string]*CdreCfg - loaderCfg []*LoaderSCfg // configuration for Loader + CdreProfiles map[string]*CdreCfg // Cdre config profiles + CdrcProfiles map[string][]*CdrcCfg // Number of CDRC instances running imports, format map[dirPath][]{Configs} + loaderCfg []*LoaderSCfg // LoaderS configurations - CdrcProfiles map[string][]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath][]{Configs} sessionSCfg *SessionSCfg fsAgentCfg *FsAgentConfig // FreeSWITCHAgent configuration kamAgentCfg *KamAgentCfg // KamailioAgent Configuration @@ -289,8 +290,8 @@ type CGRConfig struct { MigratorCgrConfig *MigratorCgrCfg // Cache defaults loaded from json and needing clones - dfltCdreProfile *CdreCfg // Default cdreConfig profile - dfltCdrcProfile *CdrcConfig // Default cdrcConfig profile + dfltCdreProfile *CdreCfg // Default cdreConfig profile + dfltCdrcProfile *CdrcCfg // Default cdrcConfig profile generalCfg *GeneralCfg // General config dataDbCfg *DataDbCfg // Database config @@ -962,21 +963,18 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { } if jsnCdrcCfg != nil { - if self.CdrcProfiles == nil { - self.CdrcProfiles = make(map[string][]*CdrcConfig) - } for _, jsnCrc1Cfg := range jsnCdrcCfg { if jsnCrc1Cfg.Id == nil || *jsnCrc1Cfg.Id == "" { - return errors.New("CDRC profile without an id") + return utils.ErrCDRCNoProfileID } if *jsnCrc1Cfg.Id == utils.META_DEFAULT { if self.dfltCdrcProfile == nil { - self.dfltCdrcProfile = new(CdrcConfig) + self.dfltCdrcProfile = new(CdrcCfg) } } indxFound := -1 // Will be different than -1 if an instance with same id will be found pathFound := "" // Will be populated with the path where slice of cfgs was found - var cdrcInstCfg *CdrcConfig + var cdrcInstCfg *CdrcCfg for path := range self.CdrcProfiles { for i := range self.CdrcProfiles[path] { if self.CdrcProfiles[path][i].ID == *jsnCrc1Cfg.Id { @@ -994,10 +992,10 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { return err } if cdrcInstCfg.CdrInDir == "" { - return errors.New("CDRC profile without cdr_in_dir") + return utils.ErrCDRCNoInDir } if _, hasDir := self.CdrcProfiles[cdrcInstCfg.CdrInDir]; !hasDir { - self.CdrcProfiles[cdrcInstCfg.CdrInDir] = make([]*CdrcConfig, 0) + self.CdrcProfiles[cdrcInstCfg.CdrInDir] = make([]*CdrcCfg, 0) } if indxFound != -1 { // Replace previous config so we have inheritance self.CdrcProfiles[pathFound][indxFound] = cdrcInstCfg diff --git a/config/config_test.go b/config/config_test.go index 5cef20a7c..6af525033 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -199,7 +199,7 @@ func TestCgrCfgCDRC(t *testing.T) { ], }` eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcConfig{ + eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ { ID: utils.META_DEFAULT, Enabled: true, @@ -1520,7 +1520,7 @@ func TestCgrMigratorCfgDefault(t *testing.T) { func TestCDRCWithDefault(t *testing.T) { eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcConfig{ + eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ { ID: utils.META_DEFAULT, Enabled: false, diff --git a/config/configcdrc_test.go b/config/configcdrc_test.go index 21b5a52cb..1a6b18fe3 100644 --- a/config/configcdrc_test.go +++ b/config/configcdrc_test.go @@ -31,13 +31,13 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { t.Error(err) } eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles = make(map[string][]*CdrcConfig) + eCgrCfg.CdrcProfiles = make(map[string][]*CdrcCfg) // Default instance first - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcConfig{ - &CdrcConfig{ + eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ + { ID: utils.META_DEFAULT, Enabled: false, - CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, + CdrsConns: []*HaPoolConfig{{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, @@ -54,69 +54,69 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { PartialCacheExpiryAction: utils.MetaDumpToFile, HeaderFields: make([]*FCTemplate, 0), ContentFields: []*FCTemplate{ - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: NewRSRParsersMustCompile("~2", true), Mandatory: true}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: NewRSRParsersMustCompile("~3", true), Mandatory: true}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: NewRSRParsersMustCompile("~4", true), Mandatory: true}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: NewRSRParsersMustCompile("~6", true), Mandatory: true}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: NewRSRParsersMustCompile("~7", true), Mandatory: true}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: NewRSRParsersMustCompile("~8", true), Mandatory: true}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, + {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, Value: NewRSRParsersMustCompile("~9", true), Mandatory: true}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: NewRSRParsersMustCompile("~10", true), Mandatory: true}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: NewRSRParsersMustCompile("~11", true), Mandatory: true}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: NewRSRParsersMustCompile("~12", true), Mandatory: true}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, + {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), CacheDumpFields: []*FCTemplate{ - &FCTemplate{Tag: "CGRID", Type: utils.META_COMPOSED, + {Tag: "CGRID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true)}, - &FCTemplate{Tag: "RunID", Type: utils.META_COMPOSED, + {Tag: "RunID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true)}, - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, + {Tag: "TOR", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true)}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, + {Tag: "OriginID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true)}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, + {Tag: "RequestType", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true)}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, + {Tag: "Tenant", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true)}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, + {Tag: "Category", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true)}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, + {Tag: "Account", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true)}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, + {Tag: "Subject", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true)}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, + {Tag: "Destination", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true)}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, + {Tag: "SetupTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, + {Tag: "Usage", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true)}, - &FCTemplate{Tag: "Cost", Type: utils.META_COMPOSED, + {Tag: "Cost", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true)}, }, }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = []*CdrcConfig{ - &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = []*CdrcCfg{ + { ID: "CDRC-CSV1", Enabled: true, - CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, + CdrsConns: []*HaPoolConfig{{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, @@ -124,6 +124,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { MaxOpenFiles: 1024, CdrInDir: "/tmp/cgrates/cdrc1/in", CdrOutDir: "/tmp/cgrates/cdrc1/out", + FailedCallsPrefix: "missed_calls", CDRPath: utils.HierarchyPath([]string{""}), CdrSourceId: "csv1", Filters: []string{}, @@ -132,69 +133,69 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { PartialCacheExpiryAction: utils.MetaDumpToFile, HeaderFields: make([]*FCTemplate, 0), ContentFields: []*FCTemplate{ - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: NewRSRParsersMustCompile("~2", true), Mandatory: true}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: NewRSRParsersMustCompile("~3", true), Mandatory: true}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: NewRSRParsersMustCompile("~4", true), Mandatory: true}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: NewRSRParsersMustCompile("~6", true), Mandatory: true}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: NewRSRParsersMustCompile("~7", true), Mandatory: true}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: NewRSRParsersMustCompile("~8", true), Mandatory: true}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, + {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, Value: NewRSRParsersMustCompile("~9", true), Mandatory: true}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: NewRSRParsersMustCompile("~10", true), Mandatory: true}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: NewRSRParsersMustCompile("~11", true), Mandatory: true}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: NewRSRParsersMustCompile("~12", true), Mandatory: true}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, + {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), CacheDumpFields: []*FCTemplate{ - &FCTemplate{Tag: "CGRID", Type: utils.META_COMPOSED, + {Tag: "CGRID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true)}, - &FCTemplate{Tag: "RunID", Type: utils.META_COMPOSED, + {Tag: "RunID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true)}, - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, + {Tag: "TOR", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true)}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, + {Tag: "OriginID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true)}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, + {Tag: "RequestType", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true)}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, + {Tag: "Tenant", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true)}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, + {Tag: "Category", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true)}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, + {Tag: "Account", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true)}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, + {Tag: "Subject", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true)}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, + {Tag: "Destination", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true)}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, + {Tag: "SetupTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, + {Tag: "Usage", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true)}, - &FCTemplate{Tag: "Cost", Type: utils.META_COMPOSED, + {Tag: "Cost", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true)}, }, }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = []*CdrcConfig{ - &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = []*CdrcCfg{ + { ID: "CDRC-CSV2", Enabled: true, - CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, + CdrsConns: []*HaPoolConfig{{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 0.000976563, @@ -202,6 +203,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { MaxOpenFiles: 1024, CdrInDir: "/tmp/cgrates/cdrc2/in", CdrOutDir: "/tmp/cgrates/cdrc2/out", + FailedCallsPrefix: "missed_calls", CDRPath: utils.HierarchyPath([]string{""}), CdrSourceId: "csv2", Filters: []string{}, @@ -210,53 +212,53 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { PartialCacheExpiryAction: utils.MetaDumpToFile, HeaderFields: make([]*FCTemplate, 0), ContentFields: []*FCTemplate{ - &FCTemplate{FieldId: utils.ToR, + {FieldId: utils.ToR, Value: NewRSRParsersMustCompile("~7:s/^(voice|data|sms|mms|generic)$/*$1/", true)}, - &FCTemplate{Tag: "", Type: "", FieldId: utils.AnswerTime, + {Tag: "", Type: "", FieldId: utils.AnswerTime, Value: NewRSRParsersMustCompile("~1", true)}, - &FCTemplate{FieldId: utils.Usage, + {FieldId: utils.Usage, Value: NewRSRParsersMustCompile("~9:s/^(\\d+)$/${1}s/", true)}, }, TrailerFields: make([]*FCTemplate, 0), CacheDumpFields: []*FCTemplate{ - &FCTemplate{Tag: "CGRID", Type: utils.META_COMPOSED, + {Tag: "CGRID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true)}, - &FCTemplate{Tag: "RunID", Type: utils.META_COMPOSED, + {Tag: "RunID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true)}, - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, + {Tag: "TOR", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true)}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, + {Tag: "OriginID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true)}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, + {Tag: "RequestType", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true)}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, + {Tag: "Tenant", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true)}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, + {Tag: "Category", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true)}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, + {Tag: "Account", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true)}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, + {Tag: "Subject", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true)}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, + {Tag: "Destination", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true)}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, + {Tag: "SetupTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, + {Tag: "Usage", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true)}, - &FCTemplate{Tag: "Cost", Type: utils.META_COMPOSED, + {Tag: "Cost", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true)}, }, }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = []*CdrcConfig{ - &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = []*CdrcCfg{ + { ID: "CDRC-CSV3", Enabled: true, - CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, + CdrsConns: []*HaPoolConfig{{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, @@ -264,6 +266,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { MaxOpenFiles: 1024, CdrInDir: "/tmp/cgrates/cdrc3/in", CdrOutDir: "/tmp/cgrates/cdrc3/out", + FailedCallsPrefix: "missed_calls", CDRPath: utils.HierarchyPath([]string{""}), CdrSourceId: "csv3", Filters: []string{}, @@ -272,60 +275,60 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { PartialCacheExpiryAction: utils.MetaDumpToFile, HeaderFields: make([]*FCTemplate, 0), ContentFields: []*FCTemplate{ - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, + {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: NewRSRParsersMustCompile("~2", true), Mandatory: true}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, + {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, Value: NewRSRParsersMustCompile("~3", true), Mandatory: true}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, + {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: NewRSRParsersMustCompile("~4", true), Mandatory: true}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, + {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, Value: NewRSRParsersMustCompile("~6", true), Mandatory: true}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, + {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: NewRSRParsersMustCompile("~7", true), Mandatory: true}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, + {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, Value: NewRSRParsersMustCompile("~8", true), Mandatory: true}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, + {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, Value: NewRSRParsersMustCompile("~9", true), Mandatory: true}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, + {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: NewRSRParsersMustCompile("~10", true), Mandatory: true}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, + {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, Value: NewRSRParsersMustCompile("~11", true), Mandatory: true}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, Value: NewRSRParsersMustCompile("~12", true), Mandatory: true}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, + {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: NewRSRParsersMustCompile("~13", true), Mandatory: true}, }, TrailerFields: make([]*FCTemplate, 0), CacheDumpFields: []*FCTemplate{ - &FCTemplate{Tag: "CGRID", Type: utils.META_COMPOSED, + {Tag: "CGRID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true)}, - &FCTemplate{Tag: "RunID", Type: utils.META_COMPOSED, + {Tag: "RunID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true)}, - &FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, + {Tag: "TOR", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true)}, - &FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, + {Tag: "OriginID", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true)}, - &FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, + {Tag: "RequestType", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true)}, - &FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, + {Tag: "Tenant", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true)}, - &FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, + {Tag: "Category", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true)}, - &FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, + {Tag: "Account", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true)}, - &FCTemplate{Tag: "Subject", Type: utils.META_COMPOSED, + {Tag: "Subject", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true)}, - &FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, + {Tag: "Destination", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true)}, - &FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, + {Tag: "SetupTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, + {Tag: "AnswerTime", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true), Layout: "2006-01-02T15:04:05Z07:00"}, - &FCTemplate{Tag: "Usage", Type: utils.META_COMPOSED, + {Tag: "Usage", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true)}, - &FCTemplate{Tag: "Cost", Type: utils.META_COMPOSED, + {Tag: "Cost", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true)}, }, }, diff --git a/utils/errors.go b/utils/errors.go index b97521a54..0fa17f7d5 100644 --- a/utils/errors.go +++ b/utils/errors.go @@ -59,6 +59,8 @@ var ( RalsErrorPrfx = "RALS_ERROR" ErrJsonIncompleteComment = errors.New("JSON_INCOMPLETE_COMMENT") + ErrCDRCNoProfileID = errors.New("CDRC_PROFILE_WITHOUT_ID") + ErrCDRCNoInDir = errors.New("CDRC_PROFILE_WITHOUT_IN_DIR") ) // NewCGRError initialises a new CGRError