diff --git a/agents/agentreq.go b/agents/agentreq.go index 48f460cbc..108a38967 100644 --- a/agents/agentreq.go +++ b/agents/agentreq.go @@ -149,17 +149,17 @@ func (aReq *AgentRequest) ParseField( out, err = cfgFld.Value.ParseValue(utils.EmptyString) isString = true case utils.META_COMPOSED: - out, err = cfgFld.Value.ParseDataProvider(aReq) + out, err = cfgFld.Value.ParseDataProvider(aReq, utils.NestingSep) isString = true case utils.META_USAGE_DIFFERENCE: if len(cfgFld.Value) != 2 { return nil, fmt.Errorf("invalid arguments <%s>", utils.ToJSON(cfgFld.Value)) } else { - strVal1, err := cfgFld.Value[0].ParseDataProvider(aReq) + strVal1, err := cfgFld.Value[0].ParseDataProvider(aReq, utils.NestingSep) if err != nil { return "", err } - strVal2, err := cfgFld.Value[1].ParseDataProvider(aReq) + strVal2, err := cfgFld.Value[1].ParseDataProvider(aReq, utils.NestingSep) if err != nil { return "", err } diff --git a/agents/librad.go b/agents/librad.go index f3261b8a4..1dee40fba 100644 --- a/agents/librad.go +++ b/agents/librad.go @@ -44,7 +44,7 @@ func attrVendorFromPath(path string) (attrName, vendorName string) { func radComposedFieldValue(pkt *radigo.Packet, agReq *AgentRequest, outTpl config.RSRParsers) (outVal string) { for _, rsrTpl := range outTpl { - if out, err := rsrTpl.ParseDataProvider(agReq); err != nil { + if out, err := rsrTpl.ParseDataProvider(agReq, utils.NestingSep); err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> %s", utils.RadiusAgent, err.Error())) diff --git a/cdrc/csv.go b/cdrc/csv.go index a42c1926f..54c2e29a6 100644 --- a/cdrc/csv.go +++ b/cdrc/csv.go @@ -164,13 +164,13 @@ func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *con var fieldVal string switch cdrFldCfg.Type { case utils.META_COMPOSED: - out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider) + out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider, utils.NestingSep) if err != nil { return nil, err } fieldVal = out case utils.MetaUnixTimestamp: - out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider) + out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider, utils.NestingSep) if err != nil { return nil, err } diff --git a/cdrc/fwv.go b/cdrc/fwv.go index 64b776f7c..1464a697e 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -174,7 +174,7 @@ func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *confi var fieldVal string switch cdrFldCfg.Type { case utils.META_COMPOSED: - out, err := cdrFldCfg.Value.ParseDataProvider(fwvProvider) + out, err := cdrFldCfg.Value.ParseDataProvider(fwvProvider, utils.NestingSep) if err != nil { return nil, err } diff --git a/cdrc/xml.go b/cdrc/xml.go index 8670f933c..8a78c2362 100644 --- a/cdrc/xml.go +++ b/cdrc/xml.go @@ -65,10 +65,16 @@ func handlerSubstractUsage(xmlElement *etree.Element, argsTpl config.RSRParsers, if err != nil { return time.Duration(0), err } + if tEnd.IsZero() { + return time.Duration(0), fmt.Errorf("EndTime is 0") + } tStart, err := utils.ParseTimeDetectLayout(handlerArgs[1], timezone) if err != nil { return time.Duration(0), err } + if tStart.IsZero() { + return time.Duration(0), fmt.Errorf("StartTime is 0") + } return tEnd.Sub(tStart), nil } @@ -150,7 +156,7 @@ func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity *etree.Element, cdrcCf } } if cdrFldCfg.Type == utils.META_COMPOSED { - out, err := cdrFldCfg.Value.ParseDataProvider(xmlProvider) + out, err := cdrFldCfg.Value.ParseDataProvider(xmlProvider, utils.NestingSep) if err != nil { return nil, err } @@ -225,16 +231,15 @@ func (xP *xmlProvider) String() string { // FieldAsInterface is part of engine.DataProvider interface func (xP *xmlProvider) FieldAsInterface(fldPath []string) (data interface{}, err error) { - if len(fldPath) != 1 { + if len(fldPath) == 0 { return nil, utils.ErrNotFound } if data, err = xP.cache.FieldAsInterface(fldPath); err == nil || err != utils.ErrNotFound { // item found in cache return } - err = nil // cancel previous err - absolutePath := utils.ParseHierarchyPath(fldPath[0], "") - relPath := utils.HierarchyPath(absolutePath[len(xP.cdrPath):]) // Need relative path to the xmlElmnt + err = nil // cancel previous err + relPath := utils.HierarchyPath(fldPath[len(xP.cdrPath):]) // Need relative path to the xmlElmnt data, err = elementText(xP.req, relPath.AsString("/", false)) xP.cache.Set(fldPath, data, false) return diff --git a/cdrc/xml_it_test.go b/cdrc/xml_it_test.go index 3f660a304..a53392360 100644 --- a/cdrc/xml_it_test.go +++ b/cdrc/xml_it_test.go @@ -292,7 +292,7 @@ func TestXmlIT3AnalyseCDRs(t *testing.T) { var reply []*engine.ExternalCDR if err := cdrcXmlRPC.Call("ApierV2.GetCdrs", utils.RPCCDRsFilter{}, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 3 { + } else if len(reply) != 2 { t.Error("Unexpected number of CDRs returned: ", len(reply)) } } diff --git a/cdrc/xml_test.go b/cdrc/xml_test.go index 44c87a470..1e58edf3b 100644 --- a/cdrc/xml_test.go +++ b/cdrc/xml_test.go @@ -205,7 +205,7 @@ func TestXMLHandlerSubstractUsage(t *testing.T) { cdrs := doc.FindElements(path.Join("/broadWorksCDR/cdrData/")) cdrWithUsage := cdrs[1] if usage, err := handlerSubstractUsage(cdrWithUsage, - config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>releaseTime;|;~broadWorksCDR>cdrData>basicModule>answerTime", true), + config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", true), utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), "UTC"); err != nil { t.Error(err) } else if usage != time.Duration(13483000000) { @@ -226,24 +226,24 @@ func TestXMLRPProcess(t *testing.T) { &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>localCallId", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.localCallId", true), Mandatory: true}, &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>userId:s/.*@(.*)/${1}/", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true), Mandatory: true}, &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>userNumber", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userNumber", true), Mandatory: true}, &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>calledNumber", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.calledNumber", true), Mandatory: true}, &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>startTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.startTime", true), Mandatory: true}, &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>answerTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>releaseTime;|;~broadWorksCDR>cdrData>basicModule>answerTime", + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, &config.FCTemplate{Tag: "UsageSeconds", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: config.NewRSRParsersMustCompile("s", true), Mandatory: true}, @@ -289,29 +289,29 @@ func TestXMLRPProcessWithNewFilters(t *testing.T) { DataUsageMultiplyFactor: 1024, CDRPath: utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), CdrSourceId: "XMLWithFilters", - Filters: []string{"*string:broadWorksCDR>cdrData>headerModule>type:Normal"}, + Filters: []string{"*string:broadWorksCDR.cdrData.headerModule.type:Normal"}, ContentFields: []*config.FCTemplate{ &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>localCallId", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.localCallId", true), Mandatory: true}, &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>userId:s/.*@(.*)/${1}/", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true), Mandatory: true}, &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>userNumber", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.userNumber", true), Mandatory: true}, &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>calledNumber", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.calledNumber", true), Mandatory: true}, &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>startTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.startTime", true), Mandatory: true}, &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>answerTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~broadWorksCDR>cdrData>basicModule>releaseTime;|;~broadWorksCDR>cdrData>basicModule>answerTime", + Value: config.NewRSRParsersMustCompile("~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", true), Mandatory: true}, &config.FCTemplate{Tag: "UsageSeconds", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: config.NewRSRParsersMustCompile("s", true), Mandatory: true}, @@ -519,12 +519,12 @@ var xmlContent = ` ` -func TestXMLElementText2(t *testing.T) { +func TestXMLElementText3(t *testing.T) { doc := etree.NewDocument() if err := doc.ReadFromString(xmlContent); err != nil { t.Error(err) } - hPath2 := utils.ParseHierarchyPath("File>CDRs>Call", "") + hPath2 := utils.ParseHierarchyPath("File.CDRs.Call", "") cdrs := doc.Root().FindElements(hPath2.AsString("/", true)) if len(cdrs) != 3 { t.Errorf("Expecting: 3, received: %+v", len(cdrs)) @@ -534,7 +534,7 @@ func TestXMLElementText2(t *testing.T) { t.Error(err) } - absolutePath := utils.ParseHierarchyPath("File>CDRs>Call>SignalingInfo>PChargingVector>icidvalue", "") + absolutePath := utils.ParseHierarchyPath("File.CDRs.Call.SignalingInfo.PChargingVector.icidvalue", "") hPath := utils.ParseHierarchyPath(cdrs[1].GetPath(), "") relPath := utils.HierarchyPath(absolutePath[len(hPath):]) // Need relative path to the xmlElmnt @@ -545,7 +545,7 @@ func TestXMLElementText2(t *testing.T) { } } -func TestXMLRPProcessWithNewFilters2(t *testing.T) { +func TestXMLRPNestingSeparator(t *testing.T) { cdrcCfgs := []*config.CdrcConfig{ &config.CdrcConfig{ ID: "msw_xml", @@ -559,7 +559,7 @@ func TestXMLRPProcessWithNewFilters2(t *testing.T) { &config.FCTemplate{Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, Value: config.NewRSRParsersMustCompile("*voice", true), Mandatory: true}, &config.FCTemplate{Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>SignalingInfo>PChargingVector>icidvalue", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.SignalingInfo.PChargingVector.icidvalue", true), Mandatory: true}, &config.FCTemplate{Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, Value: config.NewRSRParsersMustCompile("*rated", true), Mandatory: true}, &config.FCTemplate{Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, @@ -567,16 +567,16 @@ func TestXMLRPProcessWithNewFilters2(t *testing.T) { &config.FCTemplate{Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, Value: config.NewRSRParsersMustCompile("call", true), Mandatory: true}, &config.FCTemplate{Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>OrigParty>SubscriberAddr", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.OrigParty.SubscriberAddr", true), Mandatory: true}, &config.FCTemplate{Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>RoutingInfo>DestAddr", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.RoutingInfo.DestAddr", true), Mandatory: true}, &config.FCTemplate{Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>RingingTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.RingingTime", true), Mandatory: true}, &config.FCTemplate{Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>ConnectTime", true), Mandatory: true}, + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.ConnectTime", true), Mandatory: true}, &config.FCTemplate{Tag: "Usage", Type: utils.META_HANDLER, FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~File>CDRs>Call>ReleaseTime;|;~File>CDRs>Call>ConnectTime", + Value: config.NewRSRParsersMustCompile("~File.CDRs.Call.ReleaseTime;|;~File.CDRs.Call.ConnectTime", true), Mandatory: true}, &config.FCTemplate{Tag: "UsageSeconds", Type: utils.META_COMPOSED, FieldId: utils.Usage, Value: config.NewRSRParsersMustCompile("s", true), Mandatory: true}, diff --git a/config/rsrparser.go b/config/rsrparser.go index a7af9117e..196932285 100644 --- a/config/rsrparser.go +++ b/config/rsrparser.go @@ -91,9 +91,9 @@ func (prsrs RSRParsers) ParseEvent(ev map[string]interface{}) (out string, err e return } -func (prsrs RSRParsers) ParseDataProvider(dP DataProvider) (out string, err error) { +func (prsrs RSRParsers) ParseDataProvider(dP DataProvider, separator string) (out string, err error) { for _, prsr := range prsrs { - if outPrsr, err := prsr.ParseDataProvider(dP); err != nil { + if outPrsr, err := prsr.ParseDataProvider(dP, separator); err != nil { return "", err } else { out += outPrsr @@ -113,17 +113,6 @@ func (prsrs RSRParsers) ParseCDR(dP DataProvider) (out string, err error) { return } -func (prsrs RSRParsers) ParseLoaderSDP(dP DataProvider) (out string, err error) { - for _, prsr := range prsrs { - if outPrsr, err := prsr.ParseLoaderSDP(dP); err != nil { - return "", err - } else { - out += outPrsr - } - } - return -} - func NewRSRParser(parserRules string, allFiltersMatch bool) (rsrParser *RSRParser, err error) { if len(parserRules) == 0 { return @@ -275,11 +264,11 @@ func (prsr *RSRParser) ParseEvent(ev map[string]interface{}) (out string, err er return prsr.ParseValue(val) } -func (prsr *RSRParser) ParseDataProvider(dP DataProvider) (out string, err error) { +func (prsr *RSRParser) ParseDataProvider(dP DataProvider, separator string) (out string, err error) { var outStr string if prsr.attrValue == "" { if outStr, err = dP.FieldAsString( - strings.Split(prsr.attrName, utils.NestingSep)); err != nil { + strings.Split(prsr.attrName, separator)); err != nil { return } } @@ -296,16 +285,3 @@ func (prsr *RSRParser) ParseCDR(dP DataProvider) (out string, err error) { } return prsr.ParseValue(outStr) } - -// ParseLoaderSDP Parse a DataProvider from LoaderS -// data there can be FileName:ColNumber and need to be splitted by ":" -func (prsr *RSRParser) ParseLoaderSDP(dP DataProvider) (out string, err error) { - var outStr string - if prsr.attrValue == "" { - if outStr, err = dP.FieldAsString( - strings.Split(prsr.attrName, utils.InInFieldSep)); err != nil { - return - } - } - return prsr.ParseValue(outStr) -} diff --git a/config/rsrparser_test.go b/config/rsrparser_test.go index cbddf75a4..627ca72eb 100644 --- a/config/rsrparser_test.go +++ b/config/rsrparser_test.go @@ -121,3 +121,17 @@ func TestRSRParserNotConstant(t *testing.T) { t.Errorf("expecting: EmptyString , received: %+v", out) } } + +func TestRSRParsersParseEvent2(t *testing.T) { + prsrs := NewRSRParsersMustCompile("~Header1.Test;|;~Header2.Test", true) + ev := map[string]interface{}{ + "Header1.Test": "Value1", + "Header2.Test": "Value2", + } + eOut := "Value1|Value2" + if out, err := prsrs.ParseEvent(ev); err != nil { + t.Error(err) + } else if eOut != out { + t.Errorf("expecting: %s, received: %s", eOut, out) + } +} diff --git a/data/conf/samples/cdrcxml/cgrates.json b/data/conf/samples/cdrcxml/cgrates.json index c558ef7d4..6e910e597 100644 --- a/data/conf/samples/cdrcxml/cgrates.json +++ b/data/conf/samples/cdrcxml/cgrates.json @@ -24,20 +24,20 @@ "cdr_format": "xml", "cdr_in_dir": "/tmp/cdrctests/xmlit1/in", "cdr_out_dir": "/tmp/cdrctests/xmlit1/out", - "cdr_path": "broadWorksCDR>cdrData", + "cdr_path": "broadWorksCDR.cdrData", "cdr_source_id": "xmlit1", "content_fields":[ // import content_fields template, id will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>localCallId", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>userId:s/.*@(.*)/${1}/", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", "mandatory": true}, {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>userNumber", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>calledNumber", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>startTime", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>answerTime", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~broadWorksCDR>cdrData>basicModule>releaseTime;|;~broadWorksCDR>cdrData>basicModule>answerTime", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.userNumber", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.calledNumber", "mandatory": true}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.startTime", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, ], }, ], diff --git a/data/conf/samples/cdrcxmlwithfilter/cgrates.json b/data/conf/samples/cdrcxmlwithfilter/cgrates.json index 3994255ac..580107b22 100755 --- a/data/conf/samples/cdrcxmlwithfilter/cgrates.json +++ b/data/conf/samples/cdrcxmlwithfilter/cgrates.json @@ -24,44 +24,44 @@ "cdr_format": "xml", "cdr_in_dir": "/tmp/cdrcxmlwithfilters/xmlit1/in", "cdr_out_dir": "/tmp/cdrcxmlwithfilters/xmlit1/out", - "cdr_path": "broadWorksCDR>cdrData", + "cdr_path": "broadWorksCDR.cdrData", "cdr_source_id": "xmlit1", - "filters": ["*string:broadWorksCDR>cdrData>basicModule>userNumber:1002","*string:broadWorksCDR>cdrData>headerModule>type:Normal"], + "filters": ["*string:broadWorksCDR.cdrData.basicModule.userNumber:1002","*string:broadWorksCDR.cdrData.headerModule.type:Normal"], "content_fields":[ // import content_fields template, id will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>localCallId", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>userId:s/.*@(.*)/${1}/", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", "mandatory": true}, {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>userNumber", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>calledNumber", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>startTime", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~broadWorksCDR>cdrData>basicModule>answerTime", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~broadWorksCDR>cdrData>basicModule>releaseTime;|;~broadWorksCDR>cdrData>basicModule>answerTime", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.userNumber", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.calledNumber", "mandatory": true}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.startTime", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~broadWorksCDR.cdrData.basicModule.releaseTime;|;~broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, ], }, { "id": "msw_xml", // identifier of the CDRC runner "enabled": true, // enable CDR client functionality - "cdr_format": "xml", // CDR file format + "cdr_format": "xml", // CDR file format CDRs>Call", // path towards one CDR element in case of XML CDRs + "cdr_path": "File.CDRs.Call", // path towards one CDR element in case of XML CDRs "cdr_source_id": "zw_cfs1", // free form field, tag identifying the source of the CDRs within CDRS database "content_fields":[ // import content_fields template, id will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~File>CDRs>Call>SignalingInfo>PChargingVector>icidvalue", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~File.CDRs.Call.SignalingInfo.PChargingVector.icidvalue", "mandatory": true}, {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "XX.liquid.tel", "mandatory": true}, {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~File>CDRs>Call>OrigParty>SubscriberAddr", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~File>CDRs>Call>OrigParty>SubscriberAddr", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~File>CDRs>Call>RoutingInfo>DestAddr", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~File>CDRs>Call>RingingTime", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~File>CDRs>Call>ConnectTime", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~File>CDRs>Call>ReleaseTime;|;~File>CDRs>Call>ConnectTime", "mandatory": true} + {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~File.CDRs.Call.OrigParty.SubscriberAddr", "mandatory": true}, + {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~File.CDRs.Call.OrigParty.SubscriberAddr", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~File.CDRs.Call.RoutingInfo.DestAddr", "mandatory": true}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~File.CDRs.Call.RingingTime", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~File.CDRs.Call.ConnectTime", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~File.CDRs.Call.ReleaseTime;|;~File.CDRs.Call.ConnectTime", "mandatory": true} ], }, ], diff --git a/engine/action.go b/engine/action.go index aba30eea1..161a22623 100644 --- a/engine/action.go +++ b/engine/action.go @@ -180,7 +180,7 @@ func cdrLogAction(acc *Account, sq *CDRStatsQueueTriggered, a *Action, acs Actio cdr.Usage = time.Duration(1) elem := reflect.ValueOf(cdr).Elem() for key, rsrFlds := range defaultTemplate { - parsedValue, err := rsrFlds.ParseDataProvider(newCdrLogProvider(acc, action)) + parsedValue, err := rsrFlds.ParseDataProvider(newCdrLogProvider(acc, action), utils.NestingSep) field := elem.FieldByName(key) if err != nil { return err diff --git a/engine/mapevent.go b/engine/mapevent.go index 1d5e3824d..3a3dbac17 100644 --- a/engine/mapevent.go +++ b/engine/mapevent.go @@ -219,6 +219,8 @@ func (me MapEvent) AsCDR(cfg *config.CGRConfig, tmz string) (cdr *CDR, err error } } } - cdr.AddDefaults(cfg) + if cfg != nil { + cdr.AddDefaults(cfg) + } return } diff --git a/loaders/libloader.go b/loaders/libloader.go index 29bfea201..bcbceb9ea 100644 --- a/loaders/libloader.go +++ b/loaders/libloader.go @@ -52,7 +52,7 @@ func (ld LoaderData) UpdateFromCSV(fileName string, record []string, continue // Not passes filters, ignore this CDR } } - out, err := cfgFld.Value.ParseLoaderSDP(csvProvider) + out, err := cfgFld.Value.ParseDataProvider(csvProvider, utils.InInFieldSep) if err != nil { return err } diff --git a/utils/coreutils.go b/utils/coreutils.go index 18eeaff0d..46d0b3053 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -642,7 +642,7 @@ func TimeIs0h(t time.Time) bool { func ParseHierarchyPath(path string, sep string) HierarchyPath { if sep == "" { - for _, sep = range []string{HIERARCHY_SEP, "/"} { + for _, sep = range []string{"/", NestingSep} { if idx := strings.Index(path, sep); idx != -1 { break } diff --git a/utils/coreutils_test.go b/utils/coreutils_test.go index c5e6b9023..d3b7ef3cc 100644 --- a/utils/coreutils_test.go +++ b/utils/coreutils_test.go @@ -604,10 +604,10 @@ func TestEndOfMonth(t *testing.T) { func TestParseHierarchyPath(t *testing.T) { eHP := HierarchyPath([]string{"Root", "CGRateS"}) - if hp := ParseHierarchyPath("Root>CGRateS", ""); !reflect.DeepEqual(hp, eHP) { + if hp := ParseHierarchyPath("/Root/CGRateS/", ""); !reflect.DeepEqual(hp, eHP) { t.Errorf("Expecting: %+v, received: %+v", eHP, hp) } - if hp := ParseHierarchyPath("/Root/CGRateS/", ""); !reflect.DeepEqual(hp, eHP) { + if hp := ParseHierarchyPath("Root.CGRateS", ""); !reflect.DeepEqual(hp, eHP) { t.Errorf("Expecting: %+v, received: %+v", eHP, hp) } }