From 91e1ea0d5e39dcc5c6afefedbac16f54c1d1eb4e Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 29 Aug 2018 05:28:41 -0400 Subject: [PATCH] Update integration tests --- apier/v1/cdre_it_test.go | 4 +- cdrc/partialcsv_it_test.go | 6 +- config/multifiles_it_test.go | 12 +- data/conf/samples/cdrc_partcsv/cgrates.json | 4 +- data/conf/samples/cdrcfwv/cgrates.json | 4 + .../samples/cdrcfwvwithfilter/cgrates.json | 4 + data/conf/samples/multifiles/a.json | 24 +-- data/conf/samples/multifiles/b/b.json | 4 +- data/conf/samples/multifiles/c.json | 18 +- engine/cdrs.go | 158 +++++++++--------- 10 files changed, 122 insertions(+), 116 deletions(-) diff --git a/apier/v1/cdre_it_test.go b/apier/v1/cdre_it_test.go index 965e6e3da..7f198416f 100755 --- a/apier/v1/cdre_it_test.go +++ b/apier/v1/cdre_it_test.go @@ -151,8 +151,8 @@ func testCDReExportCDRs(t *testing.T) { var rply *RplExportedCDRs if err := cdreRPC.Call("ApierV1.ExportCDRs", attr, &rply); err != nil { t.Error("Unexpected error: ", err.Error()) - } else if len(rply.ExportedCGRIDs) != 4 { - t.Error("Unexpected number of CDR exported: ", len(rply.ExportedCGRIDs)) + } else if len(rply.ExportedCGRIDs) != 2 { + t.Errorf("Unexpected number of CDR exported: %s ", utils.ToJSON(rply)) } } diff --git a/cdrc/partialcsv_it_test.go b/cdrc/partialcsv_it_test.go index 587b37ed2..dca7edbd7 100644 --- a/cdrc/partialcsv_it_test.go +++ b/cdrc/partialcsv_it_test.go @@ -104,7 +104,7 @@ func TestPartcsvITRpcConn(t *testing.T) { func TestPartcsvITHandleCdr1File(t *testing.T) { fileName := "file1.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent1), 0755); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent1), 0644); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { @@ -116,7 +116,7 @@ func TestPartcsvITHandleCdr1File(t *testing.T) { func TestPartcsvITHandleCdr2File(t *testing.T) { fileName := "file2.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent2), 0755); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent2), 0644); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { @@ -128,7 +128,7 @@ func TestPartcsvITHandleCdr2File(t *testing.T) { func TestPartcsvITHandleCdr3File(t *testing.T) { fileName := "file3.csv" tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent3), 0755); err != nil { + if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent3), 0644); err != nil { t.Fatal(err.Error()) } if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn2, fileName)); err != nil { diff --git a/config/multifiles_it_test.go b/config/multifiles_it_test.go index 25a0ea364..4a106709a 100644 --- a/config/multifiles_it_test.go +++ b/config/multifiles_it_test.go @@ -62,8 +62,8 @@ func TestMfCdreDefaultInstance(t *testing.T) { if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 12 { t.Error("Default instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields)) } - if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Direction" { - t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag) + if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].ID != "Direction" { + t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].ID) } } @@ -78,13 +78,13 @@ func TestMfCdreExport1Instance(t *testing.T) { if len(mfCgrCfg.CdreProfiles[prfl].HeaderFields) != 2 { t.Error("Export1 instance has number of header fields: ", len(mfCgrCfg.CdreProfiles[prfl].HeaderFields)) } - if mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag != "RunId" { - t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag) + if mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].ID != "RunId" { + t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].ID) } if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 9 { t.Error("Export1 instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields)) } - if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Account" { - t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag) + if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].ID != "Account" { + t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].ID) } } diff --git a/data/conf/samples/cdrc_partcsv/cgrates.json b/data/conf/samples/cdrc_partcsv/cgrates.json index f5fce304d..bcb134cbb 100644 --- a/data/conf/samples/cdrc_partcsv/cgrates.json +++ b/data/conf/samples/cdrc_partcsv/cgrates.json @@ -32,7 +32,7 @@ "partial_record_cache": "1s", // duration to cache partial records when not pairing "partial_cache_expiry_action": "*dump_to_file", "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"id": "ToR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, {"id": "AccId1", "field_id": "OriginID", "type": "*composed", "value": "~0"}, {"id": "AccId2", "field_id": "OriginID", "type": "*composed", "value": "_"}, {"id": "AccId3", "field_id": "OriginID", "type": "*composed", "value": "~1"}, @@ -72,7 +72,7 @@ "partial_record_cache": "1s", // duration to cache partial records when not pairing "partial_cache_expiry_action": "*post_cdr", "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"id": "ToR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, {"id": "AccId1", "field_id": "OriginID", "type": "*composed", "value": "~0"}, {"id": "AccId2", "field_id": "OriginID", "type": "*composed", "value": "_"}, {"id": "AccId3", "field_id": "OriginID", "type": "*composed", "value": "~1"}, diff --git a/data/conf/samples/cdrcfwv/cgrates.json b/data/conf/samples/cdrcfwv/cgrates.json index e75d88ab2..3e4799988 100644 --- a/data/conf/samples/cdrcfwv/cgrates.json +++ b/data/conf/samples/cdrcfwv/cgrates.json @@ -6,6 +6,10 @@ // This file contains the default configuration hardcoded into CGRateS. // This is what you get when you load CGRateS with an empty configuration file. +"general": { + "log_level": 7, +}, + "stor_db": { // database used to store offline tariff plans and CDRs "db_password": "CGRateS.org", // password to use when connecting to stordb }, diff --git a/data/conf/samples/cdrcfwvwithfilter/cgrates.json b/data/conf/samples/cdrcfwvwithfilter/cgrates.json index ccf138822..d3168eb68 100755 --- a/data/conf/samples/cdrcfwvwithfilter/cgrates.json +++ b/data/conf/samples/cdrcfwvwithfilter/cgrates.json @@ -6,6 +6,10 @@ // This file contains the default configuration hardcoded into CGRateS. // This is what you get when you load CGRateS with an empty configuration file. +"general": { + "log_level": 7, +}, + "stor_db": { // database used to store offline tariff plans and CDRs "db_password": "CGRateS.org", // password to use when connecting to stordb }, diff --git a/data/conf/samples/multifiles/a.json b/data/conf/samples/multifiles/a.json index e34da5304..65941428b 100644 --- a/data/conf/samples/multifiles/a.json +++ b/data/conf/samples/multifiles/a.json @@ -10,18 +10,18 @@ "cdre": { "*default": { "content_fields": [ // template of the exported content fields - {"tag":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"}, - {"tag":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"}, - {"tag":"Direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "direction"}, - {"tag":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, - {"tag":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, - {"tag":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, - {"tag":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, - {"tag":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, - {"tag":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, - {"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, + {"id":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"}, + {"id":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"}, + {"id":"Direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "direction"}, + {"id":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, + {"id":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, + {"id":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, + {"id":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, + {"id":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, + {"id":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, + {"id":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, + {"id":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, + {"id":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, ], } }, diff --git a/data/conf/samples/multifiles/b/b.json b/data/conf/samples/multifiles/b/b.json index d7f1e7003..fe7c0ed5b 100644 --- a/data/conf/samples/multifiles/b/b.json +++ b/data/conf/samples/multifiles/b/b.json @@ -14,8 +14,8 @@ }, "export1": { "header_fields": [ - {"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"}, - {"tag":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"}, + {"id": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"}, + {"id":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"}, ], // template of the exported header fields } }, diff --git a/data/conf/samples/multifiles/c.json b/data/conf/samples/multifiles/c.json index e182d2b81..848769e24 100644 --- a/data/conf/samples/multifiles/c.json +++ b/data/conf/samples/multifiles/c.json @@ -7,15 +7,15 @@ "export1": { "cost_rounding_decimals": 3, // rounding decimals for Cost values. -1 to disable rounding "content_fields": [ // template of the exported content fields - {"tag":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, - {"tag":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, - {"tag":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, - {"tag":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, - {"tag":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, - {"tag":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, - {"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, + {"id":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, + {"id":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, + {"id":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, + {"id":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, + {"id":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, + {"id":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, + {"id":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, + {"id":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, + {"id":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, ], // template of the exported header fields } }, diff --git a/engine/cdrs.go b/engine/cdrs.go index 847630d2b..1935ddb2a 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -307,87 +307,85 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR, store, cdrstats, } func (self *CdrServer) deriveCdrs(cdr *CDR) (drvdCDRs []*CDR, err error) { - /* - dfltCDRRun := cdr.Clone() - cdrRuns := []*CDR{dfltCDRRun} - if cdr.RunID != utils.MetaRaw { // Only derive *raw CDRs - return cdrRuns, nil - } - dfltCDRRun.RunID = utils.META_DEFAULT // Rewrite *raw with *default since we have it as first run - if self.attrS != nil { - var rplyEv AttrSProcessEventReply - if err = self.attrS.Call(utils.AttributeSv1ProcessEvent, - cdr.AsCGREvent(), &rplyEv); err != nil { - return - } - if err = cdr.UpdateFromCGREvent(rplyEv.CGREvent, - rplyEv.AlteredFields); err != nil { - return - } - } - if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { - return nil, err - } - if err := LoadAlias(&AttrMatchingAlias{ - Destination: cdr.Destination, - Direction: utils.OUT, - Tenant: cdr.Tenant, - Category: cdr.Category, - Account: cdr.Account, - Subject: cdr.Subject, - Context: utils.MetaRating, - }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { - return nil, err - } - attrsDC := &utils.AttrDerivedChargers{Tenant: cdr.Tenant, Category: cdr.Category, Direction: utils.OUT, - Account: cdr.Account, Subject: cdr.Subject, Destination: cdr.Destination} - var dcs utils.DerivedChargers - if err := self.rals.Call("Responder.GetDerivedChargers", attrsDC, &dcs); err != nil { - utils.Logger.Err(fmt.Sprintf("Could not get derived charging for cgrid %s, error: %s", cdr.CGRID, err.Error())) - return nil, err - } - for _, dc := range dcs.Chargers { - runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) - matchingAllFilters := true - for _, dcRunFilter := range runFilters { - if _, err := cdr.FieldAsString(dcRunFilter); err != nil { - matchingAllFilters = false - break - } - } - if !matchingAllFilters { // Do not process the derived charger further if not all filters were matched - continue - } - dcRequestTypeFld, _ := utils.NewRSRField(dc.RequestTypeField) - dcTenantFld, _ := utils.NewRSRField(dc.TenantField) - dcCategoryFld, _ := utils.NewRSRField(dc.CategoryField) - dcAcntFld, _ := utils.NewRSRField(dc.AccountField) - dcSubjFld, _ := utils.NewRSRField(dc.SubjectField) - dcDstFld, _ := utils.NewRSRField(dc.DestinationField) - dcSTimeFld, _ := utils.NewRSRField(dc.SetupTimeField) - dcATimeFld, _ := utils.NewRSRField(dc.AnswerTimeField) - dcDurFld, _ := utils.NewRSRField(dc.UsageField) - dcRatedFld, _ := utils.NewRSRField(dc.PreRatedField) - dcCostFld, _ := utils.NewRSRField(dc.CostField) - - dcExtraFields := []*utils.RSRField{} - for key, _ := range cdr.ExtraFields { - dcExtraFields = append(dcExtraFields, &utils.RSRField{Id: key}) - } - - forkedCdr, err := cdr.ForkCdr(dc.RunID, dcRequestTypeFld, dcTenantFld, dcCategoryFld, dcAcntFld, dcSubjFld, dcDstFld, - dcSTimeFld, dcATimeFld, dcDurFld, dcRatedFld, dcCostFld, dcExtraFields, true, self.cgrCfg.DefaultTimezone) - if err != nil { - utils.Logger.Err(fmt.Sprintf("Could not fork CGR with cgrid %s, run: %s, error: %s", cdr.CGRID, dc.RunID, err.Error())) - continue // do not add it to the forked CDR list - } - if !forkedCdr.PreRated { - forkedCdr.Cost = -1.0 // Make sure that un-rated CDRs start with Cost -1 - } - cdrRuns = append(cdrRuns, forkedCdr) - } + dfltCDRRun := cdr.Clone() + cdrRuns := []*CDR{dfltCDRRun} + if cdr.RunID != utils.MetaRaw { // Only derive *raw CDRs return cdrRuns, nil - */ + } + dfltCDRRun.RunID = utils.META_DEFAULT // Rewrite *raw with *default since we have it as first run + if self.attrS != nil { + var rplyEv AttrSProcessEventReply + if err = self.attrS.Call(utils.AttributeSv1ProcessEvent, + cdr.AsCGREvent(), &rplyEv); err != nil { + return + } + if err = cdr.UpdateFromCGREvent(rplyEv.CGREvent, + rplyEv.AlteredFields); err != nil { + return + } + } + if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { + return nil, err + } + if err := LoadAlias(&AttrMatchingAlias{ + Destination: cdr.Destination, + Direction: utils.OUT, + Tenant: cdr.Tenant, + Category: cdr.Category, + Account: cdr.Account, + Subject: cdr.Subject, + Context: utils.MetaRating, + }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + return nil, err + } + attrsDC := &utils.AttrDerivedChargers{Tenant: cdr.Tenant, Category: cdr.Category, Direction: utils.OUT, + Account: cdr.Account, Subject: cdr.Subject, Destination: cdr.Destination} + var dcs utils.DerivedChargers + if err := self.rals.Call("Responder.GetDerivedChargers", attrsDC, &dcs); err != nil { + utils.Logger.Err(fmt.Sprintf("Could not get derived charging for cgrid %s, error: %s", cdr.CGRID, err.Error())) + return nil, err + } + for _, dc := range dcs.Chargers { + runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + matchingAllFilters := true + for _, dcRunFilter := range runFilters { + if _, err := cdr.FieldAsStringWithRSRField(dcRunFilter); err != nil { + matchingAllFilters = false + break + } + } + if !matchingAllFilters { // Do not process the derived charger further if not all filters were matched + continue + } + dcRequestTypeFld, _ := utils.NewRSRField(dc.RequestTypeField) + dcTenantFld, _ := utils.NewRSRField(dc.TenantField) + dcCategoryFld, _ := utils.NewRSRField(dc.CategoryField) + dcAcntFld, _ := utils.NewRSRField(dc.AccountField) + dcSubjFld, _ := utils.NewRSRField(dc.SubjectField) + dcDstFld, _ := utils.NewRSRField(dc.DestinationField) + dcSTimeFld, _ := utils.NewRSRField(dc.SetupTimeField) + dcATimeFld, _ := utils.NewRSRField(dc.AnswerTimeField) + dcDurFld, _ := utils.NewRSRField(dc.UsageField) + dcRatedFld, _ := utils.NewRSRField(dc.PreRatedField) + dcCostFld, _ := utils.NewRSRField(dc.CostField) + + dcExtraFields := []*utils.RSRField{} + for key, _ := range cdr.ExtraFields { + dcExtraFields = append(dcExtraFields, &utils.RSRField{Id: key}) + } + + forkedCdr, err := cdr.ForkCdr(dc.RunID, dcRequestTypeFld, dcTenantFld, dcCategoryFld, dcAcntFld, dcSubjFld, dcDstFld, + dcSTimeFld, dcATimeFld, dcDurFld, dcRatedFld, dcCostFld, dcExtraFields, true, self.cgrCfg.DefaultTimezone) + if err != nil { + utils.Logger.Err(fmt.Sprintf("Could not fork CGR with cgrid %s, run: %s, error: %s", cdr.CGRID, dc.RunID, err.Error())) + continue // do not add it to the forked CDR list + } + if !forkedCdr.PreRated { + forkedCdr.Cost = -1.0 // Make sure that un-rated CDRs start with Cost -1 + } + cdrRuns = append(cdrRuns, forkedCdr) + } + return cdrRuns, nil return }