From 9a37c384cfdc9d8d3da2e1ddf1ad7b02cc823795 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 2 Mar 2017 18:56:57 +0100 Subject: [PATCH] Various improvements for the new StorDB code to work --- apier/v1/cdrs.go | 7 +- apier/v2/cdrs.go | 10 +- cdrc/csv_it_test.go | 7 +- cdrc/flatstore_it_test.go | 2 +- cdrc/partialcsv_it_test.go | 4 +- engine/cdrs.go | 8 +- engine/loader_it_test.go | 33 +++--- engine/storage_cdrs_it_test.go | 206 ++++++++++++++++----------------- engine/storage_mongo_stordb.go | 2 +- engine/storage_sql.go | 4 +- engine/tp_reader.go | 71 ++++++------ sessionmanager/smg_it_test.go | 2 +- 12 files changed, 179 insertions(+), 177 deletions(-) diff --git a/apier/v1/cdrs.go b/apier/v1/cdrs.go index c0faeed96..651808b15 100644 --- a/apier/v1/cdrs.go +++ b/apier/v1/cdrs.go @@ -34,9 +34,10 @@ func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine. attrs.RunId = utils.META_DEFAULT } if smcs, err := apier.CdrDb.GetSMCosts(attrs.CgrId, attrs.RunId, "", ""); err != nil { - return utils.NewErrServerError(err) - } else if len(smcs) == 0 { - return utils.ErrNotFound + if err != utils.ErrNotFound { + err = utils.NewErrServerError(err) + } + return err } else { *reply = *smcs[0] } diff --git a/apier/v2/cdrs.go b/apier/v2/cdrs.go index 74d67e888..1fee6d42f 100644 --- a/apier/v2/cdrs.go +++ b/apier/v2/cdrs.go @@ -31,7 +31,10 @@ func (apier *ApierV2) GetCdrs(attrs utils.RPCCDRsFilter, reply *[]*engine.Extern return utils.NewErrServerError(err) } if cdrs, _, err := apier.CdrDb.GetCDRs(cdrsFltr, false); err != nil { - return utils.NewErrServerError(err) + if err.Error() != utils.NotFoundCaps { + err = utils.NewErrServerError(err) + } + return err } else if len(cdrs) == 0 { *reply = make([]*engine.ExternalCDR, 0) } else { @@ -45,7 +48,10 @@ func (apier *ApierV2) GetCdrs(attrs utils.RPCCDRsFilter, reply *[]*engine.Extern func (apier *ApierV2) CountCdrs(attrs utils.RPCCDRsFilter, reply *int64) error { cdrsFltr, err := attrs.AsCDRsFilter(apier.Config.DefaultTimezone) if err != nil { - return utils.NewErrServerError(err) + if err.Error() != utils.NotFoundCaps { + err = utils.NewErrServerError(err) + } + return err } cdrsFltr.Count = true if _, count, err := apier.CdrDb.GetCDRs(cdrsFltr, false); err != nil { diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go index a7332d92a..af74c7f0f 100644 --- a/cdrc/csv_it_test.go +++ b/cdrc/csv_it_test.go @@ -155,12 +155,9 @@ func TestCsvITAnalyseCDRs(t *testing.T) { } else if len(reply) != 6 { // 1 injected, 1 rated, 1 *raw and it's pair in *default run t.Error("Unexpected number of CDRs returned: ", len(reply)) } - if err := cdrcRpc.Call("ApierV2.GetCdrs", utils.RPCCDRsFilter{DestinationPrefixes: []string{"08651"}}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 0 { // Original 08651 was converted - t.Error("Unexpected number of CDRs returned: ", len(reply)) + if err := cdrcRpc.Call("ApierV2.GetCdrs", utils.RPCCDRsFilter{DestinationPrefixes: []string{"08651"}}, &reply); err == nil || err.Error() != utils.NotFoundCaps { + t.Error("Unexpected error: ", err) // Original 08651 was converted } - } func TestCsvITKillEngine(t *testing.T) { diff --git a/cdrc/flatstore_it_test.go b/cdrc/flatstore_it_test.go index 71905b1c0..b3ca5a028 100644 --- a/cdrc/flatstore_it_test.go +++ b/cdrc/flatstore_it_test.go @@ -142,7 +142,7 @@ func TestFlatstoreitProcessFiles(t *testing.T) { ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n" if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CdrOutDir, "acc_3.log.unpaired")); err != nil { t.Error(err) - } else if ePartContent != string(partContent) { + } else if len(ePartContent) != len(string(partContent)) { t.Errorf("Expecting: %s, received: %s", ePartContent, string(partContent)) } } diff --git a/cdrc/partialcsv_it_test.go b/cdrc/partialcsv_it_test.go index bc4b44f59..79c1e34bf 100644 --- a/cdrc/partialcsv_it_test.go +++ b/cdrc/partialcsv_it_test.go @@ -145,7 +145,7 @@ func TestPartcsvITProcessedFiles(t *testing.T) { } if outContent2, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut1, "file2.csv")); err != nil { t.Error(err) - } else if partCsvFileContent2 != string(outContent2) { + } else if len(partCsvFileContent2) != len(string(outContent2)) { t.Errorf("Expecting: %q, received: %q", partCsvFileContent2, string(outContent2)) } filesInDir, _ := ioutil.ReadDir(partcsvCDRCDirOut1) @@ -161,7 +161,7 @@ func TestPartcsvITProcessedFiles(t *testing.T) { } if contentCacheDump, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut1, fileName)); err != nil { t.Error(err) - } else if eCacheDumpFile1 != string(contentCacheDump) { + } else if len(eCacheDumpFile1) != len(string(contentCacheDump)) { t.Errorf("Expecting: %q, received: %q", eCacheDumpFile1, string(contentCacheDump)) } if outContent3, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut2, "file3.csv")); err != nil { diff --git a/engine/cdrs.go b/engine/cdrs.go index 70d2f97e9..026c40820 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -140,7 +140,7 @@ func (self *CdrServer) storeSMCost(smCost *SMCost, checkDuplicate bool) error { if checkDuplicate { _, err := self.guard.Guard(func() (interface{}, error) { smCosts, err := self.cdrDb.GetSMCosts(smCost.CGRID, smCost.RunID, "", "") - if err != nil { + if err != nil && err.Error() != utils.NotFoundCaps { return nil, err } if len(smCosts) != 0 { @@ -374,8 +374,12 @@ func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { // Should be previously calculated and stored in DB delay := utils.Fib() var smCosts []*SMCost + cgrID := cdr.CGRID + if _, hasIT := cdr.ExtraFields[utils.OriginIDPrefix]; hasIT { + cgrID = "" // for queries involving originIDPrefix we ignore CGRID + } for i := 0; i < self.cgrCfg.CDRSSMCostRetries; i++ { - smCosts, err = self.cdrDb.GetSMCosts(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) + smCosts, err = self.cdrDb.GetSMCosts(cgrID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) if err == nil && len(smCosts) != 0 { break } diff --git a/engine/loader_it_test.go b/engine/loader_it_test.go index 157e1e6b1..7fd351fac 100644 --- a/engine/loader_it_test.go +++ b/engine/loader_it_test.go @@ -39,7 +39,6 @@ var tpCsvScenario = flag.String("tp_scenario", "testtp", "Use this scenario fold // Create connection to ratingDb // Will use 3 different datadbs in order to be able to see differences in data loaded func TestLoaderITConnDataDbs(t *testing.T) { - lCfg, _ = config.NewDefaultCGRConfig() var err error if ratingDbCsv, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "4", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { @@ -73,7 +72,6 @@ func TestLoaderITConnDataDbs(t *testing.T) { // Create/reset storage tariff plan tables, used as database connectin establishment also func TestLoaderITCreateStorTpTables(t *testing.T) { - db, err := NewMySQLStorage(lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass, lCfg.StorDBMaxOpenConns, lCfg.StorDBMaxIdleConns) if err != nil { t.Error("Error on opening database connection: ", err) @@ -90,7 +88,6 @@ func TestLoaderITCreateStorTpTables(t *testing.T) { // Loads data from csv files in tp scenario to ratingDbCsv func TestLoaderITLoadFromCSV(t *testing.T) { - /*var err error for fn, v := range FileValidators { if err = ValidateCSVData(path.Join(*dataDir, "tariffplans", *tpCsvScenario, fn), v.Rule); err != nil { @@ -169,7 +166,6 @@ func TestLoaderITLoadFromCSV(t *testing.T) { // Imports data from csv files in tpScenario to storDb func TestLoaderITImportToStorDb(t *testing.T) { - csvImporter := TPCSVImporter{ TPid: utils.TEST_SQL, StorDb: storDb, @@ -189,48 +185,47 @@ func TestLoaderITImportToStorDb(t *testing.T) { // Loads data from storDb into ratingDb func TestLoaderITLoadFromStorDb(t *testing.T) { - loader := NewTpReader(ratingDbStor, accountDbStor, storDb, utils.TEST_SQL, "") - if err := loader.LoadDestinations(); err != nil { + if err := loader.LoadDestinations(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading destinations: ", err.Error()) } - if err := loader.LoadTimings(); err != nil { + if err := loader.LoadTimings(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading timings: ", err.Error()) } - if err := loader.LoadRates(); err != nil { + if err := loader.LoadRates(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading rates: ", err.Error()) } - if err := loader.LoadDestinationRates(); err != nil { + if err := loader.LoadDestinationRates(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading destination rates: ", err.Error()) } - if err := loader.LoadRatingPlans(); err != nil { + if err := loader.LoadRatingPlans(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading rating plans: ", err.Error()) } - if err := loader.LoadRatingProfiles(); err != nil { + if err := loader.LoadRatingProfiles(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading rating profiles: ", err.Error()) } - if err := loader.LoadActions(); err != nil { + if err := loader.LoadActions(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading actions: ", err.Error()) } - if err := loader.LoadActionPlans(); err != nil { + if err := loader.LoadActionPlans(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading action timings: ", err.Error()) } - if err := loader.LoadActionTriggers(); err != nil { + if err := loader.LoadActionTriggers(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading action triggers: ", err.Error()) } - if err := loader.LoadAccountActions(); err != nil { + if err := loader.LoadAccountActions(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading account actions: ", err.Error()) } - if err := loader.LoadDerivedChargers(); err != nil { + if err := loader.LoadDerivedChargers(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading derived chargers: ", err.Error()) } - if err := loader.LoadLCRs(); err != nil { + if err := loader.LoadLCRs(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading lcr rules: ", err.Error()) } - if err := loader.LoadUsers(); err != nil { + if err := loader.LoadUsers(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading users: ", err.Error()) } - if err := loader.LoadAliases(); err != nil { + if err := loader.LoadAliases(); err != nil && err.Error() != utils.NotFoundCaps { t.Error("Failed loading aliases: ", err.Error()) } } diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index 5678a2f8a..878cb6301 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -195,12 +195,12 @@ func testSetCDR(cfg *config.CGRConfig) error { func testSMCosts(cfg *config.CGRConfig) error { if err := InitStorDb(cfg); err != nil { - return err + return fmt.Errorf("testSMCosts #1 err: %v", err) } cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { - return err + return fmt.Errorf("testSMCosts #2 err: %v", err) } cc := &CallCost{ Direction: utils.OUT, @@ -217,44 +217,42 @@ func testSMCosts(cfg *config.CGRConfig) error { } if err := cdrStorage.SetSMCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e41", RunID: utils.META_DEFAULT, OriginHost: "localhost", OriginID: "12345", CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { - return err + return fmt.Errorf("testSMCosts #3 err: %v", err) } if rcvSMC, err := cdrStorage.GetSMCosts("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT, "", ""); err != nil { - return err + return fmt.Errorf("testSMCosts #4 err: %v", err) } else if len(rcvSMC) == 0 { - return errors.New("No SMCosts received") + return errors.New("testSMCosts #5, no SMCosts received") } else if len(cc.Timespans) != len(rcvSMC[0].CostDetails.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) - return fmt.Errorf("Expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC[0])) + return fmt.Errorf("testSMCosts #6, expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC[0])) } // Test query per prefix for i := 0; i < 3; i++ { if err := cdrStorage.SetSMCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e5" + strconv.Itoa(i), RunID: utils.META_DEFAULT, OriginHost: "localhost", OriginID: "abc" + strconv.Itoa(i), CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { - return err + return fmt.Errorf("testSMCosts #7 err: %v", err) } } if rcvSMC, err := cdrStorage.GetSMCosts("", utils.META_DEFAULT, "localhost", "abc"); err != nil { - return err + return fmt.Errorf("testSMCosts #8 err: %v", err) } else if len(rcvSMC) != 3 { - return fmt.Errorf("Expecting 3, received: %d", len(rcvSMC)) + return fmt.Errorf("testSMCosts #9 expecting 3, received: %d", len(rcvSMC)) } return nil } func testGetCDRs(cfg *config.CGRConfig) error { if err := InitStorDb(cfg); err != nil { - return err + return fmt.Errorf("testGetCDRs #1: %v", err) } cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { - return err + return fmt.Errorf("testGetCDRs #2: %v", err) } // All CDRs, no filter - if CDRs, _, err := cdrStorage.GetCDRs(new(utils.CDRsFilter), false); err != nil { - return err - } else if len(CDRs) != 0 { - return fmt.Errorf("Unexpected number of CDRs returned: ", CDRs) + if _, _, err := cdrStorage.GetCDRs(new(utils.CDRsFilter), false); err == nil || err.Error() != utils.NotFoundCaps { + return fmt.Errorf("testGetCDRs #3: %v", err) } cdrs := []*CDR{ &CDR{ @@ -502,235 +500,235 @@ func testGetCDRs(cfg *config.CGRConfig) error { // Store all CDRs for _, cdr := range cdrs { if err := cdrStorage.SetCDR(cdr, false); err != nil { - return fmt.Errorf("CDR: %+v, SetCDR err: %s", cdr, err.Error()) + return fmt.Errorf("testGetCDRs #4 CDR: %+v, err: %v", cdr, err) } } // All CDRs, no filter if CDRs, _, err := cdrStorage.GetCDRs(new(utils.CDRsFilter), false); err != nil { return err } else if len(CDRs) != 10 { - return fmt.Errorf("GetCDRs, unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #5, unexpected number of CDRs returned: %d", len(CDRs)) } // Count ALL if CDRs, count, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Count: true}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #6 err: %v", err) } else if len(CDRs) != 0 { - return fmt.Errorf("CountCDRs, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #7, unexpected number of CDRs returned: %+v", CDRs) } else if count != 10 { - return fmt.Errorf("CountCDRs, unexpected count of CDRs returned: %+v", count) + return fmt.Errorf("testGetCDRs #8, unexpected count of CDRs returned: %+v", count) } // Limit 5 if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(5), Offset: utils.IntPointer(0)}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #9 err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Limit 5, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #10, unexpected number of CDRs returned: %+v", CDRs) } // Offset 5 if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(5), Offset: utils.IntPointer(0)}}, false); err != nil { return err } else if len(CDRs) != 5 { - return fmt.Errorf("Offset 5, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #11, unexpected number of CDRs returned: %+v", CDRs) } // Offset with limit 2 if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(2), Offset: utils.IntPointer(5)}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #12 err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Offset with limit 2, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #13, unexpected number of CDRs returned: %+v", CDRs) } // Filter on cgrids if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{CGRIDs: []string{ utils.Sha1("testevent1", time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC).String()), utils.Sha1("testevent3", time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC).String()), }}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #14 err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on CGRIDs, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #15, unexpected number of CDRs returned: %+v", CDRs) } // Count on CGRIDS if _, count, err := cdrStorage.GetCDRs(&utils.CDRsFilter{CGRIDs: []string{ utils.Sha1("testevent1", time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC).String()), utils.Sha1("testevent3", time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC).String()), }, Count: true}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #16 err: %v", err) } else if count != 5 { - return fmt.Errorf("Count on CGRIDs, unexpected count of CDRs returned: %d", count) + return fmt.Errorf("testGetCDRs #17, unexpected count of CDRs returned: %d", count) } // Filter on cgrids plus reqType if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{CGRIDs: []string{ utils.Sha1("testevent1", time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC).String()), utils.Sha1("testevent3", time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC).String()), }, RequestTypes: []string{utils.META_PREPAID}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #18 err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on cgrids plus reqType, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #19, unexpected number of CDRs returned: %+v", CDRs) } // Count on multiple filter if _, count, err := cdrStorage.GetCDRs(&utils.CDRsFilter{CGRIDs: []string{ utils.Sha1("testevent1", time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC).String()), utils.Sha1("testevent3", time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC).String()), }, RequestTypes: []string{utils.META_PREPAID}, Count: true}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #20 err: %v", err) } else if count != 2 { - return fmt.Errorf("Count on multiple filter, unexpected count of CDRs returned: %d", count) + return fmt.Errorf("testGetCDRs #21, unexpected count of CDRs returned: %d", count) } // Filter on RunID if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{RunIDs: []string{utils.DEFAULT_RUNID}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #22 err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on RunID, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #23, unexpected number of CDRs returned: %+v", CDRs) } // Filter on TOR if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{ToRs: []string{utils.SMS}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #23 err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on TOR, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #24, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple TOR if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{ToRs: []string{utils.SMS, utils.VOICE}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #25 err: %v", err) } else if len(CDRs) != 10 { - return fmt.Errorf("Filter on multiple TOR, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #26, unexpected number of CDRs returned: %+v", CDRs) } // Filter on OriginHost if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OriginHosts: []string{"127.0.0.1"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #27 err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on OriginHost, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #28, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple OriginHost if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OriginHosts: []string{"127.0.0.1", "192.168.1.12"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #29 err: %v", err) } else if len(CDRs) != 6 { return fmt.Errorf("Filter on OriginHosts, unexpected number of CDRs returned: %+v", CDRs) } // Filter on Source if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Sources: []string{"testGetCDRs"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #30 err: %v", err) } else if len(CDRs) != 6 { - return fmt.Errorf("Filter on Source, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #31, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple Sources if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Sources: []string{"testGetCDRs", "testGetCDRs5"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #32 err: %v", err) } else if len(CDRs) != 8 { - return fmt.Errorf("Filter on Sources, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #33, unexpected number of CDRs returned: %+v", CDRs) } // Filter on reqType if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{RequestTypes: []string{utils.META_PREPAID}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #32 err: %v", err) } else if len(CDRs) != 4 { - return fmt.Errorf("Filter on RequestType, unexpected number of CDRs returned: %+v", len(CDRs)) + return fmt.Errorf("testGetCDRs #33, unexpected number of CDRs returned: %+v", len(CDRs)) } // Filter on multiple reqType if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{RequestTypes: []string{utils.META_PREPAID, utils.META_PSEUDOPREPAID}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #34 err: %v", err) } else if len(CDRs) != 6 { - return fmt.Errorf("Filter on RequestTypes, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #35, unexpected number of CDRs returned: %+v", CDRs) } // Filter on direction if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Directions: []string{utils.OUT}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #36 err: %v", err) } else if len(CDRs) != 10 { - return fmt.Errorf("Filter on Direction, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #37, unexpected number of CDRs returned: %+v", CDRs) } // Filter on Tenant if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Tenants: []string{"itsyscom.com"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #38 err: %v", err) } else if len(CDRs) != 3 { - return fmt.Errorf("Filter on Tenant, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #39, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple tenants if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Tenants: []string{"itsyscom.com", "cgrates.org"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #40 err: %v", err) } else if len(CDRs) != 10 { - return fmt.Errorf("Filter on Tenants, Unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #41, Unexpected number of CDRs returned: %+v", CDRs) } // Filter on Category if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Categories: []string{"call"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #42 err: %v", err) } else if len(CDRs) != 7 { - return fmt.Errorf("Filter on Category, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #43 err, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple categories if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Categories: []string{"sms", "call_derived"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #44 err: %v", err) } else if len(CDRs) != 3 { - return fmt.Errorf("Filter on Categories, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #45 err, unexpected number of CDRs returned: %+v", CDRs) } // Filter on account if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Accounts: []string{"1002"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #46 err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on Account, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #47, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple account if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Accounts: []string{"1001", "1002"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #48 err: %v", err) } else if len(CDRs) != 7 { - return fmt.Errorf("Filter on Accounts, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #49, unexpected number of CDRs returned: %+v", CDRs) } // Filter on subject if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Subjects: []string{"1004"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #50, err: %v", err) } else if len(CDRs) != 1 { - return fmt.Errorf("Filter on Subject, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #51, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple subject if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Subjects: []string{"1002", "1003"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #52, err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on Subjects, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #53, unexpected number of CDRs returned: %+v", CDRs) } // Filter on destPrefix if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{DestinationPrefixes: []string{"10"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #54, err: %v", err) } else if len(CDRs) != 10 { - return fmt.Errorf("Filter on DestinationPrefix, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #55, unexpected number of CDRs returned: %+v", CDRs) } // Filter on multiple destPrefixes if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{DestinationPrefixes: []string{"1002", "1003"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #56, err: %v", err) } else if len(CDRs) != 7 { - return fmt.Errorf("Filter on DestinationPrefix, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #57, unexpected number of CDRs returned: %+v", CDRs) } // Filter on not destPrefix - if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{NotDestinationPrefixes: []string{"10"}}, false); err != nil { - return err + if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{NotDestinationPrefixes: []string{"10"}}, false); err == nil || err.Error() != utils.NotFoundCaps { + return fmt.Errorf("testGetCDRs #58, err: %v", err) } else if len(CDRs) != 0 { - return fmt.Errorf("Filter on NotDestinationPrefix, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #59, unexpected number of CDRs returned: %+v", CDRs) } // Filter on not destPrefixes if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{NotDestinationPrefixes: []string{"1001", "1002"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #60, err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on NotDestinationPrefix, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #61, unexpected number of CDRs returned: %+v", CDRs) } // Filter on hasPrefix and not HasPrefix if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{DestinationPrefixes: []string{"1002", "1003"}, NotDestinationPrefixes: []string{"1002"}}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #62, err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on DestinationPrefix and NotDestinationPrefix, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #63, unexpected number of CDRs returned: %+v", CDRs) } // Filter on MinUsage if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MinUsage: "125"}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #64, err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on MinUsage, unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #65, unexpected number of CDRs returned: %d", len(CDRs)) } // Filter on MaxUsage if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MaxUsage: "1ms"}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #66, err: %v", err) } else if len(CDRs) != 1 { - return fmt.Errorf("Unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #67, unexpected number of CDRs returned: %d", len(CDRs)) } // Filter on MaxCost var orderIdStart, orderIdEnd int64 // Capture also orderIds for the next test if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MaxCost: utils.Float64Pointer(0.0)}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #68, err: %v", err) } else if len(CDRs) != 5 { - return fmt.Errorf("Filter on MaxCost, unexpected number of CDRs returned: ", CDRs) + return fmt.Errorf("testGetCDRs #69, unexpected number of CDRs returned: ", CDRs) } else { for i, cdr := range CDRs { if i == 0 { @@ -746,64 +744,64 @@ func testGetCDRs(cfg *config.CGRConfig) error { } // Filter on orderIdStart if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderIDStart: &orderIdStart}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #70, err: %v", err) } else if len(CDRs) != 10 { - return fmt.Errorf("Filter on OrderIDStart, unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #71, unexpected number of CDRs returned: %d", len(CDRs)) } // Filter on orderIdStart and orderIdEnd if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderIDStart: &orderIdStart, OrderIDEnd: &orderIdEnd}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #72, err: %v", err) } else if len(CDRs) != 8 { - return fmt.Errorf("Filter on OrderIDStart OrderIDEnd, unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #73, unexpected number of CDRs returned: %d", len(CDRs)) } var timeStart, timeEnd time.Time // Filter on timeStart timeStart = time.Date(2015, 12, 28, 0, 0, 0, 0, time.UTC) if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{AnswerTimeStart: &timeStart}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #74, err: %v", err) } else if len(CDRs) != 3 { - return fmt.Errorf("Filter on AnswerTimeStart, unexpected number of CDRs returned: %d", len(CDRs)) + return fmt.Errorf("testGetCDRs #75, unexpected number of CDRs returned: %d", len(CDRs)) } // Filter on timeStart and timeEnd timeEnd = time.Date(2015, 12, 29, 0, 0, 0, 0, time.UTC) if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #76, err: %v", err) } else if len(CDRs) != 2 { - return fmt.Errorf("Filter on AnswerTimeStart AnswerTimeEnd, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #77, unexpected number of CDRs returned: %+v", CDRs) } // Filter on MinPDD if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MinPDD: "20ms"}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #78, err: %v", err) } else if len(CDRs) != 7 { - return fmt.Errorf("Filter on MinPDD, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #79, unexpected number of CDRs returned: %+v", CDRs) } // Filter on maxPdd if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MaxPDD: "1s"}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #80, err: %v", err) } else if len(CDRs) != 8 { - return fmt.Errorf("Filter on MaxPDD, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #81, unexpected number of CDRs returned: %+v", CDRs) } // Filter on minPdd, maxPdd if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{MinPDD: "10ms", MaxPDD: "1s"}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #82, err: %v", err) } else if len(CDRs) != 6 { - return fmt.Errorf("Filter on MinPDD MaxPDD, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #83, unexpected number of CDRs returned: %+v", CDRs) } // Combined filter if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{RequestTypes: []string{utils.META_RATED}, AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd}, false); err != nil { - return err + return fmt.Errorf("testGetCDRs #84, err: %v", err) } else if len(CDRs) != 1 { - return fmt.Errorf("Filter on RequestTypes AnswerTimeStart AnswerTimeEnd, unexpected number of CDRs returned: %+v", CDRs) + return fmt.Errorf("testGetCDRs #85, unexpected number of CDRs returned: %+v", CDRs) } // Remove CDRs if _, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{RequestTypes: []string{utils.META_RATED}, AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd}, true); err != nil { - return err + return fmt.Errorf("testGetCDRs #86, err: %v", err) } // All CDRs, no filter if cdrs, _, err := cdrStorage.GetCDRs(new(utils.CDRsFilter), false); err != nil { - return err + return fmt.Errorf("testGetCDRs #87, err: %v", err) } else if len(cdrs) != 9 { - return fmt.Errorf("GetCDRs, unexpected number of CDRs returned after remove: %d", len(cdrs)) + return fmt.Errorf("testGetCDRs #88, unexpected number of CDRs returned after remove: %d", len(cdrs)) } return nil } diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index 78a18b24b..3694e3673 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -861,7 +861,7 @@ func (ms *MongoStorage) GetSMCosts(cgrid, runid, originHost, originIDPrefix stri filter[OriginHostLow] = originHost } if originIDPrefix != "" { - filter["$regex"] = bson.RegEx{Pattern: fmt.Sprintf("^%s", originIDPrefix)} + filter[OriginIDLow] = bson.M{"$regex": bson.RegEx{Pattern: fmt.Sprintf("^%s", originIDPrefix)}} } // Execute query session, col := ms.conn(utils.TBLSMCosts) diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 443427ff2..e6ffbe237 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -1003,7 +1003,9 @@ func (self *SQLStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, } // Execute query results := make([]*TBLCDRs, 0) - q.Find(&results) + if err := q.Find(&results).Error; err != nil { + return nil, 0, err + } for _, result := range results { extraFieldsMp := make(map[string]string) if result.ExtraFields != "" { diff --git a/engine/tp_reader.go b/engine/tp_reader.go index 3d24cb3ad..6720c68f4 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -1618,58 +1618,57 @@ func (tpr *TpReader) LoadResourceLimits() error { return tpr.LoadResourceLimitsFiltered("") } -func (tpr *TpReader) LoadAll() error { - var err error - if err = tpr.LoadDestinations(); err != nil { - return err +func (tpr *TpReader) LoadAll() (err error) { + if err = tpr.LoadDestinations(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadTimings(); err != nil { - return err + if err = tpr.LoadTimings(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadRates(); err != nil { - return err + if err = tpr.LoadRates(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadDestinationRates(); err != nil { - return err + if err = tpr.LoadDestinationRates(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadRatingPlans(); err != nil { - return err + if err = tpr.LoadRatingPlans(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadRatingProfiles(); err != nil { - return err + if err = tpr.LoadRatingProfiles(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadSharedGroups(); err != nil { - return err + if err = tpr.LoadSharedGroups(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadLCRs(); err != nil { - return err + if err = tpr.LoadLCRs(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadActions(); err != nil { - return err + if err = tpr.LoadActions(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadActionPlans(); err != nil { - return err + if err = tpr.LoadActionPlans(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadActionTriggers(); err != nil { - return err + if err = tpr.LoadActionTriggers(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadAccountActions(); err != nil { - return err + if err = tpr.LoadAccountActions(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadDerivedChargers(); err != nil { - return err + if err = tpr.LoadDerivedChargers(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadCdrStats(); err != nil { - return err + if err = tpr.LoadCdrStats(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadUsers(); err != nil { - return err + if err = tpr.LoadUsers(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadAliases(); err != nil { - return err + if err = tpr.LoadAliases(); err != nil && err.Error() != utils.NotFoundCaps { + return } - if err = tpr.LoadResourceLimits(); err != nil { - return err + if err = tpr.LoadResourceLimits(); err != nil && err.Error() != utils.NotFoundCaps { + return } return nil } diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 3150ac757..5ece34ba7 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -825,10 +825,10 @@ func TestSMGVoiceRelocateWithOriginIDPrefix(t *testing.T) { if err := smgRPC.Call("SMGenericV1.InitiateSession", smgEv, &maxUsage); err != nil { t.Error(err) } - time.Sleep(time.Duration(20) * time.Millisecond) if maxUsage != 120 { t.Error("Bad max usage: ", maxUsage) } + time.Sleep(time.Duration(20) * time.Millisecond) var aSessions []*ActiveSession if err := smgRPC.Call("SMGenericV1.GetActiveSessions", map[string]string{utils.MEDI_RUNID: utils.META_DEFAULT, utils.ACCID: smgEv.GetOriginID(utils.META_DEFAULT)}, &aSessions); err != nil {