From a9f0a9429be70876e93349b8be1a301a88828e37 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Wed, 12 Aug 2020 15:28:38 +0300 Subject: [PATCH] Added tests for GetCDRs with internalDB --- apier/v1/config_it_test.go | 1 + cmd/cgr-loader/cgr-loader.go | 2 +- cmd/cgr-loader/cgr-loader_it_test.go | 8 +- console/threshold.go | 2 +- data/conf/samples/cdrsv2internal/cgrates.json | 1 + engine/storage_internal_stordb.go | 112 ++++++++---------- engine/storage_sql.go | 7 +- ers/sql_it_test.go | 24 ++-- general_tests/cdrs_it_test.go | 4 +- general_tests/cdrs_onlexp_it_test.go | 18 +++ integration_test.sh | 11 +- 11 files changed, 106 insertions(+), 84 deletions(-) diff --git a/apier/v1/config_it_test.go b/apier/v1/config_it_test.go index 31e576c12..8d351193e 100644 --- a/apier/v1/config_it_test.go +++ b/apier/v1/config_it_test.go @@ -225,6 +225,7 @@ func testConfigSReloadConfigFromJSONEEs(t *testing.T) { "AttributeSConns": []interface{}{}, "Cache": map[string]interface{}{"*file_csv": map[string]interface{}{"Limit": -1., "Precache": false, "Replicate": false, "StaticTTL": false, "TTL": 5000000000.}}, "Exporters": []interface{}{eporter}, + "Templates": map[string]interface{}{}, } var rpl map[string]interface{} if err := configRPC.Call(utils.ConfigSv1GetJSONSection, &config.StringWithOpts{ diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 3a9dadf05..288be5205 100755 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -239,7 +239,7 @@ func importData(cfg *config.CGRConfig) (err error) { } if *flushStorDB { if err = storDB.RemTpData(utils.EmptyString, cfg.LoaderCgrCfg().TpID, map[string]string{}); err != nil { - return err + return } } csvImporter := engine.TPCSVImporter{ diff --git a/cmd/cgr-loader/cgr-loader_it_test.go b/cmd/cgr-loader/cgr-loader_it_test.go index 0b86d4062..f5a58df17 100644 --- a/cmd/cgr-loader/cgr-loader_it_test.go +++ b/cmd/cgr-loader/cgr-loader_it_test.go @@ -203,7 +203,7 @@ var ( func TestLoadIt(t *testing.T) { switch *dbType { case utils.MetaInternal: - ldrItCfgDir = "tutinternal" + t.SkipNow() case utils.MetaMySQL: ldrItCfgDir = "tutmysql" case utils.MetaMongo: @@ -303,7 +303,7 @@ func testLoadItStartLoaderRemove(t *testing.T) { } func testLoadItCheckAttributes2(t *testing.T) { - if _, err := db.GetAttributeProfileDrv("cgrates.org", "ATTR_1001_SIMPLEAUTH"); err != utils.ErrNotFound { + if _, err := db.GetAttributeProfileDrv("cgrates.org", "ATTR_1001_SESSIONAUTH"); err != utils.ErrNotFound { t.Fatal(err) } } @@ -323,7 +323,7 @@ func testLoadItStartLoaderToStorDB(t *testing.T) { } func testLoadItStartLoaderFromStorDB(t *testing.T) { - cmd := exec.Command("cgr-loader", "-config_path="+ldrItCfgPath, "-path="+path.Join(*dataDir, "tariffplans", "tutorial"), "-caches_address=", "-scheduler_address=", "-from_stordb", "-tpid=TPID") + cmd := exec.Command("cgr-loader", "-config_path="+ldrItCfgPath, "-caches_address=", "-scheduler_address=", "-from_stordb", "-tpid=TPID") output := bytes.NewBuffer(nil) outerr := bytes.NewBuffer(nil) cmd.Stdout = output @@ -337,7 +337,7 @@ func testLoadItStartLoaderFromStorDB(t *testing.T) { } func testLoadItStartLoaderFlushStorDB(t *testing.T) { - cmd := exec.Command("cgr-loader", "-config_path="+ldrItCfgPath, "-path="+path.Join(*dataDir, "tariffplans", "tutorial"), "-caches_address=", "-scheduler_address=", "-flush_stordb", "-tpid=TPID") + cmd := exec.Command("cgr-loader", "-config_path="+ldrItCfgPath, "-path="+path.Join(*dataDir, "tariffplans", "dispatchers"), "-caches_address=", "-scheduler_address=", "-to_stordb", "-flush_stordb", "-tpid=TPID") output := bytes.NewBuffer(nil) outerr := bytes.NewBuffer(nil) cmd.Stdout = output diff --git a/console/threshold.go b/console/threshold.go index 12225a4cf..1dbc963f7 100644 --- a/console/threshold.go +++ b/console/threshold.go @@ -69,6 +69,6 @@ func (self *CmdGetThreshold) RpcResult() interface{} { func (self *CmdGetThreshold) GetFormatedResult(result interface{}) string { return GetFormatedResult(result, map[string]struct{}{ - "MinSleep": struct{}{}, + "MinSleep": {}, }) } diff --git a/data/conf/samples/cdrsv2internal/cgrates.json b/data/conf/samples/cdrsv2internal/cgrates.json index c4183b9e5..e30280fdf 100644 --- a/data/conf/samples/cdrsv2internal/cgrates.json +++ b/data/conf/samples/cdrsv2internal/cgrates.json @@ -10,6 +10,7 @@ "stor_db": { "db_type": "*internal", + "string_indexed_fields": ["RunID"] }, diff --git a/engine/storage_internal_stordb.go b/engine/storage_internal_stordb.go index 484643dfc..030957029 100644 --- a/engine/storage_internal_stordb.go +++ b/engine/storage_internal_stordb.go @@ -1166,14 +1166,15 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C for _, id := range fltrSlc.ids { grpIDs := Cache.tCache.GetGroupItemIDs(utils.CacheCDRsTBL, utils.ConcatenatedKey(fltrSlc.key, id)) for _, id := range grpIDs { - if cdrMpIDs.Has(id) { - cdrMpIDs.Remove(id) - if cdrMpIDs.Size() == 0 { - if filter.Count { - return nil, 0, nil - } - return nil, 0, utils.ErrNotFound + if !cdrMpIDs.Has(id) { + continue + } + cdrMpIDs.Remove(id) + if cdrMpIDs.Size() == 0 { + if filter.Count { + return nil, 0, nil } + return nil, 0, utils.ErrNotFound } } } @@ -1190,14 +1191,12 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C var minUsage time.Duration var maxUsage time.Duration if len(filter.MinUsage) != 0 { - minUsage, err = utils.ParseDurationWithNanosecs(filter.MinUsage) - if err != nil { + if minUsage, err = utils.ParseDurationWithNanosecs(filter.MinUsage); err != nil { return nil, 0, err } } if len(filter.MaxUsage) != 0 { - maxUsage, err = utils.ParseDurationWithNanosecs(filter.MaxUsage) - if err != nil { + if maxUsage, err = utils.ParseDurationWithNanosecs(filter.MaxUsage); err != nil { return nil, 0, err } } @@ -1528,46 +1527,42 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C } } - if filter.OrderIDStart != nil { - if cdr.OrderID < *filter.OrderIDStart { - continue - } + if filter.OrderIDStart != nil && + cdr.OrderID < *filter.OrderIDStart { + continue } - if filter.OrderIDEnd != nil { - if cdr.OrderID >= *filter.OrderIDEnd { - continue - } + if filter.OrderIDEnd != nil && + cdr.OrderID >= *filter.OrderIDEnd { + continue } - if filter.AnswerTimeStart != nil && !filter.AnswerTimeStart.IsZero() { // With IsZero we keep backwards compatible with APIerSv1 - if cdr.AnswerTime.Before(*filter.AnswerTimeStart) { - continue - } + if filter.AnswerTimeStart != nil && + !filter.AnswerTimeStart.IsZero() && // With IsZero we keep backwards compatible with APIerSv1 + cdr.AnswerTime.Before(*filter.AnswerTimeStart) { + continue } - if filter.AnswerTimeEnd != nil && !filter.AnswerTimeEnd.IsZero() { - if cdr.AnswerTime.After(*filter.AnswerTimeEnd) { - continue - } + if filter.AnswerTimeEnd != nil && + !filter.AnswerTimeEnd.IsZero() && + cdr.AnswerTime.After(*filter.AnswerTimeEnd) { + continue } - if filter.SetupTimeStart != nil && !filter.SetupTimeStart.IsZero() { - if cdr.SetupTime.Before(*filter.SetupTimeStart) { - continue - } + if filter.SetupTimeStart != nil && + !filter.SetupTimeStart.IsZero() && + cdr.SetupTime.Before(*filter.SetupTimeStart) { + continue } - if filter.SetupTimeEnd != nil && !filter.SetupTimeEnd.IsZero() { - if cdr.SetupTime.Before(*filter.SetupTimeEnd) { - continue - } + if filter.SetupTimeEnd != nil && + !filter.SetupTimeEnd.IsZero() && + cdr.SetupTime.Before(*filter.SetupTimeEnd) { + continue } - if len(filter.MinUsage) != 0 { - if cdr.Usage < minUsage { - continue - } + if len(filter.MinUsage) != 0 && + cdr.Usage < minUsage { + continue } - if len(filter.MaxUsage) != 0 { - if cdr.Usage > maxUsage { - continue - } + if len(filter.MaxUsage) != 0 && + cdr.Usage > maxUsage { + continue } if filter.MinCost != nil { @@ -1579,20 +1574,16 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C if cdr.Cost < 0 { continue } - } else { - if cdr.Cost < *filter.MinCost || cdr.Cost > *filter.MaxCost { - continue - } + } else if cdr.Cost < *filter.MinCost || cdr.Cost > *filter.MaxCost { + continue } } else if filter.MaxCost != nil { if *filter.MaxCost == -1.0 { // Non-rated CDRs if cdr.Cost < 0 { continue } - } else { // Above limited CDRs, since MinCost is empty, make sure we query also NULL cost - if cdr.Cost >= *filter.MaxCost { - continue - } + } else if cdr.Cost >= *filter.MaxCost { // Above limited CDRs, since MinCost is empty, make sure we query also NULL cost + continue } } if len(filter.ExtraFields) != 0 { @@ -1631,16 +1622,14 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C } } - if filter.Paginator.Offset != nil { - if paginatorOffsetCounter <= *filter.Paginator.Offset { - paginatorOffsetCounter++ - continue - } + if filter.Paginator.Offset != nil && + paginatorOffsetCounter <= *filter.Paginator.Offset { + paginatorOffsetCounter++ + continue } - if filter.Paginator.Limit != nil { - if len(cdrs) >= *filter.Paginator.Limit { - break - } + if filter.Paginator.Limit != nil && + len(cdrs) >= *filter.Paginator.Limit { + break } //pass all filters and append to slice cdrs = append(cdrs, cdr) @@ -1655,6 +1644,9 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C } return nil, 0, nil } + if len(cdrs) == 0 { + return nil, 0, utils.ErrNotFound + } if filter.OrderBy != "" { separateVals := strings.Split(filter.OrderBy, utils.INFIELD_SEP) ascendent := true diff --git a/engine/storage_sql.go b/engine/storage_sql.go index b29b56d1e..10ad62817 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -235,10 +235,9 @@ func (self *SQLStorage) RemTpData(table, tpid string, args map[string]string) er if len(table) == 0 { // Remove tpid out of all tables for _, tblName := range []string{utils.TBLTPTimings, utils.TBLTPDestinations, utils.TBLTPRates, utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, utils.TBLTPRatingProfiles, - utils.TBLTPSharedGroups, utils.TBLTPActions, utils.TBLTPActionPlans, - utils.TBLTPActionTriggers, utils.TBLTPAccountActions, - utils.TBLTPResources, utils.TBLTPStats, utils.TBLTPFilters, - utils.TBLTPRoutes, utils.TBLTPAttributes, utils.TBLTPRateProfiles, + utils.TBLTPSharedGroups, utils.TBLTPActions, utils.TBLTPActionTriggers, + utils.TBLTPAccountActions, utils.TBLTPResources, utils.TBLTPStats, utils.TBLTPThresholds, + utils.TBLTPFilters, utils.TBLTPActionPlans, utils.TBLTPRoutes, utils.TBLTPAttributes, utils.TBLTPChargers, utils.TBLTPDispatchers, utils.TBLTPDispatcherHosts} { if err := tx.Table(tblName).Where("tpid = ?", tpid).Delete(nil).Error; err != nil { tx.Rollback() diff --git a/ers/sql_it_test.go b/ers/sql_it_test.go index 601d6bdb1..5b829cc0b 100644 --- a/ers/sql_it_test.go +++ b/ers/sql_it_test.go @@ -151,14 +151,22 @@ func testSQLInitDB(t *testing.T) { if err != nil { t.Fatal(err) } - if !db.HasTable("cdrs") { - db = db.CreateTable(new(engine.CDRsql)) - } - if !db.HasTable("cdrs2") { - db = db.CreateTable(new(testModelSql)) - } - db = db.Table(utils.CDRsTBL) tx := db.Begin() + if !tx.HasTable("cdrs") { + tx = tx.CreateTable(new(engine.CDRsql)) + if err = tx.Error; err != nil { + tx.Rollback() + t.Fatal(err) + } + } + if !tx.HasTable("cdrs2") { + tx = tx.CreateTable(new(testModelSql)) + if err = tx.Error; err != nil { + tx.Rollback() + t.Fatal(err) + } + } + tx = tx.Table(utils.CDRsTBL) cdrSql := cdr.AsCDRsql() cdrSql.CreatedAt = time.Now() saved := tx.Save(cdrSql) @@ -167,6 +175,7 @@ func testSQLInitDB(t *testing.T) { t.Fatal(err) } tx.Commit() + time.Sleep(10 * time.Millisecond) } func testSQLReader(t *testing.T) { @@ -285,6 +294,7 @@ func testSQLPoster(t *testing.T) { func testSQLStop(t *testing.T) { rdrExit <- struct{}{} db = db.DropTable("cdrs2") + db = db.DropTable("cdrs") if err := db.Close(); err != nil { t.Error(err) } diff --git a/general_tests/cdrs_it_test.go b/general_tests/cdrs_it_test.go index 845a841a0..bcd79eeae 100644 --- a/general_tests/cdrs_it_test.go +++ b/general_tests/cdrs_it_test.go @@ -364,7 +364,7 @@ func testV2CDRsGetCdrs3(t *testing.T) { } args = utils.RPCCDRsFilter{RunIDs: []string{"CustomerCharges"}, OriginIDs: []string{"testV2CDRsProcessCDR3"}} if err := cdrsRpc.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err == nil || err.Error() != utils.ErrNotFound.Error() { - t.Error("Unexpected error: ", err.Error()) + t.Error("Unexpected error: ", err) } } @@ -461,7 +461,7 @@ func testV2CDRsGetCdrs5(t *testing.T) { OriginIDs: []string{"testV2CDRsProcessCDR5"}, } if err := cdrsRpc.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err == nil || err.Error() != utils.ErrNotFound.Error() { - t.Fatal("Unexpected error: ", err.Error()) + t.Fatal("Unexpected error: ", err) } args = utils.RPCCDRsFilter{ RunIDs: []string{"CustomerCharges"}, diff --git a/general_tests/cdrs_onlexp_it_test.go b/general_tests/cdrs_onlexp_it_test.go index 377580f99..72dd11e16 100644 --- a/general_tests/cdrs_onlexp_it_test.go +++ b/general_tests/cdrs_onlexp_it_test.go @@ -571,4 +571,22 @@ func testCDRsOnExpStopEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { t.Error(err) } + conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/") + if err != nil { + t.Fatal(err) + } + + ch, err := conn.Channel() + if err != nil { + t.Fatal(err) + } + defer ch.Close() + + if _, err = ch.QueueDelete("cgrates_cdrs", false, false, true); err != nil { + t.Fatal(err) + } + + if _, err = ch.QueueDelete("queue1", false, false, true); err != nil { + t.Fatal(err) + } } diff --git a/integration_test.sh b/integration_test.sh index 182670b0c..e481365b1 100755 --- a/integration_test.sh +++ b/integration_test.sh @@ -37,7 +37,7 @@ results+=($?) echo "go test github.com/cgrates/cgrates/apier/v1 -tags=offline $@" go test github.com/cgrates/cgrates/apier/v1 -tags=offline $@ results+=($?) -echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=offline $@" +echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration $@" go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration $@ results+=($?) else @@ -72,7 +72,7 @@ results+=($?) echo 'go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*internal' go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*internal results+=($?) -echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=offline -dbtype=*internal" +echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*internal" go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*internal results+=($?) # SQL @@ -106,7 +106,7 @@ results+=($?) echo 'go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*mysql' go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*mysql results+=($?) -echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=offline -dbtype=*mysql" +echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*mysql" go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*mysql results+=($?) # Mongo @@ -140,7 +140,7 @@ results+=($?) echo 'go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*mongo' go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*mongo results+=($?) -echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=offline -dbtype=*mongo" +echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*mongo" go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*mongo results+=($?) # Postgres @@ -174,9 +174,10 @@ results+=($?) echo 'go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*postgres' go test github.com/cgrates/cgrates/apier/v1 -tags=offline -dbtype=*postgres results+=($?) -echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=offline -dbtype=*postgres" +echo "go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*postgres" go test github.com/cgrates/cgrates/cmd/cgr-loader -tags=integration -dbtype=*postgres results+=($?) + fi echo 'go test github.com/cgrates/cgrates/config -tags=integration'