From 45f638102dc3673820fed52d2202e7602f2dfcac Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 1 Mar 2018 17:40:27 +0100 Subject: [PATCH] MatchFilterIndexDrv properly prepending prefix instead of concatenating it --- engine/datamanager_it_test.go | 12 +++++------ engine/filterindexer_it_test.go | 35 ++++++++++++------------------- engine/onstor_it_test.go | 37 ++++++++++++++++----------------- engine/storage_mongo_datadb.go | 2 +- engine/storage_redis.go | 5 ++--- 5 files changed, 39 insertions(+), 52 deletions(-) diff --git a/engine/datamanager_it_test.go b/engine/datamanager_it_test.go index e86c48261..3caad4ad1 100644 --- a/engine/datamanager_it_test.go +++ b/engine/datamanager_it_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/cgrates/cgrates/cache" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) @@ -74,7 +73,7 @@ func testDMitDataFlush(t *testing.T) { if err := dm2.dataDB.Flush(""); err != nil { t.Error(err) } - cache.Flush() + Cache.Clear(nil) } func testDMitCRUDStatQueue(t *testing.T) { @@ -100,17 +99,16 @@ func testDMitCRUDStatQueue(t *testing.T) { }, }, } - cacheKey := utils.StatQueuePrefix + sq.TenantID() if _, rcvErr := dm2.GetStatQueue(sq.Tenant, sq.ID, false, ""); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if _, ok := cache.Get(cacheKey); ok != false { + if _, ok := Cache.Get(utils.CacheStatQueues, sq.TenantID()); ok != false { t.Error("should not be in cache") } if err := dm2.SetStatQueue(sq); err != nil { t.Error(err) } - if _, ok := cache.Get(cacheKey); ok != false { + if _, ok := Cache.Get(utils.CacheStatQueues, sq.TenantID()); ok != false { t.Error("should not be in cache") } if rcv, err := dm2.GetStatQueue(sq.Tenant, sq.ID, false, ""); err != nil { @@ -118,13 +116,13 @@ func testDMitCRUDStatQueue(t *testing.T) { } else if !reflect.DeepEqual(sq, rcv) { t.Errorf("expecting: %v, received: %v", sq, rcv) } - if _, ok := cache.Get(cacheKey); ok != true { + if _, ok := Cache.Get(utils.CacheStatQueues, sq.TenantID()); ok != true { t.Error("should be in cache") } if err := dm2.RemoveStatQueue(sq.Tenant, sq.ID, ""); err != nil { t.Error(err) } - if _, ok := cache.Get(cacheKey); ok != false { + if _, ok := Cache.Get(utils.CacheStatQueues, sq.TenantID()); ok != false { t.Error("should not be in cache") } if _, rcvErr := dm2.GetStatQueue(sq.Tenant, sq.ID, false, ""); rcvErr != utils.ErrNotFound { diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index 7ff418291..c7d8ee942 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -26,14 +26,11 @@ import ( "testing" "time" - "github.com/cgrates/cgrates/cache" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) var ( - redisDB *RedisStorage - mongoDB *MongoStorage dataManager *DataManager cfgDBName string ) @@ -45,7 +42,7 @@ var sTests = []func(t *testing.T){ testITSetFilterIndexes, testITGetFilterIndexes, testITMatchFilterIndex, - testITFlush, + /*testITFlush, testITIsDBEmpty, testITTestThresholdFilterIndexes, testITTestAttributeProfileFilterIndexes, @@ -58,40 +55,36 @@ var sTests = []func(t *testing.T){ testITIsDBEmpty, testITTestIndexingWithEmptyFltrID, testITTestIndexingWithEmptyFltrID2, + */ } -func TestITRedisConnect(t *testing.T) { +func TestFilterIndexerITRedis(t *testing.T) { cfg, _ := config.NewDefaultCGRConfig() - redisDB, err = NewRedisStorage(fmt.Sprintf("%s:%s", cfg.DataDbHost, cfg.DataDbPort), 4, + redisDB, err := NewRedisStorage(fmt.Sprintf("%s:%s", cfg.DataDbHost, cfg.DataDbPort), 4, cfg.DataDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, nil, 1) if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } cfgDBName = cfg.DataDbName -} - -func TestITRedis(t *testing.T) { dataManager = NewDataManager(redisDB) for _, stest := range sTests { t.Run("TestITRedis", stest) } } -func TestITMongoConnect(t *testing.T) { +func TestFilterIndexerITMongo(t *testing.T) { cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "cdrsv2mongo") mgoITCfg, err := config.NewCGRConfigFromFolder(cdrsMongoCfgPath) if err != nil { t.Fatal(err) } - if mongoDB, err = NewMongoStorage(mgoITCfg.StorDBHost, mgoITCfg.StorDBPort, + mongoDB, err := NewMongoStorage(mgoITCfg.StorDBHost, mgoITCfg.StorDBPort, mgoITCfg.StorDBName, mgoITCfg.StorDBUser, mgoITCfg.StorDBPass, - utils.StorDB, nil, mgoITCfg.CacheCfg(), mgoITCfg.LoadHistorySize); err != nil { + utils.StorDB, nil, mgoITCfg.CacheCfg(), mgoITCfg.LoadHistorySize) + if err != nil { t.Fatal(err) } cfgDBName = mgoITCfg.StorDBName -} - -func TestITMongo(t *testing.T) { dataManager = NewDataManager(mongoDB) for _, stest := range sTests { t.Run("TestITMongo", stest) @@ -102,7 +95,7 @@ func testITFlush(t *testing.T) { if err := dataManager.DataDB().Flush(""); err != nil { t.Error(err) } - cache.Flush() + Cache.Clear(nil) } func testITIsDBEmpty(t *testing.T) { @@ -213,14 +206,14 @@ func testITMatchFilterIndex(t *testing.T) { "RL2": true, } if rcvMp, err := dataManager.MatchFilterIndex( - GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), + utils.CacheResourceFilterIndexes, "cgrates.org", MetaString, "Account", "1002"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { t.Errorf("Expecting: %+v, received: %+v", eMp, rcvMp) } if _, err := dataManager.MatchFilterIndex( - GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), + utils.CacheResourceFilterIndexes, "cgrates.org", MetaString, "NonexistentField", "1002"); err == nil || err != utils.ErrNotFound { t.Error(err) } @@ -885,8 +878,7 @@ func testITTestIndexingWithEmptyFltrID(t *testing.T) { eMp := utils.StringMap{ "THD_Test": true, } - if rcvMp, err := dataManager.MatchFilterIndex( - GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), + if rcvMp, err := dataManager.MatchFilterIndex(utils.CacheThresholdFilterIndexes, th.Tenant, utils.MetaDefault, utils.META_ANY, utils.META_ANY); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { @@ -954,8 +946,7 @@ func testITTestIndexingWithEmptyFltrID2(t *testing.T) { eMp := utils.StringMap{ "SPL_Weight": true, } - if rcvMp, err := dataManager.MatchFilterIndex( - GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), + if rcvMp, err := dataManager.MatchFilterIndex(utils.CacheSupplierFilterIndexes, splProfile.Tenant, utils.MetaDefault, utils.META_ANY, utils.META_ANY); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index ce07f736e..6f00d16dc 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/cgrates/cgrates/cache" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) @@ -133,7 +132,7 @@ func testOnStorITFlush(t *testing.T) { if err := onStor.DataDB().Flush(""); err != nil { t.Error(err) } - cache.Flush() + Cache.Clear(nil) } func testOnStorITIsDBEmpty(t *testing.T) { @@ -194,19 +193,19 @@ func testOnStorITCacheDestinations(t *testing.T) { t.Error(err) } - if _, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); hasIt { + if _, hasIt := Cache.Get(utils.CacheDestinations, dst.Id); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, true); err != nil { // Should not cache due to mustBeCached t.Error(err) } - if _, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); hasIt { + if _, hasIt := Cache.Get(utils.CacheDestinations, dst.Id); hasIt { t.Error("Should not be in cache") } if err := onStor.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheDestinations, dst.Id); !hasIt { t.Error("Did not cache") } else if !reflect.DeepEqual(dst, itm.(*Destination)) { t.Error("Wrong item in the cache") @@ -219,7 +218,7 @@ func testOnStorITCacheReverseDestinations(t *testing.T) { t.Error(err) } for _, prfx := range dst.Prefixes { - if _, hasIt := cache.Get(utils.REVERSE_DESTINATION_PREFIX + dst.Id); hasIt { + if _, hasIt := Cache.Get(utils.CacheReverseDestinations, dst.Id); hasIt { t.Errorf("Prefix: %s already in cache", prfx) } } @@ -227,7 +226,7 @@ func testOnStorITCacheReverseDestinations(t *testing.T) { t.Error(err) } for _, prfx := range dst.Prefixes { - if itm, hasIt := cache.Get(utils.REVERSE_DESTINATION_PREFIX + prfx); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheReverseDestinations, prfx); !hasIt { t.Error("Did not cache") } else if !reflect.DeepEqual([]string{dst.Id}, itm.([]string)) { t.Error("Wrong item in the cache") @@ -279,13 +278,13 @@ func testOnStorITCacheActionPlan(t *testing.T) { } else if !reflect.DeepEqual(expectedCAp, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCAp, itm) } - if _, hasIt := cache.Get(utils.ACTION_PLAN_PREFIX + ap.Id); hasIt { + if _, hasIt := Cache.Get(utils.CacheActionPlans, ap.Id); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, []string{ap.Id}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.ACTION_PLAN_PREFIX + ap.Id); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheActionPlans, ap.Id); !hasIt { t.Error("Did not cache") } else if rcv := itm.(*ActionPlan); !reflect.DeepEqual(ap, rcv) { t.Errorf("Expecting: %+v, received: %+v", ap, rcv) @@ -304,13 +303,13 @@ func testOnStorITCacheAccountActionPlans(t *testing.T) { if err := onStor.DataDB().SetAccountActionPlans(acntID, aAPs, true); err != nil { t.Error(err) } - if _, hasIt := cache.Get(utils.AccountActionPlansPrefix + acntID); hasIt { + if _, hasIt := Cache.Get(utils.CacheAccountActionPlans, acntID); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.AccountActionPlansPrefix, []string{acntID}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.AccountActionPlansPrefix + acntID); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheAccountActionPlans, acntID); !hasIt { t.Error("Did not cache") } else if rcv := itm.([]string); !reflect.DeepEqual(aAPs, rcv) { t.Errorf("Expecting: %+v, received: %+v", aAPs, rcv) @@ -339,13 +338,13 @@ func testOnStorITCacheActionTriggers(t *testing.T) { } else if !reflect.DeepEqual(expectedCAt, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCAt, itm) } - if _, hasIt := cache.Get(utils.ACTION_TRIGGER_PREFIX + atsID); hasIt { + if _, hasIt := Cache.Get(utils.CacheActionTriggers, atsID); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, []string{atsID}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.ACTION_TRIGGER_PREFIX + atsID); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheActionTriggers, atsID); !hasIt { t.Error("Did not cache") } else if rcv := itm.(ActionTriggers); !reflect.DeepEqual(ats, rcv) { t.Errorf("Expecting: %+v, received: %+v", ats, rcv) @@ -369,13 +368,13 @@ func testOnStorITCacheDerivedChargers(t *testing.T) { if err := onStor.DataDB().SetDerivedChargers(keyDCS, dcs, utils.NonTransactional); err != nil { t.Error(err) } - if _, hasIt := cache.Get(utils.DERIVEDCHARGERS_PREFIX + keyDCS); hasIt { + if _, hasIt := Cache.Get(utils.CacheDerivedChargers, keyDCS); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{keyDCS}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.DERIVEDCHARGERS_PREFIX + keyDCS); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheDerivedChargers, keyDCS); !hasIt { t.Error("Did not cache") } else if rcv := itm.(*utils.DerivedChargers); !reflect.DeepEqual(dcs, rcv) { t.Errorf("Expecting: %+v, received: %+v", dcs, rcv) @@ -421,13 +420,13 @@ func testOnStorITCacheAlias(t *testing.T) { } else if !reflect.DeepEqual(expectedCA, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCA, itm) } - if _, hasIt := cache.Get(utils.ALIASES_PREFIX + als.GetId()); hasIt { + if _, hasIt := Cache.Get(utils.CacheAliases, als.GetId()); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.ALIASES_PREFIX, []string{als.GetId()}, false); err != nil { t.Error(err) } - if itm, hasIt := cache.Get(utils.ALIASES_PREFIX + als.GetId()); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheAliases, als.GetId()); !hasIt { t.Error("Did not cache") } else if rcv := itm.(*Alias); !reflect.DeepEqual(als, rcv) { t.Errorf("Expecting: %+v, received: %+v", als, rcv) @@ -468,14 +467,14 @@ func testOnStorITCacheReverseAlias(t *testing.T) { t.Error(err) } rvAlsID := strings.Join([]string{als.Values[1].Pairs["Account"]["dan"], "Account", als.Context}, "") - if _, hasIt := cache.Get(utils.REVERSE_ALIASES_PREFIX + rvAlsID); hasIt { + if _, hasIt := Cache.Get(utils.CacheReverseAliases, rvAlsID); hasIt { t.Error("Already in cache") } if err := onStor.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, []string{rvAlsID}, false); err != nil { t.Error(err) } eRvrsAls := []string{utils.ConcatenatedKey(als.GetId(), als.Values[1].DestinationId)} - if itm, hasIt := cache.Get(utils.REVERSE_ALIASES_PREFIX + rvAlsID); !hasIt { + if itm, hasIt := Cache.Get(utils.CacheReverseAliases, rvAlsID); !hasIt { t.Error("Did not cache") } else if rcv := itm.([]string); !reflect.DeepEqual(eRvrsAls, rcv) { t.Errorf("Expecting: %+v, received: %+v", eRvrsAls, rcv) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 6617989c2..ddda83eb6 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -2145,7 +2145,7 @@ func (ms *MongoStorage) MatchFilterIndexDrv(cacheID, itemIDPrefix, fldKey := fmt.Sprintf("value.%s", utils.ConcatenatedKey(filterType, fldName, fldVal)) if err = col.Find( - bson.M{"key": utils.ConcatenatedKey(utils.CacheInstanceToPrefix[cacheID], itemIDPrefix), + bson.M{"key": utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix, fldKey: bson.M{"$exists": true}}).Select(bson.M{fldKey: true}).One(&result); err != nil { if err == mgo.ErrNotFound { err = utils.ErrNotFound diff --git a/engine/storage_redis.go b/engine/storage_redis.go index cf2e482bf..3d8c8e7a9 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1221,7 +1221,7 @@ func (rs *RedisStorage) SetDerivedChargers(key string, if marshaled, err = rs.ms.Marshal(dcs); err != nil { return } - if err = rs.Cmd("SET", key, marshaled).Err; err != nil { + if err = rs.Cmd("SET", utils.DERIVEDCHARGERS_PREFIX+key, marshaled).Err; err != nil { return } return @@ -1524,8 +1524,7 @@ func (rs *RedisStorage) MatchFilterIndexDrv(cacheID, itemIDPrefix, filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) { fieldValKey := utils.ConcatenatedKey(filterType, fldName, fldVal) fldValBytes, err := rs.Cmd("HGET", - utils.ConcatenatedKey(utils.CacheInstanceToPrefix[cacheID], - itemIDPrefix), fieldValKey).Bytes() + utils.CacheInstanceToPrefix[cacheID]+itemIDPrefix, fieldValKey).Bytes() if err != nil { if err == redis.ErrRespNil { // did not find the destination err = utils.ErrNotFound