From d95c5e85b3612bb554e2f258055b6e89d5f6bd55 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 2 Mar 2018 18:06:33 +0100 Subject: [PATCH] DataManager.GetFilterIndexes with cacheID and itemIDPrefix, utils.CachePrefixToInstance, utils.PrefixToRevIndexCache --- apier/v1/filter_indexes.go | 4 +-- apier/v1/filterindexecache_it_test.go | 40 ++++++++++++++------------- apier/v1/resourcesv1.go | 5 +++- engine/account_test.go | 6 ++-- engine/attributes_test.go | 15 ++++------ engine/datamanager.go | 5 ++-- engine/filterindexer.go | 2 +- engine/resources.go | 2 ++ engine/storage_interface.go | 2 +- engine/storage_map.go | 9 +++--- engine/storage_mongo_datadb.go | 6 ++-- engine/storage_redis.go | 3 +- utils/consts.go | 14 ++++++++++ 13 files changed, 68 insertions(+), 45 deletions(-) diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index a8f186a7d..f634a800e 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -59,8 +59,8 @@ func (self *ApierV1) GetFilterIndexes(arg AttrGetFilterIndexes, reply *[]string) arg.ItemType = utils.AttributeProfilePrefix key = utils.ConcatenatedKey(arg.Tenant, arg.Context) } - if indexes, err = self.DataManager.GetFilterIndexes(engine.GetDBIndexKey(arg.ItemType, key, false), - "", nil); err != nil { + if indexes, err = self.DataManager.GetFilterIndexes( + utils.PrefixToIndexCache[arg.ItemType], key, "", nil); err != nil { return err } if arg.FilterType != "" { diff --git a/apier/v1/filterindexecache_it_test.go b/apier/v1/filterindexecache_it_test.go index 9984e92e9..aa98a2d8e 100644 --- a/apier/v1/filterindexecache_it_test.go +++ b/apier/v1/filterindexecache_it_test.go @@ -79,18 +79,14 @@ var sTestsFilterIndexesSV1Ca = []func(t *testing.T){ testV1FIdxCaRemoveResourceProfile, } -func TestFIdxCaV1ITMySQLConnect(t *testing.T) { +// Test start here +func TestFIdxCaV1ITMySQL(t *testing.T) { cfg, _ := config.NewDefaultCGRConfig() - rdsITdb, err = engine.NewRedisStorage(fmt.Sprintf("%s:%s", cfg.DataDbHost, cfg.DataDbPort), 10, + rdsITdb, err := engine.NewRedisStorage(fmt.Sprintf("%s:%s", cfg.DataDbHost, cfg.DataDbPort), 10, cfg.DataDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, nil, 1) - if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } -} - -// Test start here -func TestFIdxCaV1ITMySQL(t *testing.T) { onStor = engine.NewDataManager(rdsITdb) tSv1ConfDIR = "tutmysql" for _, stest := range sTestsFilterIndexesSV1Ca { @@ -98,20 +94,18 @@ func TestFIdxCaV1ITMySQL(t *testing.T) { } } -func TestFIdxCaV1ITMongoConnect(t *testing.T) { +func TestFIdxCaV1ITMongo(t *testing.T) { cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "tutmongo") mgoITCfg, err := config.NewCGRConfigFromFolder(cdrsMongoCfgPath) if err != nil { t.Fatal(err) } - if mgoITdb, err = engine.NewMongoStorage(mgoITCfg.DataDbHost, mgoITCfg.DataDbPort, + mgoITdb, err := engine.NewMongoStorage(mgoITCfg.DataDbHost, mgoITCfg.DataDbPort, mgoITCfg.DataDbName, mgoITCfg.DataDbUser, mgoITCfg.DataDbPass, - utils.DataDB, nil, mgoITCfg.CacheCfg(), mgoITCfg.LoadHistorySize); err != nil { + utils.DataDB, nil, mgoITCfg.CacheCfg(), mgoITCfg.LoadHistorySize) + if err != nil { t.Fatal(err) } -} - -func TestFIdxCaV1ITMongo(t *testing.T) { onStor = engine.NewDataManager(mgoITdb) tSv1ConfDIR = "tutmongo" time.Sleep(time.Duration(2 * time.Second)) // give time for engine to start @@ -413,7 +407,6 @@ func testV1FIdxCaUpdateThresholdProfileFromTP(t *testing.T) { } var reply *engine.ThresholdProfile - if err := tFIdxCaRpc.Call("ApierV1.GetThresholdProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "THD_ACNT_BALANCE_1"}, &reply); err != nil { t.Error(err) @@ -539,7 +532,8 @@ func testV1FIdxCaRemoveThresholdProfile(t *testing.T) { } //test to make sure indexes are made as expected fldNameVal2 := map[string]string{"THD_ACNT_BALANCE_1": "", "TEST_PROFILE1": ""} - if _, err = onStor.GetFilterReverseIndexes(engine.GetDBIndexKey(utils.ThresholdProfilePrefix, "cgrates.org", true), + if _, err = onStor.GetFilterReverseIndexes( + engine.GetDBIndexKey(utils.ThresholdProfilePrefix, "cgrates.org", true), fldNameVal2); err == nil || err != utils.ErrNotFound { t.Error(err) } @@ -1263,7 +1257,8 @@ func testV1FIdxCaRemoveAttributeProfile(t *testing.T) { } //test to make sure indexes are made as expected fldNameVal2 := map[string]string{"ATTR_1": "", "TEST_PROFILE1": ""} - if _, err = onStor.GetFilterReverseIndexes(engine.GetDBIndexKey(utils.AttributeProfilePrefix, "cgrates.org:*rating", true), + if _, err = onStor.GetFilterReverseIndexes( + engine.GetDBIndexKey(utils.AttributeProfilePrefix, "cgrates.org:*rating", true), fldNameVal2); err == nil || err != utils.ErrNotFound { t.Error(err) } @@ -1423,9 +1418,15 @@ func testV1FIdxCaGetResourceProfileFromTP(t *testing.T) { t.Error("Unexpected reply returned", reply) } - idx := map[string]utils.StringMap{"ResGroup1": {"*default:*any:*any": true, "*prefix:Destination:10": true, "*prefix:Destination:20": true, "*string:Account:1001": true, "*string:Account:1002": true}} + idx := map[string]utils.StringMap{ + "ResGroup1": {"*default:*any:*any": true, + "*prefix:Destination:10": true, + "*prefix:Destination:20": true, + "*string:Account:1001": true, + "*string:Account:1002": true}} fldNameVal := map[string]string{"ResGroup1": ""} - if indexes, err = onStor.GetFilterReverseIndexes(engine.GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", true), + if indexes, err = onStor.GetFilterReverseIndexes( + engine.GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", true), fldNameVal); err != nil { t.Error(err) } @@ -1480,7 +1481,8 @@ func testV1FIdxCaUpdateResourceProfile(t *testing.T) { Weight: 20, ThresholdIDs: []string{"Val1", "Val2"}, } - if err := tFIdxCaRpc.Call("ApierV1.SetResourceProfile", rlsConfig, &result); err != nil { + if err := tFIdxCaRpc.Call("ApierV1.SetResourceProfile", + rlsConfig, &result); err != nil { t.Error(err) } else if result != utils.OK { t.Error("Unexpected reply returned", result) diff --git a/apier/v1/resourcesv1.go b/apier/v1/resourcesv1.go index e7d286425..0ba8e8566 100644 --- a/apier/v1/resourcesv1.go +++ b/apier/v1/resourcesv1.go @@ -81,7 +81,10 @@ func (apierV1 *ApierV1) SetResourceProfile(res *engine.ResourceProfile, reply *s if err := apierV1.DataManager.SetResourceProfile(res, true); err != nil { return utils.APIErrorHandler(err) } - if err := apierV1.DataManager.SetResource(&engine.Resource{Tenant: res.Tenant, ID: res.ID, Usages: make(map[string]*engine.ResourceUsage)}); err != nil { + if err := apierV1.DataManager.SetResource( + &engine.Resource{Tenant: res.Tenant, + ID: res.ID, + Usages: make(map[string]*engine.ResourceUsage)}); err != nil { return utils.APIErrorHandler(err) } *reply = utils.OK diff --git a/engine/account_test.go b/engine/account_test.go index a4c698cdd..881fccaeb 100644 --- a/engine/account_test.go +++ b/engine/account_test.go @@ -26,8 +26,10 @@ import ( ) var ( - NAT = &Destination{Id: "NAT", Prefixes: []string{"0257", "0256", "0723"}} - RET = &Destination{Id: "RET", Prefixes: []string{"0723", "0724"}} + NAT = &Destination{Id: "NAT", + Prefixes: []string{"0257", "0256", "0723"}} + RET = &Destination{Id: "RET", + Prefixes: []string{"0723", "0724"}} ) func TestBalanceStoreRestore(t *testing.T) { diff --git a/engine/attributes_test.go b/engine/attributes_test.go index 77019cd88..12f70f52f 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -519,9 +519,8 @@ func TestAttributeIndexer(t *testing.T) { } rfi1 := NewFilterIndexer(dmAtr, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrPrf.Tenant, utils.META_ANY)) - if rcvIdx, err := dmAtr.GetFilterIndexes( - GetDBIndexKey(rfi1.itemType, rfi1.dbKeySuffix, false), MetaString, - nil); err != nil { + if rcvIdx, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType], + rfi1.dbKeySuffix, MetaString, nil); err != nil { t.Error(err) } else { if !reflect.DeepEqual(eIdxes, rcvIdx) { @@ -544,9 +543,8 @@ func TestAttributeIndexer(t *testing.T) { } rfi2 := NewFilterIndexer(dmAtr, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrPrf.Tenant, utils.MetaSessionS)) - if rcvIdx, err := dmAtr.GetFilterIndexes( - GetDBIndexKey(rfi2.itemType, rfi2.dbKeySuffix, false), MetaString, - nil); err != nil { + if rcvIdx, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi2.itemType], + rfi2.dbKeySuffix, MetaString, nil); err != nil { t.Error(err) } else { if !reflect.DeepEqual(eIdxes, rcvIdx) { @@ -563,9 +561,8 @@ func TestAttributeIndexer(t *testing.T) { } } //verify if old index was deleted ( context *any) - if _, err := dmAtr.GetFilterIndexes( - GetDBIndexKey(rfi1.itemType, rfi1.dbKeySuffix, false), MetaString, - nil); err != utils.ErrNotFound { + if _, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType], + rfi1.dbKeySuffix, MetaString, nil); err != utils.ErrNotFound { t.Error(err) } if _, err := dmAtr.GetFilterReverseIndexes( diff --git a/engine/datamanager.go b/engine/datamanager.go index 24c32c7da..0c1c84873 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -1107,8 +1107,9 @@ func (dm *DataManager) HasData(category, subject, tenant string) (has bool, err return dm.DataDB().HasDataDrv(category, subject, tenant) } -func (dm *DataManager) GetFilterIndexes(dbKey, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { - return dm.DataDB().GetFilterIndexesDrv(dbKey, filterType, fldNameVal) +func (dm *DataManager) GetFilterIndexes(cacheID, itemIDPrefix, filterType string, + fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { + return dm.DataDB().GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType, fldNameVal) } func (dm *DataManager) SetFilterIndexes(dbKey string, indexes map[string]utils.StringMap, commit bool, transactionID string) (err error) { diff --git a/engine/filterindexer.go b/engine/filterindexer.go index 3e83e8e9a..72f6225be 100644 --- a/engine/filterindexer.go +++ b/engine/filterindexer.go @@ -159,7 +159,7 @@ func (rfi *FilterIndexer) loadItemReverseIndex(filterType, itemID string) (err e //Populate FilterIndexer.indexes with specific fieldName:fieldValue , item func (rfi *FilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal string) error { rcvIdx, err := rfi.dm.GetFilterIndexes( - GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), filterType, + utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, filterType, map[string]string{fldName: fldVal}) if err != nil { return err diff --git a/engine/resources.go b/engine/resources.go index b0725afcc..978f2d266 100755 --- a/engine/resources.go +++ b/engine/resources.go @@ -255,6 +255,8 @@ func (rs Resources) allocateResource(ru *ResourceUsage, dryRun bool) (alcMessage r.removeExpiredUnits() if _, hasID := r.Usages[ru.ID]; hasID { // update r.clearUsage(ru.ID) + if r.rPrf == nil { + return "", fmt.Errorf("empty configuration for resourceID: %s", r.TenantID()) } if r.rPrf.Limit >= r.totalUsage()+ru.Units { if alcMessage == "" { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 6c9b2cbae..0c79f5c3c 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -118,7 +118,7 @@ type DataDB interface { RemoveTimingDrv(string) error GetLoadHistory(int, bool, string) ([]*utils.LoadInstance, error) AddLoadHistory(*utils.LoadInstance, int, string) error - GetFilterIndexesDrv(dbKey, filterType string, + GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) SetFilterIndexesDrv(dbKey string, indexes map[string]utils.StringMap, commit bool, transactionID string) (err error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 9024c74a9..bf9047381 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1231,18 +1231,18 @@ func (ms *MapStorage) RemoveTimingDrv(id string) error { } //GetFilterIndexesDrv retrieves Indexes from dataDB -func (ms *MapStorage) GetFilterIndexesDrv(dbKey, filterType string, +func (ms *MapStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { ms.mu.RLock() defer ms.mu.RUnlock() + dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix values, ok := ms.dict[dbKey] if !ok { return nil, utils.ErrNotFound } if len(fldNameVal) != 0 { rcvidx := make(map[string]utils.StringMap) - err = ms.ms.Unmarshal(values, &rcvidx) - if err != nil { + if err = ms.ms.Unmarshal(values, &rcvidx); err != nil { return nil, err } indexes = make(map[string]utils.StringMap) @@ -1256,8 +1256,7 @@ func (ms *MapStorage) GetFilterIndexesDrv(dbKey, filterType string, } return } else { - err = ms.ms.Unmarshal(values, &indexes) - if err != nil { + if err = ms.ms.Unmarshal(values, &indexes); err != nil { return nil, err } if len(indexes) == 0 { diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index ddda83eb6..88ed07cd5 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1934,7 +1934,7 @@ func (ms *MongoStorage) RemoveTimingDrv(id string) (err error) { } // GetFilterIndexesDrv retrieves Indexes from dataDB -func (ms *MongoStorage) GetFilterIndexesDrv(dbKey, filterType string, +func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { session, col := ms.conn(colRFI) defer session.Close() @@ -1942,10 +1942,12 @@ func (ms *MongoStorage) GetFilterIndexesDrv(dbKey, filterType string, Key string Value map[string][]string } + dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix findParam := bson.M{"key": dbKey} if len(fldNameVal) != 0 { for fldName, fldValue := range fldNameVal { - qryFltr := fmt.Sprintf("value.%s", utils.ConcatenatedKey(filterType, fldName, fldValue)) + qryFltr := fmt.Sprintf("value.%s", + utils.ConcatenatedKey(filterType, fldName, fldValue)) if err = col.Find(bson.M{"key": dbKey, qryFltr: bson.M{"$exists": true}}).Select( bson.M{qryFltr: true}).One(&result); err != nil { if err == mgo.ErrNotFound { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 3d8c8e7a9..340ef33c5 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1369,9 +1369,10 @@ func (rs *RedisStorage) RemoveTimingDrv(id string) (err error) { } //GetFilterIndexesDrv retrieves Indexes from dataDB -func (rs *RedisStorage) GetFilterIndexesDrv(dbKey, filterType string, +func (rs *RedisStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { mp := make(map[string]string) + dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix if len(fldNameVal) == 0 { mp, err = rs.Cmd("HGETALL", dbKey).Map() if err != nil { diff --git a/utils/consts.go b/utils/consts.go index 12f035dbf..c382a4cde 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -78,6 +78,20 @@ var ( CacheAttributeFilterRevIndexes: AttributeFilterRevIndexes, } CachePrefixToInstance map[string]string // will be built on init + PrefixToIndexCache = map[string]string{ + ThresholdProfilePrefix: CacheThresholdFilterIndexes, + ResourceProfilesPrefix: CacheResourceFilterIndexes, + StatQueueProfilePrefix: CacheStatFilterIndexes, + SupplierProfilePrefix: CacheSupplierFilterIndexes, + AttributeProfilePrefix: CacheAttributeFilterIndexes, + } + PrefixToRevIndexCache = map[string]string{ + ThresholdProfilePrefix: CacheThresholdFilterRevIndexes, + ResourceProfilesPrefix: CacheResourceFilterRevIndexes, + StatQueueProfilePrefix: CacheStatFilterRevIndexes, + SupplierProfilePrefix: CacheSupplierFilterRevIndexes, + AttributeProfilePrefix: CacheAttributeFilterRevIndexes, + } ) const (