From aeb1cf349a51f3835f0cdc32f3c6b08f7b3a3118 Mon Sep 17 00:00:00 2001 From: adi Date: Mon, 5 Dec 2022 17:12:35 +0200 Subject: [PATCH] Reverse filter indexes updates + tests --- config/config_defaults.go | 1 + config/config_json_test.go | 2 + config/config_test.go | 2 + engine/datamanager.go | 8 ++ engine/libindex.go | 66 ++++++++++++---- general_tests/filter_indexes_cases_it_test.go | 77 ++++++++++++++++++- 6 files changed, 142 insertions(+), 14 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 7917ee84d..51c0ac1ee 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -216,6 +216,7 @@ const CGRATES_CFG_JSON = ` "*attribute_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control attribute filter indexes caching "*charger_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control charger filter indexes caching "*dispatcher_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control dispatcher filter indexes caching + "*reverse_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control reverse filter indexes caching "*dispatcher_routes": {"limit": -1, "ttl": "", "static_ttl": false}, // control dispatcher routes caching "*diameter_messages": {"limit": -1, "ttl": "3h", "static_ttl": false}, // diameter messages caching "*rpc_responses": {"limit": 0, "ttl": "2s", "static_ttl": false}, // RPC responses caching diff --git a/config/config_json_test.go b/config/config_json_test.go index 7ef45456a..b23ecc770 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -74,6 +74,8 @@ func TestDfGeneralJsonCfg(t *testing.T) { func TestCacheJsonCfg(t *testing.T) { eCfg := &CacheJsonCfg{ + utils.CacheReverseFilterIndexes: &CacheParamJsonCfg{Limit: utils.IntPointer(-1), + Ttl: utils.StringPointer(""), Static_ttl: utils.BoolPointer(false)}, utils.CacheDestinations: &CacheParamJsonCfg{Limit: utils.IntPointer(-1), Ttl: utils.StringPointer(""), Static_ttl: utils.BoolPointer(false), Precache: utils.BoolPointer(false)}, diff --git a/config/config_test.go b/config/config_test.go index 9641d2aa2..9be2d21ce 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -624,6 +624,8 @@ func TestCgrCfgJSONDefaultsSMGenericCfg(t *testing.T) { func TestCgrCfgJSONDefaultsCacheCFG(t *testing.T) { eCacheCfg := CacheCfg{ + utils.CacheReverseFilterIndexes: &CacheParamCfg{Limit: -1, + TTL: time.Duration(0), StaticTTL: false, Precache: false}, utils.CacheDestinations: &CacheParamCfg{Limit: -1, TTL: time.Duration(0), StaticTTL: false, Precache: false}, utils.CacheReverseDestinations: &CacheParamCfg{Limit: -1, diff --git a/engine/datamanager.go b/engine/datamanager.go index 614f18022..002b8543a 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -2152,12 +2152,20 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil { return } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString, + cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil { + return + } } } if err = createAndIndex(utils.ChargerProfilePrefix, cpp.Tenant, utils.EmptyString, cpp.ID, cpp.FilterIDs, dm); err != nil { return } + if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString, + cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil { + return + } } if config.CgrConfig().DataDbCfg().Items[utils.MetaChargerProfiles].Replicate { var reply string diff --git a/engine/libindex.go b/engine/libindex.go index ab33463c2..a6caf2823 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -34,7 +34,55 @@ func UpdateFilterIndexes(dm *DataManager, oldFltr *Filter, newFltr *Filter) (err } // addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes -func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) { +func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, + itemID string, filterIDs []string) (err error) { + for _, fltrID := range filterIDs { + if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters + continue + } + + tntFltrID := utils.ConcatenatedKey(tnt, fltrID) + refID := guardian.Guardian.GuardIDs(utils.EmptyString, + config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntFltrID) + var indexes map[string]utils.StringMap + if indexes, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntFltrID, + utils.EmptyString, nil); err != nil { + + if err != utils.ErrNotFound { + guardian.Guardian.UnguardIDs(refID) + return + } + err = nil + indexes = map[string]utils.StringMap{ + idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps + } + } + indexes[idxItmType] = map[string]bool{ + itemID: true, + } + // it is removed in StoreIndexes + /* // remove the old reference from cache in case + for idxKeyItmType := range indexes { + Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType), + true, utils.NonTransactional) + } */ + indexerKey := utils.ConcatenatedKey(tnt, fltrID) + if ctx != utils.EmptyString { + indexerKey = utils.ConcatenatedKey(tnt, ctx) + } + fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey) + fltrIndexer.indexes = indexes + if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil { + guardian.Guardian.UnguardIDs(refID) + return + } + guardian.Guardian.UnguardIDs(refID) + } + return +} + +// removeReverseFilterIndexForFilter will remove a reference for the filter in reverse filter indexes +func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) { for _, fltrID := range filterIDs { if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters continue @@ -50,19 +98,11 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemI return } err = nil - indexes = map[string]utils.StringMap{ - idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps - } + continue // already removed } - indexes[idxItmType] = map[string]bool{ - itemID: true, - } - // IT IS REMOVED IN StoreIndexes - /* // remove the old reference from cache in case - for idxKeyItmType := range indexes { - Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType), - true, utils.NonTransactional) - } */ + + delete(indexes[idxItmType], itemID) // delete index from map + indexerKey := tnt if ctx != utils.EmptyString { indexerKey = utils.ConcatenatedKey(tnt, ctx) diff --git a/general_tests/filter_indexes_cases_it_test.go b/general_tests/filter_indexes_cases_it_test.go index 696eb6952..9a74692af 100644 --- a/general_tests/filter_indexes_cases_it_test.go +++ b/general_tests/filter_indexes_cases_it_test.go @@ -480,6 +480,65 @@ func testFilterIndexesCasesSetIndexedFilter(t *testing.T) { } else if result != utils.OK { t.Error("Unexpected reply returned", result) } + + /* + filter1 = &v1.FilterWithCache{ + Filter: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_Charger12312", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.CGRID", + Values: []string{"tester_id"}, + }, + { + Type: utils.MetaPrefix, + Element: "~*req.AnswerTime", + Values: []string{"2022"}, + }, + { + Type: utils.MetaSuffix, + Element: "~*req.AnswerTime", + Values: []string{"202"}, + }, + }, + }, + } + filter2 := &v1.FilterWithCache{ + Filter: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_Charger4564", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.CGRID", + Values: []string{"tester_id"}, + }, + { + Type: utils.MetaPrefix, + Element: "~*req.AnswerTime", + Values: []string{"2022"}, + }, + { + Type: utils.MetaSuffix, + Element: "~*req.AnswerTime", + Values: []string{"202"}, + }, + }, + }, + } + if err := fIdxCasesRPC.Call(utils.APIerSv1SetFilter, filter1, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } + if err := fIdxCasesRPC.Call(utils.APIerSv1SetFilter, filter2, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } + */ } func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) { @@ -487,7 +546,7 @@ func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) { ChargerProfile: &engine.ChargerProfile{ Tenant: "cgrates.org", ID: "ChrgerIndexable", - FilterIDs: []string{"FLTR_Charger"}, + FilterIDs: []string{"FLTR_Charger" /*"FLTR_Charger12312", "FLTR_Charger4564"*/}, RunID: utils.MetaRaw, AttributeIDs: []string{"ATTR_FLTR1"}, Weight: 20, @@ -499,6 +558,22 @@ func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) { } else if result != utils.OK { t.Error("Unexpected reply returned", result) } + /* chargerProfile = + &v1.ChargerWithCache{ + ChargerProfile: &engine.ChargerProfile{ + Tenant: "cgrates.org", + ID: "ChrgerIndexable222", + FilterIDs: []string{"FLTR_Charger", "FLTR_Charger4564"}, + RunID: utils.MetaRaw, + AttributeIDs: []string{"ATTR_FLTR1"}, + Weight: 20, + }, + } + if err := fIdxCasesRPC.Call(utils.APIerSv1SetChargerProfile, chargerProfile, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } */ } func testFilterIndexesCasesGetChargerIndexes(t *testing.T) {