From 98a044b3fe2babbf6da175b5f464a9d16d3ed868 Mon Sep 17 00:00:00 2001 From: adi Date: Tue, 29 Nov 2022 17:04:46 +0200 Subject: [PATCH] Started to add reverse filter indexes --- engine/datamanager.go | 11 +++ engine/filterindexer.go | 9 ++- engine/libindex.go | 76 +++++++++++++++++++ general_tests/filter_indexes_cases_it_test.go | 4 +- utils/consts.go | 8 +- 5 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 engine/libindex.go diff --git a/engine/datamanager.go b/engine/datamanager.go index 14de35d12..7fadd537e 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -627,9 +627,20 @@ func GetFilter(dm *DataManager, tenant, id string, cacheRead, cacheWrite bool, } func (dm *DataManager) SetFilter(fltr *Filter) (err error) { + var oldFltr *Filter + oldFltr, err = GetFilter(dm, fltr.Tenant, fltr.ID, true, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetFilterDrv(fltr); err != nil { return } + if oldFltr != nil { + /* if err = UpdateFilterIndexes(dm, oldFltr, fltr); err != nil { + return + } */ + + } if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaFilters]; itm.Replicate { err = replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, config.CgrConfig().DataDbCfg().RplFiltered, diff --git a/engine/filterindexer.go b/engine/filterindexer.go index b1c5a5fbf..24cf03227 100644 --- a/engine/filterindexer.go +++ b/engine/filterindexer.go @@ -28,7 +28,10 @@ import ( // NewFilterIndexer creates the FilterIndexes without any indexes in it func NewFilterIndexer(dm *DataManager, itemType, dbKeySuffix string) *FilterIndexer { - return &FilterIndexer{dm: dm, itemType: itemType, dbKeySuffix: dbKeySuffix, + return &FilterIndexer{ + dm: dm, + itemType: itemType, + dbKeySuffix: dbKeySuffix, indexes: make(map[string]utils.StringMap), chngdIndxKeys: make(utils.StringMap)} } @@ -109,6 +112,9 @@ func (rfi *FilterIndexer) cacheRemItemType() { // ToDo: tune here by removing pe case utils.DispatcherProfilePrefix: Cache.Clear([]string{utils.CacheDispatcherFilterIndexes}) + + case utils.ReverseFilterIndexes: + Cache.Clear([]string{utils.CacheReverseFilterIndexes}) } } @@ -279,6 +285,7 @@ func (rfi *FilterIndexer) RemoveItemFromIndex(tenant, itemID string, oldFilters } } } + return rfi.StoreIndexes(false, utils.NonTransactional) } diff --git a/engine/libindex.go b/engine/libindex.go new file mode 100644 index 000000000..5bcd7da8f --- /dev/null +++ b/engine/libindex.go @@ -0,0 +1,76 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package engine + +import ( + "strings" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/guardian" + "github.com/cgrates/cgrates/utils" +) + +// UpdateFilterIndexes will update the indexes for every reference of a filter that exists in a profile. +// Every profile that contains the filters from oldFltr will be updated with the new values for newFltr. +// oldFltr and newFltr has the same tenant and ID. +func UpdateFilterIndexes(dm *DataManager, oldFltr *Filter, newFltr *Filter) (err error) { + return nil +} + +// addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes +func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) { + for _, fltrID := range filterIDs { + if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters + continue + } + tntCtx := utils.ConcatenatedKey(tnt, fltrID) + refID := guardian.Guardian.GuardIDs(utils.EmptyString, + config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntCtx) + var indexes map[string]utils.StringMap + if indexes, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntCtx, + utils.EmptyString, nil); err != nil { + if err != utils.ErrNotFound { + guardian.Guardian.UnguardIDs(refID) + return + } + err = nil + indexes = map[string]utils.StringMap{ + idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps + } + } + indexes[idxItmType] = map[string]bool{ + itemID: true, + } + // IT IS REMOVED IN StoreIndexes + /* // remove the old reference from cache in case + for idxKeyItmType := range indexes { + Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType), + true, utils.NonTransactional) + } */ + indexerKey := tnt + if ctx != utils.EmptyString { + indexerKey = utils.ConcatenatedKey(tnt, ctx) + } + fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey) + if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil { + return + } + } + return +} diff --git a/general_tests/filter_indexes_cases_it_test.go b/general_tests/filter_indexes_cases_it_test.go index 247e5f70b..696eb6952 100644 --- a/general_tests/filter_indexes_cases_it_test.go +++ b/general_tests/filter_indexes_cases_it_test.go @@ -100,6 +100,7 @@ var ( testFilterIndexesCasesSetDifferentFilters, testFilterIndexesCasesOverwriteAttributes, + testFilterIndexesCasesComputeAttributesIndexes, testFilterIndexesCasesGetIndexesAnyContextChanged, testFilterIndexesCasesGetIndexesSessionsContextChanged, @@ -266,7 +267,8 @@ func testFilterIndexesCasesGetIndexesAnyContext(t *testing.T) { "*prefix:~*req.Missed:1:ATTR_NO_FLTR2", "*string:~*req.Account:1001:ATTR_NO_FLTR1", "*string:~*req.Account:1002:ATTR_NO_FLTR1", - "*string:~*req.Category:call:ATTR_NO_FLTR2"} + "*string:~*req.Category:call:ATTR_NO_FLTR2", + } sort.Strings(expectedIndexes) var reply []string if err := fIdxCasesRPC.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil { diff --git a/utils/consts.go b/utils/consts.go index 7bff91b4d..6516ddf04 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -69,7 +69,7 @@ var ( CacheSupplierProfiles, CacheAttributeProfiles, CacheChargerProfiles, CacheDispatcherProfiles, CacheDispatcherHosts, CacheResourceFilterIndexes, CacheStatFilterIndexes, CacheThresholdFilterIndexes, CacheSupplierFilterIndexes, - CacheAttributeFilterIndexes, CacheChargerFilterIndexes, CacheDispatcherFilterIndexes, + CacheAttributeFilterIndexes, CacheChargerFilterIndexes, CacheDispatcherFilterIndexes, CacheReverseFilterIndexes, CacheDispatcherRoutes, CacheDiameterMessages, CacheRPCResponses, CacheClosedSessions, CacheCDRIDs, CacheLoadIDs, CacheRPCConnections, CacheRatingProfilesTmp}) CacheInstanceToPrefix = map[string]string{ @@ -102,6 +102,7 @@ var ( CacheAttributeFilterIndexes: AttributeFilterIndexes, CacheChargerFilterIndexes: ChargerFilterIndexes, CacheDispatcherFilterIndexes: DispatcherFilterIndexes, + CacheReverseFilterIndexes: ReverseFilterIndexes, CacheLoadIDs: LoadIDPrefix, CacheAccounts: ACCOUNT_PREFIX, } @@ -114,6 +115,7 @@ var ( AttributeProfilePrefix: CacheAttributeFilterIndexes, ChargerProfilePrefix: CacheChargerFilterIndexes, DispatcherProfilePrefix: CacheDispatcherFilterIndexes, + ReverseFilterIndexes: CacheReverseFilterIndexes, } CacheIndexesToPrefix map[string]string // will be built on init @@ -129,7 +131,7 @@ var ( CacheTimings, CacheStatQueueProfiles, CacheStatQueues, CacheThresholdProfiles, CacheThresholds, CacheFilters, CacheSupplierProfiles, CacheAttributeProfiles, CacheChargerProfiles, CacheDispatcherProfiles, CacheDispatcherHosts, CacheResourceFilterIndexes, CacheStatFilterIndexes, - CacheThresholdFilterIndexes, CacheSupplierFilterIndexes, CacheAttributeFilterIndexes, + CacheThresholdFilterIndexes, CacheSupplierFilterIndexes, CacheAttributeFilterIndexes, CacheReverseFilterIndexes, CacheChargerFilterIndexes, CacheDispatcherFilterIndexes, CacheLoadIDs, CacheAccounts}) // ProtectedSFlds are the fields that sessions should not alter @@ -1518,6 +1520,7 @@ const ( CacheAttributeFilterIndexes = "*attribute_filter_indexes" CacheChargerFilterIndexes = "*charger_filter_indexes" CacheDispatcherFilterIndexes = "*dispatcher_filter_indexes" + CacheReverseFilterIndexes = "*reverse_filter_indexes" CacheDiameterMessages = "*diameter_messages" CacheRPCResponses = "*rpc_responses" CacheClosedSessions = "*closed_sessions" @@ -1541,6 +1544,7 @@ const ( ChargerFilterIndexes = "cfi_" DispatcherFilterIndexes = "dfi_" ActionPlanIndexes = "api_" + ReverseFilterIndexes = "fii_" ) // Agents