Started to add reverse filter indexes

engine/datamanager.go

@@ -627,9 +627,20 @@ func GetFilter(dm *DataManager, tenant, id string, cacheRead, cacheWrite bool,
 }
 
 func (dm *DataManager) SetFilter(fltr *Filter) (err error) {
+	var oldFltr *Filter
+	oldFltr, err = GetFilter(dm, fltr.Tenant, fltr.ID, true, false, utils.NonTransactional)
+	if err != nil && err != utils.ErrNotFound {
+		return err
+	}
 	if err = dm.DataDB().SetFilterDrv(fltr); err != nil {
 		return
 	}
+	if oldFltr != nil {
+		/* if err = UpdateFilterIndexes(dm, oldFltr, fltr); err != nil {
+			return
+		} */
+
+	}
 	if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaFilters]; itm.Replicate {
 		err = replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns,
 			config.CgrConfig().DataDbCfg().RplFiltered,

engine/filterindexer.go

@@ -28,7 +28,10 @@ import (
 
 // NewFilterIndexer creates the FilterIndexes without any indexes in it
 func NewFilterIndexer(dm *DataManager, itemType, dbKeySuffix string) *FilterIndexer {
-	return &FilterIndexer{dm: dm, itemType: itemType, dbKeySuffix: dbKeySuffix,
+	return &FilterIndexer{
+		dm:            dm,
+		itemType:      itemType,
+		dbKeySuffix:   dbKeySuffix,
 		indexes:       make(map[string]utils.StringMap),
 		chngdIndxKeys: make(utils.StringMap)}
 }
@@ -109,6 +112,9 @@ func (rfi *FilterIndexer) cacheRemItemType() { // ToDo: tune here by removing pe
 
 	case utils.DispatcherProfilePrefix:
 		Cache.Clear([]string{utils.CacheDispatcherFilterIndexes})
+
+	case utils.ReverseFilterIndexes:
+		Cache.Clear([]string{utils.CacheReverseFilterIndexes})
 	}
 }
 
@@ -279,6 +285,7 @@ func (rfi *FilterIndexer) RemoveItemFromIndex(tenant, itemID string, oldFilters
 			}
 		}
 	}
+
 	return rfi.StoreIndexes(false, utils.NonTransactional)
 }
 

engine/libindex.go

@@ -0,0 +1,76 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package engine
+
+import (
+	"strings"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/guardian"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// UpdateFilterIndexes will update the indexes for every reference of a filter that exists in a profile.
+// Every profile that contains the filters from oldFltr will be updated with the new values for newFltr.
+// oldFltr and newFltr has the same tenant and ID.
+func UpdateFilterIndexes(dm *DataManager, oldFltr *Filter, newFltr *Filter) (err error) {
+	return nil
+}
+
+// addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes
+func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) {
+	for _, fltrID := range filterIDs {
+		if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters
+			continue
+		}
+		tntCtx := utils.ConcatenatedKey(tnt, fltrID)
+		refID := guardian.Guardian.GuardIDs(utils.EmptyString,
+			config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntCtx)
+		var indexes map[string]utils.StringMap
+		if indexes, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntCtx,
+			utils.EmptyString, nil); err != nil {
+			if err != utils.ErrNotFound {
+				guardian.Guardian.UnguardIDs(refID)
+				return
+			}
+			err = nil
+			indexes = map[string]utils.StringMap{
+				idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps
+			}
+		}
+		indexes[idxItmType] = map[string]bool{
+			itemID: true,
+		}
+		// IT IS REMOVED IN StoreIndexes
+		/* // remove the old reference from cache in case
+		for idxKeyItmType := range indexes {
+			Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType),
+				true, utils.NonTransactional)
+		} */
+		indexerKey := tnt
+		if ctx != utils.EmptyString {
+			indexerKey = utils.ConcatenatedKey(tnt, ctx)
+		}
+		fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey)
+		if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil {
+			return
+		}
+	}
+	return
+}

general_tests/filter_indexes_cases_it_test.go

@@ -100,6 +100,7 @@ var (
 
 		testFilterIndexesCasesSetDifferentFilters,
 		testFilterIndexesCasesOverwriteAttributes,
+		testFilterIndexesCasesComputeAttributesIndexes,
 		testFilterIndexesCasesGetIndexesAnyContextChanged,
 		testFilterIndexesCasesGetIndexesSessionsContextChanged,
 
@@ -266,7 +267,8 @@ func testFilterIndexesCasesGetIndexesAnyContext(t *testing.T) {
 		"*prefix:~*req.Missed:1:ATTR_NO_FLTR2",
 		"*string:~*req.Account:1001:ATTR_NO_FLTR1",
 		"*string:~*req.Account:1002:ATTR_NO_FLTR1",
-		"*string:~*req.Category:call:ATTR_NO_FLTR2"}
+		"*string:~*req.Category:call:ATTR_NO_FLTR2",
+	}
 	sort.Strings(expectedIndexes)
 	var reply []string
 	if err := fIdxCasesRPC.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil {

utils/consts.go

@@ -69,7 +69,7 @@ var (
 		CacheSupplierProfiles, CacheAttributeProfiles, CacheChargerProfiles,
 		CacheDispatcherProfiles, CacheDispatcherHosts, CacheResourceFilterIndexes,
 		CacheStatFilterIndexes, CacheThresholdFilterIndexes, CacheSupplierFilterIndexes,
-		CacheAttributeFilterIndexes, CacheChargerFilterIndexes, CacheDispatcherFilterIndexes,
+		CacheAttributeFilterIndexes, CacheChargerFilterIndexes, CacheDispatcherFilterIndexes, CacheReverseFilterIndexes,
 		CacheDispatcherRoutes, CacheDiameterMessages, CacheRPCResponses, CacheClosedSessions,
 		CacheCDRIDs, CacheLoadIDs, CacheRPCConnections, CacheRatingProfilesTmp})
 	CacheInstanceToPrefix = map[string]string{
@@ -102,6 +102,7 @@ var (
 		CacheAttributeFilterIndexes:  AttributeFilterIndexes,
 		CacheChargerFilterIndexes:    ChargerFilterIndexes,
 		CacheDispatcherFilterIndexes: DispatcherFilterIndexes,
+		CacheReverseFilterIndexes:    ReverseFilterIndexes,
 		CacheLoadIDs:                 LoadIDPrefix,
 		CacheAccounts:                ACCOUNT_PREFIX,
 	}
@@ -114,6 +115,7 @@ var (
 		AttributeProfilePrefix:  CacheAttributeFilterIndexes,
 		ChargerProfilePrefix:    CacheChargerFilterIndexes,
 		DispatcherProfilePrefix: CacheDispatcherFilterIndexes,
+		ReverseFilterIndexes:    CacheReverseFilterIndexes,
 	}
 	CacheIndexesToPrefix map[string]string // will be built on init
 
@@ -129,7 +131,7 @@ var (
 		CacheTimings, CacheStatQueueProfiles, CacheStatQueues, CacheThresholdProfiles, CacheThresholds,
 		CacheFilters, CacheSupplierProfiles, CacheAttributeProfiles, CacheChargerProfiles,
 		CacheDispatcherProfiles, CacheDispatcherHosts, CacheResourceFilterIndexes, CacheStatFilterIndexes,
-		CacheThresholdFilterIndexes, CacheSupplierFilterIndexes, CacheAttributeFilterIndexes,
+		CacheThresholdFilterIndexes, CacheSupplierFilterIndexes, CacheAttributeFilterIndexes, CacheReverseFilterIndexes,
 		CacheChargerFilterIndexes, CacheDispatcherFilterIndexes, CacheLoadIDs, CacheAccounts})
 
 	// ProtectedSFlds are the fields that sessions should not alter
@@ -1518,6 +1520,7 @@ const (
 	CacheAttributeFilterIndexes  = "*attribute_filter_indexes"
 	CacheChargerFilterIndexes    = "*charger_filter_indexes"
 	CacheDispatcherFilterIndexes = "*dispatcher_filter_indexes"
+	CacheReverseFilterIndexes    = "*reverse_filter_indexes"
 	CacheDiameterMessages        = "*diameter_messages"
 	CacheRPCResponses            = "*rpc_responses"
 	CacheClosedSessions          = "*closed_sessions"
@@ -1541,6 +1544,7 @@ const (
 	ChargerFilterIndexes    = "cfi_"
 	DispatcherFilterIndexes = "dfi_"
 	ActionPlanIndexes       = "api_"
+	ReverseFilterIndexes    = "fii_"
 )
 
 // Agents