Merge branch 'master' into feat/replicator-async

engine/attributes.go

@@ -72,6 +72,7 @@ func (alS *AttributeService) attributeProfileForEvent(tnt string, ctx *string, a
 			alS.cgrcfg.AttributeSCfg().StringIndexedFields,
 			alS.cgrcfg.AttributeSCfg().PrefixIndexedFields,
 			alS.cgrcfg.AttributeSCfg().SuffixIndexedFields,
+			alS.cgrcfg.AttributeSCfg().ExistsIndexedFields,
 			alS.dm, utils.CacheAttributeFilterIndexes, attrIdxKey,
 			alS.cgrcfg.AttributeSCfg().IndexedSelects,
 			alS.cgrcfg.AttributeSCfg().NestedFields,
@@ -86,6 +87,7 @@ func (alS *AttributeService) attributeProfileForEvent(tnt string, ctx *string, a
 				alS.cgrcfg.AttributeSCfg().StringIndexedFields,
 				alS.cgrcfg.AttributeSCfg().PrefixIndexedFields,
 				alS.cgrcfg.AttributeSCfg().SuffixIndexedFields,
+				alS.cgrcfg.AttributeSCfg().ExistsIndexedFields,
 				alS.dm, utils.CacheAttributeFilterIndexes,
 				utils.ConcatenatedKey(tnt, utils.MetaAny),
 				alS.cgrcfg.AttributeSCfg().IndexedSelects,

engine/chargers.go

@@ -56,6 +56,7 @@ func (cS *ChargerService) matchingChargerProfilesForEvent(tnt string, cgrEv *uti
 		cS.cfg.ChargerSCfg().StringIndexedFields,
 		cS.cfg.ChargerSCfg().PrefixIndexedFields,
 		cS.cfg.ChargerSCfg().SuffixIndexedFields,
+		cS.cfg.ChargerSCfg().ExistsIndexedFields,
 		cS.dm, utils.CacheChargerFilterIndexes, tnt,
 		cS.cfg.ChargerSCfg().IndexedSelects,
 		cS.cfg.ChargerSCfg().NestedFields,

engine/filterhelpers.go

@@ -36,11 +36,11 @@ import (
 // MatchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event
 // fieldIDs limits the fields which are checked against indexes
 // helper on top of dataDB.GetIndexes, adding utils.MetaAny to list of fields queried
-func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, suffixFldIDs *[]string,
+func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, suffixFldIDs, existsFldIDs *[]string,
 	dm *DataManager, cacheID, itemIDPrefix string, indexedSelects, nestedFields bool) (itemIDs utils.StringSet, err error) {
 	itemIDs = make(utils.StringSet)
 	var allFieldIDs []string
-	if indexedSelects && (stringFldIDs == nil || prefixFldIDs == nil || suffixFldIDs == nil) {
+	if indexedSelects && (stringFldIDs == nil || prefixFldIDs == nil || suffixFldIDs == nil || existsFldIDs == nil) {
 		allFieldIDs = ev.GetKeys(nestedFields, 2, utils.EmptyString)
 	}
 	// Guard will protect the function with automatic locking
@@ -58,13 +58,32 @@ func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, su
 			itemIDs = utils.NewStringSet(sliceIDs)
 			return
 		}
-		stringFieldVals := map[string]string{utils.MetaAny: utils.MetaAny}                                 // cache here field string values, start with default one
-		filterIndexTypes := []string{utils.MetaString, utils.MetaPrefix, utils.MetaSuffix, utils.MetaNone} // the MetaNone is used for all items that do not have filters
-		for i, fieldIDs := range []*[]string{stringFldIDs, prefixFldIDs, suffixFldIDs, {utils.MetaAny}} {  // same routine for both string and prefix filter types
+		stringFieldVals := map[string]string{utils.MetaAny: utils.MetaAny}                                                   // cache here field string values, start with default one
+		filterIndexTypes := []string{utils.MetaString, utils.MetaPrefix, utils.MetaSuffix, utils.MetaExists, utils.MetaNone} // the MetaNone is used for all items that do not have filters
+		for i, fieldIDs := range []*[]string{stringFldIDs, prefixFldIDs, suffixFldIDs, existsFldIDs, {utils.MetaAny}} {      // same routine for both string and prefix filter types
 			if fieldIDs == nil {
 				fieldIDs = &allFieldIDs
 			}
 			for _, fldName := range *fieldIDs {
+				var dbItemIDs utils.StringSet // list of items matched in DB
+				if filterIndexTypes[i] == utils.MetaExists {
+					var dbIndexes map[string]utils.StringSet // list of items matched in DB
+					key := utils.ConcatenatedKey(filterIndexTypes[i], fldName)
+					if dbIndexes, err = dm.GetIndexes(cacheID, itemIDPrefix, key, true, true); err != nil {
+						if err == utils.ErrNotFound {
+							err = nil
+							continue
+						}
+						return
+					}
+					dbItemIDs = dbIndexes[key]
+					for itemID := range dbItemIDs {
+						if _, hasIt := itemIDs[itemID]; !hasIt { // Add it to list if not already there
+							itemIDs[itemID] = dbItemIDs[itemID]
+						}
+					}
+					continue // no need to look at values for *exists indexes
+				}
 				var fieldValIf any
 				fieldValIf, err = ev.FieldAsInterface(utils.SplitPath(fldName, utils.NestingSep[0], -1))
 				if err != nil && filterIndexTypes[i] != utils.MetaNone {
@@ -81,7 +100,6 @@ func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, su
 				} else if filterIndexTypes[i] == utils.MetaSuffix {
 					fldVals = utils.SplitSuffix(fldVal) // all suffix till first digit
 				}
-				var dbItemIDs utils.StringSet // list of items matched in DB
 				for _, val := range fldVals {
 					var dbIndexes map[string]utils.StringSet // list of items matched in DB
 					key := utils.ConcatenatedKey(filterIndexTypes[i], fldName, val)

engine/libindex.go

@@ -28,7 +28,7 @@ import (
 )
 
 var (
-	FilterIndexTypes = utils.NewStringSet([]string{utils.MetaPrefix, utils.MetaString, utils.MetaSuffix})
+	FilterIndexTypes = utils.NewStringSet([]string{utils.MetaPrefix, utils.MetaString, utils.MetaSuffix, utils.MetaExists})
 	// Element or values of a filter that starts with one of this should not be indexed
 	ToNotBeIndexed = []string{
 		utils.DynamicDataPrefix + utils.MetaAccounts,
@@ -86,6 +86,23 @@ func newFilterIndex(dm *DataManager, idxItmType, tnt, ctx, itemID string, filter
 				continue
 			}
 			isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix)
+			if isDyn && flt.Type == utils.MetaExists {
+				idxKey := utils.ConcatenatedKey(flt.Type, flt.Element[1:])
+				var rcvIndx map[string]utils.StringSet
+				// only read from cache in case if we do not find the index to not cache the negative response
+				if rcvIndx, err = dm.GetIndexes(idxItmType, tntCtx,
+					idxKey, true, false); err != nil {
+					if err != utils.ErrNotFound {
+						return
+					}
+					err = nil
+					indexes[idxKey] = make(utils.StringSet) // create an empty index if is not found in DB in case we add them later
+					continue
+				}
+				for idxKey, idx := range rcvIndx { // parse the received indexes
+					indexes[idxKey] = idx
+				}
+			}
 			for _, fldVal := range flt.Values {
 				if IsDynamicDPPath(fldVal) {
 					continue

engine/libindex_test.go

@@ -20,6 +20,8 @@ package engine
 
 import (
 	"errors"
+	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/cgrates/cgrates/config"
@@ -207,3 +209,590 @@ func TestLibIndexNewFilterIndexGetFilterErrNotFound(t *testing.T) {
 		t.Fatalf("Expected error %v, got %v", expectedErr, err)
 	}
 }
+func TestLibIndex_newFilterIndex(t *testing.T) {
+	cfg := config.NewDefaultCGRConfig()
+	dataDB, err := NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dm := NewDataManager(dataDB, cfg.CacheCfg(), nil)
+
+	for i := range 2 {
+		idx := strconv.Itoa(i + 1)
+		flt := &Filter{
+			Tenant: "cgrates.org",
+			ID:     "FLTR_" + idx,
+			Rules: []*FilterRule{
+				{
+					Type:    utils.MetaString,
+					Element: "~*req.Field" + idx,
+					Values:  []string{"val" + idx},
+				},
+				{
+					Type:    utils.MetaPrefix,
+					Element: "~*req.Field" + idx,
+					Values:  []string{"val"},
+				},
+				{
+					Type:    utils.MetaSuffix,
+					Element: "~*req.Field" + idx,
+					Values:  []string{idx},
+				},
+				{
+					Type:    utils.MetaExists,
+					Element: "~*req.Field" + idx,
+				},
+			},
+		}
+		if err := dm.SetFilter(flt, true); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	filterIDsList := [][]string{
+		{"*prefix:~*req.Field2:val", "*empty:~*req.Field3:", "FLTR_1"},
+		{"*suffix:~*req.Field1:1", "*notstring:~*req.Field2:val1", "FLTR_2"},
+		{"*exists:~*req.Field2:", "*suffix:~*req.Field1:1", "FLTR_1"},
+	}
+
+	for i, filterIDs := range filterIDsList {
+		idx := strconv.Itoa(i + 1)
+		if err := dm.SetAttributeProfile(&AttributeProfile{
+			Tenant:    "cgrates.org",
+			ID:        "ATTR_" + idx,
+			FilterIDs: filterIDs,
+			Contexts:  []string{utils.MetaCDRs},
+		}, true); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	wantIndexes := map[string]utils.StringSet{
+		"*exists:*req.Field1": {
+			"ATTR_1": {},
+			"ATTR_3": {},
+		},
+		"*exists:*req.Field2": {
+			"ATTR_2": {},
+			"ATTR_3": {},
+		},
+		"*prefix:*req.Field1:val": {
+			"ATTR_1": {},
+			"ATTR_3": {},
+		},
+		"*prefix:*req.Field2:val": {
+			"ATTR_1": {},
+			"ATTR_2": {},
+		},
+		"*string:*req.Field1:val1": {
+			"ATTR_1": {},
+			"ATTR_3": {},
+		},
+		"*string:*req.Field2:val2": {
+			"ATTR_2": {},
+		},
+		"*suffix:*req.Field1:1": {
+			"ATTR_1": {},
+			"ATTR_2": {},
+			"ATTR_3": {},
+		},
+		"*suffix:*req.Field2:2": {
+			"ATTR_2": {},
+		},
+	}
+	tntCtx := utils.ConcatenatedKey("cgrates.org", utils.MetaCDRs)
+	gotIndexes, err := dm.GetIndexes(utils.CacheAttributeFilterIndexes, tntCtx, "", false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(gotIndexes, wantIndexes) {
+		t.Errorf("dm.GetIndexes() = %s, want %s", utils.ToJSON(gotIndexes), utils.ToJSON(wantIndexes))
+	}
+
+	tests := []struct {
+		name       string
+		idxItmType string
+		tnt        string
+		ctx        string
+		itemID     string
+		filterIDs  []string
+		newFlt     *Filter
+		want       map[string]utils.StringSet
+		wantErr    bool
+	}{
+		{
+			name:       "no filterIDs",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			want: map[string]utils.StringSet{
+				"*none:*any:*any": {},
+			},
+		},
+		{
+			name:       "newFlt with no filterIDs",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			newFlt: &Filter{
+				Tenant: "cgrates.org",
+				ID:     "FLTR_TEST",
+				Rules: []*FilterRule{
+					{
+						Type:    utils.MetaString,
+						Element: "~*req.Field",
+						Values:  []string{"val"},
+					},
+				},
+			},
+			want: map[string]utils.StringSet{
+				"*none:*any:*any": {},
+			},
+		},
+		{
+			name:       "broken reference",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			itemID:     "ATTR_1",
+			filterIDs:  []string{"FLTR_1"},
+			wantErr:    true,
+		},
+		{
+			name:       "unindexable filter",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			filterIDs:  []string{"*notstring:~*req.Field1:val2"},
+			want:       make(map[string]utils.StringSet),
+		},
+		{
+			name:       "dynamic element, constant value",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			filterIDs:  []string{"*string:~*req.Field1:val1"},
+			want: map[string]utils.StringSet{
+				"*string:*req.Field1:val1": {},
+			},
+		},
+		{
+			name:       "constant element, dynamic value",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			filterIDs:  []string{"*string:val1:~*req.Field1"},
+			want: map[string]utils.StringSet{
+				"*string:*req.Field1:val1": {},
+			},
+		},
+		{
+			name:       "dynamic element, dynamic value",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			filterIDs:  []string{"*string:~*req.Field1:~*req.Field1"},
+			want:       make(map[string]utils.StringSet),
+		},
+		{
+			name:       "constant element, constant value",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			filterIDs:  []string{"*string:val1:val1"},
+			want:       make(map[string]utils.StringSet),
+		},
+		{
+			name:       "filter and tenant without context",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			// ctx:        utils.MetaCDRs,
+			filterIDs: []string{"*string:~*req.Field1:val1"},
+			want: map[string]utils.StringSet{
+				"*string:*req.Field1:val1": {},
+			},
+		},
+		{
+			name:       "filter and tenant with context (without references)",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        utils.MetaCDRs,
+			filterIDs:  []string{"*string:~*req.Random:val"},
+			want: map[string]utils.StringSet{
+				"*string:*req.Random:val": {},
+			},
+		},
+		{
+			name:       "filter and tenant with context (with references)",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        utils.MetaCDRs,
+			filterIDs:  []string{"*string:~*req.Field1:val1"},
+			want: map[string]utils.StringSet{
+				"*string:*req.Field1:val1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+			},
+		},
+		{
+			name:       "filterID of newFlt",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        utils.MetaCDRs,
+			filterIDs:  []string{"FLTR_TEST"},
+			newFlt: &Filter{
+				Tenant: "cgrates.org",
+				ID:     "FLTR_TEST",
+				Rules: []*FilterRule{
+					{
+						Type:    utils.MetaString,
+						Element: "~*req.Random",
+						Values:  []string{"val"},
+					},
+					{
+						Type:    utils.MetaString,
+						Element: "~*req.Field1",
+						Values:  []string{"val1"},
+					},
+				},
+			},
+			want: map[string]utils.StringSet{
+				"*string:*req.Field1:val1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*string:*req.Random:val": {},
+			},
+		},
+		{
+			name:       "all indexable filters with references",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        "*cdrs",
+			filterIDs: []string{
+				"FLTR_1",
+				"FLTR_2",
+				"*prefix:~*req.Field2:val",
+				"*suffix:~*req.Field1:1",
+				"*suffix:~*req.Field1:1",
+			},
+			want: map[string]utils.StringSet{
+				"*exists:*req.Field1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*exists:*req.Field2": {
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field1:val": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field2:val": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+				},
+				"*string:*req.Field1:val1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*string:*req.Field2:val2": {
+					"ATTR_2": {},
+				},
+				"*suffix:*req.Field1:1": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*suffix:*req.Field2:2": {
+					"ATTR_2": {},
+				},
+			},
+		},
+		{
+			name:       "all filters",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        "*cdrs",
+			filterIDs: []string{
+				"FLTR_1",
+				"FLTR_2",
+				"*prefix:~*req.Field2:val",
+				"*suffix:~*req.Field1:1",
+				"*exists:~*req.Field2:",
+				"*empty:~*req.Field3:",
+				"*notstring:~*req.Field2:val1",
+			},
+			want: map[string]utils.StringSet{
+				"*exists:*req.Field1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*exists:*req.Field2": {
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field1:val": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field2:val": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+				},
+				"*string:*req.Field1:val1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*string:*req.Field2:val2": {
+					"ATTR_2": {},
+				},
+				"*suffix:*req.Field1:1": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*suffix:*req.Field2:2": {
+					"ATTR_2": {},
+				},
+			},
+		},
+		{
+			name:       "all filters (with extra unreferenced filters)",
+			idxItmType: utils.CacheAttributeFilterIndexes,
+			tnt:        "cgrates.org",
+			ctx:        "*cdrs",
+			filterIDs: []string{
+				"FLTR_1",
+				"FLTR_2",
+				"*prefix:~*req.Field2:val",
+				"*suffix:~*req.Field1:1",
+				"*exists:~*req.Field2:",
+				"*empty:~*req.Field3:",
+				"*notstring:~*req.Field2:val1",
+				"*prefix:~*req.Field4:val",
+				"*string:~*req.Field5:val5",
+			},
+			want: map[string]utils.StringSet{
+				"*exists:*req.Field1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*exists:*req.Field2": {
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field1:val": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*prefix:*req.Field2:val": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+				},
+				"*string:*req.Field1:val1": {
+					"ATTR_1": {},
+					"ATTR_3": {},
+				},
+				"*string:*req.Field2:val2": {
+					"ATTR_2": {},
+				},
+				"*suffix:*req.Field1:1": {
+					"ATTR_1": {},
+					"ATTR_2": {},
+					"ATTR_3": {},
+				},
+				"*suffix:*req.Field2:2": {
+					"ATTR_2": {},
+				},
+				"*prefix:*req.Field4:val":  {},
+				"*string:*req.Field5:val5": {},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newFilterIndex(dm, tt.idxItmType, tt.tnt, tt.ctx, tt.itemID, tt.filterIDs, tt.newFlt)
+			if gotErr != nil {
+				if !tt.wantErr {
+					t.Errorf("newFilterIndex() failed: %v", gotErr)
+				}
+				return
+			}
+			if tt.wantErr {
+				t.Fatal("newFilterIndex() succeeded unexpectedly")
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("newFilterIndex() = %s, want %s", utils.ToJSON(got), utils.ToJSON(tt.want))
+			}
+		})
+	}
+}
+
+// func TestLibIndex_prepareFilterIndexMap(t *testing.T) {
+// 	cfg := config.NewDefaultCGRConfig()
+// 	dataDB := NewInternalDB(nil, nil, true, true, cfg.DataDbCfg().Items)
+// 	dm := NewDataManager(dataDB, cfg.CacheCfg(), nil)
+//
+// 	var flt *Filter // to be used as newFlt
+// 	for i := range 2 {
+// 		idx := strconv.Itoa(i + 1)
+// 		flt = &Filter{
+// 			Tenant: "cgrates.org",
+// 			ID:     "FLTR_" + idx,
+// 			Rules: []*FilterRule{
+// 				{
+// 					Type:    utils.MetaString,
+// 					Element: "~*req.Field" + idx,
+// 					Values:  []string{"val" + idx},
+// 				},
+// 				{
+// 					Type:    utils.MetaPrefix,
+// 					Element: "~*req.Field" + idx,
+// 					Values:  []string{"val"},
+// 				},
+// 				{
+// 					Type:    utils.MetaSuffix,
+// 					Element: "~*req.Field" + idx,
+// 					Values:  []string{idx},
+// 				},
+// 				{
+// 					Type:    utils.MetaExists,
+// 					Element: "~*req.Field" + idx,
+// 				},
+// 			},
+// 		}
+// 		if err := dm.SetFilter(flt, true); err != nil {
+// 			t.Fatal(err)
+// 		}
+// 	}
+//
+// 	filterIDsList := [][]string{
+// 		{"*prefix:~*req.Field2:val", "FLTR_1"},
+// 		{"*suffix:~*req.Field1:1", "FLTR_2"},
+// 		{"*exists:~*req.Field2:", "*suffix:~*req.Field1:1", "FLTR_1"},
+// 	}
+//
+// 	for i, filterIDs := range filterIDsList {
+// 		idx := strconv.Itoa(i + 1)
+// 		if err := dm.SetChargerProfile(&ChargerProfile{
+// 			Tenant:       "cgrates.org",
+// 			ID:           "CP_" + idx,
+// 			FilterIDs:    filterIDs,
+// 			RunID:        "DEFAULT" + idx,
+// 			AttributeIDs: []string{"*none"},
+// 		}, true); err != nil {
+// 			t.Fatal(err)
+// 		}
+// 		if err := dm.SetAttributeProfile(&AttributeProfile{
+// 			Tenant:    "cgrates.org",
+// 			ID:        "ATTR_CDRS_" + idx,
+// 			FilterIDs: filterIDs,
+// 			Contexts:  []string{utils.MetaCDRs},
+// 		}, true); err != nil {
+// 			t.Fatal(err)
+// 		}
+// 		if err := dm.SetAttributeProfile(&AttributeProfile{
+// 			Tenant:    "cgrates.org",
+// 			ID:        "ATTR_SESSIONS_" + idx,
+// 			FilterIDs: filterIDs,
+// 			Contexts:  []string{utils.MetaSessionS},
+// 		}, true); err != nil {
+// 			t.Fatal(err)
+// 		}
+// 	}
+//
+// 	wantTemplate := func(prefix string) map[string]utils.StringSet {
+// 		return map[string]utils.StringSet{
+// 			"*prefix:*req.Field1:val": {
+// 				prefix + "_1": {},
+// 				prefix + "_3": {},
+// 			},
+// 			"*prefix:*req.Field2:val": {
+// 				prefix + "_1": {},
+// 				prefix + "_2": {},
+// 			},
+// 			"*string:*req.Field1:val1": {
+// 				prefix + "_1": {},
+// 				prefix + "_3": {},
+// 			},
+// 			"*string:*req.Field2:val2": {
+// 				prefix + "_2": {},
+// 			},
+// 			"*suffix:*req.Field1:1": {
+// 				prefix + "_1": {},
+// 				prefix + "_2": {},
+// 				prefix + "_3": {},
+// 			},
+// 			"*suffix:*req.Field2:2": {
+// 				prefix + "_2": {},
+// 			},
+// 		}
+// 	}
+//
+// 	argsList := []struct {
+// 		itemType string
+// 		ctx      string
+// 		prefix   string // of the profile (CP, ATTR_CDRS, ATTR_SESSIONS)
+// 	}{
+// 		{
+// 			itemType: utils.CacheChargerFilterIndexes,
+// 			ctx:      "",
+// 			prefix:   "CP",
+// 		},
+// 		{
+// 			itemType: utils.CacheAttributeFilterIndexes,
+// 			ctx:      utils.MetaCDRs,
+// 			prefix:   "ATTR_CDRS",
+// 		},
+// 		{
+// 			itemType: utils.CacheAttributeFilterIndexes,
+// 			ctx:      utils.MetaSessionS,
+// 			prefix:   "ATTR_SESSIONS",
+// 		},
+// 	}
+//
+// 	for _, args := range argsList {
+// 		wantIndexes := wantTemplate(args.prefix)
+// 		tntCtx := "cgrates.org"
+// 		if args.ctx != "" {
+// 			tntCtx = utils.ConcatenatedKey(tntCtx, args.ctx)
+// 		}
+// 		gotIndexes, err := dm.GetIndexes(args.itemType, tntCtx, "", false, false)
+// 		if err != nil {
+// 			t.Fatal(err)
+// 		}
+// 		if !reflect.DeepEqual(gotIndexes, wantIndexes) {
+// 			t.Errorf("dm.GetIndexes() = %s, want %s", utils.ToJSON(gotIndexes), utils.ToJSON(wantIndexes))
+// 		}
+//
+// 	}
+//
+// 	tests := []struct {
+// 		name       string
+// 		idxItmType string
+// 		tnt        string
+// 		ctx        string
+// 		itemID     string
+// 		filterIDs  []string
+// 		newFlt     *Filter
+// 		want       map[string]utils.StringSet
+// 		wantErr    bool
+// 	}{
+// 		{
+// 			name:       "test1",
+// 			idxItmType: utils.CacheAttributeFilterIndexes,
+// 			tnt:        "cgrates.org",
+// 			ctx:        "*cdrs",
+// 			itemID:     "ATTR_CDRS",
+// 			filterIDs:  []string{"FLTR_1"},
+// 			newFlt:     flt,
+// 			want:       map[string]utils.StringSet{},
+// 		},
+// 	}
+//
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			got, gotErr := prepareFilterIndexMap(dm, tt.idxItmType, tt.tnt, tt.ctx, tt.itemID, tt.filterIDs, tt.newFlt)
+// 			if gotErr != nil {
+// 				if !tt.wantErr {
+// 					t.Errorf("prepareFilterIndexMap() failed: %v", gotErr)
+// 				}
+// 				return
+// 			}
+// 			if tt.wantErr {
+// 				t.Fatal("prepareFilterIndexMap() succeeded unexpectedly")
+// 			}
+// 			if !reflect.DeepEqual(got, tt.want) {
+// 				t.Errorf("prepareFilterIndexMap() = %s, want %s", utils.ToJSON(got), utils.ToJSON(tt.want))
+// 			}
+// 		})
+// 	}
+// }

engine/libtest.go

@@ -630,15 +630,23 @@ func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, grace
 	t.Cleanup(func() {
 		if gracefulShutdown {
 			if err := engine.Process.Signal(syscall.SIGTERM); err != nil {
-				t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
+				t.Errorf("failed to terminate cgr-engine process (%d): %v",
+					engine.Process.Pid, err)
 			}
+
+			// TODO: check if it's a good idea to wait for the Kill signal
+			// also. Might prevent scenarios where engine from previous test is
+			// still running (even though kill should be almost instant)
 			if err := engine.Wait(); err != nil {
-				t.Errorf("cgr-engine process failed to exit cleanly: %v", err)
-			}
-		} else {
-			if err := engine.Process.Kill(); err != nil {
-				t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
+				t.Errorf("could not wait for cgr-engine process (%d) to shut down: %v",
+					engine.Process.Pid, err)
 			}
+
+			return
+		}
+		if err := engine.Process.Kill(); err != nil {
+			t.Errorf("failed to kill cgr-engine process (%d): %v",
+				engine.Process.Pid, err)
 		}
 	})
 	backoff := utils.FibDuration(time.Millisecond, 0)

engine/resources.go

@@ -659,6 +659,7 @@ func (rS *ResourceService) matchingResourcesForEvent(tnt string, ev *utils.CGREv
 			rS.cgrcfg.ResourceSCfg().StringIndexedFields,
 			rS.cgrcfg.ResourceSCfg().PrefixIndexedFields,
 			rS.cgrcfg.ResourceSCfg().SuffixIndexedFields,
+			rS.cgrcfg.ResourceSCfg().ExistsIndexedFields,
 			rS.dm, utils.CacheResourceFilterIndexes, tnt,
 			rS.cgrcfg.ResourceSCfg().IndexedSelects,
 			rS.cgrcfg.ResourceSCfg().NestedFields,

engine/routes.go

@@ -230,6 +230,7 @@ func (rpS *RouteService) matchingRouteProfilesForEvent(tnt string, ev *utils.CGR
 		rpS.cgrcfg.RouteSCfg().StringIndexedFields,
 		rpS.cgrcfg.RouteSCfg().PrefixIndexedFields,
 		rpS.cgrcfg.RouteSCfg().SuffixIndexedFields,
+		rpS.cgrcfg.RouteSCfg().ExistsIndexedFields,
 		rpS.dm, utils.CacheRouteFilterIndexes, tnt,
 		rpS.cgrcfg.RouteSCfg().IndexedSelects,
 		rpS.cgrcfg.RouteSCfg().NestedFields,

engine/stats.go

@@ -165,6 +165,7 @@ func (sS *StatService) matchingStatQueuesForEvent(tnt string, statsIDs []string,
 			sS.cgrcfg.StatSCfg().StringIndexedFields,
 			sS.cgrcfg.StatSCfg().PrefixIndexedFields,
 			sS.cgrcfg.StatSCfg().SuffixIndexedFields,
+			sS.cgrcfg.StatSCfg().ExistsIndexedFields,
 			sS.dm, utils.CacheStatFilterIndexes, tnt,
 			sS.cgrcfg.StatSCfg().IndexedSelects,
 			sS.cgrcfg.StatSCfg().NestedFields,

engine/thresholds.go

@@ -411,6 +411,7 @@ func (tS *ThresholdService) matchingThresholdsForEvent(tnt string, args *utils.C
 			tS.cgrcfg.ThresholdSCfg().StringIndexedFields,
 			tS.cgrcfg.ThresholdSCfg().PrefixIndexedFields,
 			tS.cgrcfg.ThresholdSCfg().SuffixIndexedFields,
+			tS.cgrcfg.ThresholdSCfg().ExistsIndexedFields,
 			tS.dm, utils.CacheThresholdFilterIndexes, tnt,
 			tS.cgrcfg.ThresholdSCfg().IndexedSelects,
 			tS.cgrcfg.ThresholdSCfg().NestedFields,

engine/z_filterhelpers_test.go

@@ -103,7 +103,7 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) {
 		utils.AnswerTime: time.Date(2014, 7, 14, 14, 30, 0, 0, time.UTC),
 		"Field":          "profile",
 	}}
-	aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, nil,
+	aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, nil, nil,
 		dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, false)
 	if err != nil {
 		t.Errorf("Error: %+v", err)
@@ -116,7 +116,7 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) {
 	matchEV = utils.MapStorage{utils.MetaReq: map[string]any{
 		"Field": "profilePrefix",
 	}}
-	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil,
+	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil, nil,
 		dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, false)
 	if err != nil {
 		t.Errorf("Error: %+v", err)
@@ -129,7 +129,7 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) {
 	matchEV = utils.MapStorage{utils.MetaReq: map[string]any{
 		"Field": "profilePrefix",
 	}}
-	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil,
+	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil, nil,
 		dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, false)
 	if err != nil {
 		t.Errorf("Error: %+v", err)
@@ -196,7 +196,7 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) {
 		utils.AnswerTime: time.Date(2014, 7, 14, 14, 30, 0, 0, time.UTC),
 		"CallCost":       map[string]any{"Account": 1001},
 	}}
-	aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, nil,
+	aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, nil, nil,
 		dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, true)
 	if err != nil {
 		t.Errorf("Error: %+v", err)
@@ -208,7 +208,7 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) {
 	matchEV = utils.MapStorage{utils.MetaReq: map[string]any{
 		"CallCost": map[string]any{"Field": "profilePrefix"},
 	}}
-	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil,
+	aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, nil, nil,
 		dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, true)
 	if err != nil {
 		t.Errorf("Error: %+v", err)