Add warning log regarding too many profiles matched for a request

engine/filterhelpers.go

@@ -33,6 +33,8 @@ import (
 	"github.com/cgrates/cgrates/utils"
 )
 
+const matchedItemsWarningThreshold int = 100
+
 // MatchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event
 // fieldIDs limits the fields which are checked against indexes
 // helper on top of dataDB.GetIndexes, adding utils.MetaAny to list of fields queried
@@ -128,6 +130,10 @@ func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, su
 	if len(itemIDs) == 0 {
 		return nil, utils.ErrNotFound
 	}
+	if len(itemIDs) > matchedItemsWarningThreshold {
+		utils.Logger.Warning(fmt.Sprintf(
+			"Matched %d %s items. Performance may be affected", len(itemIDs), cacheID))
+	}
 	return
 }
 

engine/z_filterhelpers_test.go

@@ -18,6 +18,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package engine
 
 import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -218,3 +223,63 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) {
 		t.Errorf("Expecting: %+v, received: %+v", prefixFilterID, aPrflIDs)
 	}
 }
+
+func TestMatchingItemIDsForEventWarningThresholds(t *testing.T) {
+	utils.Logger.SetLogLevel(4)
+	utils.Logger.SetSyslog(nil)
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	defer func() { log.SetOutput(os.Stderr) }()
+	cfg := config.NewDefaultCGRConfig()
+	data, dErr := NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items)
+	if dErr != nil {
+		t.Fatal(dErr)
+	}
+	dm := NewDataManager(data, cfg.CacheCfg(), nil)
+	tnt := cfg.GeneralCfg().DefaultTenant
+	contextKey := utils.MetaRating
+	tntCtx := utils.ConcatenatedKey(tnt, contextKey)
+	for i := 0; i < matchedItemsWarningThreshold+5; i++ {
+		fid := fmt.Sprintf("filter_%d", i)
+		rule := &FilterRule{
+			Type:    utils.MetaString,
+			Element: "~*req.Field",
+			Values:  []string{"trigger"},
+		}
+		if err := rule.CompileValues(); err != nil {
+			t.Fatal(err)
+		}
+		f := &Filter{
+			Tenant: tnt,
+			ID:     fid,
+			Rules:  []*FilterRule{rule},
+		}
+		if err := dm.SetFilter(f, true); err != nil {
+			t.Fatal(err)
+		}
+		if err := addItemToFilterIndex(dm, utils.CacheAttributeFilterIndexes,
+			tnt, contextKey, fid, []string{fid}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ev := utils.MapStorage{
+		utils.MetaReq: map[string]any{
+			"Field": "trigger",
+		},
+	}
+	ids, err := MatchingItemIDsForEvent(ev, nil, nil, nil, nil,
+		dm, utils.CacheAttributeFilterIndexes, tntCtx, true, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ids) <= matchedItemsWarningThreshold {
+		t.Errorf("Expected more than %d matched items, got %d",
+			matchedItemsWarningThreshold, len(ids))
+	}
+	expectedLog := fmt.Sprintf("Matched %d *attribute_filter_indexes items. Performance may be affected", len(ids))
+	logContent := buf.String()
+	if !strings.Contains(logContent, expectedLog) {
+		t.Errorf("Expected warning not found in logs:\n%s", logContent)
+	}
+	utils.Logger.SetLogLevel(0)
+}