diff --git a/engine/filterhelpers.go b/engine/filterhelpers.go index 8399fe608..0f6c693ab 100644 --- a/engine/filterhelpers.go +++ b/engine/filterhelpers.go @@ -33,6 +33,8 @@ import ( "github.com/cgrates/cgrates/utils" ) +const matchedItemsWarningThreshold int = 100 + // MatchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event // fieldIDs limits the fields which are checked against indexes // helper on top of dataDB.GetIndexes, adding utils.MetaAny to list of fields queried @@ -128,6 +130,10 @@ func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs, su if len(itemIDs) == 0 { return nil, utils.ErrNotFound } + if len(itemIDs) > matchedItemsWarningThreshold { + utils.Logger.Warning(fmt.Sprintf( + "Matched %d %s items. Performance may be affected", len(itemIDs), cacheID)) + } return } diff --git a/engine/z_filterhelpers_test.go b/engine/z_filterhelpers_test.go index aaa196bd8..7e9dbe12e 100644 --- a/engine/z_filterhelpers_test.go +++ b/engine/z_filterhelpers_test.go @@ -18,6 +18,11 @@ along with this program. If not, see package engine import ( + "bytes" + "fmt" + "log" + "os" + "strings" "testing" "time" @@ -218,3 +223,63 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", prefixFilterID, aPrflIDs) } } + +func TestMatchingItemIDsForEventWarningThresholds(t *testing.T) { + utils.Logger.SetLogLevel(4) + utils.Logger.SetSyslog(nil) + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { log.SetOutput(os.Stderr) }() + cfg := config.NewDefaultCGRConfig() + data, dErr := NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items) + if dErr != nil { + t.Fatal(dErr) + } + dm := NewDataManager(data, cfg.CacheCfg(), nil) + tnt := cfg.GeneralCfg().DefaultTenant + contextKey := utils.MetaRating + tntCtx := utils.ConcatenatedKey(tnt, contextKey) + for i := 0; i < matchedItemsWarningThreshold+5; i++ { + fid := fmt.Sprintf("filter_%d", i) + rule := &FilterRule{ + Type: utils.MetaString, + Element: "~*req.Field", + Values: []string{"trigger"}, + } + if err := rule.CompileValues(); err != nil { + t.Fatal(err) + } + f := &Filter{ + Tenant: tnt, + ID: fid, + Rules: []*FilterRule{rule}, + } + if err := dm.SetFilter(f, true); err != nil { + t.Fatal(err) + } + if err := addItemToFilterIndex(dm, utils.CacheAttributeFilterIndexes, + tnt, contextKey, fid, []string{fid}); err != nil { + t.Fatal(err) + } + } + ev := utils.MapStorage{ + utils.MetaReq: map[string]any{ + "Field": "trigger", + }, + } + ids, err := MatchingItemIDsForEvent(ev, nil, nil, nil, nil, + dm, utils.CacheAttributeFilterIndexes, tntCtx, true, false) + if err != nil { + t.Fatal(err) + } + if len(ids) <= matchedItemsWarningThreshold { + t.Errorf("Expected more than %d matched items, got %d", + matchedItemsWarningThreshold, len(ids)) + } + expectedLog := fmt.Sprintf("Matched %d *attribute_filter_indexes items. Performance may be affected", len(ids)) + logContent := buf.String() + if !strings.Contains(logContent, expectedLog) { + t.Errorf("Expected warning not found in logs:\n%s", logContent) + } + utils.Logger.SetLogLevel(0) +}