diff --git a/engine/filterhelpers.go b/engine/filterhelpers.go index 5cbbd46e4..79d6e9543 100644 --- a/engine/filterhelpers.go +++ b/engine/filterhelpers.go @@ -34,6 +34,8 @@ import ( "github.com/cgrates/guardian" ) +const matchedItemsWarningThreshold int = 100 + // MatchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event // fieldIDs limits the fields which are checked against indexes // helper on top of dataDB.GetIndexes, adding utils.MetaAny to list of fields queried @@ -116,6 +118,11 @@ func MatchingItemIDsForEvent(ctx *context.Context, ev utils.MapStorage, stringFl if len(itemIDs) == 0 { return nil, utils.ErrNotFound } + if len(itemIDs) > matchedItemsWarningThreshold { + utils.Logger.Warning(fmt.Sprintf( + "Matched %d %s items. Performance may be affected.\nevent = %s", + len(itemIDs), cacheID, utils.ToJSON(ev))) + } return } diff --git a/engine/filterhelpers_test.go b/engine/filterhelpers_test.go index a8b07e9db..f3e04f91a 100644 --- a/engine/filterhelpers_test.go +++ b/engine/filterhelpers_test.go @@ -19,7 +19,10 @@ along with this program. If not, see package engine import ( + "bytes" + "fmt" "reflect" + "strings" "testing" "time" @@ -193,3 +196,70 @@ func TestFilterHelpersExtractURLFromHTTPType(t *testing.T) { }) } } + +func TestMatchingItemIDsForEventWarningThresholds(t *testing.T) { + tmpLogger := utils.Logger + defer func() { + utils.Logger = tmpLogger + }() + var buf bytes.Buffer + utils.Logger = utils.NewStdLoggerWithWriter(&buf, "", 7) + + cfg := config.NewDefaultCGRConfig() + data, dErr := NewInternalDB(nil, nil, nil, cfg.DataDbCfg().Items) + if dErr != nil { + t.Fatal(dErr) + } + dm := NewDataManager(data, cfg, nil) + + tnt := cfg.GeneralCfg().DefaultTenant + contextKey := utils.MetaRating + tntCtx := utils.ConcatenatedKey(tnt, contextKey) + + for i := 0; i < matchedItemsWarningThreshold+5; i++ { + fid := fmt.Sprintf("filter_%d", i) + rule := &FilterRule{ + Type: utils.MetaString, + Element: "~*req.Field", + Values: []string{"trigger"}, + } + if err := rule.CompileValues(); err != nil { + t.Fatal(err) + } + f := &Filter{ + Tenant: tnt, + ID: fid, + Rules: []*FilterRule{rule}, + } + if err := dm.SetFilter(context.Background(), f, true); err != nil { + t.Fatal(err) + } + if err := addItemToFilterIndex(context.Background(), dm, utils.CacheAttributeFilterIndexes, + tnt, contextKey, fid, []string{fid}); err != nil { + t.Fatal(err) + } + } + + ev := utils.MapStorage{ + utils.MetaReq: map[string]any{ + "Field": "trigger", + }, + } + + ids, err := MatchingItemIDsForEvent(context.Background(), ev, nil, nil, nil, nil, nil, + dm, utils.CacheAttributeFilterIndexes, tntCtx, true, false) + if err != nil { + t.Fatal(err) + } + + if len(ids) <= matchedItemsWarningThreshold { + t.Errorf("Expected more than %d matched items, got %d", + matchedItemsWarningThreshold, len(ids)) + } + + expectedLog := "Matched 105 *attribute_filter_indexes items. Performance may be affected." + logContent := buf.String() + if !strings.Contains(logContent, expectedLog) { + t.Errorf("Expected warning not found in logs:\n%s", logContent) + } +}