From 24834ba933986e6bb375a906c0bf6683efa399fc Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Fri, 25 Mar 2022 18:12:13 +0200 Subject: [PATCH] Add additional tests for indexes --- apis/attributes_it_test.go | 320 +++++++++++++++++++++++++++++++++++++ engine/libindex_test.go | 254 +++++++++++++++++++++++++++++ 2 files changed, 574 insertions(+) diff --git a/apis/attributes_it_test.go b/apis/attributes_it_test.go index 2fdcb280e..43444b66e 100644 --- a/apis/attributes_it_test.go +++ b/apis/attributes_it_test.go @@ -75,6 +75,18 @@ var ( testAttributeGetAttributeProfileAllCount, testAttributeRemoveRemainAttributeProfiles, testAttributeGetAttributeProfileAfterRemove, + + // Testing index behaviour + testAttributeSSetNonIndexedTypeFilter, + testAttributeSSetIndexedTypeFilter, + testAttributeSClearIndexes, + testAttributeSCheckIndexesSetAttributeProfileWithoutFilters, + // testAttributeSCheckIndexesAddNonIndexedFilter, + testAttributeSCheckIndexesAddIndexedFilters, + testAttributeSCheckIndexesModifyIndexedFilter, + testAttributeSCheckIndexesRemoveAnIndexedFilter, + testAttributeSCheckIndexesRemoveAttributeProfile, + testAttributeSKillEngine, } ) @@ -1352,6 +1364,314 @@ func testAttributeGetAttributeProfileAfterRemove(t *testing.T) { } } +func testAttributeSSetNonIndexedTypeFilter(t *testing.T) { + filter := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: "cgrates.org", + ID: "NONINDEXED_FLTR_TYPE", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaGreaterThan, + Element: utils.Cost, + Values: []string{"10"}, + }, + }, + }, + } + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply: ", reply) + } +} + +func testAttributeSSetIndexedTypeFilter(t *testing.T) { + filter := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: "cgrates.org", + ID: "INDEXED_FLTR_TYPE", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaPrefix, + Element: "~*req.Account", + Values: []string{"10"}, + }, + }, + }, + } + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply: ", reply) + } +} + +func testAttributeSClearIndexes(t *testing.T) { + args := &AttrRemFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes, + args, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply: ", reply) + } +} + +func testAttributeSCheckIndexesSetAttributeProfileWithoutFilters(t *testing.T) { + attrPrf := &engine.APIAttributeProfileWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + Attributes: []*engine.ExternalAttribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: "1002", + }, + }, + }, + } + + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + expIdx := []string{"*none:*any:*any:ATTR_TEST"} + args := &AttrGetFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var replyIdx []string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(replyIdx, expIdx) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) + } +} + +// func testAttributeSCheckIndexesAddNonIndexedFilter(t *testing.T) { +// attrPrf := &engine.APIAttributeProfileWithAPIOpts{ +// APIAttributeProfile: &engine.APIAttributeProfile{ +// Tenant: "cgrates.org", +// ID: "ATTR_TEST", +// FilterIDs: []string{"NONINDEXED_FLTR_TYPE"}, +// Attributes: []*engine.ExternalAttribute{ +// { +// Type: utils.MetaConstant, +// Path: "~*req.Account", +// Value: "1002", +// }, +// }, +// }, +// } + +// var reply string +// if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile, +// attrPrf, &reply); err != nil { +// t.Error(err) +// } else if reply != utils.OK { +// t.Error(err) +// } + +// expIdx := []string{"*none:*any:*any:ATTR_TEST"} +// args := &AttrGetFilterIndexes{ +// Tenant: "cgrates.org", +// ItemType: utils.MetaAttributes, +// } +// var replyIdx []string +// if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, +// args, &replyIdx); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(replyIdx, expIdx) { +// t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) +// } +// } + +func testAttributeSCheckIndexesAddIndexedFilters(t *testing.T) { + attrPrf := &engine.APIAttributeProfileWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE"}, + Attributes: []*engine.ExternalAttribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: "1002", + }, + }, + }, + } + + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + expIdx := []string{"*prefix:*req.Account:10:ATTR_TEST"} + args := &AttrGetFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var replyIdx []string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(replyIdx, expIdx) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) + } + + attrPrf = &engine.APIAttributeProfileWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE", "*string:~*req.Category:call"}, + Attributes: []*engine.ExternalAttribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: "1002", + }, + }, + }, + } + + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + expIdx = []string{"*prefix:*req.Account:10:ATTR_TEST", "*string:*req.Category:call:ATTR_TEST"} + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + if !reflect.DeepEqual(replyIdx, expIdx) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) + } + } +} + +func testAttributeSCheckIndexesModifyIndexedFilter(t *testing.T) { + filter := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: "cgrates.org", + ID: "INDEXED_FLTR_TYPE", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaSuffix, + Element: "~*req.Subject", + Values: []string{"01"}, + }, + }, + }, + } + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply: ", reply) + } + + expIdx := []string{"*string:*req.Category:call:ATTR_TEST", "*suffix:*req.Subject:01:ATTR_TEST"} + args := &AttrGetFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var replyIdx []string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + if !reflect.DeepEqual(replyIdx, expIdx) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) + } + } +} + +func testAttributeSCheckIndexesRemoveAnIndexedFilter(t *testing.T) { + attrPrf := &engine.APIAttributeProfileWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE"}, + Attributes: []*engine.ExternalAttribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: "1002", + }, + }, + }, + } + + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + expIdx := []string{"*suffix:*req.Subject:01:ATTR_TEST"} + args := &AttrGetFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var replyIdx []string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(replyIdx, expIdx) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx)) + } +} +func testAttributeSCheckIndexesRemoveAttributeProfile(t *testing.T) { + argsRem := &utils.TenantIDWithAPIOpts{ + TenantID: &utils.TenantID{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + }, + } + var reply string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile, + argsRem, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + args := &AttrGetFilterIndexes{ + Tenant: "cgrates.org", + ItemType: utils.MetaAttributes, + } + var replyIdx []string + if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + args, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ErrNotFound, err) + } +} + //Kill the engine when it is about to be finished func testAttributeSKillEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { diff --git a/engine/libindex_test.go b/engine/libindex_test.go index 7b6259e6c..4cf791a55 100644 --- a/engine/libindex_test.go +++ b/engine/libindex_test.go @@ -171,3 +171,257 @@ func TestRemoveFilterIndexesForFilter(t *testing.T) { t.Error(err) } } + +func TestLibIndexSetUpdateRemAttributeProfile(t *testing.T) { + cfg := config.NewDefaultCGRConfig() + dataDB := NewInternalDB(nil, nil, cfg.DataDbCfg().Items) + dm := NewDataManager(dataDB, cfg.CacheCfg(), nil) + + // Set an AttributeProfile without filterIDs + attrPrf := &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err := dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } + expIndexes := map[string]utils.StringSet{ + "*none:*any:*any": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err := dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", + utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + } + + // Add a non-indexed filter type + attrPrf = &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"*gt:~*req.Element:10"}, + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err = dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + // if err != nil { + // t.Error(err) + // } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + // t.Errorf("expected: <%+v>, \nreceived: <%+v>", + // utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + // } + + // Add an indexed filter type + attrPrf = &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10"}, + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err = dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } + expIndexes = map[string]utils.StringSet{ + "*prefix:*req.Account:10": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", + utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + } + + // Add another indexed filter type + attrPrf = &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10", "*string:~*req.Category:call"}, + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err = dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } + expIndexes = map[string]utils.StringSet{ + "*prefix:*req.Account:10": { + "ATTR_TEST": {}, + }, + "*string:*req.Category:call": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", + utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + } + + // Remove an indexed filter type + attrPrf = &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10"}, + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err = dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", + utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + } + expIndexes = map[string]utils.StringSet{ + "*prefix:*req.Account:10": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err != nil { + t.Error(err) + } + + // Remove the attribute profile + err = dm.RemoveAttributeProfile(context.Background(), attrPrf.Tenant, attrPrf.ID, true) + if err != nil { + t.Error(err) + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ErrNotFound, err) + } +} + +func TestLibIndexModifyAttrPrfFilter(t *testing.T) { + cfg := config.NewDefaultCGRConfig() + dataDB := NewInternalDB(nil, nil, cfg.DataDbCfg().Items) + dm := NewDataManager(dataDB, cfg.CacheCfg(), nil) + + // Set a non-indexed type filter + fltr := &Filter{ + Tenant: "cgrates.org", + ID: "fltr_test", + Rules: []*FilterRule{ + { + Type: utils.MetaGreaterThan, + Element: "~*req.Cost", + Values: []string{"10"}, + }, + }, + } + err := dm.SetFilter(context.Background(), fltr, true) + if err != nil { + t.Error(err) + } + + // Create an AttributeProfile using the previously created filter + attrPrf := &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_TEST", + FilterIDs: []string{"fltr_test"}, + Attributes: []*Attribute{ + { + Type: utils.MetaConstant, + Path: "~*req.Account", + Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep), + }, + }, + } + err = dm.SetAttributeProfile(context.Background(), attrPrf, true) + if err != nil { + t.Error(err) + } + expIndexes := map[string]utils.StringSet{ + "*none:*any:*any": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err := dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + // if err != nil { + // t.Error(err) + // } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + // t.Errorf("expected: <%+v>, \nreceived: <%+v>", + // utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + // } + + // Make the filter indexable + fltr = &Filter{ + Tenant: "cgrates.org", + ID: "fltr_test", + Rules: []*FilterRule{ + { + Type: utils.MetaPrefix, + Element: "~*req.Account", + Values: []string{"10"}, + }, + }, + } + err = dm.SetFilter(context.Background(), fltr, true) + if err != nil { + t.Error(err) + } + + expIndexes = map[string]utils.StringSet{ + "*prefix:*req.Account:10": { + "ATTR_TEST": {}, + }, + } + rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant, + utils.EmptyString, utils.NonTransactional, false, false) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcvIndexes, expIndexes) { + t.Errorf("expected: <%+v>, \nreceived: <%+v>", + utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes)) + } +}