diff --git a/apis/filter_indexes_it_test.go b/apis/filter_indexes_it_test.go new file mode 100644 index 000000000..5d8a0ce8b --- /dev/null +++ b/apis/filter_indexes_it_test.go @@ -0,0 +1,726 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package apis + +import ( + "path" + "reflect" + "sort" + "testing" + "time" + + "github.com/cgrates/birpc" + "github.com/cgrates/birpc/context" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + tFltrIdxCfgPath string + tFIdxRpc *birpc.Client + tFltrIdxCfg *config.CGRConfig + tFltrIdxConfDIR string + + sTestsFilterIndexesSV1 = []func(t *testing.T){ + testV1FIdxLoadConfig, + testV1FIdxdxInitDataDb, + testV1FIdxResetStorDb, + testV1FIdxStartEngine, + testV1FIdxRpcConn, + + testV1FIdxSetAttributeSProfileWithFltr, + testV1FIdxSetAttributeSMoreFltrsMoreIndexing, + testV1FIdxAttributesRemoveIndexes, + testV1FIdxAttributeComputeIndexes, + testV1FIdxAttributeMoreProfilesForFilters, + testV1FIdxAttributeSRemoveComputedIndexesIDs, + testV1FIdxAttributesRemoveProfilesNoIndexes, + testV1IndexClearCache, + + testV1FIdxStopEngine, + } +) + +// Test start here +func TestFltrIdxV1IT(t *testing.T) { + switch *dbType { + case utils.MetaInternal: + tFltrIdxConfDIR = "filter_indexes_internal" + case utils.MetaMySQL: + tFltrIdxConfDIR = "filter_indexes_mysql" + case utils.MetaMongo: + tFltrIdxConfDIR = "filter_indexes_mongo" + case utils.MetaPostgres: + t.SkipNow() + default: + t.Fatal("Unknown Database type") + } + for _, stest := range sTestsFilterIndexesSV1 { + t.Run(tFltrIdxConfDIR, stest) + } +} + +func testV1FIdxLoadConfig(t *testing.T) { + tFltrIdxCfgPath = path.Join(*dataDir, "conf", "samples", tFltrIdxConfDIR) + var err error + if tFltrIdxCfg, err = config.NewCGRConfigFromPath(tFltrIdxCfgPath); err != nil { + t.Error(err) + } +} + +func testV1FIdxdxInitDataDb(t *testing.T) { + if err := engine.InitDataDB(tFltrIdxCfg); err != nil { + t.Fatal(err) + } +} + +func testV1IndexClearCache(t *testing.T) { + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.CacheSv1Clear, &utils.AttrCacheIDsWithAPIOpts{}, &reply); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func testV1FIdxResetStorDb(t *testing.T) { + if err := engine.InitStorDB(tFltrIdxCfg); err != nil { + t.Fatal(err) + } +} + +func testV1FIdxStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(tFltrIdxCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +func testV1FIdxRpcConn(t *testing.T) { + var err error + tFIdxRpc, err = newRPCClient(tFltrIdxCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal("Could not connect to rater: ", err.Error()) + } +} + +func testV1FIdxSetAttributeSProfileWithFltr(t *testing.T) { + // First we will set a filter for usage + fltr := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: utils.CGRateSorg, + ID: "fltr_for_attr", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.Subject", + Values: []string{"1004", "6774", "22312"}, + }, + { + Type: utils.MetaString, + Element: "~*opts.Subsystems", + Values: []string{"*attributes"}, + }, + { + Type: utils.MetaPrefix, + Element: "~*req.Destinations", + Values: []string{"+0775", "+442"}, + }, + }, + }, + } + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter, + fltr, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply result", reply) + } + + // Get filter for checking it's existence + var resultFltr *engine.Filter + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilter, + &utils.TenantID{Tenant: utils.CGRateSorg, ID: "fltr_for_attr"}, &resultFltr); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(resultFltr, fltr.Filter) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(fltr.Filter), utils.ToJSON(resultFltr)) + } + + //we will set an AttributeProfile with our filter and check the indexes + attrPrf := &engine.AttributeWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: utils.CGRateSorg, + ID: "TEST_ATTRIBUTES_IT_TEST", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"fltr_for_attr"}, + Attributes: []*engine.ExternalAttribute{ + { + Path: utils.AccountField, + Type: utils.MetaConstant, + Value: "1002", + }, + { + Path: "*tenant", + Type: utils.MetaConstant, + Value: "cgrates.itsyscom", + }, + }, + }, + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + //check indexes + var replyIdx []string + expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes}, + &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + sort.Strings(expectedIDx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx) + } + } + + //update the filter for checking the indexes + fltr = &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: utils.CGRateSorg, + ID: "fltr_for_attr", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.CGRID", + Values: []string{"QWEASDZXC", "IOPJKLBNM"}, + }, + }, + }, + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter, + fltr, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply result", reply) + } + + // check the updated indexes + expectedIDx = []string{"*string:*req.CGRID:QWEASDZXC:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.CGRID:IOPJKLBNM:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes}, + &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + sort.Strings(expectedIDx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx) + } + } + + // context changed, not gonna match any indexes + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, ItemType: utils.MetaAttributes}, + &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } + + //back to our initial filter + fltr = &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: utils.CGRateSorg, + ID: "fltr_for_attr", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.Subject", + Values: []string{"1004", "6774", "22312"}, + }, + { + Type: utils.MetaString, + Element: "~*opts.Subsystems", + Values: []string{"*attributes"}, + }, + { + Type: utils.MetaPrefix, + Element: "~*req.Destinations", + Values: []string{"+0775", "+442"}, + }, + }, + }, + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter, + fltr, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply result", reply) + } +} + +func testV1FIdxSetAttributeSMoreFltrsMoreIndexing(t *testing.T) { + // More filters for our AttributeProfile + fltr1 := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: utils.CGRateSorg, + ID: "fltr_for_attr2", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.Usage", + Values: []string{"123s"}, + }, + }, + }, + } + fltr2 := &engine.FilterWithAPIOpts{ + Filter: &engine.Filter{ + Tenant: utils.CGRateSorg, + ID: "fltr_for_attr3", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaPrefix, + Element: "~*req.AnswerTime", + Values: []string{"12", "33"}, + }, + }, + }, + } + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter, + fltr1, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply result", reply) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter, + fltr2, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply result", reply) + } + + // update our Attribute with our filters + attrPrf := &engine.AttributeWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: utils.CGRateSorg, + ID: "TEST_ATTRIBUTES_IT_TEST", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"fltr_for_attr", "fltr_for_attr2", "fltr_for_attr3"}, + Attributes: []*engine.ExternalAttribute{ + { + Path: utils.AccountField, + Type: utils.MetaConstant, + Value: "1002", + }, + { + Path: "*tenant", + Type: utils.MetaConstant, + Value: "cgrates.itsyscom", + }, + }, + }, + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + //check indexes + var replyIdx []string + expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes}, + &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + sort.Strings(expectedIDx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx) + } + } +} + +func testV1FIdxAttributesRemoveIndexes(t *testing.T) { + //indexes will not be removed because of this different context + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes, + &AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("UNexpected reply returned") + } + + //these are not removed, so the indexes are not removed + var replyIdx []string + expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes}, + &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(replyIdx) + sort.Strings(expectedIDx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx) + } + } + + //indexes will be removed for this specific context + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes, + &AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("UNexpected reply returned") + } + + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes}, + &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %T, received %T", utils.ErrNotFound, err) + } +} + +func testV1FIdxAttributeComputeIndexes(t *testing.T) { + // compute our indexes + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexes, + &utils.ArgsComputeFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + AttributeS: true}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply returned") + } + + // our indexes are sotred again, so we can get them + // firstly, not gonna get for a different context + var replyIdx []string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %T, received %T", utils.ErrNotFound, err) + } + + //matching for our context + expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expectedIDx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx) + } + } +} + +func testV1FIdxAttributeMoreProfilesForFilters(t *testing.T) { + //we will more attributes with different context for matching filters + attrPrf2 := &engine.AttributeWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: utils.CGRateSorg, + ID: "TEST_ATTRIBUTES_new_fltr", + Contexts: []string{utils.MetaChargers}, + FilterIDs: []string{"fltr_for_attr2", "fltr_for_attr3"}, + Attributes: []*engine.ExternalAttribute{ + { + Path: utils.AccountField, + Type: utils.MetaConstant, + Value: "1002", + }, + }, + }, + } + attrPrf3 := &engine.AttributeWithAPIOpts{ + APIAttributeProfile: &engine.APIAttributeProfile{ + Tenant: utils.CGRateSorg, + ID: "TEST_ATTRIBUTE3", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"fltr_for_attr3"}, + Attributes: []*engine.ExternalAttribute{ + { + Path: utils.Destinations, + Type: utils.MetaConstant, + Value: "1008", + }, + }, + }, + } + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf2, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile, + attrPrf3, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error(err) + } + + // now we will match indexes for *chargers + var replyIdx []string + expectedIDx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr"} + //"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_TEST_ATTRIBUTE3", + //"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_TEST_ATTRIBUTE"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expectedIDx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx) + } + } + + // now we will match indexes for *sessions + expectedIDx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expectedIDx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expectedIDx, replyIdx) { + t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx) + } + } +} + +func testV1FIdxAttributeSRemoveComputedIndexesIDs(t *testing.T) { + //indexes will ne removed for both contexts + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes, + &AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("UNexpected reply returned") + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes, + &AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, + &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("UNexpected reply returned") + } + + //not found for both cases + var replyIdx []string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } + + // now we will ComputeFilterIndexes by IDs for *chargers context + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs, + &utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + AttributeIDs: []string{"TEST_ATTRIBUTES_new_fltr"}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply returned") + } + + //able to get indexes with context *chargers + expIdx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expIdx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expIdx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expIdx, replyIdx) + } + } + + // now we will ComputeFilterIndexes by IDs for *sessions context(but just only 1 profile, not both) + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs, + &utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + AttributeIDs: []string{"TEST_ATTRIBUTE3"}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply returned") + } + + //able to get indexes with context *sessions + expIdx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expIdx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expIdx, replyIdx) { + t.Errorf("Expected %+v, received %+v", expIdx, replyIdx) + } + } + + // compute for the last profile remain + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs, + &utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + AttributeIDs: []string{"TEST_ATTRIBUTES_IT_TEST"}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply returned") + } + expIdx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3", + "*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3", + "*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST", + "*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST", + "*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST", + "*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"} + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err != nil { + t.Error(err) + } else { + sort.Strings(expIdx) + sort.Strings(replyIdx) + if !reflect.DeepEqual(expIdx, replyIdx) { + t.Errorf("Expected %+v \n, received %+v", expIdx, replyIdx) + } + } + time.Sleep(100) +} + +func testV1FIdxAttributesRemoveProfilesNoIndexes(t *testing.T) { + //as we delete our profiles, indexes will be deleted too + var reply string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTES_IT_TEST", + Tenant: utils.CGRateSorg}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("Expected %+v \n, received %+v", utils.OK, reply) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTE3", + Tenant: utils.CGRateSorg}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("Expected %+v \n, received %+v", utils.OK, reply) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTES_new_fltr", + Tenant: utils.CGRateSorg}}, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("Expected %+v \n, received %+v", utils.OK, reply) + } + + // Check indexes as we removed, not found for both indexes + var replyIdx []string + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, + ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } + if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes, + &AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, + ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } +} + +func testV1FIdxStopEngine(t *testing.T) { + if err := engine.KillEngine(100); err != nil { + t.Error(err) + } +} diff --git a/console/filter.go b/console/filter.go index 8306c54a5..e61ea8fec 100644 --- a/console/filter.go +++ b/console/filter.go @@ -26,7 +26,7 @@ import ( func init() { c := &CmdGetFilter{ name: "filter", - rpcMethod: utils.APIerSv1GetFilter, + rpcMethod: utils.AdminSv1GetFilter, rpcParams: &utils.TenantID{}, } commands[c.Name()] = c diff --git a/console/filter_ids.go b/console/filter_ids.go index 725f64c47..dfdcba82c 100644 --- a/console/filter_ids.go +++ b/console/filter_ids.go @@ -25,7 +25,7 @@ import ( func init() { c := &CmdGetFilterIDs{ name: "filter_ids", - rpcMethod: utils.APIerSv1GetFilterIDs, + rpcMethod: utils.AdminSv1GetFilterIDs, rpcParams: &utils.PaginatorWithTenant{}, } commands[c.Name()] = c diff --git a/console/filter_remove.go b/console/filter_remove.go index a3f4911fe..679c8d785 100644 --- a/console/filter_remove.go +++ b/console/filter_remove.go @@ -23,7 +23,7 @@ import "github.com/cgrates/cgrates/utils" func init() { c := &CmdRemoveFilter{ name: "filter_remove", - rpcMethod: utils.APIerSv1RemoveFilter, + rpcMethod: utils.AdminSv1RemoveFilter, rpcParams: &utils.TenantIDWithAPIOpts{}, } commands[c.Name()] = c diff --git a/console/filter_set.go b/console/filter_set.go index 488a0f99c..1f71017e1 100644 --- a/console/filter_set.go +++ b/console/filter_set.go @@ -26,7 +26,7 @@ import ( func init() { c := &CmdSetFilter{ name: "filter_set", - rpcMethod: utils.APIerSv1SetFilter, + rpcMethod: utils.AdminSv1SetFilter, rpcParams: &engine.FilterWithAPIOpts{}, } commands[c.Name()] = c diff --git a/data/conf/samples/filter_indexes_internal/cgrates.json b/data/conf/samples/filter_indexes_internal/cgrates.json new file mode 100644 index 000000000..2c5a6e1ae --- /dev/null +++ b/data/conf/samples/filter_indexes_internal/cgrates.json @@ -0,0 +1,27 @@ +{ + // CGRateS Configuration file + // will be used in apis/filter_indexes_it_test.go + + + "data_db": { + "db_type": "*internal", + }, + + "stor_db": { + "db_type": "*internal", + }, + + "attributes": { + "enabled": true, + //"string_indexes_fields": ["*req.Subject"], + //"prefix_indexes_fields": ["*req.Destinations"] + }, + + "admins": { + "enabled": true, + }, + + "filters": { + "admins_conns": ["*internal"] + } +}, \ No newline at end of file diff --git a/data/conf/samples/filter_indexes_mongo/cgrates.json b/data/conf/samples/filter_indexes_mongo/cgrates.json new file mode 100644 index 000000000..e10f5b874 --- /dev/null +++ b/data/conf/samples/filter_indexes_mongo/cgrates.json @@ -0,0 +1,32 @@ +{ + // CGRateS Configuration file + // will be used in apis/filter_indexes_it_test.go + + + "data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017, + }, + + + "stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017, + }, + + "attributes": { + "enabled": true, + //"string_indexes_fields": ["*req.Subject"], + //"prefix_indexes_fields": ["*req.Destinations"] + }, + + "admins": { + "enabled": true, + }, + + "filters": { + "admins_conns": ["*internal"] + } +}, \ No newline at end of file diff --git a/data/conf/samples/filter_indexes_mysql/cgrates.json b/data/conf/samples/filter_indexes_mysql/cgrates.json new file mode 100644 index 000000000..d6ac6216f --- /dev/null +++ b/data/conf/samples/filter_indexes_mysql/cgrates.json @@ -0,0 +1,29 @@ +{ + // CGRateS Configuration file + // will be used in apis/filter_indexes_it_test.go + + + "data_db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to + }, + + "stor_db": { + "db_password": "CGRateS.org", + }, + + "attributes": { + "enabled": true, + //"string_indexes_fields": ["*req.Subject"], + //"prefix_indexes_fields": ["*req.Destinations"] + }, + + "admins": { + "enabled": true, + }, + + "filters": { + "admins_conns": ["*internal"] + } +}, \ No newline at end of file diff --git a/engine/datamanager.go b/engine/datamanager.go index 6a3978275..4f95fe5ab 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -766,6 +766,7 @@ func (dm *DataManager) SetFilter(ctx *context.Context, fltr *Filter, withIndex b if err = dm.DataDB().SetFilterDrv(ctx, fltr); err != nil { return } + fmt.Println("set fltr da", withIndex, utils.ToJSON(oldFlt), utils.ToJSON(fltr)) if withIndex { if err = UpdateFilterIndex(ctx, dm, oldFlt, fltr); err != nil { return @@ -1743,6 +1744,7 @@ func (dm *DataManager) SetAttributeProfile(ctx *context.Context, ap *AttributePr oldContexes = &oldAP.Contexts oldFiltersIDs = &oldAP.FilterIDs } + fmt.Println(oldContexes, oldFiltersIDs) if err = updatedIndexesWithContexts(ctx, dm, utils.CacheAttributeFilterIndexes, ap.Tenant, ap.ID, oldContexes, oldFiltersIDs, ap.Contexts, ap.FilterIDs); err != nil { return diff --git a/engine/libattributes_test.go b/engine/libattributes_test.go index 51f44b974..4c2b5ec21 100644 --- a/engine/libattributes_test.go +++ b/engine/libattributes_test.go @@ -34,10 +34,6 @@ func TestConvertExternalToProfile(t *testing.T) { ID: "ATTR_ID", Contexts: []string{utils.MetaSessionS, utils.MetaCDRs}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - ExpiryTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - }, Attributes: []*ExternalAttribute{ { Path: utils.MetaReq + utils.NestingSep + "Account", @@ -52,10 +48,6 @@ func TestConvertExternalToProfile(t *testing.T) { ID: "ATTR_ID", Contexts: []string{utils.MetaSessionS, utils.MetaCDRs}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - ExpiryTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - }, Attributes: []*Attribute{ { Path: utils.MetaReq + utils.NestingSep + "Account", diff --git a/engine/libtest.go b/engine/libtest.go index bf5a6fc9f..daee08145 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -20,7 +20,6 @@ package engine import ( "bytes" - "context" "fmt" "io" "net/rpc/jsonrpc" @@ -29,6 +28,8 @@ import ( "path" "time" + "github.com/cgrates/birpc/context" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/cgrates/ltcache" diff --git a/migrator/stats.go b/migrator/stats.go index f5670606b..9d49c1fd9 100644 --- a/migrator/stats.go +++ b/migrator/stats.go @@ -63,7 +63,7 @@ type v1Stats []*v1Stat func (m *Migrator) moveStatQueueProfile() (err error) { //StatQueueProfile var ids []string - if ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(context.TODO(), utils.StatQueueProfilePrefix); err != nil { + if ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(context.Background(), utils.StatQueueProfilePrefix); err != nil { return err } for _, id := range ids { diff --git a/services/loaders.go b/services/loaders.go index d5ad22a27..2681ddc3f 100644 --- a/services/loaders.go +++ b/services/loaders.go @@ -65,7 +65,7 @@ type LoaderService struct { srvDep map[string]*sync.WaitGroup } -// Start should handle the sercive start +// Start should handle the service start func (ldrs *LoaderService) Start() (err error) { if ldrs.IsRunning() { return utils.ErrServiceAlreadyRunning diff --git a/utils/consts.go b/utils/consts.go index 4b26bf890..5d4e756ab 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -1169,8 +1169,8 @@ const ( // APIerSv1 APIs const ( - APIerSv1ComputeFilterIndexes = "APIerSv1.ComputeFilterIndexes" - APIerSv1ComputeFilterIndexIDs = "APIerSv1.ComputeFilterIndexIDs" + AdminSv1ComputeFilterIndexes = "AdminSv1.ComputeFilterIndexes" + AdminSv1ComputeFilterIndexIDs = "AdminSv1.ComputeFilterIndexIDs" APIerSv1Ping = "APIerSv1.Ping" APIerSv1SetDispatcherProfile = "APIerSv1.SetDispatcherProfile" APIerSv1GetDispatcherProfile = "APIerSv1.GetDispatcherProfile" @@ -1185,12 +1185,12 @@ const ( APIerSv1LoadTariffPlanFromFolder = "APIerSv1.LoadTariffPlanFromFolder" APIerSv1ExportToFolder = "APIerSv1.ExportToFolder" APIerSv1GetCost = "APIerSv1.GetCost" - APIerSv1GetFilter = "APIerSv1.GetFilter" - APIerSv1GetFilterIndexes = "APIerSv1.GetFilterIndexes" - APIerSv1RemoveFilterIndexes = "APIerSv1.RemoveFilterIndexes" - APIerSv1RemoveFilter = "APIerSv1.RemoveFilter" - APIerSv1SetFilter = "APIerSv1.SetFilter" - APIerSv1GetFilterIDs = "APIerSv1.GetFilterIDs" + AdminSv1GetFilter = "AdminSv1.GetFilter" + AdminSv1GetFilterIndexes = "AdminSv1.GetFilterIndexes" + AdminSv1RemoveFilterIndexes = "AdminSv1.RemoveFilterIndexes" + AdminSv1RemoveFilter = "AdminSv1.RemoveFilter" + AdminSv1SetFilter = "AdminSv1.SetFilter" + AdminSv1GetFilterIDs = "AdminSv1.GetFilterIDs" APIerSv1SetDataDBVersions = "APIerSv1.SetDataDBVersions" APIerSv1SetStorDBVersions = "APIerSv1.SetStorDBVersions" APIerSv1GetActions = "APIerSv1.GetActions"