Tested APIs for indexes + their constants and configs

apis/filter_indexes_it_test.go

@@ -0,0 +1,726 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package apis
+
+import (
+	"path"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/cgrates/birpc"
+	"github.com/cgrates/birpc/context"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	tFltrIdxCfgPath string
+	tFIdxRpc        *birpc.Client
+	tFltrIdxCfg     *config.CGRConfig
+	tFltrIdxConfDIR string
+
+	sTestsFilterIndexesSV1 = []func(t *testing.T){
+		testV1FIdxLoadConfig,
+		testV1FIdxdxInitDataDb,
+		testV1FIdxResetStorDb,
+		testV1FIdxStartEngine,
+		testV1FIdxRpcConn,
+
+		testV1FIdxSetAttributeSProfileWithFltr,
+		testV1FIdxSetAttributeSMoreFltrsMoreIndexing,
+		testV1FIdxAttributesRemoveIndexes,
+		testV1FIdxAttributeComputeIndexes,
+		testV1FIdxAttributeMoreProfilesForFilters,
+		testV1FIdxAttributeSRemoveComputedIndexesIDs,
+		testV1FIdxAttributesRemoveProfilesNoIndexes,
+		testV1IndexClearCache,
+
+		testV1FIdxStopEngine,
+	}
+)
+
+// Test start here
+func TestFltrIdxV1IT(t *testing.T) {
+	switch *dbType {
+	case utils.MetaInternal:
+		tFltrIdxConfDIR = "filter_indexes_internal"
+	case utils.MetaMySQL:
+		tFltrIdxConfDIR = "filter_indexes_mysql"
+	case utils.MetaMongo:
+		tFltrIdxConfDIR = "filter_indexes_mongo"
+	case utils.MetaPostgres:
+		t.SkipNow()
+	default:
+		t.Fatal("Unknown Database type")
+	}
+	for _, stest := range sTestsFilterIndexesSV1 {
+		t.Run(tFltrIdxConfDIR, stest)
+	}
+}
+
+func testV1FIdxLoadConfig(t *testing.T) {
+	tFltrIdxCfgPath = path.Join(*dataDir, "conf", "samples", tFltrIdxConfDIR)
+	var err error
+	if tFltrIdxCfg, err = config.NewCGRConfigFromPath(tFltrIdxCfgPath); err != nil {
+		t.Error(err)
+	}
+}
+
+func testV1FIdxdxInitDataDb(t *testing.T) {
+	if err := engine.InitDataDB(tFltrIdxCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testV1IndexClearCache(t *testing.T) {
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.CacheSv1Clear, &utils.AttrCacheIDsWithAPIOpts{}, &reply); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Wipe out the cdr database
+func testV1FIdxResetStorDb(t *testing.T) {
+	if err := engine.InitStorDB(tFltrIdxCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testV1FIdxStartEngine(t *testing.T) {
+	if _, err := engine.StopStartEngine(tFltrIdxCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testV1FIdxRpcConn(t *testing.T) {
+	var err error
+	tFIdxRpc, err = newRPCClient(tFltrIdxCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func testV1FIdxSetAttributeSProfileWithFltr(t *testing.T) {
+	// First we will set a filter for usage
+	fltr := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: utils.CGRateSorg,
+			ID:     "fltr_for_attr",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaString,
+					Element: "~*req.Subject",
+					Values:  []string{"1004", "6774", "22312"},
+				},
+				{
+					Type:    utils.MetaString,
+					Element: "~*opts.Subsystems",
+					Values:  []string{"*attributes"},
+				},
+				{
+					Type:    utils.MetaPrefix,
+					Element: "~*req.Destinations",
+					Values:  []string{"+0775", "+442"},
+				},
+			},
+		},
+	}
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
+		fltr, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply result", reply)
+	}
+
+	// Get filter for checking it's existence
+	var resultFltr *engine.Filter
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilter,
+		&utils.TenantID{Tenant: utils.CGRateSorg, ID: "fltr_for_attr"}, &resultFltr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(resultFltr, fltr.Filter) {
+		t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(fltr.Filter), utils.ToJSON(resultFltr))
+	}
+
+	//we will set an AttributeProfile with our filter and check the indexes
+	attrPrf := &engine.AttributeWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    utils.CGRateSorg,
+			ID:        "TEST_ATTRIBUTES_IT_TEST",
+			Contexts:  []string{utils.MetaSessionS},
+			FilterIDs: []string{"fltr_for_attr"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Path:  utils.AccountField,
+					Type:  utils.MetaConstant,
+					Value: "1002",
+				},
+				{
+					Path:  "*tenant",
+					Type:  utils.MetaConstant,
+					Value: "cgrates.itsyscom",
+				},
+			},
+		},
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	//check indexes
+	var replyIdx []string
+	expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes},
+		&replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		sort.Strings(expectedIDx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
+		}
+	}
+
+	//update the filter for checking the indexes
+	fltr = &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: utils.CGRateSorg,
+			ID:     "fltr_for_attr",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaString,
+					Element: "~*req.CGRID",
+					Values:  []string{"QWEASDZXC", "IOPJKLBNM"},
+				},
+			},
+		},
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
+		fltr, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply result", reply)
+	}
+
+	// check the updated indexes
+	expectedIDx = []string{"*string:*req.CGRID:QWEASDZXC:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.CGRID:IOPJKLBNM:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes},
+		&replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		sort.Strings(expectedIDx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
+		}
+	}
+
+	// context changed, not gonna match any indexes
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers, ItemType: utils.MetaAttributes},
+		&replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Error(err)
+	}
+
+	//back to our initial filter
+	fltr = &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: utils.CGRateSorg,
+			ID:     "fltr_for_attr",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaString,
+					Element: "~*req.Subject",
+					Values:  []string{"1004", "6774", "22312"},
+				},
+				{
+					Type:    utils.MetaString,
+					Element: "~*opts.Subsystems",
+					Values:  []string{"*attributes"},
+				},
+				{
+					Type:    utils.MetaPrefix,
+					Element: "~*req.Destinations",
+					Values:  []string{"+0775", "+442"},
+				},
+			},
+		},
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
+		fltr, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply result", reply)
+	}
+}
+
+func testV1FIdxSetAttributeSMoreFltrsMoreIndexing(t *testing.T) {
+	// More filters for our AttributeProfile
+	fltr1 := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: utils.CGRateSorg,
+			ID:     "fltr_for_attr2",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaString,
+					Element: "~*req.Usage",
+					Values:  []string{"123s"},
+				},
+			},
+		},
+	}
+	fltr2 := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: utils.CGRateSorg,
+			ID:     "fltr_for_attr3",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaPrefix,
+					Element: "~*req.AnswerTime",
+					Values:  []string{"12", "33"},
+				},
+			},
+		},
+	}
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
+		fltr1, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply result", reply)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
+		fltr2, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply result", reply)
+	}
+
+	// update our Attribute with our filters
+	attrPrf := &engine.AttributeWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    utils.CGRateSorg,
+			ID:        "TEST_ATTRIBUTES_IT_TEST",
+			Contexts:  []string{utils.MetaSessionS},
+			FilterIDs: []string{"fltr_for_attr", "fltr_for_attr2", "fltr_for_attr3"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Path:  utils.AccountField,
+					Type:  utils.MetaConstant,
+					Value: "1002",
+				},
+				{
+					Path:  "*tenant",
+					Type:  utils.MetaConstant,
+					Value: "cgrates.itsyscom",
+				},
+			},
+		},
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	//check indexes
+	var replyIdx []string
+	expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes},
+		&replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		sort.Strings(expectedIDx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
+		}
+	}
+}
+
+func testV1FIdxAttributesRemoveIndexes(t *testing.T) {
+	//indexes will not be removed because of this different context
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
+		&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes},
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("UNexpected reply returned")
+	}
+
+	//these are not removed, so the indexes are not removed
+	var replyIdx []string
+	expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes},
+		&replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		sort.Strings(expectedIDx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
+		}
+	}
+
+	//indexes will be removed for this specific context
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
+		&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes},
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("UNexpected reply returned")
+	}
+
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS, ItemType: utils.MetaAttributes},
+		&replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Errorf("Expected %T, received %T", utils.ErrNotFound, err)
+	}
+}
+
+func testV1FIdxAttributeComputeIndexes(t *testing.T) {
+	// compute our indexes
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexes,
+		&utils.ArgsComputeFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			AttributeS: true}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned")
+	}
+
+	// our indexes are sotred again, so we can get them
+	// firstly, not gonna get for a different context
+	var replyIdx []string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Errorf("Expected %T, received %T", utils.ErrNotFound, err)
+	}
+
+	//matching for our context
+	expectedIDx := []string{"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expectedIDx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx)
+		}
+	}
+}
+
+func testV1FIdxAttributeMoreProfilesForFilters(t *testing.T) {
+	//we will more attributes with different context for matching filters
+	attrPrf2 := &engine.AttributeWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    utils.CGRateSorg,
+			ID:        "TEST_ATTRIBUTES_new_fltr",
+			Contexts:  []string{utils.MetaChargers},
+			FilterIDs: []string{"fltr_for_attr2", "fltr_for_attr3"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Path:  utils.AccountField,
+					Type:  utils.MetaConstant,
+					Value: "1002",
+				},
+			},
+		},
+	}
+	attrPrf3 := &engine.AttributeWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    utils.CGRateSorg,
+			ID:        "TEST_ATTRIBUTE3",
+			Contexts:  []string{utils.MetaSessionS},
+			FilterIDs: []string{"fltr_for_attr3"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Path:  utils.Destinations,
+					Type:  utils.MetaConstant,
+					Value: "1008",
+				},
+			},
+		},
+	}
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf2, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf3, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	// now we will match indexes for *chargers
+	var replyIdx []string
+	expectedIDx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr"}
+	//"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_TEST_ATTRIBUTE3",
+	//"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_TEST_ATTRIBUTE"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expectedIDx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx)
+		}
+	}
+
+	// now we will match indexes for *sessions
+	expectedIDx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expectedIDx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expectedIDx, replyIdx) {
+			t.Errorf("Expected %+v \n, received %+v", expectedIDx, replyIdx)
+		}
+	}
+}
+
+func testV1FIdxAttributeSRemoveComputedIndexesIDs(t *testing.T) {
+	//indexes will ne removed for both contexts
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
+		&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes},
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("UNexpected reply returned")
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
+		&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes},
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("UNexpected reply returned")
+	}
+
+	//not found for both cases
+	var replyIdx []string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Error(err)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Error(err)
+	}
+
+	// now we will ComputeFilterIndexes by IDs for *chargers context
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs,
+		&utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			AttributeIDs: []string{"TEST_ATTRIBUTES_new_fltr"}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned")
+	}
+
+	//able to get indexes with context *chargers
+	expIdx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expIdx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expIdx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expIdx, replyIdx)
+		}
+	}
+
+	// now we will ComputeFilterIndexes by IDs for *sessions context(but just only 1 profile, not both)
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs,
+		&utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			AttributeIDs: []string{"TEST_ATTRIBUTE3"}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned")
+	}
+
+	//able to get indexes with context *sessions
+	expIdx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expIdx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expIdx, replyIdx) {
+			t.Errorf("Expected %+v, received %+v", expIdx, replyIdx)
+		}
+	}
+
+	// compute for the last profile remain
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs,
+		&utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			AttributeIDs: []string{"TEST_ATTRIBUTES_IT_TEST"}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned")
+	}
+	expIdx = []string{"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3",
+		"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3",
+		"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
+		"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
+		"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(expIdx)
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(expIdx, replyIdx) {
+			t.Errorf("Expected %+v \n, received %+v", expIdx, replyIdx)
+		}
+	}
+	time.Sleep(100)
+}
+
+func testV1FIdxAttributesRemoveProfilesNoIndexes(t *testing.T) {
+	//as we delete our profiles, indexes will  be deleted too
+	var reply string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile,
+		&utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTES_IT_TEST",
+			Tenant: utils.CGRateSorg}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expected %+v \n, received %+v", utils.OK, reply)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile,
+		&utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTE3",
+			Tenant: utils.CGRateSorg}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expected %+v \n, received %+v", utils.OK, reply)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile,
+		&utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{ID: "TEST_ATTRIBUTES_new_fltr",
+			Tenant: utils.CGRateSorg}}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expected %+v \n, received %+v", utils.OK, reply)
+	}
+
+	// Check indexes as we removed, not found for both indexes
+	var replyIdx []string
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaSessionS,
+			ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Error(err)
+	}
+	if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, Context: utils.MetaChargers,
+			ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Error(err)
+	}
+}
+
+func testV1FIdxStopEngine(t *testing.T) {
+	if err := engine.KillEngine(100); err != nil {
+		t.Error(err)
+	}
+}

console/filter.go

@@ -26,7 +26,7 @@ import (
 func init() {
 	c := &CmdGetFilter{
 		name:      "filter",
-		rpcMethod: utils.APIerSv1GetFilter,
+		rpcMethod: utils.AdminSv1GetFilter,
 		rpcParams: &utils.TenantID{},
 	}
 	commands[c.Name()] = c

console/filter_ids.go

@@ -25,7 +25,7 @@ import (
 func init() {
 	c := &CmdGetFilterIDs{
 		name:      "filter_ids",
-		rpcMethod: utils.APIerSv1GetFilterIDs,
+		rpcMethod: utils.AdminSv1GetFilterIDs,
 		rpcParams: &utils.PaginatorWithTenant{},
 	}
 	commands[c.Name()] = c

console/filter_remove.go

@@ -23,7 +23,7 @@ import "github.com/cgrates/cgrates/utils"
 func init() {
 	c := &CmdRemoveFilter{
 		name:      "filter_remove",
-		rpcMethod: utils.APIerSv1RemoveFilter,
+		rpcMethod: utils.AdminSv1RemoveFilter,
 		rpcParams: &utils.TenantIDWithAPIOpts{},
 	}
 	commands[c.Name()] = c

console/filter_set.go

@@ -26,7 +26,7 @@ import (
 func init() {
 	c := &CmdSetFilter{
 		name:      "filter_set",
-		rpcMethod: utils.APIerSv1SetFilter,
+		rpcMethod: utils.AdminSv1SetFilter,
 		rpcParams: &engine.FilterWithAPIOpts{},
 	}
 	commands[c.Name()] = c

data/conf/samples/filter_indexes_internal/cgrates.json

@@ -0,0 +1,27 @@
+{
+  // CGRateS Configuration file
+  // will be used in apis/filter_indexes_it_test.go
+
+
+  "data_db": {
+    "db_type": "*internal",
+  },
+
+  "stor_db": {
+    "db_type": "*internal",
+  },
+
+  "attributes": {
+    "enabled": true,
+    //"string_indexes_fields": ["*req.Subject"],
+    //"prefix_indexes_fields": ["*req.Destinations"]
+  },
+
+  "admins": {
+    "enabled": true,
+  },
+
+  "filters": {
+    "admins_conns": ["*internal"]
+  }
+},

data/conf/samples/filter_indexes_mongo/cgrates.json

@@ -0,0 +1,32 @@
+{
+  // CGRateS Configuration file
+  // will be used in apis/filter_indexes_it_test.go
+
+
+  "data_db": {
+    "db_type": "mongo",
+    "db_name": "10",
+    "db_port": 27017,
+  },
+
+
+  "stor_db": {
+    "db_type": "mongo",
+    "db_name": "cgrates",
+    "db_port": 27017,
+  },
+
+  "attributes": {
+    "enabled": true,
+    //"string_indexes_fields": ["*req.Subject"],
+    //"prefix_indexes_fields": ["*req.Destinations"]
+  },
+
+  "admins": {
+    "enabled": true,
+  },
+
+  "filters": {
+    "admins_conns": ["*internal"]
+  }
+},

data/conf/samples/filter_indexes_mysql/cgrates.json

@@ -0,0 +1,29 @@
+{
+  // CGRateS Configuration file
+  // will be used in apis/filter_indexes_it_test.go
+
+
+  "data_db": {								// database used to store runtime data (eg: accounts, cdr stats)
+    "db_type": "redis",						// data_db type: <redis|mongo>
+    "db_port": 6379, 						// data_db port to reach the database
+    "db_name": "10", 						// data_db database name to connect to
+  },
+
+  "stor_db": {
+    "db_password": "CGRateS.org",
+  },
+
+  "attributes": {
+    "enabled": true,
+    //"string_indexes_fields": ["*req.Subject"],
+    //"prefix_indexes_fields": ["*req.Destinations"]
+  },
+
+  "admins": {
+    "enabled": true,
+  },
+
+  "filters": {
+    "admins_conns": ["*internal"]
+  }
+},

engine/datamanager.go

@@ -766,6 +766,7 @@ func (dm *DataManager) SetFilter(ctx *context.Context, fltr *Filter, withIndex b
 	if err = dm.DataDB().SetFilterDrv(ctx, fltr); err != nil {
 		return
 	}
+	fmt.Println("set fltr da", withIndex, utils.ToJSON(oldFlt), utils.ToJSON(fltr))
 	if withIndex {
 		if err = UpdateFilterIndex(ctx, dm, oldFlt, fltr); err != nil {
 			return
@@ -1743,6 +1744,7 @@ func (dm *DataManager) SetAttributeProfile(ctx *context.Context, ap *AttributePr
 			oldContexes = &oldAP.Contexts
 			oldFiltersIDs = &oldAP.FilterIDs
 		}
+		fmt.Println(oldContexes, oldFiltersIDs)
 		if err = updatedIndexesWithContexts(ctx, dm, utils.CacheAttributeFilterIndexes, ap.Tenant, ap.ID,
 			oldContexes, oldFiltersIDs, ap.Contexts, ap.FilterIDs); err != nil {
 			return

engine/libattributes_test.go

@@ -34,10 +34,6 @@ func TestConvertExternalToProfile(t *testing.T) {
 		ID:        "ATTR_ID",
 		Contexts:  []string{utils.MetaSessionS, utils.MetaCDRs},
 		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
-		ActivationInterval: &utils.ActivationInterval{
-			ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
-			ExpiryTime:     time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
-		},
 		Attributes: []*ExternalAttribute{
 			{
 				Path:  utils.MetaReq + utils.NestingSep + "Account",
@@ -52,10 +48,6 @@ func TestConvertExternalToProfile(t *testing.T) {
 		ID:        "ATTR_ID",
 		Contexts:  []string{utils.MetaSessionS, utils.MetaCDRs},
 		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
-		ActivationInterval: &utils.ActivationInterval{
-			ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
-			ExpiryTime:     time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
-		},
 		Attributes: []*Attribute{
 			{
 				Path:  utils.MetaReq + utils.NestingSep + "Account",

engine/libtest.go

@@ -20,7 +20,6 @@ package engine
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"net/rpc/jsonrpc"
@@ -29,6 +28,8 @@ import (
 	"path"
 	"time"
 
+	"github.com/cgrates/birpc/context"
+
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/utils"
 	"github.com/cgrates/ltcache"

migrator/stats.go

@@ -63,7 +63,7 @@ type v1Stats []*v1Stat
 func (m *Migrator) moveStatQueueProfile() (err error) {
 	//StatQueueProfile
 	var ids []string
-	if ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(context.TODO(), utils.StatQueueProfilePrefix); err != nil {
+	if ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(context.Background(), utils.StatQueueProfilePrefix); err != nil {
 		return err
 	}
 	for _, id := range ids {

services/loaders.go

@@ -65,7 +65,7 @@ type LoaderService struct {
 	srvDep   map[string]*sync.WaitGroup
 }
 
-// Start should handle the sercive start
+// Start should handle the service start
 func (ldrs *LoaderService) Start() (err error) {
 	if ldrs.IsRunning() {
 		return utils.ErrServiceAlreadyRunning

utils/consts.go

@@ -1169,8 +1169,8 @@ const (
 
 // APIerSv1 APIs
 const (
-	APIerSv1ComputeFilterIndexes        = "APIerSv1.ComputeFilterIndexes"
-	APIerSv1ComputeFilterIndexIDs       = "APIerSv1.ComputeFilterIndexIDs"
+	AdminSv1ComputeFilterIndexes        = "AdminSv1.ComputeFilterIndexes"
+	AdminSv1ComputeFilterIndexIDs       = "AdminSv1.ComputeFilterIndexIDs"
 	APIerSv1Ping                        = "APIerSv1.Ping"
 	APIerSv1SetDispatcherProfile        = "APIerSv1.SetDispatcherProfile"
 	APIerSv1GetDispatcherProfile        = "APIerSv1.GetDispatcherProfile"
@@ -1185,12 +1185,12 @@ const (
 	APIerSv1LoadTariffPlanFromFolder    = "APIerSv1.LoadTariffPlanFromFolder"
 	APIerSv1ExportToFolder              = "APIerSv1.ExportToFolder"
 	APIerSv1GetCost                     = "APIerSv1.GetCost"
-	APIerSv1GetFilter                   = "APIerSv1.GetFilter"
-	APIerSv1GetFilterIndexes            = "APIerSv1.GetFilterIndexes"
-	APIerSv1RemoveFilterIndexes         = "APIerSv1.RemoveFilterIndexes"
-	APIerSv1RemoveFilter                = "APIerSv1.RemoveFilter"
-	APIerSv1SetFilter                   = "APIerSv1.SetFilter"
-	APIerSv1GetFilterIDs                = "APIerSv1.GetFilterIDs"
+	AdminSv1GetFilter                   = "AdminSv1.GetFilter"
+	AdminSv1GetFilterIndexes            = "AdminSv1.GetFilterIndexes"
+	AdminSv1RemoveFilterIndexes         = "AdminSv1.RemoveFilterIndexes"
+	AdminSv1RemoveFilter                = "AdminSv1.RemoveFilter"
+	AdminSv1SetFilter                   = "AdminSv1.SetFilter"
+	AdminSv1GetFilterIDs                = "AdminSv1.GetFilterIDs"
 	APIerSv1SetDataDBVersions           = "APIerSv1.SetDataDBVersions"
 	APIerSv1SetStorDBVersions           = "APIerSv1.SetStorDBVersions"
 	APIerSv1GetActions                  = "APIerSv1.GetActions"