Add additional tests for indexes

apis/attributes_it_test.go

@@ -75,6 +75,18 @@ var (
 		testAttributeGetAttributeProfileAllCount,
 		testAttributeRemoveRemainAttributeProfiles,
 		testAttributeGetAttributeProfileAfterRemove,
+
+		// Testing index behaviour
+		testAttributeSSetNonIndexedTypeFilter,
+		testAttributeSSetIndexedTypeFilter,
+		testAttributeSClearIndexes,
+		testAttributeSCheckIndexesSetAttributeProfileWithoutFilters,
+		// testAttributeSCheckIndexesAddNonIndexedFilter,
+		testAttributeSCheckIndexesAddIndexedFilters,
+		testAttributeSCheckIndexesModifyIndexedFilter,
+		testAttributeSCheckIndexesRemoveAnIndexedFilter,
+		testAttributeSCheckIndexesRemoveAttributeProfile,
+
 		testAttributeSKillEngine,
 	}
 )
@@ -1352,6 +1364,314 @@ func testAttributeGetAttributeProfileAfterRemove(t *testing.T) {
 	}
 }
 
+func testAttributeSSetNonIndexedTypeFilter(t *testing.T) {
+	filter := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: "cgrates.org",
+			ID:     "NONINDEXED_FLTR_TYPE",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaGreaterThan,
+					Element: utils.Cost,
+					Values:  []string{"10"},
+				},
+			},
+		},
+	}
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter,
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply: ", reply)
+	}
+}
+
+func testAttributeSSetIndexedTypeFilter(t *testing.T) {
+	filter := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: "cgrates.org",
+			ID:     "INDEXED_FLTR_TYPE",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaPrefix,
+					Element: "~*req.Account",
+					Values:  []string{"10"},
+				},
+			},
+		},
+	}
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter,
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply: ", reply)
+	}
+}
+
+func testAttributeSClearIndexes(t *testing.T) {
+	args := &AttrRemFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
+		args, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply: ", reply)
+	}
+}
+
+func testAttributeSCheckIndexesSetAttributeProfileWithoutFilters(t *testing.T) {
+	attrPrf := &engine.APIAttributeProfileWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant: "cgrates.org",
+			ID:     "ATTR_TEST",
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Type:  utils.MetaConstant,
+					Path:  "~*req.Account",
+					Value: "1002",
+				},
+			},
+		},
+	}
+
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	expIdx := []string{"*none:*any:*any:ATTR_TEST"}
+	args := &AttrGetFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var replyIdx []string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(replyIdx, expIdx) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+	}
+}
+
+// func testAttributeSCheckIndexesAddNonIndexedFilter(t *testing.T) {
+// 	attrPrf := &engine.APIAttributeProfileWithAPIOpts{
+// 		APIAttributeProfile: &engine.APIAttributeProfile{
+// 			Tenant:    "cgrates.org",
+// 			ID:        "ATTR_TEST",
+// 			FilterIDs: []string{"NONINDEXED_FLTR_TYPE"},
+// 			Attributes: []*engine.ExternalAttribute{
+// 				{
+// 					Type:  utils.MetaConstant,
+// 					Path:  "~*req.Account",
+// 					Value: "1002",
+// 				},
+// 			},
+// 		},
+// 	}
+
+// 	var reply string
+// 	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+// 		attrPrf, &reply); err != nil {
+// 		t.Error(err)
+// 	} else if reply != utils.OK {
+// 		t.Error(err)
+// 	}
+
+// 	expIdx := []string{"*none:*any:*any:ATTR_TEST"}
+// 	args := &AttrGetFilterIndexes{
+// 		Tenant:   "cgrates.org",
+// 		ItemType: utils.MetaAttributes,
+// 	}
+// 	var replyIdx []string
+// 	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+// 		args, &replyIdx); err != nil {
+// 		t.Error(err)
+// 	} else if !reflect.DeepEqual(replyIdx, expIdx) {
+// 		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+// 	}
+// }
+
+func testAttributeSCheckIndexesAddIndexedFilters(t *testing.T) {
+	attrPrf := &engine.APIAttributeProfileWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    "cgrates.org",
+			ID:        "ATTR_TEST",
+			FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Type:  utils.MetaConstant,
+					Path:  "~*req.Account",
+					Value: "1002",
+				},
+			},
+		},
+	}
+
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	expIdx := []string{"*prefix:*req.Account:10:ATTR_TEST"}
+	args := &AttrGetFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var replyIdx []string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(replyIdx, expIdx) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+	}
+
+	attrPrf = &engine.APIAttributeProfileWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    "cgrates.org",
+			ID:        "ATTR_TEST",
+			FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE", "*string:~*req.Category:call"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Type:  utils.MetaConstant,
+					Path:  "~*req.Account",
+					Value: "1002",
+				},
+			},
+		},
+	}
+
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	expIdx = []string{"*prefix:*req.Account:10:ATTR_TEST", "*string:*req.Category:call:ATTR_TEST"}
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(replyIdx, expIdx) {
+			t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+		}
+	}
+}
+
+func testAttributeSCheckIndexesModifyIndexedFilter(t *testing.T) {
+	filter := &engine.FilterWithAPIOpts{
+		Filter: &engine.Filter{
+			Tenant: "cgrates.org",
+			ID:     "INDEXED_FLTR_TYPE",
+			Rules: []*engine.FilterRule{
+				{
+					Type:    utils.MetaSuffix,
+					Element: "~*req.Subject",
+					Values:  []string{"01"},
+				},
+			},
+		},
+	}
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetFilter, filter,
+		&reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply: ", reply)
+	}
+
+	expIdx := []string{"*string:*req.Category:call:ATTR_TEST", "*suffix:*req.Subject:01:ATTR_TEST"}
+	args := &AttrGetFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var replyIdx []string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err != nil {
+		t.Error(err)
+	} else {
+		sort.Strings(replyIdx)
+		if !reflect.DeepEqual(replyIdx, expIdx) {
+			t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+		}
+	}
+}
+
+func testAttributeSCheckIndexesRemoveAnIndexedFilter(t *testing.T) {
+	attrPrf := &engine.APIAttributeProfileWithAPIOpts{
+		APIAttributeProfile: &engine.APIAttributeProfile{
+			Tenant:    "cgrates.org",
+			ID:        "ATTR_TEST",
+			FilterIDs: []string{"NONINDEXED_FLTR_TYPE", "INDEXED_FLTR_TYPE"},
+			Attributes: []*engine.ExternalAttribute{
+				{
+					Type:  utils.MetaConstant,
+					Path:  "~*req.Account",
+					Value: "1002",
+				},
+			},
+		},
+	}
+
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1SetAttributeProfile,
+		attrPrf, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	expIdx := []string{"*suffix:*req.Subject:01:ATTR_TEST"}
+	args := &AttrGetFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var replyIdx []string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(replyIdx, expIdx) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expIdx), utils.ToJSON(replyIdx))
+	}
+}
+func testAttributeSCheckIndexesRemoveAttributeProfile(t *testing.T) {
+	argsRem := &utils.TenantIDWithAPIOpts{
+		TenantID: &utils.TenantID{
+			Tenant: "cgrates.org",
+			ID:     "ATTR_TEST",
+		},
+	}
+	var reply string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1RemoveAttributeProfile,
+		argsRem, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(err)
+	}
+
+	args := &AttrGetFilterIndexes{
+		Tenant:   "cgrates.org",
+		ItemType: utils.MetaAttributes,
+	}
+	var replyIdx []string
+	if err := attrSRPC.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
+		args, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ErrNotFound, err)
+	}
+}
+
 //Kill the engine when it is about to be finished
 func testAttributeSKillEngine(t *testing.T) {
 	if err := engine.KillEngine(100); err != nil {

engine/libindex_test.go

@@ -171,3 +171,257 @@ func TestRemoveFilterIndexesForFilter(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestLibIndexSetUpdateRemAttributeProfile(t *testing.T) {
+	cfg := config.NewDefaultCGRConfig()
+	dataDB := NewInternalDB(nil, nil, cfg.DataDbCfg().Items)
+	dm := NewDataManager(dataDB, cfg.CacheCfg(), nil)
+
+	// Set an AttributeProfile without filterIDs
+	attrPrf := &AttributeProfile{
+		Tenant: "cgrates.org",
+		ID:     "ATTR_TEST",
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err := dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	}
+	expIndexes := map[string]utils.StringSet{
+		"*none:*any:*any": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err := dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+			utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	}
+
+	// Add a non-indexed filter type
+	attrPrf = &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ATTR_TEST",
+		FilterIDs: []string{"*gt:~*req.Element:10"},
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err = dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	// if err != nil {
+	// 	t.Error(err)
+	// } else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+	// 	t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+	// 		utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	// }
+
+	// Add an indexed filter type
+	attrPrf = &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ATTR_TEST",
+		FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10"},
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err = dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	}
+	expIndexes = map[string]utils.StringSet{
+		"*prefix:*req.Account:10": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+			utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	}
+
+	// Add another indexed filter type
+	attrPrf = &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ATTR_TEST",
+		FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10", "*string:~*req.Category:call"},
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err = dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	}
+	expIndexes = map[string]utils.StringSet{
+		"*prefix:*req.Account:10": {
+			"ATTR_TEST": {},
+		},
+		"*string:*req.Category:call": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+			utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	}
+
+	// Remove an indexed filter type
+	attrPrf = &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ATTR_TEST",
+		FilterIDs: []string{"*gt:~*req.Element:10", "*prefix:~*req.Account:10"},
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err = dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+			utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	}
+	expIndexes = map[string]utils.StringSet{
+		"*prefix:*req.Account:10": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Remove the attribute profile
+	err = dm.RemoveAttributeProfile(context.Background(), attrPrf.Tenant, attrPrf.ID, true)
+	if err != nil {
+		t.Error(err)
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ErrNotFound, err)
+	}
+}
+
+func TestLibIndexModifyAttrPrfFilter(t *testing.T) {
+	cfg := config.NewDefaultCGRConfig()
+	dataDB := NewInternalDB(nil, nil, cfg.DataDbCfg().Items)
+	dm := NewDataManager(dataDB, cfg.CacheCfg(), nil)
+
+	// Set a non-indexed type filter
+	fltr := &Filter{
+		Tenant: "cgrates.org",
+		ID:     "fltr_test",
+		Rules: []*FilterRule{
+			{
+				Type:    utils.MetaGreaterThan,
+				Element: "~*req.Cost",
+				Values:  []string{"10"},
+			},
+		},
+	}
+	err := dm.SetFilter(context.Background(), fltr, true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Create an AttributeProfile using the previously created filter
+	attrPrf := &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ATTR_TEST",
+		FilterIDs: []string{"fltr_test"},
+		Attributes: []*Attribute{
+			{
+				Type:  utils.MetaConstant,
+				Path:  "~*req.Account",
+				Value: config.NewRSRParsersMustCompile("1002", cfg.GeneralCfg().RSRSep),
+			},
+		},
+	}
+	err = dm.SetAttributeProfile(context.Background(), attrPrf, true)
+	if err != nil {
+		t.Error(err)
+	}
+	expIndexes := map[string]utils.StringSet{
+		"*none:*any:*any": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err := dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	// if err != nil {
+	// 	t.Error(err)
+	// } else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+	// 	t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+	// 		utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	// }
+
+	// Make the filter indexable
+	fltr = &Filter{
+		Tenant: "cgrates.org",
+		ID:     "fltr_test",
+		Rules: []*FilterRule{
+			{
+				Type:    utils.MetaPrefix,
+				Element: "~*req.Account",
+				Values:  []string{"10"},
+			},
+		},
+	}
+	err = dm.SetFilter(context.Background(), fltr, true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	expIndexes = map[string]utils.StringSet{
+		"*prefix:*req.Account:10": {
+			"ATTR_TEST": {},
+		},
+	}
+	rcvIndexes, err = dm.GetIndexes(context.Background(), utils.CacheAttributeFilterIndexes, attrPrf.Tenant,
+		utils.EmptyString, utils.NonTransactional, false, false)
+	if err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(rcvIndexes, expIndexes) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>",
+			utils.ToJSON(expIndexes), utils.ToJSON(rcvIndexes))
+	}
+}