uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Added integration tests for reverse filter indexes

Browse Source
This commit is contained in:
Trial97
2020-06-16 11:51:04 +03:00
committed by Dan Christian Bogos
parent ba09feb3d6
commit f608a58e36
5 changed files with 118 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
engine/filterindexer_it_test.go
View File

@@ -50,6 +50,8 @@ var sTests = []func(t *testing.T){
testITFlush,
testITIsDBEmpty,
testITTestStoreFilterIndexesWithTransID,
testITFlush,
testITIsDBEmpty,
testITTestStoreFilterIndexesWithTransID2,
testITFlush,
testITIsDBEmpty,
@@ -387,6 +389,51 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
//replace old filter with two different filters
fp3 = &Filter{
Tenant: "cgrates.org",
ID: "Filter3",
Rules: []*FilterRule{
{
Element: "Destination",
Type: utils.MetaString,
Values: []string{"30", "50"},
},
},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
},
}
if err := dataManager.SetFilter(fp3, true); err != nil {
t.Error(err)
}
eIdxes = map[string]utils.StringSet{
"*string:Destination:30": {
"THD_Test": struct{}{},
},
"*string:Destination:50": {
"THD_Test": struct{}{},
},
"*string:EventType:Event1": {
"THD_Test": struct{}{},
"THD_Test2": struct{}{},
},
"*string:EventType:Event2": {
"THD_Test": struct{}{},
"THD_Test2": struct{}{},
},
}
if rcvIdx, err := dataManager.GetIndexes(
utils.CacheThresholdFilterIndexes, th.Tenant,
utils.EmptyString, false, false); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
//remove thresholds
if err := dataManager.RemoveThresholdProfile(th.Tenant,
th.ID, utils.NonTransactional, true); err != nil {
@@ -485,6 +532,57 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
}
}
fp = &Filter{
Tenant: "cgrates.org",
ID: "AttrFilter",
Rules: []*FilterRule{
{
Element: "EventType",
Type: utils.MetaString,
Values: []string{"Event3", "Event4"},
},
},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
},
}
if err := dataManager.SetFilter(fp, true); err != nil {
t.Error(err)
}
eIdxes = map[string]utils.StringSet{
"*string:EventType:Event3": {
"AttrPrf": struct{}{},
},
"*string:EventType:Event4": {
"AttrPrf": struct{}{},
},
}
for _, ctx := range attrProfile.Contexts {
if rcvIdx, err := dataManager.GetIndexes(
utils.CacheAttributeFilterIndexes,
utils.ConcatenatedKey(attrProfile.Tenant, ctx),
utils.EmptyString, false, false); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
eIdxes = map[string]utils.StringSet{
"*attribute_filter_indexes": {
"AttrPrf": struct{}{},
},
}
if rcvIdx, err := dataManager.GetIndexes(
utils.CacheFilterIndexes,
fp.TenantID(),
utils.EmptyString, false, false); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if err := dataManager.RemoveAttributeProfile(attrProfile.Tenant,
attrProfile.ID, utils.NonTransactional, true); err != nil {
t.Error(err)
@@ -665,7 +763,7 @@ func testITTestStoreFilterIndexesWithTransID2(t *testing.T) {
}
//commit transaction
if err := dataManager.SetIndexes(utils.CacheResourceFilterIndexes,
"cgrates.org", idxes, true, transID); err != nil {
"cgrates.org", nil, true, transID); err != nil {
t.Error(err)
}
//verify if old key was deleted
@@ -681,7 +779,7 @@ func testITTestStoreFilterIndexesWithTransID2(t *testing.T) {
"cgrates.org", utils.EmptyString, false, false); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(idxes, rcv) {
t.Errorf("Expecting: %+v, received: %+v", idxes, rcv)
t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(idxes), utils.ToJSON(rcv))
}
}

8
engine/libindex.go
View File

@@ -407,7 +407,7 @@ func addIndexFiltersItem(dm *DataManager, idxItmType, tnt, itemID string, filter
}
indexes[idxItmType].Add(itemID)
if err = dm.SetIndexes(idxItmType, tntCtx, indexes, true, utils.NonTransactional); err != nil {
if err = dm.SetIndexes(utils.CacheFilterIndexes, tntCtx, indexes, true, utils.NonTransactional); err != nil {
return
}
for indxKey := range indexes {
@@ -437,7 +437,7 @@ func removeIndexFiltersItem(dm *DataManager, idxItmType, tnt, itemID string, fil
}
indexes[idxItmType].Remove(itemID)
if err = dm.SetIndexes(idxItmType, tntCtx, indexes, true, utils.NonTransactional); err != nil {
if err = dm.SetIndexes(utils.CacheFilterIndexes, tntCtx, indexes, true, utils.NonTransactional); err != nil {
return
}
for indxKey := range indexes {
@@ -626,7 +626,7 @@ func updateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) {
for _, idx := range updIdx {
idx.Add(itemID)
}
if err = dm.SetIndexes(idxItmType, newFlt.TenantID(),
if err = dm.SetIndexes(idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx),
updIdx, false, utils.NonTransactional); err != nil {
return
}
@@ -653,7 +653,7 @@ func updateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) {
for _, idx := range updIdx {
idx.Add(itemID)
}
if err = dm.SetIndexes(idxItmType, newFlt.TenantID(),
if err = dm.SetIndexes(idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx),
updIdx, false, utils.NonTransactional); err != nil {
return
}

7
engine/storage_mongo_datadb.go
View File

@@ -2224,12 +2224,7 @@ func (ms *MongoStorage) GetIndexesDrv(idxItmType, tntCtx, idxKey string) (indexe
if len(elem.Value) == 0 {
continue
}
keys := strings.Split(elem.Key, ":")
indexKey := utils.ConcatenatedKey(keys[1], keys[2], keys[3])
//check here if key has context
if len(strings.Split(tntCtx, ":")) == 2 {
indexKey = utils.ConcatenatedKey(keys[2], keys[3], keys[4])
}
indexKey := strings.TrimPrefix(elem.Key, utils.CacheInstanceToPrefix[idxItmType]+tntCtx+utils.CONCATENATED_KEY_SEP)
indexes[indexKey] = utils.NewStringSet(elem.Value)
}
return cur.Close(sctx)

10
migrator/action_plan.go
View File

@@ -116,22 +116,16 @@ func (m *Migrator) migrateActionPlans() (err error) {
for {
switch version {
case current[utils.ActionPlans]:
migrated = false
if m.sameDataDB {
migrated = false
break
}
migrated = false
if err = m.migrateCurrentActionPlans(); err != nil && err != utils.ErrNoMoreData {
return err
} else if err == utils.ErrNoMoreData {
break
}
version = 3
case 1:
if v3, err = m.migrateV1ActionPlans(); err != nil && err != utils.ErrNoMoreData {
return err
} else if err == utils.ErrNoMoreData {
break
}
version = 3
case 2: // neded to rebuild action plan indexes for redis
@@ -162,7 +156,7 @@ func (m *Migrator) migrateActionPlans() (err error) {
}
}
}
m.stats[utils.ActionPlans] += 1
m.stats[utils.ActionPlans]++
}
if m.dryRun || !migrated {
return nil

22
migrator/filters_it_test.go
View File

@@ -133,7 +133,7 @@ func testFltrITFlush(t *testing.T) {
}
func testFltrITMigrateAndMove(t *testing.T) {
Filters := &v1Filter{
fltr := &v1Filter{
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*v1FilterRule{{
@@ -186,10 +186,10 @@ func testFltrITMigrateAndMove(t *testing.T) {
attrProf.Compile()
switch fltrAction {
case utils.Migrate:
if err := fltrMigrator.dmIN.setV1Filter(Filters); err != nil {
if err := fltrMigrator.dmIN.setV1Filter(fltr); err != nil {
t.Error("Error when setting v1 Filters ", err.Error())
}
if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, true); err != nil {
t.Error("Error when setting attribute profile for v1 Filters ", err.Error())
}
currentVersion := engine.Versions{utils.RQF: 1}
@@ -215,7 +215,7 @@ func testFltrITMigrateAndMove(t *testing.T) {
t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
}
//check if Filters was migrate correctly
result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
result, err := fltrMigrator.dmOut.DataManager().GetFilter(fltr.Tenant, fltr.ID, false, false, utils.NonTransactional)
if err != nil {
t.Fatalf("Error when getting Attributes %v", err.Error())
}
@@ -358,7 +358,7 @@ func testFltrITMigratev2(t *testing.T) {
if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil {
t.Error("Error when setting v1 Filters ", err.Error())
}
if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, true); err != nil {
t.Error("Error when setting attribute profile for v1 Filters ", err.Error())
}
currentVersion := engine.Versions{utils.RQF: 2}
@@ -424,17 +424,17 @@ func testFltrITMigratev3(t *testing.T) {
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*v1FilterRule{
&v1FilterRule{
{
Type: utils.MetaString,
FieldName: "~*req.Account",
Values: []string{"1001"},
},
&v1FilterRule{
{
Type: utils.MetaString,
FieldName: "~*req.Subject",
Values: []string{"1001"},
},
&v1FilterRule{
{
Type: utils.MetaRSR,
FieldName: utils.EmptyString,
Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"},
@@ -445,17 +445,17 @@ func testFltrITMigratev3(t *testing.T) {
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*engine.FilterRule{
&engine.FilterRule{
{
Type: utils.MetaString,
Element: utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Account,
Values: []string{"1001"},
},
&engine.FilterRule{
{
Type: utils.MetaString,
Element: "~*req.Subject",
Values: []string{"1001"},
},
&engine.FilterRule{
{
Type: utils.MetaRSR,
Element: utils.EmptyString,
Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"},