diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index 731bc8ce0..4c2f5f0f9 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -50,6 +50,8 @@ var sTests = []func(t *testing.T){ testITFlush, testITIsDBEmpty, testITTestStoreFilterIndexesWithTransID, + testITFlush, + testITIsDBEmpty, testITTestStoreFilterIndexesWithTransID2, testITFlush, testITIsDBEmpty, @@ -387,6 +389,51 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } + + //replace old filter with two different filters + fp3 = &Filter{ + Tenant: "cgrates.org", + ID: "Filter3", + Rules: []*FilterRule{ + { + Element: "Destination", + Type: utils.MetaString, + Values: []string{"30", "50"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + } + if err := dataManager.SetFilter(fp3, true); err != nil { + t.Error(err) + } + + eIdxes = map[string]utils.StringSet{ + "*string:Destination:30": { + "THD_Test": struct{}{}, + }, + "*string:Destination:50": { + "THD_Test": struct{}{}, + }, + "*string:EventType:Event1": { + "THD_Test": struct{}{}, + "THD_Test2": struct{}{}, + }, + "*string:EventType:Event2": { + "THD_Test": struct{}{}, + "THD_Test2": struct{}{}, + }, + } + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheThresholdFilterIndexes, th.Tenant, + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + //remove thresholds if err := dataManager.RemoveThresholdProfile(th.Tenant, th.ID, utils.NonTransactional, true); err != nil { @@ -485,6 +532,57 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { } } + fp = &Filter{ + Tenant: "cgrates.org", + ID: "AttrFilter", + Rules: []*FilterRule{ + { + Element: "EventType", + Type: utils.MetaString, + Values: []string{"Event3", "Event4"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + } + if err := dataManager.SetFilter(fp, true); err != nil { + t.Error(err) + } + eIdxes = map[string]utils.StringSet{ + "*string:EventType:Event3": { + "AttrPrf": struct{}{}, + }, + "*string:EventType:Event4": { + "AttrPrf": struct{}{}, + }, + } + for _, ctx := range attrProfile.Contexts { + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProfile.Tenant, ctx), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + } + + eIdxes = map[string]utils.StringSet{ + "*attribute_filter_indexes": { + "AttrPrf": struct{}{}, + }, + } + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheFilterIndexes, + fp.TenantID(), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + if err := dataManager.RemoveAttributeProfile(attrProfile.Tenant, attrProfile.ID, utils.NonTransactional, true); err != nil { t.Error(err) @@ -665,7 +763,7 @@ func testITTestStoreFilterIndexesWithTransID2(t *testing.T) { } //commit transaction if err := dataManager.SetIndexes(utils.CacheResourceFilterIndexes, - "cgrates.org", idxes, true, transID); err != nil { + "cgrates.org", nil, true, transID); err != nil { t.Error(err) } //verify if old key was deleted @@ -681,7 +779,7 @@ func testITTestStoreFilterIndexesWithTransID2(t *testing.T) { "cgrates.org", utils.EmptyString, false, false); err != nil { t.Error(err) } else if !reflect.DeepEqual(idxes, rcv) { - t.Errorf("Expecting: %+v, received: %+v", idxes, rcv) + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(idxes), utils.ToJSON(rcv)) } } diff --git a/engine/libindex.go b/engine/libindex.go index 11d4bc21f..a55773545 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -407,7 +407,7 @@ func addIndexFiltersItem(dm *DataManager, idxItmType, tnt, itemID string, filter } indexes[idxItmType].Add(itemID) - if err = dm.SetIndexes(idxItmType, tntCtx, indexes, true, utils.NonTransactional); err != nil { + if err = dm.SetIndexes(utils.CacheFilterIndexes, tntCtx, indexes, true, utils.NonTransactional); err != nil { return } for indxKey := range indexes { @@ -437,7 +437,7 @@ func removeIndexFiltersItem(dm *DataManager, idxItmType, tnt, itemID string, fil } indexes[idxItmType].Remove(itemID) - if err = dm.SetIndexes(idxItmType, tntCtx, indexes, true, utils.NonTransactional); err != nil { + if err = dm.SetIndexes(utils.CacheFilterIndexes, tntCtx, indexes, true, utils.NonTransactional); err != nil { return } for indxKey := range indexes { @@ -626,7 +626,7 @@ func updateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { for _, idx := range updIdx { idx.Add(itemID) } - if err = dm.SetIndexes(idxItmType, newFlt.TenantID(), + if err = dm.SetIndexes(idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx), updIdx, false, utils.NonTransactional); err != nil { return } @@ -653,7 +653,7 @@ func updateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { for _, idx := range updIdx { idx.Add(itemID) } - if err = dm.SetIndexes(idxItmType, newFlt.TenantID(), + if err = dm.SetIndexes(idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx), updIdx, false, utils.NonTransactional); err != nil { return } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 742387e23..631e56b9c 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -2224,12 +2224,7 @@ func (ms *MongoStorage) GetIndexesDrv(idxItmType, tntCtx, idxKey string) (indexe if len(elem.Value) == 0 { continue } - keys := strings.Split(elem.Key, ":") - indexKey := utils.ConcatenatedKey(keys[1], keys[2], keys[3]) - //check here if key has context - if len(strings.Split(tntCtx, ":")) == 2 { - indexKey = utils.ConcatenatedKey(keys[2], keys[3], keys[4]) - } + indexKey := strings.TrimPrefix(elem.Key, utils.CacheInstanceToPrefix[idxItmType]+tntCtx+utils.CONCATENATED_KEY_SEP) indexes[indexKey] = utils.NewStringSet(elem.Value) } return cur.Close(sctx) diff --git a/migrator/action_plan.go b/migrator/action_plan.go index 48773da05..10bfb5690 100644 --- a/migrator/action_plan.go +++ b/migrator/action_plan.go @@ -116,22 +116,16 @@ func (m *Migrator) migrateActionPlans() (err error) { for { switch version { case current[utils.ActionPlans]: + migrated = false if m.sameDataDB { - migrated = false break } - migrated = false if err = m.migrateCurrentActionPlans(); err != nil && err != utils.ErrNoMoreData { return err - } else if err == utils.ErrNoMoreData { - break } - version = 3 case 1: if v3, err = m.migrateV1ActionPlans(); err != nil && err != utils.ErrNoMoreData { return err - } else if err == utils.ErrNoMoreData { - break } version = 3 case 2: // neded to rebuild action plan indexes for redis @@ -162,7 +156,7 @@ func (m *Migrator) migrateActionPlans() (err error) { } } } - m.stats[utils.ActionPlans] += 1 + m.stats[utils.ActionPlans]++ } if m.dryRun || !migrated { return nil diff --git a/migrator/filters_it_test.go b/migrator/filters_it_test.go index d0d0b227b..d6d76d5c1 100644 --- a/migrator/filters_it_test.go +++ b/migrator/filters_it_test.go @@ -133,7 +133,7 @@ func testFltrITFlush(t *testing.T) { } func testFltrITMigrateAndMove(t *testing.T) { - Filters := &v1Filter{ + fltr := &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_2", Rules: []*v1FilterRule{{ @@ -186,10 +186,10 @@ func testFltrITMigrateAndMove(t *testing.T) { attrProf.Compile() switch fltrAction { case utils.Migrate: - if err := fltrMigrator.dmIN.setV1Filter(Filters); err != nil { + if err := fltrMigrator.dmIN.setV1Filter(fltr); err != nil { t.Error("Error when setting v1 Filters ", err.Error()) } - if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil { + if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, true); err != nil { t.Error("Error when setting attribute profile for v1 Filters ", err.Error()) } currentVersion := engine.Versions{utils.RQF: 1} @@ -215,7 +215,7 @@ func testFltrITMigrateAndMove(t *testing.T) { t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) } //check if Filters was migrate correctly - result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + result, err := fltrMigrator.dmOut.DataManager().GetFilter(fltr.Tenant, fltr.ID, false, false, utils.NonTransactional) if err != nil { t.Fatalf("Error when getting Attributes %v", err.Error()) } @@ -358,7 +358,7 @@ func testFltrITMigratev2(t *testing.T) { if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil { t.Error("Error when setting v1 Filters ", err.Error()) } - if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil { + if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, true); err != nil { t.Error("Error when setting attribute profile for v1 Filters ", err.Error()) } currentVersion := engine.Versions{utils.RQF: 2} @@ -424,17 +424,17 @@ func testFltrITMigratev3(t *testing.T) { Tenant: "cgrates.org", ID: "FLTR_2", Rules: []*v1FilterRule{ - &v1FilterRule{ + { Type: utils.MetaString, FieldName: "~*req.Account", Values: []string{"1001"}, }, - &v1FilterRule{ + { Type: utils.MetaString, FieldName: "~*req.Subject", Values: []string{"1001"}, }, - &v1FilterRule{ + { Type: utils.MetaRSR, FieldName: utils.EmptyString, Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"}, @@ -445,17 +445,17 @@ func testFltrITMigratev3(t *testing.T) { Tenant: "cgrates.org", ID: "FLTR_2", Rules: []*engine.FilterRule{ - &engine.FilterRule{ + { Type: utils.MetaString, Element: utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Account, Values: []string{"1001"}, }, - &engine.FilterRule{ + { Type: utils.MetaString, Element: "~*req.Subject", Values: []string{"1001"}, }, - &engine.FilterRule{ + { Type: utils.MetaRSR, Element: utils.EmptyString, Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"},