From b34821c4ccf479a283cc18598e5d615fa5c34d0f Mon Sep 17 00:00:00 2001 From: Trial97 Date: Mon, 20 Jan 2020 18:08:10 +0200 Subject: [PATCH] Added Filters migration from v3 to v4 --- engine/version.go | 2 +- migrator/filters.go | 207 +++++++++++++++++++++---------- migrator/filters_it_test.go | 146 +++++++++++++++++----- migrator/filters_test.go | 94 +++++++------- migrator/migrator_datadb.go | 4 + migrator/storage_map_datadb.go | 16 +++ migrator/storage_mongo_datadb.go | 41 +++++- migrator/storage_redis.go | 56 ++++++++- 8 files changed, 415 insertions(+), 151 deletions(-) diff --git a/engine/version.go b/engine/version.go index df4459c36..93afbbae3 100644 --- a/engine/version.go +++ b/engine/version.go @@ -153,7 +153,7 @@ func CurrentDataDBVersions() Versions { utils.Suppliers: 1, utils.Attributes: 5, utils.Timing: 1, - utils.RQF: 3, + utils.RQF: 4, utils.Resource: 1, utils.Subscribers: 1, utils.Destinations: 1, diff --git a/migrator/filters.go b/migrator/filters.go index 9466152e2..816db9381 100644 --- a/migrator/filters.go +++ b/migrator/filters.go @@ -57,47 +57,86 @@ func (m *Migrator) migrateCurrentRequestFilter() (err error) { var filterTypes = utils.NewStringSet([]string{utils.MetaRSR, utils.MetaStatS, utils.MetaResources, utils.MetaNotRSR, utils.MetaNotStatS, utils.MetaNotResources}) -func migrateFilterV1(fl *engine.Filter) *engine.Filter { +func migrateFilterV1(fl *v1Filter) (fltr *engine.Filter) { + fltr = &engine.Filter{ + Tenant: fl.Tenant, + ID: fl.ID, + Rules: make([]*engine.FilterRule, len(fl.Rules)), + ActivationInterval: fl.ActivationInterval, + } for i, rule := range fl.Rules { - if rule.Element == "" || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix) || + fltr.Rules[i] = &engine.FilterRule{ + Type: rule.Type, + Element: rule.FieldName, + Values: rule.Values, + } + if rule.FieldName == "" || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) || filterTypes.Has(rule.Type) { continue } - fl.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.Element + fltr.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.FieldName } - return fl + return } -func migrateFilterV2(fl *engine.Filter) *engine.Filter { +func migrateFilterV2(fl *v1Filter) (fltr *engine.Filter) { + fltr = &engine.Filter{ + Tenant: fl.Tenant, + ID: fl.ID, + Rules: make([]*engine.FilterRule, len(fl.Rules)), + ActivationInterval: fl.ActivationInterval, + } for i, rule := range fl.Rules { - if (rule.Element == "" && rule.Type != utils.MetaRSR) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaReq) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaVars) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCgreq) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCgrep) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaRep) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCGRAReq) || - strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaAct) { + fltr.Rules[i] = &engine.FilterRule{ + Type: rule.Type, + Element: rule.FieldName, + Values: rule.Values, + } + if (rule.FieldName == "" && rule.Type != utils.MetaRSR) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaReq) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaVars) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgreq) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgrep) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaRep) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCGRAReq) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaAct) { continue } if rule.Type != utils.MetaRSR { // in case we found dynamic data prefix we remove it - if strings.HasPrefix(rule.Element, utils.DynamicDataPrefix) { - fl.Rules[i].Element = fl.Rules[i].Element[1:] + if strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) { + fl.Rules[i].FieldName = fl.Rules[i].FieldName[1:] } - fl.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.Element + fltr.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.FieldName } else { for idx, val := range rule.Values { if strings.HasPrefix(val, utils.DynamicDataPrefix) { // remove dynamic data prefix from fieldName val = val[1:] } - fl.Rules[i].Values[idx] = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + val + fltr.Rules[i].Values[idx] = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + val } } } - return fl + return +} + +func migrateFilterV3(fl *v1Filter) (fltr *engine.Filter) { + fltr = &engine.Filter{ + Tenant: fl.Tenant, + ID: fl.ID, + Rules: make([]*engine.FilterRule, len(fl.Rules)), + ActivationInterval: fl.ActivationInterval, + } + for i, rule := range fl.Rules { + fltr.Rules[i] = &engine.FilterRule{ + Type: rule.Type, + Element: rule.FieldName, + Values: rule.Values, + } + } + return } func migrateInlineFilter(fl string) string { @@ -141,36 +180,30 @@ func migrateInlineFilterV2(fl string) string { ruleSplt[1] = ruleSplt[1][1:] } return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep)) - } else { // in case of *rsr filter we need to add the prefix at fieldValue - if strings.HasPrefix(ruleSplt[2], utils.DynamicDataPrefix) { - // remove dynamic data prefix from fieldName - ruleSplt[2] = ruleSplt[2][1:] - } - return fmt.Sprintf("%s::~%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+strings.Join(ruleSplt[2:], utils.InInFieldSep)) + } // in case of *rsr filter we need to add the prefix at fieldValue + if strings.HasPrefix(ruleSplt[2], utils.DynamicDataPrefix) { + // remove dynamic data prefix from fieldName + ruleSplt[2] = ruleSplt[2][1:] } - + return fmt.Sprintf("%s::~%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+strings.Join(ruleSplt[2:], utils.InInFieldSep)) } func (m *Migrator) migrateRequestFilterV1() (err error) { - var ids []string - tenant := config.CgrConfig().GeneralCfg().DefaultTenant - ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix) - if err != nil { - return err - } - for _, id := range ids { - idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":") - fl, err := engine.GetFilter(m.dmIN.DataManager(), tenant, idg, false, false, utils.NonTransactional) - if err != nil { + for { + fl, err := m.dmIN.getV1Filter() + if err != nil && err != utils.ErrNoMoreData { return err } + if err == utils.ErrNoMoreData { + break + } if m.dryRun || fl == nil { continue } if err := m.dmOut.DataManager().SetFilter(migrateFilterV1(fl)); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } if err = m.migrateResourceProfileFiltersV1(); err != nil { return err @@ -193,7 +226,7 @@ func (m *Migrator) migrateRequestFilterV1() (err error) { if err = m.migrateDispatcherProfileFiltersV1(); err != nil { return err } - vrs := engine.Versions{utils.RQF: 2} + vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]} if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, @@ -204,27 +237,22 @@ func (m *Migrator) migrateRequestFilterV1() (err error) { } func (m *Migrator) migrateRequestFilterV2() (err error) { - var ids []string - tenant := config.CgrConfig().GeneralCfg().DefaultTenant - ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix) - if err != nil { - return fmt.Errorf("Error: <%s> when getting filter IDs for migration", err.Error()) - } - for _, id := range ids { - idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":") - fl, err := engine.GetFilter(m.dmIN.DataManager(), tenant, idg, false, false, utils.NonTransactional) - if err != nil { - return fmt.Errorf("Error: <%s> when getting filter with tenant: <%s> and id: <%s> for migration", - err.Error(), tenant, idg) + for { + fl, err := m.dmIN.getV1Filter() + if err != nil && err != utils.ErrNoMoreData { + return err + } + if err == utils.ErrNoMoreData { + break } if m.dryRun || fl == nil { continue } if err := m.dmOut.DataManager().SetFilter(migrateFilterV2(fl)); err != nil { return fmt.Errorf("Error: <%s> when setting filter with tenant: <%s> and id: <%s> after migration", - err.Error(), tenant, idg) + err.Error(), fl.Tenant, fl.ID) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } if err = m.migrateResourceProfileFiltersV2(); err != nil { return fmt.Errorf("Error: <%s> when trying to migrate filter for ResourceProfiles", @@ -264,6 +292,34 @@ func (m *Migrator) migrateRequestFilterV2() (err error) { return } +func (m *Migrator) migrateRequestFilterV3() (err error) { + for { + fl, err := m.dmIN.getV1Filter() + if err != nil && err != utils.ErrNoMoreData { + return err + } + if err == utils.ErrNoMoreData { + break + } + if m.dryRun || fl == nil { + continue + } + if err := m.dmOut.DataManager().SetFilter(migrateFilterV3(fl)); err != nil { + return fmt.Errorf("Error: <%s> when setting filter with tenant: <%s> and id: <%s> after migration", + err.Error(), fl.Tenant, fl.ID) + } + m.stats[utils.RQF]++ + } + vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]} + if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating Filters version into dataDB", err.Error())) + } + return +} + func (m *Migrator) migrateFilters() (err error) { var vrs engine.Versions current := engine.CurrentDataDBVersions() @@ -280,6 +336,10 @@ func (m *Migrator) migrateFilters() (err error) { "version number is not defined for ActionTriggers model") } switch vrs[utils.RQF] { + case 3: + if err = m.migrateRequestFilterV3(); err != nil { + return err + } case 2: if err = m.migrateRequestFilterV2(); err != nil { return err @@ -321,7 +381,7 @@ func (m *Migrator) migrateResourceProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetResourceProfile(res, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -348,7 +408,7 @@ func (m *Migrator) migrateStatQueueProfileFiltersV1() (err error) { if err = m.dmOut.DataManager().SetStatQueueProfile(sgs, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -375,7 +435,7 @@ func (m *Migrator) migrateThresholdsProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetThresholdProfile(ths, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -402,7 +462,7 @@ func (m *Migrator) migrateSupplierProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetSupplierProfile(splp, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -434,7 +494,7 @@ func (m *Migrator) migrateAttributeProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -461,7 +521,7 @@ func (m *Migrator) migrateChargerProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetChargerProfile(cpp, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -488,7 +548,7 @@ func (m *Migrator) migrateDispatcherProfileFiltersV1() (err error) { if err := m.dmOut.DataManager().SetDispatcherProfile(dpp, true); err != nil { return err } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -518,7 +578,7 @@ func (m *Migrator) migrateResourceProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting resource profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -547,7 +607,7 @@ func (m *Migrator) migrateStatQueueProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting statQueue profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -576,7 +636,7 @@ func (m *Migrator) migrateThresholdsProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting threshold profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -605,7 +665,7 @@ func (m *Migrator) migrateSupplierProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting supplier profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -639,7 +699,7 @@ func (m *Migrator) migrateAttributeProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting attribute profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -668,7 +728,7 @@ func (m *Migrator) migrateChargerProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting charger profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } @@ -697,7 +757,22 @@ func (m *Migrator) migrateDispatcherProfileFiltersV2() (err error) { return fmt.Errorf("error: <%s> when setting dispatcher profile with tenant: <%s> and id: <%s>", err.Error(), tenant, idg) } - m.stats[utils.RQF] += 1 + m.stats[utils.RQF]++ } return } + +type v1Filter struct { + Tenant string + ID string + Rules []*v1FilterRule + ActivationInterval *utils.ActivationInterval +} + +type v1FilterRule struct { + Type string // Filter type (*string, *timing, *rsr_filters, *stats, *lt, *lte, *gt, *gte) + FieldName string // Name of the field providing us the Values to check (used in case of some ) + Values []string // Filter definition + rsrFields config.RSRParsers // Cache here the RSRFilter Values + negative *bool +} diff --git a/migrator/filters_it_test.go b/migrator/filters_it_test.go index 0ebc7eb78..7cb1140f2 100644 --- a/migrator/filters_it_test.go +++ b/migrator/filters_it_test.go @@ -42,7 +42,10 @@ var sTestsFltrIT = []func(t *testing.T){ testFltrITConnect, testFltrITFlush, testFltrITMigrateAndMove, + testFltrITFlush, testFltrITMigratev2, + testFltrITFlush, + testFltrITMigratev3, } func TestFiltersMigrateITRedis(t *testing.T) { @@ -130,14 +133,14 @@ func testFltrITFlush(t *testing.T) { } func testFltrITMigrateAndMove(t *testing.T) { - Filters := &engine.Filter{ + Filters := &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_2", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "Account", - Values: []string{"1001"}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "Account", + Values: []string{"1001"}, }, }, } @@ -187,7 +190,7 @@ func testFltrITMigrateAndMove(t *testing.T) { attrProf.Compile() switch fltrAction { case utils.Migrate: - if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil { + if err := fltrMigrator.dmIN.setV1Filter(Filters); err != nil { t.Error("Error when setting v1 Filters ", err.Error()) } if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil { @@ -212,7 +215,7 @@ func testFltrITMigrateAndMove(t *testing.T) { //check if version was updated if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { t.Error(err) - } else if vrs[utils.RQF] != 2 { + } else if vrs[utils.RQF] != 4 { t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) } //check if Filters was migrate correctly @@ -243,7 +246,7 @@ func testFltrITMigrateAndMove(t *testing.T) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) } case utils.Move: - if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil { + if err := fltrMigrator.dmIN.DataManager().SetFilter(expFilters); err != nil { t.Error(err) } currentVersion := engine.CurrentDataDBVersions() @@ -257,16 +260,16 @@ func testFltrITMigrateAndMove(t *testing.T) { t.Error("Error when fltrMigratorrating Filters ", err.Error()) } //check if account was migrate correctly - result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), expFilters.Tenant, expFilters.ID, false, false, utils.NonTransactional) if err != nil { t.Error(err) } result.Compile() - if !reflect.DeepEqual(Filters, result) { - t.Errorf("Expecting: %+v, received: %+v", Filters, result) + if !reflect.DeepEqual(expFilters, result) { + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result)) } // check if old account was deleted - result, err = engine.GetFilter(fltrMigrator.dmIN.DataManager(), Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + result, err = engine.GetFilter(fltrMigrator.dmIN.DataManager(), expFilters.Tenant, expFilters.ID, false, false, utils.NonTransactional) if err != utils.ErrNotFound { t.Error(err) } @@ -277,24 +280,24 @@ func testFltrITMigratev2(t *testing.T) { if fltrAction != utils.Migrate { t.SkipNow() } - filters := &engine.Filter{ + filters := &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_2", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaString, - Element: "~Account", - Values: []string{"1001"}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "~Account", + Values: []string{"1001"}, }, - &engine.FilterRule{ - Type: utils.MetaString, - Element: "~*req.Subject", - Values: []string{"1001"}, + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "~*req.Subject", + Values: []string{"1001"}, }, - &engine.FilterRule{ - Type: utils.MetaRSR, - Element: utils.EmptyString, - Values: []string{"~Tenant(~^cgr.*\\.org$)"}, + &v1FilterRule{ + Type: utils.MetaRSR, + FieldName: utils.EmptyString, + Values: []string{"~Tenant(~^cgr.*\\.org$)"}, }, }, } @@ -353,7 +356,7 @@ func testFltrITMigratev2(t *testing.T) { expAttrProf.Compile() attrProf.Compile() - if err := fltrMigrator.dmIN.DataManager().SetFilter(filters); err != nil { + if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil { t.Error("Error when setting v1 Filters ", err.Error()) } if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil { @@ -378,7 +381,7 @@ func testFltrITMigratev2(t *testing.T) { //check if version was updated if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { t.Error(err) - } else if vrs[utils.RQF] != 3 { + } else if vrs[utils.RQF] != 4 { t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) } //check if Filters was migrate correctly @@ -400,7 +403,6 @@ func testFltrITMigratev2(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultAttr)) } expFltrIdx := map[string]utils.StringMap{ - "*prefix:~*req.Account:1001": utils.StringMap{"ATTR_1": true}, "*string:~*req.Account:1001": utils.StringMap{"ATTR_1": true}, "*string:~*req.Subject:1001": utils.StringMap{"ATTR_1": true}, } @@ -411,3 +413,87 @@ func testFltrITMigratev2(t *testing.T) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) } } + +func testFltrITMigratev3(t *testing.T) { + if fltrAction != utils.Migrate { + t.SkipNow() + } + filters := &v1Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "~*req.Account", + Values: []string{"1001"}, + }, + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "~*req.Subject", + Values: []string{"1001"}, + }, + &v1FilterRule{ + Type: utils.MetaRSR, + FieldName: utils.EmptyString, + Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"}, + }, + }, + } + expFilters := &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: utils.MetaString, + Element: "~*req.Account", + Values: []string{"1001"}, + }, + &engine.FilterRule{ + Type: utils.MetaString, + Element: "~*req.Subject", + Values: []string{"1001"}, + }, + &engine.FilterRule{ + Type: utils.MetaRSR, + Element: utils.EmptyString, + Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"}, + }, + }, + } + expFilters.Compile() + + if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil { + t.Error("Error when setting v1 Filters ", err.Error()) + } + currentVersion := engine.Versions{utils.RQF: 3} + err := fltrMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for Filters ", err.Error()) + } + //check if version was set correctly + if vrs, err := fltrMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.RQF] != 3 { + t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) + } + //migrate Filters + err, _ = fltrMigrator.Migrate([]string{utils.MetaFilters}) + if err != nil { + t.Error("Error when migrating Filters ", err.Error()) + } + //check if version was updated + if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.RQF] != 4 { + t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) + } + //check if Filters was migrate correctly + result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), filters.Tenant, filters.ID, false, false, utils.NonTransactional) + if err != nil { + t.Fatalf("Error when getting filters %v", err.Error()) + } + result.Compile() + if !reflect.DeepEqual(*expFilters, *result) { + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result)) + } +} diff --git a/migrator/filters_test.go b/migrator/filters_test.go index 1ec92613a..b72a2869b 100644 --- a/migrator/filters_test.go +++ b/migrator/filters_test.go @@ -58,16 +58,19 @@ func TestFiltersInlineMigrate(t *testing.T) { } func TestFiltersMigrate(t *testing.T) { - data := []struct{ in, exp *engine.Filter }{ + data := []struct { + in *v1Filter + exp *engine.Filter + }{ { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_1", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaString, - Element: "Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "Account", + Values: []string{}, }, }, }, @@ -84,14 +87,14 @@ func TestFiltersMigrate(t *testing.T) { }, }, { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_2", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "~Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "~Account", + Values: []string{}, }, }, }, @@ -116,16 +119,19 @@ func TestFiltersMigrate(t *testing.T) { } func TestFiltersMigrateV2(t *testing.T) { - data := []struct{ in, exp *engine.Filter }{ + data := []struct { + in *v1Filter + exp *engine.Filter + }{ { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_1", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaString, - Element: "~Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaString, + FieldName: "~Account", + Values: []string{}, }, }, }, @@ -142,14 +148,14 @@ func TestFiltersMigrateV2(t *testing.T) { }, }, { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_2", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "~*req.Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "~*req.Account", + Values: []string{}, }, }, }, @@ -166,14 +172,14 @@ func TestFiltersMigrateV2(t *testing.T) { }, }, { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_3", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "~*act.Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "~*act.Account", + Values: []string{}, }, }, }, @@ -190,14 +196,14 @@ func TestFiltersMigrateV2(t *testing.T) { }, }, { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_4", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "~*act.Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "~*act.Account", + Values: []string{}, }, }, }, @@ -214,14 +220,14 @@ func TestFiltersMigrateV2(t *testing.T) { }, }, { - in: &engine.Filter{ + in: &v1Filter{ Tenant: "cgrates.org", ID: "FLTR_5", - Rules: []*engine.FilterRule{ - &engine.FilterRule{ - Type: utils.MetaPrefix, - Element: "~*vars.Account", - Values: []string{}, + Rules: []*v1FilterRule{ + &v1FilterRule{ + Type: utils.MetaPrefix, + FieldName: "~*vars.Account", + Values: []string{}, }, }, }, diff --git a/migrator/migrator_datadb.go b/migrator/migrator_datadb.go index 9f8372439..82fad78ff 100644 --- a/migrator/migrator_datadb.go +++ b/migrator/migrator_datadb.go @@ -66,6 +66,10 @@ type MigratorDataDB interface { setV4AttributeProfile(x *v4AttributeProfile) (err error) remV4AttributeProfile(tenant, id string) (err error) + getV1Filter() (v1Fltr *v1Filter, err error) + setV1Filter(x *v1Filter) (err error) + remV1Filter(tenant, id string) (err error) + DataManager() *engine.DataManager close() } diff --git a/migrator/storage_map_datadb.go b/migrator/storage_map_datadb.go index 43ee0cb52..079aac1cd 100755 --- a/migrator/storage_map_datadb.go +++ b/migrator/storage_map_datadb.go @@ -263,4 +263,20 @@ func (iDBMig *internalMigrator) remV4AttributeProfile(tenant, id string) (err er return utils.ErrNotImplemented } +// Filter Methods +//get +func (iDBMig *internalMigrator) getV1Filter() (v1Fltr *v1Filter, err error) { + return nil, utils.ErrNotImplemented +} + +//set +func (iDBMig *internalMigrator) setV1Filter(x *v1Filter) (err error) { + return utils.ErrNotImplemented +} + +//rem +func (iDBMig *internalMigrator) remV1Filter(tenant, id string) (err error) { + return utils.ErrNotImplemented +} + func (iDBMig *internalMigrator) close() {} diff --git a/migrator/storage_mongo_datadb.go b/migrator/storage_mongo_datadb.go index f3d6fcea9..f4bbe86bc 100644 --- a/migrator/storage_mongo_datadb.go +++ b/migrator/storage_mongo_datadb.go @@ -59,12 +59,12 @@ func newMongoMigrator(dm *engine.DataManager) (mgoMig *mongoMigrator) { } } -func (mgoMig *mongoMigrator) close() { - mgoMig.mgoDB.Close() +func (v1ms *mongoMigrator) close() { + v1ms.mgoDB.Close() } -func (mgoMig *mongoMigrator) DataManager() *engine.DataManager { - return mgoMig.dm +func (v1ms *mongoMigrator) DataManager() *engine.DataManager { + return v1ms.dm } //Account methods @@ -621,3 +621,36 @@ func (v1ms *mongoMigrator) remV4AttributeProfile(tenant, id string) (err error) _, err = v1ms.mgoDB.DB().Collection(v1AttributeProfilesCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id}) return } + +// Filter Methods +//get +func (v1ms *mongoMigrator) getV1Filter() (v1Fltr *v1Filter, err error) { + if v1ms.cursor == nil { + v1ms.cursor, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).Find(v1ms.mgoDB.GetContext(), bson.D{}) + if err != nil { + return nil, err + } + } + if !(*v1ms.cursor).Next(v1ms.mgoDB.GetContext()) { + (*v1ms.cursor).Close(v1ms.mgoDB.GetContext()) + v1ms.cursor = nil + return nil, utils.ErrNoMoreData + } + v1Fltr = new(v1Filter) + if err := (*v1ms.cursor).Decode(v1Fltr); err != nil { + return nil, err + } + return +} + +//set +func (v1ms *mongoMigrator) setV1Filter(x *v1Filter) (err error) { + _, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).InsertOne(v1ms.mgoDB.GetContext(), x) + return +} + +//rem +func (v1ms *mongoMigrator) remV1Filter(tenant, id string) (err error) { + _, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id}) + return +} diff --git a/migrator/storage_redis.go b/migrator/storage_redis.go index 5931e3067..345af4cd1 100644 --- a/migrator/storage_redis.go +++ b/migrator/storage_redis.go @@ -34,7 +34,7 @@ type redisMigrator struct { } var ( - REVERSE_ALIASES_PREFIX = "rls_" + reverseAliasesPrefix = "rls_" ) func newRedisMigrator(dm *engine.DataManager) (rM *redisMigrator) { @@ -44,12 +44,12 @@ func newRedisMigrator(dm *engine.DataManager) (rM *redisMigrator) { } } -func (rdsMig *redisMigrator) close() { - rdsMig.rds.Close() +func (v1rs *redisMigrator) close() { + v1rs.rds.Close() } -func (rdsMig *redisMigrator) DataManager() *engine.DataManager { - return rdsMig.dm +func (v1rs *redisMigrator) DataManager() *engine.DataManager { + return v1rs.dm } //Account methods @@ -558,7 +558,7 @@ func (v1rs *redisMigrator) remV1Alias(key string) (err error) { for target, pairs := range value.Pairs { for _, alias := range pairs { revID := alias + target + al.Context - err = v1rs.rds.Cmd("SREM", REVERSE_ALIASES_PREFIX+revID, tmpKey).Err + err = v1rs.rds.Cmd("SREM", reverseAliasesPrefix+revID, tmpKey).Err if err != nil { return err } @@ -802,3 +802,47 @@ func (v1rs *redisMigrator) remV4AttributeProfile(tenant, id string) (err error) key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(tenant, id) return v1rs.rds.Cmd("DEL", key).Err } + +// Filter Methods +//get +func (v1rs *redisMigrator) getV1Filter() (v1Fltr *v1Filter, err error) { + if v1rs.qryIdx == nil { + v1rs.dataKeys, err = v1rs.rds.GetKeysForPrefix(utils.FilterPrefix) + if err != nil { + return + } else if len(v1rs.dataKeys) == 0 { + return nil, utils.ErrNoMoreData + } + v1rs.qryIdx = utils.IntPointer(0) + } + if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 { + strVal, err := v1rs.rds.Cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes() + if err != nil { + return nil, err + } + if err := v1rs.rds.Marshaler().Unmarshal(strVal, &v1Fltr); err != nil { + return nil, err + } + *v1rs.qryIdx = *v1rs.qryIdx + 1 + } else { + v1rs.qryIdx = nil + return nil, utils.ErrNoMoreData + } + return +} + +//set +func (v1rs *redisMigrator) setV1Filter(x *v1Filter) (err error) { + key := utils.FilterPrefix + utils.ConcatenatedKey(x.Tenant, x.ID) + bit, err := v1rs.rds.Marshaler().Marshal(x) + if err != nil { + return err + } + return v1rs.rds.Cmd("SET", key, bit).Err +} + +//rem +func (v1rs *redisMigrator) remV1Filter(tenant, id string) (err error) { + key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) + return v1rs.rds.Cmd("DEL", key).Err +}