Added Filters migration from v3 to v4

engine/version.go

@@ -153,7 +153,7 @@ func CurrentDataDBVersions() Versions {
 		utils.Suppliers:           1,
 		utils.Attributes:          5,
 		utils.Timing:              1,
-		utils.RQF:                 3,
+		utils.RQF:                 4,
 		utils.Resource:            1,
 		utils.Subscribers:         1,
 		utils.Destinations:        1,

migrator/filters.go

@@ -57,47 +57,86 @@ func (m *Migrator) migrateCurrentRequestFilter() (err error) {
 var filterTypes = utils.NewStringSet([]string{utils.MetaRSR, utils.MetaStatS, utils.MetaResources,
 	utils.MetaNotRSR, utils.MetaNotStatS, utils.MetaNotResources})
 
-func migrateFilterV1(fl *engine.Filter) *engine.Filter {
+func migrateFilterV1(fl *v1Filter) (fltr *engine.Filter) {
+	fltr = &engine.Filter{
+		Tenant:             fl.Tenant,
+		ID:                 fl.ID,
+		Rules:              make([]*engine.FilterRule, len(fl.Rules)),
+		ActivationInterval: fl.ActivationInterval,
+	}
 	for i, rule := range fl.Rules {
-		if rule.Element == "" ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix) ||
+		fltr.Rules[i] = &engine.FilterRule{
+			Type:    rule.Type,
+			Element: rule.FieldName,
+			Values:  rule.Values,
+		}
+		if rule.FieldName == "" ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) ||
 			filterTypes.Has(rule.Type) {
 			continue
 		}
-		fl.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.Element
+		fltr.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.FieldName
 	}
-	return fl
+	return
 }
 
-func migrateFilterV2(fl *engine.Filter) *engine.Filter {
+func migrateFilterV2(fl *v1Filter) (fltr *engine.Filter) {
+	fltr = &engine.Filter{
+		Tenant:             fl.Tenant,
+		ID:                 fl.ID,
+		Rules:              make([]*engine.FilterRule, len(fl.Rules)),
+		ActivationInterval: fl.ActivationInterval,
+	}
 	for i, rule := range fl.Rules {
-		if (rule.Element == "" && rule.Type != utils.MetaRSR) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaReq) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaVars) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCgreq) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCgrep) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaRep) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaCGRAReq) ||
-			strings.HasPrefix(rule.Element, utils.DynamicDataPrefix+utils.MetaAct) {
+		fltr.Rules[i] = &engine.FilterRule{
+			Type:    rule.Type,
+			Element: rule.FieldName,
+			Values:  rule.Values,
+		}
+		if (rule.FieldName == "" && rule.Type != utils.MetaRSR) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaReq) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaVars) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgreq) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgrep) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaRep) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCGRAReq) ||
+			strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaAct) {
 			continue
 		}
 		if rule.Type != utils.MetaRSR {
 			// in case we found dynamic data prefix we remove it
-			if strings.HasPrefix(rule.Element, utils.DynamicDataPrefix) {
-				fl.Rules[i].Element = fl.Rules[i].Element[1:]
+			if strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) {
+				fl.Rules[i].FieldName = fl.Rules[i].FieldName[1:]
 			}
-			fl.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.Element
+			fltr.Rules[i].Element = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.FieldName
 		} else {
 			for idx, val := range rule.Values {
 				if strings.HasPrefix(val, utils.DynamicDataPrefix) {
 					// remove dynamic data prefix from fieldName
 					val = val[1:]
 				}
-				fl.Rules[i].Values[idx] = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + val
+				fltr.Rules[i].Values[idx] = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + val
 			}
 		}
 	}
-	return fl
+	return
+}
+
+func migrateFilterV3(fl *v1Filter) (fltr *engine.Filter) {
+	fltr = &engine.Filter{
+		Tenant:             fl.Tenant,
+		ID:                 fl.ID,
+		Rules:              make([]*engine.FilterRule, len(fl.Rules)),
+		ActivationInterval: fl.ActivationInterval,
+	}
+	for i, rule := range fl.Rules {
+		fltr.Rules[i] = &engine.FilterRule{
+			Type:    rule.Type,
+			Element: rule.FieldName,
+			Values:  rule.Values,
+		}
+	}
+	return
 }
 
 func migrateInlineFilter(fl string) string {
@@ -141,36 +180,30 @@ func migrateInlineFilterV2(fl string) string {
 			ruleSplt[1] = ruleSplt[1][1:]
 		}
 		return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep))
-	} else { // in case of *rsr filter we need to add the prefix at fieldValue
-		if strings.HasPrefix(ruleSplt[2], utils.DynamicDataPrefix) {
-			// remove dynamic data prefix from fieldName
-			ruleSplt[2] = ruleSplt[2][1:]
-		}
-		return fmt.Sprintf("%s::~%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+strings.Join(ruleSplt[2:], utils.InInFieldSep))
+	} // in case of *rsr filter we need to add the prefix at fieldValue
+	if strings.HasPrefix(ruleSplt[2], utils.DynamicDataPrefix) {
+		// remove dynamic data prefix from fieldName
+		ruleSplt[2] = ruleSplt[2][1:]
 	}
-
+	return fmt.Sprintf("%s::~%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+strings.Join(ruleSplt[2:], utils.InInFieldSep))
 }
 
 func (m *Migrator) migrateRequestFilterV1() (err error) {
-	var ids []string
-	tenant := config.CgrConfig().GeneralCfg().DefaultTenant
-	ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix)
-	if err != nil {
-		return err
-	}
-	for _, id := range ids {
-		idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":")
-		fl, err := engine.GetFilter(m.dmIN.DataManager(), tenant, idg, false, false, utils.NonTransactional)
-		if err != nil {
+	for {
+		fl, err := m.dmIN.getV1Filter()
+		if err != nil && err != utils.ErrNoMoreData {
 			return err
 		}
+		if err == utils.ErrNoMoreData {
+			break
+		}
 		if m.dryRun || fl == nil {
 			continue
 		}
 		if err := m.dmOut.DataManager().SetFilter(migrateFilterV1(fl)); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	if err = m.migrateResourceProfileFiltersV1(); err != nil {
 		return err
@@ -193,7 +226,7 @@ func (m *Migrator) migrateRequestFilterV1() (err error) {
 	if err = m.migrateDispatcherProfileFiltersV1(); err != nil {
 		return err
 	}
-	vrs := engine.Versions{utils.RQF: 2}
+	vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]}
 	if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
 		return utils.NewCGRError(utils.Migrator,
 			utils.ServerErrorCaps,
@@ -204,27 +237,22 @@ func (m *Migrator) migrateRequestFilterV1() (err error) {
 }
 
 func (m *Migrator) migrateRequestFilterV2() (err error) {
-	var ids []string
-	tenant := config.CgrConfig().GeneralCfg().DefaultTenant
-	ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix)
-	if err != nil {
-		return fmt.Errorf("Error: <%s> when getting filter IDs for migration", err.Error())
-	}
-	for _, id := range ids {
-		idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":")
-		fl, err := engine.GetFilter(m.dmIN.DataManager(), tenant, idg, false, false, utils.NonTransactional)
-		if err != nil {
-			return fmt.Errorf("Error: <%s> when getting filter with tenant: <%s> and id: <%s> for migration",
-				err.Error(), tenant, idg)
+	for {
+		fl, err := m.dmIN.getV1Filter()
+		if err != nil && err != utils.ErrNoMoreData {
+			return err
+		}
+		if err == utils.ErrNoMoreData {
+			break
 		}
 		if m.dryRun || fl == nil {
 			continue
 		}
 		if err := m.dmOut.DataManager().SetFilter(migrateFilterV2(fl)); err != nil {
 			return fmt.Errorf("Error: <%s> when setting filter with tenant: <%s> and id: <%s> after migration",
-				err.Error(), tenant, idg)
+				err.Error(), fl.Tenant, fl.ID)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	if err = m.migrateResourceProfileFiltersV2(); err != nil {
 		return fmt.Errorf("Error: <%s> when trying to migrate filter for ResourceProfiles",
@@ -264,6 +292,34 @@ func (m *Migrator) migrateRequestFilterV2() (err error) {
 	return
 }
 
+func (m *Migrator) migrateRequestFilterV3() (err error) {
+	for {
+		fl, err := m.dmIN.getV1Filter()
+		if err != nil && err != utils.ErrNoMoreData {
+			return err
+		}
+		if err == utils.ErrNoMoreData {
+			break
+		}
+		if m.dryRun || fl == nil {
+			continue
+		}
+		if err := m.dmOut.DataManager().SetFilter(migrateFilterV3(fl)); err != nil {
+			return fmt.Errorf("Error: <%s> when setting filter with tenant: <%s> and id: <%s> after migration",
+				err.Error(), fl.Tenant, fl.ID)
+		}
+		m.stats[utils.RQF]++
+	}
+	vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]}
+	if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
+		return utils.NewCGRError(utils.Migrator,
+			utils.ServerErrorCaps,
+			err.Error(),
+			fmt.Sprintf("error: <%s> when updating Filters version into dataDB", err.Error()))
+	}
+	return
+}
+
 func (m *Migrator) migrateFilters() (err error) {
 	var vrs engine.Versions
 	current := engine.CurrentDataDBVersions()
@@ -280,6 +336,10 @@ func (m *Migrator) migrateFilters() (err error) {
 			"version number is not defined for ActionTriggers model")
 	}
 	switch vrs[utils.RQF] {
+	case 3:
+		if err = m.migrateRequestFilterV3(); err != nil {
+			return err
+		}
 	case 2:
 		if err = m.migrateRequestFilterV2(); err != nil {
 			return err
@@ -321,7 +381,7 @@ func (m *Migrator) migrateResourceProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetResourceProfile(res, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -348,7 +408,7 @@ func (m *Migrator) migrateStatQueueProfileFiltersV1() (err error) {
 		if err = m.dmOut.DataManager().SetStatQueueProfile(sgs, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -375,7 +435,7 @@ func (m *Migrator) migrateThresholdsProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetThresholdProfile(ths, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -402,7 +462,7 @@ func (m *Migrator) migrateSupplierProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetSupplierProfile(splp, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -434,7 +494,7 @@ func (m *Migrator) migrateAttributeProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -461,7 +521,7 @@ func (m *Migrator) migrateChargerProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetChargerProfile(cpp, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -488,7 +548,7 @@ func (m *Migrator) migrateDispatcherProfileFiltersV1() (err error) {
 		if err := m.dmOut.DataManager().SetDispatcherProfile(dpp, true); err != nil {
 			return err
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -518,7 +578,7 @@ func (m *Migrator) migrateResourceProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting resource profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -547,7 +607,7 @@ func (m *Migrator) migrateStatQueueProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting statQueue profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -576,7 +636,7 @@ func (m *Migrator) migrateThresholdsProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting threshold profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -605,7 +665,7 @@ func (m *Migrator) migrateSupplierProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting supplier profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -639,7 +699,7 @@ func (m *Migrator) migrateAttributeProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting attribute profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -668,7 +728,7 @@ func (m *Migrator) migrateChargerProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting charger profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
@@ -697,7 +757,22 @@ func (m *Migrator) migrateDispatcherProfileFiltersV2() (err error) {
 			return fmt.Errorf("error: <%s> when setting dispatcher profile with tenant: <%s> and id: <%s>",
 				err.Error(), tenant, idg)
 		}
-		m.stats[utils.RQF] += 1
+		m.stats[utils.RQF]++
 	}
 	return
 }
+
+type v1Filter struct {
+	Tenant             string
+	ID                 string
+	Rules              []*v1FilterRule
+	ActivationInterval *utils.ActivationInterval
+}
+
+type v1FilterRule struct {
+	Type      string            // Filter type (*string, *timing, *rsr_filters, *stats, *lt, *lte, *gt, *gte)
+	FieldName string            // Name of the field providing us the Values to check (used in case of some )
+	Values    []string          // Filter definition
+	rsrFields config.RSRParsers // Cache here the RSRFilter Values
+	negative  *bool
+}

migrator/filters_it_test.go

@@ -42,7 +42,10 @@ var sTestsFltrIT = []func(t *testing.T){
 	testFltrITConnect,
 	testFltrITFlush,
 	testFltrITMigrateAndMove,
+	testFltrITFlush,
 	testFltrITMigratev2,
+	testFltrITFlush,
+	testFltrITMigratev3,
 }
 
 func TestFiltersMigrateITRedis(t *testing.T) {
@@ -130,14 +133,14 @@ func testFltrITFlush(t *testing.T) {
 }
 
 func testFltrITMigrateAndMove(t *testing.T) {
-	Filters := &engine.Filter{
+	Filters := &v1Filter{
 		Tenant: "cgrates.org",
 		ID:     "FLTR_2",
-		Rules: []*engine.FilterRule{
-			&engine.FilterRule{
-				Type:    utils.MetaPrefix,
-				Element: "Account",
-				Values:  []string{"1001"},
+		Rules: []*v1FilterRule{
+			&v1FilterRule{
+				Type:      utils.MetaPrefix,
+				FieldName: "Account",
+				Values:    []string{"1001"},
 			},
 		},
 	}
@@ -187,7 +190,7 @@ func testFltrITMigrateAndMove(t *testing.T) {
 	attrProf.Compile()
 	switch fltrAction {
 	case utils.Migrate:
-		if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil {
+		if err := fltrMigrator.dmIN.setV1Filter(Filters); err != nil {
 			t.Error("Error when setting v1 Filters ", err.Error())
 		}
 		if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
@@ -212,7 +215,7 @@ func testFltrITMigrateAndMove(t *testing.T) {
 		//check if version was updated
 		if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
 			t.Error(err)
-		} else if vrs[utils.RQF] != 2 {
+		} else if vrs[utils.RQF] != 4 {
 			t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
 		}
 		//check if Filters was migrate correctly
@@ -243,7 +246,7 @@ func testFltrITMigrateAndMove(t *testing.T) {
 			t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx))
 		}
 	case utils.Move:
-		if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil {
+		if err := fltrMigrator.dmIN.DataManager().SetFilter(expFilters); err != nil {
 			t.Error(err)
 		}
 		currentVersion := engine.CurrentDataDBVersions()
@@ -257,16 +260,16 @@ func testFltrITMigrateAndMove(t *testing.T) {
 			t.Error("Error when fltrMigratorrating Filters ", err.Error())
 		}
 		//check if account was migrate correctly
-		result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
+		result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), expFilters.Tenant, expFilters.ID, false, false, utils.NonTransactional)
 		if err != nil {
 			t.Error(err)
 		}
 		result.Compile()
-		if !reflect.DeepEqual(Filters, result) {
-			t.Errorf("Expecting: %+v, received: %+v", Filters, result)
+		if !reflect.DeepEqual(expFilters, result) {
+			t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result))
 		}
 		// check if old account was deleted
-		result, err = engine.GetFilter(fltrMigrator.dmIN.DataManager(), Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
+		result, err = engine.GetFilter(fltrMigrator.dmIN.DataManager(), expFilters.Tenant, expFilters.ID, false, false, utils.NonTransactional)
 		if err != utils.ErrNotFound {
 			t.Error(err)
 		}
@@ -277,24 +280,24 @@ func testFltrITMigratev2(t *testing.T) {
 	if fltrAction != utils.Migrate {
 		t.SkipNow()
 	}
-	filters := &engine.Filter{
+	filters := &v1Filter{
 		Tenant: "cgrates.org",
 		ID:     "FLTR_2",
-		Rules: []*engine.FilterRule{
-			&engine.FilterRule{
-				Type:    utils.MetaString,
-				Element: "~Account",
-				Values:  []string{"1001"},
+		Rules: []*v1FilterRule{
+			&v1FilterRule{
+				Type:      utils.MetaString,
+				FieldName: "~Account",
+				Values:    []string{"1001"},
 			},
-			&engine.FilterRule{
-				Type:    utils.MetaString,
-				Element: "~*req.Subject",
-				Values:  []string{"1001"},
+			&v1FilterRule{
+				Type:      utils.MetaString,
+				FieldName: "~*req.Subject",
+				Values:    []string{"1001"},
 			},
-			&engine.FilterRule{
-				Type:    utils.MetaRSR,
-				Element: utils.EmptyString,
-				Values:  []string{"~Tenant(~^cgr.*\\.org$)"},
+			&v1FilterRule{
+				Type:      utils.MetaRSR,
+				FieldName: utils.EmptyString,
+				Values:    []string{"~Tenant(~^cgr.*\\.org$)"},
 			},
 		},
 	}
@@ -353,7 +356,7 @@ func testFltrITMigratev2(t *testing.T) {
 	expAttrProf.Compile()
 	attrProf.Compile()
 
-	if err := fltrMigrator.dmIN.DataManager().SetFilter(filters); err != nil {
+	if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil {
 		t.Error("Error when setting v1 Filters ", err.Error())
 	}
 	if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
@@ -378,7 +381,7 @@ func testFltrITMigratev2(t *testing.T) {
 	//check if version was updated
 	if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
 		t.Error(err)
-	} else if vrs[utils.RQF] != 3 {
+	} else if vrs[utils.RQF] != 4 {
 		t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
 	}
 	//check if Filters was migrate correctly
@@ -400,7 +403,6 @@ func testFltrITMigratev2(t *testing.T) {
 		t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultAttr))
 	}
 	expFltrIdx := map[string]utils.StringMap{
-		"*prefix:~*req.Account:1001": utils.StringMap{"ATTR_1": true},
 		"*string:~*req.Account:1001": utils.StringMap{"ATTR_1": true},
 		"*string:~*req.Subject:1001": utils.StringMap{"ATTR_1": true},
 	}
@@ -411,3 +413,87 @@ func testFltrITMigratev2(t *testing.T) {
 		t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx))
 	}
 }
+
+func testFltrITMigratev3(t *testing.T) {
+	if fltrAction != utils.Migrate {
+		t.SkipNow()
+	}
+	filters := &v1Filter{
+		Tenant: "cgrates.org",
+		ID:     "FLTR_2",
+		Rules: []*v1FilterRule{
+			&v1FilterRule{
+				Type:      utils.MetaString,
+				FieldName: "~*req.Account",
+				Values:    []string{"1001"},
+			},
+			&v1FilterRule{
+				Type:      utils.MetaString,
+				FieldName: "~*req.Subject",
+				Values:    []string{"1001"},
+			},
+			&v1FilterRule{
+				Type:      utils.MetaRSR,
+				FieldName: utils.EmptyString,
+				Values:    []string{"~*req.Tenant(~^cgr.*\\.org$)"},
+			},
+		},
+	}
+	expFilters := &engine.Filter{
+		Tenant: "cgrates.org",
+		ID:     "FLTR_2",
+		Rules: []*engine.FilterRule{
+			&engine.FilterRule{
+				Type:    utils.MetaString,
+				Element: "~*req.Account",
+				Values:  []string{"1001"},
+			},
+			&engine.FilterRule{
+				Type:    utils.MetaString,
+				Element: "~*req.Subject",
+				Values:  []string{"1001"},
+			},
+			&engine.FilterRule{
+				Type:    utils.MetaRSR,
+				Element: utils.EmptyString,
+				Values:  []string{"~*req.Tenant(~^cgr.*\\.org$)"},
+			},
+		},
+	}
+	expFilters.Compile()
+
+	if err := fltrMigrator.dmIN.setV1Filter(filters); err != nil {
+		t.Error("Error when setting v1 Filters ", err.Error())
+	}
+	currentVersion := engine.Versions{utils.RQF: 3}
+	err := fltrMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false)
+	if err != nil {
+		t.Error("Error when setting version for Filters ", err.Error())
+	}
+	//check if version was set correctly
+	if vrs, err := fltrMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil {
+		t.Error(err)
+	} else if vrs[utils.RQF] != 3 {
+		t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
+	}
+	//migrate Filters
+	err, _ = fltrMigrator.Migrate([]string{utils.MetaFilters})
+	if err != nil {
+		t.Error("Error when migrating Filters ", err.Error())
+	}
+	//check if version was updated
+	if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
+		t.Error(err)
+	} else if vrs[utils.RQF] != 4 {
+		t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
+	}
+	//check if Filters was migrate correctly
+	result, err := engine.GetFilter(fltrMigrator.dmOut.DataManager(), filters.Tenant, filters.ID, false, false, utils.NonTransactional)
+	if err != nil {
+		t.Fatalf("Error when getting filters %v", err.Error())
+	}
+	result.Compile()
+	if !reflect.DeepEqual(*expFilters, *result) {
+		t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result))
+	}
+}

migrator/filters_test.go

@@ -58,16 +58,19 @@ func TestFiltersInlineMigrate(t *testing.T) {
 }
 
 func TestFiltersMigrate(t *testing.T) {
-	data := []struct{ in, exp *engine.Filter }{
+	data := []struct {
+		in  *v1Filter
+		exp *engine.Filter
+	}{
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_1",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaString,
-						Element: "Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaString,
+						FieldName: "Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -84,14 +87,14 @@ func TestFiltersMigrate(t *testing.T) {
 			},
 		},
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_2",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaPrefix,
-						Element: "~Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaPrefix,
+						FieldName: "~Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -116,16 +119,19 @@ func TestFiltersMigrate(t *testing.T) {
 }
 
 func TestFiltersMigrateV2(t *testing.T) {
-	data := []struct{ in, exp *engine.Filter }{
+	data := []struct {
+		in  *v1Filter
+		exp *engine.Filter
+	}{
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_1",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaString,
-						Element: "~Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaString,
+						FieldName: "~Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -142,14 +148,14 @@ func TestFiltersMigrateV2(t *testing.T) {
 			},
 		},
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_2",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaPrefix,
-						Element: "~*req.Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaPrefix,
+						FieldName: "~*req.Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -166,14 +172,14 @@ func TestFiltersMigrateV2(t *testing.T) {
 			},
 		},
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_3",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaPrefix,
-						Element: "~*act.Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaPrefix,
+						FieldName: "~*act.Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -190,14 +196,14 @@ func TestFiltersMigrateV2(t *testing.T) {
 			},
 		},
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_4",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaPrefix,
-						Element: "~*act.Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaPrefix,
+						FieldName: "~*act.Account",
+						Values:    []string{},
 					},
 				},
 			},
@@ -214,14 +220,14 @@ func TestFiltersMigrateV2(t *testing.T) {
 			},
 		},
 		{
-			in: &engine.Filter{
+			in: &v1Filter{
 				Tenant: "cgrates.org",
 				ID:     "FLTR_5",
-				Rules: []*engine.FilterRule{
-					&engine.FilterRule{
-						Type:    utils.MetaPrefix,
-						Element: "~*vars.Account",
-						Values:  []string{},
+				Rules: []*v1FilterRule{
+					&v1FilterRule{
+						Type:      utils.MetaPrefix,
+						FieldName: "~*vars.Account",
+						Values:    []string{},
 					},
 				},
 			},

migrator/migrator_datadb.go

@@ -66,6 +66,10 @@ type MigratorDataDB interface {
 	setV4AttributeProfile(x *v4AttributeProfile) (err error)
 	remV4AttributeProfile(tenant, id string) (err error)
 
+	getV1Filter() (v1Fltr *v1Filter, err error)
+	setV1Filter(x *v1Filter) (err error)
+	remV1Filter(tenant, id string) (err error)
+
 	DataManager() *engine.DataManager
 	close()
 }

migrator/storage_map_datadb.go

@@ -263,4 +263,20 @@ func (iDBMig *internalMigrator) remV4AttributeProfile(tenant, id string) (err er
 	return utils.ErrNotImplemented
 }
 
+// Filter Methods
+//get
+func (iDBMig *internalMigrator) getV1Filter() (v1Fltr *v1Filter, err error) {
+	return nil, utils.ErrNotImplemented
+}
+
+//set
+func (iDBMig *internalMigrator) setV1Filter(x *v1Filter) (err error) {
+	return utils.ErrNotImplemented
+}
+
+//rem
+func (iDBMig *internalMigrator) remV1Filter(tenant, id string) (err error) {
+	return utils.ErrNotImplemented
+}
+
 func (iDBMig *internalMigrator) close() {}

migrator/storage_mongo_datadb.go

@@ -59,12 +59,12 @@ func newMongoMigrator(dm *engine.DataManager) (mgoMig *mongoMigrator) {
 	}
 }
 
-func (mgoMig *mongoMigrator) close() {
-	mgoMig.mgoDB.Close()
+func (v1ms *mongoMigrator) close() {
+	v1ms.mgoDB.Close()
 }
 
-func (mgoMig *mongoMigrator) DataManager() *engine.DataManager {
-	return mgoMig.dm
+func (v1ms *mongoMigrator) DataManager() *engine.DataManager {
+	return v1ms.dm
 }
 
 //Account methods
@@ -621,3 +621,36 @@ func (v1ms *mongoMigrator) remV4AttributeProfile(tenant, id string) (err error)
 	_, err = v1ms.mgoDB.DB().Collection(v1AttributeProfilesCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id})
 	return
 }
+
+// Filter Methods
+//get
+func (v1ms *mongoMigrator) getV1Filter() (v1Fltr *v1Filter, err error) {
+	if v1ms.cursor == nil {
+		v1ms.cursor, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).Find(v1ms.mgoDB.GetContext(), bson.D{})
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !(*v1ms.cursor).Next(v1ms.mgoDB.GetContext()) {
+		(*v1ms.cursor).Close(v1ms.mgoDB.GetContext())
+		v1ms.cursor = nil
+		return nil, utils.ErrNoMoreData
+	}
+	v1Fltr = new(v1Filter)
+	if err := (*v1ms.cursor).Decode(v1Fltr); err != nil {
+		return nil, err
+	}
+	return
+}
+
+//set
+func (v1ms *mongoMigrator) setV1Filter(x *v1Filter) (err error) {
+	_, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).InsertOne(v1ms.mgoDB.GetContext(), x)
+	return
+}
+
+//rem
+func (v1ms *mongoMigrator) remV1Filter(tenant, id string) (err error) {
+	_, err = v1ms.mgoDB.DB().Collection(engine.ColFlt).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id})
+	return
+}

migrator/storage_redis.go

@@ -34,7 +34,7 @@ type redisMigrator struct {
 }
 
 var (
-	REVERSE_ALIASES_PREFIX = "rls_"
+	reverseAliasesPrefix = "rls_"
 )
 
 func newRedisMigrator(dm *engine.DataManager) (rM *redisMigrator) {
@@ -44,12 +44,12 @@ func newRedisMigrator(dm *engine.DataManager) (rM *redisMigrator) {
 	}
 }
 
-func (rdsMig *redisMigrator) close() {
-	rdsMig.rds.Close()
+func (v1rs *redisMigrator) close() {
+	v1rs.rds.Close()
 }
 
-func (rdsMig *redisMigrator) DataManager() *engine.DataManager {
-	return rdsMig.dm
+func (v1rs *redisMigrator) DataManager() *engine.DataManager {
+	return v1rs.dm
 }
 
 //Account methods
@@ -558,7 +558,7 @@ func (v1rs *redisMigrator) remV1Alias(key string) (err error) {
 		for target, pairs := range value.Pairs {
 			for _, alias := range pairs {
 				revID := alias + target + al.Context
-				err = v1rs.rds.Cmd("SREM", REVERSE_ALIASES_PREFIX+revID, tmpKey).Err
+				err = v1rs.rds.Cmd("SREM", reverseAliasesPrefix+revID, tmpKey).Err
 				if err != nil {
 					return err
 				}
@@ -802,3 +802,47 @@ func (v1rs *redisMigrator) remV4AttributeProfile(tenant, id string) (err error)
 	key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(tenant, id)
 	return v1rs.rds.Cmd("DEL", key).Err
 }
+
+// Filter Methods
+//get
+func (v1rs *redisMigrator) getV1Filter() (v1Fltr *v1Filter, err error) {
+	if v1rs.qryIdx == nil {
+		v1rs.dataKeys, err = v1rs.rds.GetKeysForPrefix(utils.FilterPrefix)
+		if err != nil {
+			return
+		} else if len(v1rs.dataKeys) == 0 {
+			return nil, utils.ErrNoMoreData
+		}
+		v1rs.qryIdx = utils.IntPointer(0)
+	}
+	if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 {
+		strVal, err := v1rs.rds.Cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes()
+		if err != nil {
+			return nil, err
+		}
+		if err := v1rs.rds.Marshaler().Unmarshal(strVal, &v1Fltr); err != nil {
+			return nil, err
+		}
+		*v1rs.qryIdx = *v1rs.qryIdx + 1
+	} else {
+		v1rs.qryIdx = nil
+		return nil, utils.ErrNoMoreData
+	}
+	return
+}
+
+//set
+func (v1rs *redisMigrator) setV1Filter(x *v1Filter) (err error) {
+	key := utils.FilterPrefix + utils.ConcatenatedKey(x.Tenant, x.ID)
+	bit, err := v1rs.rds.Marshaler().Marshal(x)
+	if err != nil {
+		return err
+	}
+	return v1rs.rds.Cmd("SET", key, bit).Err
+}
+
+//rem
+func (v1rs *redisMigrator) remV1Filter(tenant, id string) (err error) {
+	key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id)
+	return v1rs.rds.Cmd("DEL", key).Err
+}