Add migrate from V2 to V3 for Filters

This commit is contained in:
TeoV
2019-11-28 11:30:00 +02:00
parent 76a901b4e5
commit 9be1f2b7db
4 changed files with 619 additions and 1 deletions

View File

@@ -147,7 +147,7 @@ func CurrentDataDBVersions() Versions {
utils.Suppliers: 1,
utils.Attributes: 4,
utils.Timing: 1,
utils.RQF: 2,
utils.RQF: 3,
utils.Resource: 1,
utils.Subscribers: 1,
utils.Destinations: 1,

View File

@@ -69,6 +69,37 @@ func migrateFilterV1(fl *engine.Filter) *engine.Filter {
return fl
}
func migrateFilterV2(fl *engine.Filter) *engine.Filter {
for i, rule := range fl.Rules {
if (rule.FieldName == "" && rule.Type != utils.MetaRSR) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaReq) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaVars) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgreq) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCgrep) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaRep) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaCGRAReq) ||
strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix+utils.MetaAct) {
continue
}
if rule.Type != utils.MetaRSR {
// in case we found dynamic data prefix we remove it
if strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) {
fl.Rules[i].FieldName = fl.Rules[i].FieldName[1:]
}
fl.Rules[i].FieldName = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + rule.FieldName
} else {
for idx, val := range rule.Values {
if strings.HasPrefix(val, utils.DynamicDataPrefix) {
// remove dynamic data prefix from fieldName
val = val[1:]
}
fl.Rules[i].Values[idx] = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + val
}
}
}
return fl
}
func migrateInlineFilter(fl string) string {
if fl == "" || !strings.HasPrefix(fl, utils.Meta) {
return fl
@@ -85,6 +116,41 @@ func migrateInlineFilter(fl string) string {
return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep))
}
func migrateInlineFilterV2(fl string) string {
if fl == "" || !strings.HasPrefix(fl, utils.Meta) {
return fl
}
ruleSplt := strings.Split(fl, utils.InInFieldSep)
if len(ruleSplt) < 3 {
return fl
}
if ruleSplt[1] != utils.EmptyString && // no need conversion
(strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaReq) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaVars) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaCgreq) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaCgrep) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaRep) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaCGRAReq) ||
strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix+utils.MetaAct)) {
return fl
}
if ruleSplt[0] != utils.MetaRSR {
if strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix) {
// remove dynamic data prefix from fieldName
ruleSplt[1] = ruleSplt[1][1:]
}
return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep))
} else { // in case of *rsr filter we need to add the prefix at fieldValue
if strings.HasPrefix(ruleSplt[2], utils.DynamicDataPrefix) {
// remove dynamic data prefix from fieldName
ruleSplt[2] = ruleSplt[2][1:]
}
return fmt.Sprintf("%s::~%s", ruleSplt[0], utils.MetaReq+utils.NestingSep+strings.Join(ruleSplt[2:], utils.InInFieldSep))
}
}
func (m *Migrator) migrateRequestFilterV1() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
@@ -127,6 +193,58 @@ func (m *Migrator) migrateRequestFilterV1() (err error) {
if err = m.migrateDispatcherProfileFiltersV1(); err != nil {
return err
}
vrs := engine.Versions{utils.RQF: 2}
if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,
err.Error(),
fmt.Sprintf("error: <%s> when updating Filters version into dataDB", err.Error()))
}
return
}
func (m *Migrator) migrateRequestFilterV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":")
fl, err := m.dmIN.DataManager().GetFilter(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if m.dryRun || fl == nil {
continue
}
if err := m.dmOut.DataManager().SetFilter(migrateFilterV2(fl)); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
if err = m.migrateResourceProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateStatQueueProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateThresholdsProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateSupplierProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateAttributeProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateChargerProfileFiltersV2(); err != nil {
return err
}
if err = m.migrateDispatcherProfileFiltersV2(); err != nil {
return err
}
vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]}
if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
return utils.NewCGRError(utils.Migrator,
@@ -153,6 +271,10 @@ func (m *Migrator) migrateFilters() (err error) {
"version number is not defined for ActionTriggers model")
}
switch vrs[utils.RQF] {
case 2:
if err = m.migrateRequestFilterV2(); err != nil {
return err
}
case 1:
if err = m.migrateRequestFilterV1(); err != nil {
return err
@@ -361,3 +483,198 @@ func (m *Migrator) migrateDispatcherProfileFiltersV1() (err error) {
}
return
}
// migrate filters from v2 to v3 for items
func (m *Migrator) migrateResourceProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.ResourceProfilesPrefix+tenant+":")
res, err := m.dmIN.DataManager().GetResourceProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if m.dryRun || res == nil {
continue
}
for i, fl := range res.FilterIDs {
res.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err := m.dmOut.DataManager().SetResourceProfile(res, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateStatQueueProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.StatQueueProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.StatQueueProfilePrefix+tenant+":")
sgs, err := m.dmIN.DataManager().GetStatQueueProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if sgs == nil || m.dryRun {
continue
}
for i, fl := range sgs.FilterIDs {
sgs.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err = m.dmOut.DataManager().SetStatQueueProfile(sgs, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateThresholdsProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ThresholdProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.ThresholdProfilePrefix+tenant+":")
ths, err := m.dmIN.DataManager().GetThresholdProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if ths == nil || m.dryRun {
continue
}
for i, fl := range ths.FilterIDs {
ths.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err := m.dmOut.DataManager().SetThresholdProfile(ths, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateSupplierProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.SupplierProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.SupplierProfilePrefix)
splp, err := m.dmIN.DataManager().GetSupplierProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if splp == nil || m.dryRun {
continue
}
for i, fl := range splp.FilterIDs {
splp.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err := m.dmOut.DataManager().SetSupplierProfile(splp, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateAttributeProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.AttributeProfilePrefix+tenant+":")
attrPrf, err := m.dmIN.DataManager().GetAttributeProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if attrPrf == nil || m.dryRun {
continue
}
for i, fl := range attrPrf.FilterIDs {
attrPrf.FilterIDs[i] = migrateInlineFilterV2(fl)
}
for i, attr := range attrPrf.Attributes {
for j, fl := range attr.FilterIDs {
attrPrf.Attributes[i].FilterIDs[j] = migrateInlineFilterV2(fl)
}
}
if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateChargerProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ChargerProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.ChargerProfilePrefix+tenant+":")
cpp, err := m.dmIN.DataManager().GetChargerProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if cpp == nil || m.dryRun {
continue
}
for i, fl := range cpp.FilterIDs {
cpp.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err := m.dmOut.DataManager().SetChargerProfile(cpp, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}
func (m *Migrator) migrateDispatcherProfileFiltersV2() (err error) {
var ids []string
tenant := config.CgrConfig().GeneralCfg().DefaultTenant
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.DispatcherProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.DispatcherProfilePrefix+tenant+":")
dpp, err := m.dmIN.DataManager().GetDispatcherProfile(tenant, idg, false, false, utils.NonTransactional)
if err != nil {
return err
}
if dpp == nil || m.dryRun {
continue
}
for i, fl := range dpp.FilterIDs {
dpp.FilterIDs[i] = migrateInlineFilterV2(fl)
}
if err := m.dmOut.DataManager().SetDispatcherProfile(dpp, true); err != nil {
return err
}
m.stats[utils.RQF] += 1
}
return
}

View File

@@ -42,6 +42,7 @@ var sTestsFltrIT = []func(t *testing.T){
testFltrITConnect,
testFltrITFlush,
testFltrITMigrateAndMove,
testFltrITMigratev2,
}
func TestFiltersMigrateITRedis(t *testing.T) {
@@ -271,3 +272,142 @@ func testFltrITMigrateAndMove(t *testing.T) {
}
}
}
func testFltrITMigratev2(t *testing.T) {
if fltrAction != utils.Migrate {
t.SkipNow()
}
filters := &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~Account",
Values: []string{"1001"},
},
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~*req.Subject",
Values: []string{"1001"},
},
&engine.FilterRule{
Type: utils.MetaRSR,
FieldName: utils.EmptyString,
Values: []string{"~Tenant(~^cgr.*\\.org$)"},
},
},
}
expFilters := &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~*req.Account",
Values: []string{"1001"},
},
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~*req.Subject",
Values: []string{"1001"},
},
&engine.FilterRule{
Type: utils.MetaRSR,
FieldName: utils.EmptyString,
Values: []string{"~*req.Tenant(~^cgr.*\\.org$)"},
},
},
}
expFilters.Compile()
attrProf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "ATTR_1",
Contexts: []string{utils.META_ANY},
FilterIDs: []string{"*string:~Account:1001", "FLTR_2"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{
FilterIDs: []string{"*string:~Account:1001"},
FieldName: "Account",
Value: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP),
},
},
Weight: 10,
}
expAttrProf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "ATTR_1",
Contexts: []string{utils.META_ANY},
FilterIDs: []string{"*string:~*req.Account:1001", "FLTR_2"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{
FilterIDs: []string{"*string:~*req.Account:1001"},
FieldName: "Account",
Value: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP),
},
},
Weight: 10,
}
expAttrProf.Compile()
attrProf.Compile()
if err := fltrMigrator.dmIN.DataManager().SetFilter(filters); err != nil {
t.Error("Error when setting v1 Filters ", err.Error())
}
if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
t.Error("Error when setting attribute profile for v1 Filters ", err.Error())
}
currentVersion := engine.Versions{utils.RQF: 2}
err := fltrMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for Filters ", err.Error())
}
//check if version was set correctly
if vrs, err := fltrMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if vrs[utils.RQF] != 2 {
t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
}
//migrate Filters
err, _ = fltrMigrator.Migrate([]string{utils.MetaFilters})
if err != nil {
t.Error("Error when migrating Filters ", err.Error())
}
//check if version was updated
if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if vrs[utils.RQF] != 3 {
t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
}
//check if Filters was migrate correctly
result, err := fltrMigrator.dmOut.DataManager().GetFilter(filters.Tenant, filters.ID, false, false, utils.NonTransactional)
if err != nil {
t.Fatalf("Error when getting filters %v", err.Error())
}
result.Compile()
if !reflect.DeepEqual(*expFilters, *result) {
t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result))
}
resultAttr, err := fltrMigrator.dmOut.DataManager().DataDB().GetAttributeProfileDrv(attrProf.Tenant, attrProf.ID)
if err != nil {
t.Fatalf("Error when getting Attributes %v", err.Error())
}
resultAttr.Compile()
if !reflect.DeepEqual(*expAttrProf, *resultAttr) {
t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultAttr))
}
expFltrIdx := map[string]utils.StringMap{
"*prefix:~*req.Account:1001": utils.StringMap{"ATTR_1": true},
"*string:~*req.Account:1001": utils.StringMap{"ATTR_1": true},
"*string:~*req.Subject:1001": utils.StringMap{"ATTR_1": true},
}
if fltridx, err := fltrMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), utils.MetaString, nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expFltrIdx, fltridx) {
t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx))
}
}

View File

@@ -114,3 +114,164 @@ func TestFiltersMigrate(t *testing.T) {
}
}
}
func TestFiltersMigrateV2(t *testing.T) {
data := []struct{ in, exp *engine.Filter }{
{
in: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_1",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~Account",
Values: []string{},
},
},
},
exp: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_1",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaString,
FieldName: "~*req.Account",
Values: []string{},
},
},
},
},
{
in: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*req.Account",
Values: []string{},
},
},
},
exp: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_2",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*req.Account",
Values: []string{},
},
},
},
},
{
in: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_3",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*act.Account",
Values: []string{},
},
},
},
exp: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_3",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*act.Account",
Values: []string{},
},
},
},
},
{
in: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_4",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*act.Account",
Values: []string{},
},
},
},
exp: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_4",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*act.Account",
Values: []string{},
},
},
},
},
{
in: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_5",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*vars.Account",
Values: []string{},
},
},
},
exp: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_5",
Rules: []*engine.FilterRule{
&engine.FilterRule{
Type: utils.MetaPrefix,
FieldName: "~*vars.Account",
Values: []string{},
},
},
},
},
}
for _, m := range data {
if rply := migrateFilterV2(m.in); !reflect.DeepEqual(rply, m.exp) {
t.Errorf("Expected: %s, recived: %s", utils.ToJSON(m.exp), utils.ToJSON(rply))
}
}
}
func TestFiltersInlineV2Migrate(t *testing.T) {
data := []struct{ in, exp string }{
{
in: "*string:~Account:1002",
exp: "*string:~*req.Account:1002",
},
{
in: "*string:~*req.Account:1002",
exp: "*string:~*req.Account:1002",
},
{
in: "FLTR_1",
exp: "FLTR_1",
},
{
in: "",
exp: "",
},
{
in: "*rsr::~Tenant(~^cgr.*\\.org$)",
exp: "*rsr::~*req.Tenant(~^cgr.*\\.org$)",
},
}
for _, m := range data {
if rply := migrateInlineFilterV2(m.in); rply != m.exp {
t.Errorf("Expected: %s, recived: %s", m.exp, rply)
}
}
}