diff --git a/engine/version.go b/engine/version.go
index 153ab9b4e..9824361e2 100644
--- a/engine/version.go
+++ b/engine/version.go
@@ -146,7 +146,7 @@ func CurrentDataDBVersions() Versions {
utils.Suppliers: 1,
utils.Attributes: 3,
utils.Timing: 1,
- utils.RQF: 1,
+ utils.RQF: 2,
utils.Resource: 1,
utils.Subscribers: 1,
utils.Destinations: 1,
diff --git a/migrator/alias.go b/migrator/alias.go
index 4a9352f85..74533b7ec 100644
--- a/migrator/alias.go
+++ b/migrator/alias.go
@@ -87,13 +87,13 @@ func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.Attri
out.Tenant = defaultTenant
}
if len(alias.Category) != 0 && alias.Category != utils.META_ANY {
- out.FilterIDs = append(out.FilterIDs, "*string:Category:"+alias.Category)
+ out.FilterIDs = append(out.FilterIDs, "*string:~Category:"+alias.Category)
}
if len(alias.Account) != 0 && alias.Account != utils.META_ANY {
- out.FilterIDs = append(out.FilterIDs, "*string:Account:"+alias.Account)
+ out.FilterIDs = append(out.FilterIDs, "*string:~Account:"+alias.Account)
}
if len(alias.Subject) != 0 && alias.Subject != utils.META_ANY {
- out.FilterIDs = append(out.FilterIDs, "*string:Subject:"+alias.Subject)
+ out.FilterIDs = append(out.FilterIDs, "*string:~Subject:"+alias.Subject)
}
var destination string
for _, av := range alias.Values {
@@ -115,7 +115,7 @@ func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.Attri
}
}
if len(destination) != 0 && destination != utils.META_ANY {
- out.FilterIDs = append(out.FilterIDs, "*destination:Destination:"+destination)
+ out.FilterIDs = append(out.FilterIDs, "*destination:~Destination:"+destination)
}
return out
}
diff --git a/migrator/alias_it_test.go b/migrator/alias_it_test.go
index b67bca207..abc09761d 100644
--- a/migrator/alias_it_test.go
+++ b/migrator/alias_it_test.go
@@ -156,9 +156,9 @@ func testAlsITMigrateAndMove(t *testing.T) {
ID: alias.GetId(),
Contexts: []string{utils.META_ANY},
FilterIDs: []string{
- "*string:Account:1001",
- "*string:Subject:call_1001",
- "*destination:Destination:DST_1003",
+ "*string:~Account:1001",
+ "*string:~Subject:call_1001",
+ "*destination:~Destination:DST_1003",
},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
@@ -226,10 +226,10 @@ func testAlsITMigrateAndMove(t *testing.T) {
}
expAlsIdx := map[string]utils.StringMap{
- "*string:Account:1001": utils.StringMap{
+ "*string:~Account:1001": utils.StringMap{
"*out:*any:*any:1001:call_1001:*rated": true,
},
- "*string:Subject:call_1001": utils.StringMap{
+ "*string:~Subject:call_1001": utils.StringMap{
"*out:*any:*any:1001:call_1001:*rated": true,
},
}
diff --git a/migrator/alias_test.go b/migrator/alias_test.go
index ab3243c45..b5a28ee93 100644
--- a/migrator/alias_test.go
+++ b/migrator/alias_test.go
@@ -219,9 +219,9 @@ func TestAlias2AtttributeProfile(t *testing.T) {
ID: aliases[4].GetId(),
Contexts: []string{utils.META_ANY},
FilterIDs: []string{
- "*string:Category:*voice",
- "*string:Account:1001",
- "*destination:Destination:DST_1003",
+ "*string:~Category:*voice",
+ "*string:~Account:1001",
+ "*destination:~Destination:DST_1003",
},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
@@ -244,9 +244,9 @@ func TestAlias2AtttributeProfile(t *testing.T) {
ID: aliases[5].GetId(),
Contexts: []string{utils.META_ANY},
FilterIDs: []string{
- "*string:Account:1001",
- "*string:Subject:call_1001",
- "*destination:Destination:DST_1003",
+ "*string:~Account:1001",
+ "*string:~Subject:call_1001",
+ "*destination:~Destination:DST_1003",
},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
diff --git a/migrator/derived_chargers.go b/migrator/derived_chargers.go
index fa12b9384..64a97b8cf 100644
--- a/migrator/derived_chargers.go
+++ b/migrator/derived_chargers.go
@@ -143,7 +143,7 @@ func (m *Migrator) derivedChargers2Chargers(dck *v1DerivedChargersWithKey) (err
skey := utils.SplitConcatenatedKey(dck.Key)
destination := ""
if len(dck.Value.DestinationIDs) != 0 {
- destination = "*destination:Destination:"
+ destination = "*destination:~Destination:"
keys := dcGetMapKeys(dck.Value.DestinationIDs)
destination += strings.Join(keys, utils.INFIELD_SEP)
}
@@ -153,13 +153,13 @@ func (m *Migrator) derivedChargers2Chargers(dck *v1DerivedChargersWithKey) (err
filter = append(filter, destination)
}
if len(skey[2]) != 0 && skey[2] != utils.META_ANY {
- filter = append(filter, "*string:Category:"+skey[2])
+ filter = append(filter, "*string:~Category:"+skey[2])
}
if len(skey[3]) != 0 && skey[3] != utils.META_ANY {
- filter = append(filter, "*string:Account:"+skey[3])
+ filter = append(filter, "*string:~Account:"+skey[3])
}
if len(skey[4]) != 0 && skey[4] != utils.META_ANY {
- filter = append(filter, "*string:Subject:"+skey[4])
+ filter = append(filter, "*string:~Subject:"+skey[4])
}
for i, dc := range dck.Value.Chargers {
diff --git a/migrator/derived_chargers_it_test.go b/migrator/derived_chargers_it_test.go
index 4fc80a871..293fa52ea 100644
--- a/migrator/derived_chargers_it_test.go
+++ b/migrator/derived_chargers_it_test.go
@@ -162,8 +162,8 @@ func testDCITMigrateAndMove(t *testing.T) {
ID: fmt.Sprintf("%s_%v", derivch.Key, 0),
Contexts: []string{utils.META_ANY},
FilterIDs: []string{
- "*destination:Destination:1001;1002;1003",
- "*string:Account:1003",
+ "*destination:~Destination:1001;1002;1003",
+ "*string:~Account:1003",
},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
@@ -184,8 +184,8 @@ func testDCITMigrateAndMove(t *testing.T) {
Tenant: defaultTenant,
ID: fmt.Sprintf("%s_%v", derivch.Key, 0),
FilterIDs: []string{
- "*destination:Destination:1001;1002;1003",
- "*string:Account:1003",
+ "*destination:~Destination:1001;1002;1003",
+ "*string:~Account:1003",
"*rsr::~filterhdr1:s/(.+)/special_run3/",
},
ActivationInterval: nil,
@@ -246,7 +246,7 @@ func testDCITMigrateAndMove(t *testing.T) {
t.Error("Error should be not found : ", err)
}
expDcIdx := map[string]utils.StringMap{
- "*string:Account:1003": utils.StringMap{
+ "*string:~Account:1003": utils.StringMap{
"*out:cgrates.org:*any:1003:*any_0": true,
},
}
@@ -257,7 +257,7 @@ func testDCITMigrateAndMove(t *testing.T) {
t.Errorf("Expected %v, recived: %v", utils.ToJSON(expDcIdx), utils.ToJSON(dcidx))
}
expDcIdx = map[string]utils.StringMap{
- "*string:Account:1003": utils.StringMap{
+ "*string:~Account:1003": utils.StringMap{
"*out:cgrates.org:*any:1003:*any_0": true,
},
}
diff --git a/migrator/derived_chargers_test.go b/migrator/derived_chargers_test.go
index 94e864e4d..ccf90dec7 100644
--- a/migrator/derived_chargers_test.go
+++ b/migrator/derived_chargers_test.go
@@ -145,12 +145,12 @@ func TestDerivedChargers2AttributeProfile(t *testing.T) {
},
Tenant: defaultTenant,
Key: "key1",
- Filters: []string{"*string:Subject:1005"},
+ Filters: []string{"*string:~Subject:1005"},
Expected: &engine.AttributeProfile{
Tenant: defaultTenant,
ID: "key1",
Contexts: []string{utils.META_ANY},
- FilterIDs: []string{"*string:Subject:1005"},
+ FilterIDs: []string{"*string:~Subject:1005"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
&engine.Attribute{
@@ -202,15 +202,15 @@ func TestDerivedChargers2Charger(t *testing.T) {
Tenant: defaultTenant,
Key: "key2",
Filters: []string{
- "*string:Category:*voice1",
- "*string:Account:1001",
+ "*string:~Category:*voice1",
+ "*string:~Account:1001",
},
Expected: &engine.ChargerProfile{
Tenant: defaultTenant,
ID: "key2",
FilterIDs: []string{
- "*string:Category:*voice1",
- "*string:Account:1001",
+ "*string:~Category:*voice1",
+ "*string:~Account:1001",
"*rsr::~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)",
},
ActivationInterval: nil,
diff --git a/migrator/filters.go b/migrator/filters.go
index b07eaac43..b73933a8d 100644
--- a/migrator/filters.go
+++ b/migrator/filters.go
@@ -40,14 +40,98 @@ func (m *Migrator) migrateCurrentRequestFilter() (err error) {
if err != nil {
return err
}
- if fl != nil {
- if m.dryRun != true {
- if err := m.dmOut.DataManager().SetFilter(fl); err != nil {
- return err
- }
- m.stats[utils.RQF] += 1
- }
+ if m.dryRun || fl == nil {
+ continue
}
+ if err := m.dmIN.DataManager().RemoveFilter(tenant, idg, utils.NonTransactional); err != nil {
+ return err
+ }
+ if err := m.dmOut.DataManager().SetFilter(fl); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func migrateFilterV1(fl *engine.Filter) *engine.Filter {
+ for i, rule := range fl.Rules {
+ if rule.FieldName == "" ||
+ utils.IsSliceMember([]string{engine.MetaRSR, engine.MetaStatS, engine.MetaResources,
+ engine.MetaNotRSR, engine.MetaNotStatS, engine.MetaNotResources}, rule.Type) ||
+ strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) {
+ continue
+ }
+ fl.Rules[i].FieldName = utils.DynamicDataPrefix + rule.FieldName
+ }
+ return fl
+}
+
+func migrateInlineFilter(fl string) string {
+ if fl == "" || !strings.HasPrefix(fl, utils.Meta) {
+ return fl
+ }
+ ruleSplt := strings.Split(fl, utils.InInFieldSep)
+ if len(ruleSplt) < 3 {
+ return fl
+ }
+
+ if utils.IsSliceMember([]string{engine.MetaRSR, engine.MetaStatS, engine.MetaResources,
+ engine.MetaNotRSR, engine.MetaNotStatS, engine.MetaNotResources}, ruleSplt[0]) ||
+ strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix) {
+ return fl
+ }
+ return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep))
+}
+
+func (m *Migrator) migrateRequestFilterV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":")
+ fl, err := m.dmIN.DataManager().GetFilter(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if m.dryRun || fl == nil {
+ continue
+ }
+ if err := m.dmOut.DataManager().SetFilter(migrateFilterV1(fl)); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ if err = m.migrateResourceProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateStatQueueProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateThresholdsProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateSupplierProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateAttributeProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateChargerProfileFiltersV1(); err != nil {
+ return err
+ }
+ if err = m.migrateDispatcherProfileFiltersV1(); err != nil {
+ return err
+ }
+ vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]}
+ if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
+ return utils.NewCGRError(utils.Migrator,
+ utils.ServerErrorCaps,
+ err.Error(),
+ fmt.Sprintf("error: <%s> when updating Filters version into dataDB", err.Error()))
}
return
}
@@ -68,14 +152,202 @@ func (m *Migrator) migrateRequestFilter() (err error) {
"version number is not defined for ActionTriggers model")
}
switch vrs[utils.RQF] {
+ case 1:
+ return m.migrateRequestFilterV1()
case current[utils.RQF]:
if m.sameDataDB {
return
}
- if err := m.migrateCurrentRequestFilter(); err != nil {
- return err
- }
- return
+ return m.migrateCurrentRequestFilter()
+ }
+ return
+}
+
+func (m *Migrator) migrateResourceProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.ResourceProfilesPrefix+tenant+":")
+ res, err := m.dmIN.DataManager().GetResourceProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if m.dryRun || res == nil {
+ continue
+ }
+ for i, fl := range res.FilterIDs {
+ res.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetResourceProfile(res, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateStatQueueProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.StatQueueProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.StatQueueProfilePrefix+tenant+":")
+ sgs, err := m.dmIN.DataManager().GetStatQueueProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if sgs == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range sgs.FilterIDs {
+ sgs.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err = m.dmOut.DataManager().SetStatQueueProfile(sgs, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateThresholdsProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ThresholdProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.ThresholdProfilePrefix+tenant+":")
+ ths, err := m.dmIN.DataManager().GetThresholdProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if ths == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range ths.FilterIDs {
+ ths.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetThresholdProfile(ths, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateSupplierProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.SupplierProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.SupplierProfilePrefix)
+ splp, err := m.dmIN.DataManager().GetSupplierProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if splp == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range splp.FilterIDs {
+ splp.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetSupplierProfile(splp, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateAttributeProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.AttributeProfilePrefix+tenant+":")
+ attrPrf, err := m.dmIN.DataManager().GetAttributeProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if attrPrf == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range attrPrf.FilterIDs {
+ attrPrf.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateChargerProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ChargerProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.ChargerProfilePrefix+tenant+":")
+ cpp, err := m.dmIN.DataManager().GetChargerProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if cpp == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range cpp.FilterIDs {
+ cpp.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetChargerProfile(cpp, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
+ }
+ return
+}
+
+func (m *Migrator) migrateDispatcherProfileFiltersV1() (err error) {
+ var ids []string
+ tenant := config.CgrConfig().GeneralCfg().DefaultTenant
+ ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.DispatcherProfilePrefix)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ idg := strings.TrimPrefix(id, utils.DispatcherProfilePrefix+tenant+":")
+ dpp, err := m.dmIN.DataManager().GetDispatcherProfile(tenant, idg, false, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ if dpp == nil || m.dryRun {
+ continue
+ }
+ for i, fl := range dpp.FilterIDs {
+ dpp.FilterIDs[i] = migrateInlineFilter(fl)
+ }
+ if err := m.dmOut.DataManager().SetDispatcherProfile(dpp, true); err != nil {
+ return err
+ }
+ m.stats[utils.RQF] += 1
}
return
}
diff --git a/migrator/filters_it_test.go b/migrator/filters_it_test.go
new file mode 100644
index 000000000..03716b2ee
--- /dev/null
+++ b/migrator/filters_it_test.go
@@ -0,0 +1,275 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see
+*/
+
+package migrator
+
+import (
+ "log"
+ "path"
+ "reflect"
+ "testing"
+
+ "github.com/cgrates/cgrates/config"
+ "github.com/cgrates/cgrates/engine"
+ "github.com/cgrates/cgrates/utils"
+)
+
+var (
+ fltrCfgIn *config.CGRConfig
+ fltrCfgOut *config.CGRConfig
+ fltrMigrator *Migrator
+ fltrAction string
+)
+
+var sTestsFltrIT = []func(t *testing.T){
+ testFltrITConnect,
+ testFltrITFlush,
+ testFltrITMigrateAndMove,
+}
+
+func TestFiltersMigrateITRedis(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+ testFltrStart("TestFiltersMigrateITRedis", inPath, inPath, utils.Migrate, t)
+}
+
+func TestFiltersMigrateITMongo(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+ testFltrStart("TestFiltersMigrateITMongo", inPath, inPath, utils.Migrate, t)
+}
+
+func TestFiltersITMove(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+ outPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+ testFltrStart("TestFiltersITMove", inPath, outPath, utils.Move, t)
+}
+
+func TestFiltersITMigrateMongo2Redis(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+ outPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+ testFltrStart("TestFiltersITMigrateMongo2Redis", inPath, outPath, utils.Migrate, t)
+}
+
+func TestFiltersITMoveEncoding(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+ outPath := path.Join(*dataDir, "conf", "samples", "tutmongojson")
+ testFltrStart("TestFiltersITMoveEncoding", inPath, outPath, utils.Move, t)
+}
+
+func TestFiltersITMoveEncoding2(t *testing.T) {
+ inPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+ outPath := path.Join(*dataDir, "conf", "samples", "tutmysqljson")
+ testFltrStart("TestFiltersITMoveEncoding2", inPath, outPath, utils.Move, t)
+}
+
+func testFltrStart(testName, inPath, outPath, action string, t *testing.T) {
+ var err error
+ fltrAction = action
+ if fltrCfgIn, err = config.NewCGRConfigFromFolder(inPath); err != nil {
+ t.Fatal(err)
+ }
+ if fltrCfgOut, err = config.NewCGRConfigFromFolder(outPath); err != nil {
+ t.Fatal(err)
+ }
+ for _, stest := range sTestsFltrIT {
+ t.Run(testName, stest)
+ }
+}
+
+func testFltrITConnect(t *testing.T) {
+ dataDBIn, err := NewMigratorDataDB(fltrCfgIn.DataDbCfg().DataDbType,
+ fltrCfgIn.DataDbCfg().DataDbHost, fltrCfgIn.DataDbCfg().DataDbPort,
+ fltrCfgIn.DataDbCfg().DataDbName, fltrCfgIn.DataDbCfg().DataDbUser,
+ fltrCfgIn.DataDbCfg().DataDbPass, fltrCfgIn.GeneralCfg().DBDataEncoding,
+ config.CgrConfig().CacheCfg(), "")
+ if err != nil {
+ log.Fatal(err)
+ }
+ dataDBOut, err := NewMigratorDataDB(fltrCfgOut.DataDbCfg().DataDbType,
+ fltrCfgOut.DataDbCfg().DataDbHost, fltrCfgOut.DataDbCfg().DataDbPort,
+ fltrCfgOut.DataDbCfg().DataDbName, fltrCfgOut.DataDbCfg().DataDbUser,
+ fltrCfgOut.DataDbCfg().DataDbPass, fltrCfgOut.GeneralCfg().DBDataEncoding,
+ config.CgrConfig().CacheCfg(), "")
+ if err != nil {
+ log.Fatal(err)
+ }
+ fltrMigrator, err = NewMigrator(dataDBIn, dataDBOut,
+ nil, nil, false, false, false)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func testFltrITFlush(t *testing.T) {
+ fltrMigrator.dmOut.DataManager().DataDB().Flush("")
+ if err := engine.SetDBVersions(fltrMigrator.dmOut.DataManager().DataDB()); err != nil {
+ t.Error("Error ", err.Error())
+ }
+ fltrMigrator.dmIN.DataManager().DataDB().Flush("")
+ if err := engine.SetDBVersions(fltrMigrator.dmIN.DataManager().DataDB()); err != nil {
+ t.Error("Error ", err.Error())
+ }
+}
+
+func testFltrITMigrateAndMove(t *testing.T) {
+ Filters := &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_2",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaPrefix,
+ FieldName: "Account",
+ Values: []string{"1001"},
+ },
+ },
+ }
+ expFilters := &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_2",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaPrefix,
+ FieldName: "~Account",
+ Values: []string{"1001"},
+ },
+ },
+ }
+ expFilters.Compile()
+
+ attrProf := &engine.AttributeProfile{
+ Tenant: "cgrates.org",
+ ID: "ATTR_1",
+ Contexts: []string{utils.META_ANY},
+ FilterIDs: []string{"*string:Account:1001", "FLTR_2"},
+ ActivationInterval: nil,
+ Attributes: []*engine.Attribute{
+ {
+ FieldName: "Account",
+ Initial: "1001",
+ Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP),
+ Append: true,
+ },
+ },
+ Weight: 10,
+ }
+ expAttrProf := &engine.AttributeProfile{
+ Tenant: "cgrates.org",
+ ID: "ATTR_1",
+ Contexts: []string{utils.META_ANY},
+ FilterIDs: []string{"*string:~Account:1001", "FLTR_2"},
+ ActivationInterval: nil,
+ Attributes: []*engine.Attribute{
+ {
+ FieldName: "Account",
+ Initial: "1001",
+ Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP),
+ Append: true,
+ },
+ },
+ Weight: 10,
+ }
+ expAttrProf.Compile()
+ attrProf.Compile()
+ switch fltrAction {
+ case utils.Migrate:
+ if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil {
+ t.Error("Error when setting v1 Filters ", err.Error())
+ }
+ if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil {
+ t.Error("Error when setting attribute profile for v1 Filters ", err.Error())
+ }
+ currentVersion := engine.Versions{utils.RQF: 1}
+ err := fltrMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false)
+ if err != nil {
+ t.Error("Error when setting version for Filters ", err.Error())
+ }
+ //check if version was set correctly
+ if vrs, err := fltrMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil {
+ t.Error(err)
+ } else if vrs[utils.RQF] != 1 {
+ t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
+ }
+ //migrate Filters
+ err, _ = fltrMigrator.Migrate([]string{utils.MetaReqFilters})
+ if err != nil {
+ t.Error("Error when migrating Filters ", err.Error())
+ }
+ //check if version was updated
+ if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
+ t.Error(err)
+ } else if vrs[utils.RQF] != 2 {
+ t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
+ }
+ //check if Filters was migrate correctly
+ result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
+ if err != nil {
+ t.Fatalf("Error when getting Attributes %v", err.Error())
+ }
+ result.Compile()
+ if !reflect.DeepEqual(*expFilters, *result) {
+ t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result))
+ }
+
+ resultattr, err := fltrMigrator.dmOut.DataManager().DataDB().GetAttributeProfileDrv(attrProf.Tenant, attrProf.ID)
+ if err != nil {
+ t.Fatalf("Error when getting Attributes %v", err.Error())
+ }
+ resultattr.Compile()
+ if !reflect.DeepEqual(*expAttrProf, *resultattr) {
+ t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultattr))
+ }
+ expFltrIdx := map[string]utils.StringMap{
+ "*prefix:~Account:1001": utils.StringMap{"ATTR_1": true},
+ "*string:~Account:1001": utils.StringMap{"ATTR_1": true}}
+
+ if fltridx, err := fltrMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), utils.MetaString, nil); err != nil {
+ t.Error(err)
+ } else if !reflect.DeepEqual(expFltrIdx, fltridx) {
+ t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx))
+ }
+ case utils.Move:
+ if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil {
+ t.Error(err)
+ }
+ currentVersion := engine.CurrentDataDBVersions()
+ err := fltrMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false)
+ if err != nil {
+ t.Error("Error when setting version for Filters ", err.Error())
+ }
+ //migrate accounts
+ err, _ = fltrMigrator.Migrate([]string{utils.MetaReqFilters})
+ if err != nil {
+ t.Error("Error when fltrMigratorrating Filters ", err.Error())
+ }
+ //check if account was migrate correctly
+ result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
+ if err != nil {
+ t.Error(err)
+ }
+ result.Compile()
+ if !reflect.DeepEqual(Filters, result) {
+ t.Errorf("Expecting: %+v, received: %+v", Filters, result)
+ }
+ // check if old account was deleted
+ result, err = fltrMigrator.dmIN.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional)
+ if err != utils.ErrNotFound {
+ t.Error(err)
+ }
+ }
+}
diff --git a/migrator/filters_test.go b/migrator/filters_test.go
new file mode 100644
index 000000000..8c12e6885
--- /dev/null
+++ b/migrator/filters_test.go
@@ -0,0 +1,116 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see
+*/
+
+package migrator
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/cgrates/cgrates/engine"
+ "github.com/cgrates/cgrates/utils"
+)
+
+func TestFiltersInlineMigrate(t *testing.T) {
+ data := []struct{ in, exp string }{
+ {
+ in: "*string:Account:1002",
+ exp: "*string:~Account:1002",
+ },
+ {
+ in: "*string:~Account:1002",
+ exp: "*string:~Account:1002",
+ },
+ {
+ in: "FLTR_1",
+ exp: "FLTR_1",
+ },
+ {
+ in: "",
+ exp: "",
+ },
+ {
+ in: "*rsr::~Tenant(~^cgr.*\\.org$)",
+ exp: "*rsr::~Tenant(~^cgr.*\\.org$)",
+ },
+ }
+ for _, m := range data {
+ if rply := migrateInlineFilter(m.in); rply != m.exp {
+ t.Errorf("Expected: %s, recived: %s", m.exp, rply)
+ }
+ }
+
+}
+
+func TestFiltersMigrate(t *testing.T) {
+ data := []struct{ in, exp *engine.Filter }{
+ {
+ in: &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_1",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaString,
+ FieldName: "Account",
+ Values: []string{},
+ },
+ },
+ },
+ exp: &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_1",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaString,
+ FieldName: "~Account",
+ Values: []string{},
+ },
+ },
+ },
+ },
+ {
+ in: &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_2",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaPrefix,
+ FieldName: "~Account",
+ Values: []string{},
+ },
+ },
+ },
+ exp: &engine.Filter{
+ Tenant: "cgrates.org",
+ ID: "FLTR_2",
+ Rules: []*engine.FilterRule{
+ &engine.FilterRule{
+ Type: engine.MetaPrefix,
+ FieldName: "~Account",
+ Values: []string{},
+ },
+ },
+ },
+ },
+ }
+ for _, m := range data {
+ if rply := migrateFilterV1(m.in); !reflect.DeepEqual(rply, m.exp) {
+ t.Errorf("Expected: %s, recived: %s", utils.ToJSON(m.exp), utils.ToJSON(rply))
+ }
+ }
+}
diff --git a/migrator/thresholds.go b/migrator/thresholds.go
index 41aef08f3..3135b11a6 100644
--- a/migrator/thresholds.go
+++ b/migrator/thresholds.go
@@ -190,14 +190,9 @@ func (m *Migrator) migrateThresholds() (err error) {
if m.sameDataDB {
return
}
- if err := m.migrateCurrentThresholds(); err != nil {
- return err
- }
- return
-
+ return m.migrateCurrentThresholds()
case 1:
return m.migrateV2ActionTriggers()
-
case 2:
return m.migrateV2Thresholds()
}
diff --git a/migrator/user.go b/migrator/user.go
index c4ee51766..a73bc8044 100644
--- a/migrator/user.go
+++ b/migrator/user.go
@@ -72,7 +72,7 @@ func userProfile2attributeProfile(user *v1UserProfile) (attr *engine.AttributePr
fieldName = utils.RequestType
}
if utils.IsSliceMember(usrFltr, fieldName) {
- attr.FilterIDs = append(attr.FilterIDs, fmt.Sprintf("*string:%s:%s", fieldName, substitute))
+ attr.FilterIDs = append(attr.FilterIDs, fmt.Sprintf("*string:~%s:%s", fieldName, substitute))
continue
}
attr.Attributes = append(attr.Attributes, &engine.Attribute{
diff --git a/migrator/user_it_test.go b/migrator/user_it_test.go
index a97cf9072..a83ce08df 100644
--- a/migrator/user_it_test.go
+++ b/migrator/user_it_test.go
@@ -145,7 +145,7 @@ func testUsrITMigrateAndMove(t *testing.T) {
Tenant: defaultTenant,
ID: "1001",
Contexts: []string{utils.META_ANY},
- FilterIDs: []string{"*string:Account:1002"},
+ FilterIDs: []string{"*string:~Account:1002"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{
@@ -211,7 +211,7 @@ func testUsrITMigrateAndMove(t *testing.T) {
}
expUsrIdx := map[string]utils.StringMap{
- "*string:Account:1002": utils.StringMap{
+ "*string:~Account:1002": utils.StringMap{
"1001": true,
},
}
diff --git a/migrator/user_test.go b/migrator/user_test.go
index b9f94c778..d0afe6c18 100644
--- a/migrator/user_test.go
+++ b/migrator/user_test.go
@@ -88,7 +88,7 @@ func TestUserProfile2attributeProfile(t *testing.T) {
Tenant: defaultTenant,
ID: "1001",
Contexts: []string{utils.META_ANY},
- FilterIDs: []string{"*string:Account:1002"},
+ FilterIDs: []string{"*string:~Account:1002"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{
@@ -107,7 +107,7 @@ func TestUserProfile2attributeProfile(t *testing.T) {
Tenant: defaultTenant,
ID: "1001",
Contexts: []string{utils.META_ANY},
- FilterIDs: []string{"*string:Account:1002"},
+ FilterIDs: []string{"*string:~Account:1002"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{
@@ -126,7 +126,7 @@ func TestUserProfile2attributeProfile(t *testing.T) {
Tenant: defaultTenant,
ID: "1001",
Contexts: []string{utils.META_ANY},
- FilterIDs: []string{"*string:Account:1002"},
+ FilterIDs: []string{"*string:~Account:1002"},
ActivationInterval: nil,
Attributes: []*engine.Attribute{
{