From 89824ae37fb83ea172e5cfbc5d1d83df12331903 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 5 Mar 2019 13:52:27 +0200 Subject: [PATCH] Updated FilterS version --- engine/version.go | 2 +- migrator/alias.go | 8 +- migrator/alias_it_test.go | 10 +- migrator/alias_test.go | 12 +- migrator/derived_chargers.go | 8 +- migrator/derived_chargers_it_test.go | 12 +- migrator/derived_chargers_test.go | 12 +- migrator/filters.go | 294 ++++++++++++++++++++++++++- migrator/filters_it_test.go | 275 +++++++++++++++++++++++++ migrator/filters_test.go | 116 +++++++++++ migrator/thresholds.go | 7 +- migrator/user.go | 2 +- migrator/user_it_test.go | 4 +- migrator/user_test.go | 6 +- 14 files changed, 713 insertions(+), 55 deletions(-) create mode 100644 migrator/filters_it_test.go create mode 100644 migrator/filters_test.go diff --git a/engine/version.go b/engine/version.go index 153ab9b4e..9824361e2 100644 --- a/engine/version.go +++ b/engine/version.go @@ -146,7 +146,7 @@ func CurrentDataDBVersions() Versions { utils.Suppliers: 1, utils.Attributes: 3, utils.Timing: 1, - utils.RQF: 1, + utils.RQF: 2, utils.Resource: 1, utils.Subscribers: 1, utils.Destinations: 1, diff --git a/migrator/alias.go b/migrator/alias.go index 4a9352f85..74533b7ec 100644 --- a/migrator/alias.go +++ b/migrator/alias.go @@ -87,13 +87,13 @@ func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.Attri out.Tenant = defaultTenant } if len(alias.Category) != 0 && alias.Category != utils.META_ANY { - out.FilterIDs = append(out.FilterIDs, "*string:Category:"+alias.Category) + out.FilterIDs = append(out.FilterIDs, "*string:~Category:"+alias.Category) } if len(alias.Account) != 0 && alias.Account != utils.META_ANY { - out.FilterIDs = append(out.FilterIDs, "*string:Account:"+alias.Account) + out.FilterIDs = append(out.FilterIDs, "*string:~Account:"+alias.Account) } if len(alias.Subject) != 0 && alias.Subject != utils.META_ANY { - out.FilterIDs = append(out.FilterIDs, "*string:Subject:"+alias.Subject) + out.FilterIDs = append(out.FilterIDs, "*string:~Subject:"+alias.Subject) } var destination string for _, av := range alias.Values { @@ -115,7 +115,7 @@ func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.Attri } } if len(destination) != 0 && destination != utils.META_ANY { - out.FilterIDs = append(out.FilterIDs, "*destination:Destination:"+destination) + out.FilterIDs = append(out.FilterIDs, "*destination:~Destination:"+destination) } return out } diff --git a/migrator/alias_it_test.go b/migrator/alias_it_test.go index b67bca207..abc09761d 100644 --- a/migrator/alias_it_test.go +++ b/migrator/alias_it_test.go @@ -156,9 +156,9 @@ func testAlsITMigrateAndMove(t *testing.T) { ID: alias.GetId(), Contexts: []string{utils.META_ANY}, FilterIDs: []string{ - "*string:Account:1001", - "*string:Subject:call_1001", - "*destination:Destination:DST_1003", + "*string:~Account:1001", + "*string:~Subject:call_1001", + "*destination:~Destination:DST_1003", }, ActivationInterval: nil, Attributes: []*engine.Attribute{ @@ -226,10 +226,10 @@ func testAlsITMigrateAndMove(t *testing.T) { } expAlsIdx := map[string]utils.StringMap{ - "*string:Account:1001": utils.StringMap{ + "*string:~Account:1001": utils.StringMap{ "*out:*any:*any:1001:call_1001:*rated": true, }, - "*string:Subject:call_1001": utils.StringMap{ + "*string:~Subject:call_1001": utils.StringMap{ "*out:*any:*any:1001:call_1001:*rated": true, }, } diff --git a/migrator/alias_test.go b/migrator/alias_test.go index ab3243c45..b5a28ee93 100644 --- a/migrator/alias_test.go +++ b/migrator/alias_test.go @@ -219,9 +219,9 @@ func TestAlias2AtttributeProfile(t *testing.T) { ID: aliases[4].GetId(), Contexts: []string{utils.META_ANY}, FilterIDs: []string{ - "*string:Category:*voice", - "*string:Account:1001", - "*destination:Destination:DST_1003", + "*string:~Category:*voice", + "*string:~Account:1001", + "*destination:~Destination:DST_1003", }, ActivationInterval: nil, Attributes: []*engine.Attribute{ @@ -244,9 +244,9 @@ func TestAlias2AtttributeProfile(t *testing.T) { ID: aliases[5].GetId(), Contexts: []string{utils.META_ANY}, FilterIDs: []string{ - "*string:Account:1001", - "*string:Subject:call_1001", - "*destination:Destination:DST_1003", + "*string:~Account:1001", + "*string:~Subject:call_1001", + "*destination:~Destination:DST_1003", }, ActivationInterval: nil, Attributes: []*engine.Attribute{ diff --git a/migrator/derived_chargers.go b/migrator/derived_chargers.go index fa12b9384..64a97b8cf 100644 --- a/migrator/derived_chargers.go +++ b/migrator/derived_chargers.go @@ -143,7 +143,7 @@ func (m *Migrator) derivedChargers2Chargers(dck *v1DerivedChargersWithKey) (err skey := utils.SplitConcatenatedKey(dck.Key) destination := "" if len(dck.Value.DestinationIDs) != 0 { - destination = "*destination:Destination:" + destination = "*destination:~Destination:" keys := dcGetMapKeys(dck.Value.DestinationIDs) destination += strings.Join(keys, utils.INFIELD_SEP) } @@ -153,13 +153,13 @@ func (m *Migrator) derivedChargers2Chargers(dck *v1DerivedChargersWithKey) (err filter = append(filter, destination) } if len(skey[2]) != 0 && skey[2] != utils.META_ANY { - filter = append(filter, "*string:Category:"+skey[2]) + filter = append(filter, "*string:~Category:"+skey[2]) } if len(skey[3]) != 0 && skey[3] != utils.META_ANY { - filter = append(filter, "*string:Account:"+skey[3]) + filter = append(filter, "*string:~Account:"+skey[3]) } if len(skey[4]) != 0 && skey[4] != utils.META_ANY { - filter = append(filter, "*string:Subject:"+skey[4]) + filter = append(filter, "*string:~Subject:"+skey[4]) } for i, dc := range dck.Value.Chargers { diff --git a/migrator/derived_chargers_it_test.go b/migrator/derived_chargers_it_test.go index 4fc80a871..293fa52ea 100644 --- a/migrator/derived_chargers_it_test.go +++ b/migrator/derived_chargers_it_test.go @@ -162,8 +162,8 @@ func testDCITMigrateAndMove(t *testing.T) { ID: fmt.Sprintf("%s_%v", derivch.Key, 0), Contexts: []string{utils.META_ANY}, FilterIDs: []string{ - "*destination:Destination:1001;1002;1003", - "*string:Account:1003", + "*destination:~Destination:1001;1002;1003", + "*string:~Account:1003", }, ActivationInterval: nil, Attributes: []*engine.Attribute{ @@ -184,8 +184,8 @@ func testDCITMigrateAndMove(t *testing.T) { Tenant: defaultTenant, ID: fmt.Sprintf("%s_%v", derivch.Key, 0), FilterIDs: []string{ - "*destination:Destination:1001;1002;1003", - "*string:Account:1003", + "*destination:~Destination:1001;1002;1003", + "*string:~Account:1003", "*rsr::~filterhdr1:s/(.+)/special_run3/", }, ActivationInterval: nil, @@ -246,7 +246,7 @@ func testDCITMigrateAndMove(t *testing.T) { t.Error("Error should be not found : ", err) } expDcIdx := map[string]utils.StringMap{ - "*string:Account:1003": utils.StringMap{ + "*string:~Account:1003": utils.StringMap{ "*out:cgrates.org:*any:1003:*any_0": true, }, } @@ -257,7 +257,7 @@ func testDCITMigrateAndMove(t *testing.T) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expDcIdx), utils.ToJSON(dcidx)) } expDcIdx = map[string]utils.StringMap{ - "*string:Account:1003": utils.StringMap{ + "*string:~Account:1003": utils.StringMap{ "*out:cgrates.org:*any:1003:*any_0": true, }, } diff --git a/migrator/derived_chargers_test.go b/migrator/derived_chargers_test.go index 94e864e4d..ccf90dec7 100644 --- a/migrator/derived_chargers_test.go +++ b/migrator/derived_chargers_test.go @@ -145,12 +145,12 @@ func TestDerivedChargers2AttributeProfile(t *testing.T) { }, Tenant: defaultTenant, Key: "key1", - Filters: []string{"*string:Subject:1005"}, + Filters: []string{"*string:~Subject:1005"}, Expected: &engine.AttributeProfile{ Tenant: defaultTenant, ID: "key1", Contexts: []string{utils.META_ANY}, - FilterIDs: []string{"*string:Subject:1005"}, + FilterIDs: []string{"*string:~Subject:1005"}, ActivationInterval: nil, Attributes: []*engine.Attribute{ &engine.Attribute{ @@ -202,15 +202,15 @@ func TestDerivedChargers2Charger(t *testing.T) { Tenant: defaultTenant, Key: "key2", Filters: []string{ - "*string:Category:*voice1", - "*string:Account:1001", + "*string:~Category:*voice1", + "*string:~Account:1001", }, Expected: &engine.ChargerProfile{ Tenant: defaultTenant, ID: "key2", FilterIDs: []string{ - "*string:Category:*voice1", - "*string:Account:1001", + "*string:~Category:*voice1", + "*string:~Account:1001", "*rsr::~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)", }, ActivationInterval: nil, diff --git a/migrator/filters.go b/migrator/filters.go index b07eaac43..b73933a8d 100644 --- a/migrator/filters.go +++ b/migrator/filters.go @@ -40,14 +40,98 @@ func (m *Migrator) migrateCurrentRequestFilter() (err error) { if err != nil { return err } - if fl != nil { - if m.dryRun != true { - if err := m.dmOut.DataManager().SetFilter(fl); err != nil { - return err - } - m.stats[utils.RQF] += 1 - } + if m.dryRun || fl == nil { + continue } + if err := m.dmIN.DataManager().RemoveFilter(tenant, idg, utils.NonTransactional); err != nil { + return err + } + if err := m.dmOut.DataManager().SetFilter(fl); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func migrateFilterV1(fl *engine.Filter) *engine.Filter { + for i, rule := range fl.Rules { + if rule.FieldName == "" || + utils.IsSliceMember([]string{engine.MetaRSR, engine.MetaStatS, engine.MetaResources, + engine.MetaNotRSR, engine.MetaNotStatS, engine.MetaNotResources}, rule.Type) || + strings.HasPrefix(rule.FieldName, utils.DynamicDataPrefix) { + continue + } + fl.Rules[i].FieldName = utils.DynamicDataPrefix + rule.FieldName + } + return fl +} + +func migrateInlineFilter(fl string) string { + if fl == "" || !strings.HasPrefix(fl, utils.Meta) { + return fl + } + ruleSplt := strings.Split(fl, utils.InInFieldSep) + if len(ruleSplt) < 3 { + return fl + } + + if utils.IsSliceMember([]string{engine.MetaRSR, engine.MetaStatS, engine.MetaResources, + engine.MetaNotRSR, engine.MetaNotStatS, engine.MetaNotResources}, ruleSplt[0]) || + strings.HasPrefix(ruleSplt[1], utils.DynamicDataPrefix) { + return fl + } + return fmt.Sprintf("%s:~%s:%s", ruleSplt[0], ruleSplt[1], strings.Join(ruleSplt[2:], utils.InInFieldSep)) +} + +func (m *Migrator) migrateRequestFilterV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.FilterPrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":") + fl, err := m.dmIN.DataManager().GetFilter(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if m.dryRun || fl == nil { + continue + } + if err := m.dmOut.DataManager().SetFilter(migrateFilterV1(fl)); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + if err = m.migrateResourceProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateStatQueueProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateThresholdsProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateSupplierProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateAttributeProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateChargerProfileFiltersV1(); err != nil { + return err + } + if err = m.migrateDispatcherProfileFiltersV1(); err != nil { + return err + } + vrs := engine.Versions{utils.RQF: engine.CurrentDataDBVersions()[utils.RQF]} + if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating Filters version into dataDB", err.Error())) } return } @@ -68,14 +152,202 @@ func (m *Migrator) migrateRequestFilter() (err error) { "version number is not defined for ActionTriggers model") } switch vrs[utils.RQF] { + case 1: + return m.migrateRequestFilterV1() case current[utils.RQF]: if m.sameDataDB { return } - if err := m.migrateCurrentRequestFilter(); err != nil { - return err - } - return + return m.migrateCurrentRequestFilter() + } + return +} + +func (m *Migrator) migrateResourceProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.ResourceProfilesPrefix+tenant+":") + res, err := m.dmIN.DataManager().GetResourceProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if m.dryRun || res == nil { + continue + } + for i, fl := range res.FilterIDs { + res.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetResourceProfile(res, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateStatQueueProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.StatQueueProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.StatQueueProfilePrefix+tenant+":") + sgs, err := m.dmIN.DataManager().GetStatQueueProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if sgs == nil || m.dryRun { + continue + } + for i, fl := range sgs.FilterIDs { + sgs.FilterIDs[i] = migrateInlineFilter(fl) + } + if err = m.dmOut.DataManager().SetStatQueueProfile(sgs, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateThresholdsProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ThresholdProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.ThresholdProfilePrefix+tenant+":") + ths, err := m.dmIN.DataManager().GetThresholdProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if ths == nil || m.dryRun { + continue + } + for i, fl := range ths.FilterIDs { + ths.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetThresholdProfile(ths, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateSupplierProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.SupplierProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.SupplierProfilePrefix) + splp, err := m.dmIN.DataManager().GetSupplierProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if splp == nil || m.dryRun { + continue + } + for i, fl := range splp.FilterIDs { + splp.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetSupplierProfile(splp, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateAttributeProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.AttributeProfilePrefix+tenant+":") + attrPrf, err := m.dmIN.DataManager().GetAttributeProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if attrPrf == nil || m.dryRun { + continue + } + for i, fl := range attrPrf.FilterIDs { + attrPrf.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateChargerProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ChargerProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.ChargerProfilePrefix+tenant+":") + cpp, err := m.dmIN.DataManager().GetChargerProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if cpp == nil || m.dryRun { + continue + } + for i, fl := range cpp.FilterIDs { + cpp.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetChargerProfile(cpp, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 + } + return +} + +func (m *Migrator) migrateDispatcherProfileFiltersV1() (err error) { + var ids []string + tenant := config.CgrConfig().GeneralCfg().DefaultTenant + ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.DispatcherProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.DispatcherProfilePrefix+tenant+":") + dpp, err := m.dmIN.DataManager().GetDispatcherProfile(tenant, idg, false, false, utils.NonTransactional) + if err != nil { + return err + } + if dpp == nil || m.dryRun { + continue + } + for i, fl := range dpp.FilterIDs { + dpp.FilterIDs[i] = migrateInlineFilter(fl) + } + if err := m.dmOut.DataManager().SetDispatcherProfile(dpp, true); err != nil { + return err + } + m.stats[utils.RQF] += 1 } return } diff --git a/migrator/filters_it_test.go b/migrator/filters_it_test.go new file mode 100644 index 000000000..03716b2ee --- /dev/null +++ b/migrator/filters_it_test.go @@ -0,0 +1,275 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "log" + "path" + "reflect" + "testing" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + fltrCfgIn *config.CGRConfig + fltrCfgOut *config.CGRConfig + fltrMigrator *Migrator + fltrAction string +) + +var sTestsFltrIT = []func(t *testing.T){ + testFltrITConnect, + testFltrITFlush, + testFltrITMigrateAndMove, +} + +func TestFiltersMigrateITRedis(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testFltrStart("TestFiltersMigrateITRedis", inPath, inPath, utils.Migrate, t) +} + +func TestFiltersMigrateITMongo(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + testFltrStart("TestFiltersMigrateITMongo", inPath, inPath, utils.Migrate, t) +} + +func TestFiltersITMove(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testFltrStart("TestFiltersITMove", inPath, outPath, utils.Move, t) +} + +func TestFiltersITMigrateMongo2Redis(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testFltrStart("TestFiltersITMigrateMongo2Redis", inPath, outPath, utils.Migrate, t) +} + +func TestFiltersITMoveEncoding(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmongojson") + testFltrStart("TestFiltersITMoveEncoding", inPath, outPath, utils.Move, t) +} + +func TestFiltersITMoveEncoding2(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysqljson") + testFltrStart("TestFiltersITMoveEncoding2", inPath, outPath, utils.Move, t) +} + +func testFltrStart(testName, inPath, outPath, action string, t *testing.T) { + var err error + fltrAction = action + if fltrCfgIn, err = config.NewCGRConfigFromFolder(inPath); err != nil { + t.Fatal(err) + } + if fltrCfgOut, err = config.NewCGRConfigFromFolder(outPath); err != nil { + t.Fatal(err) + } + for _, stest := range sTestsFltrIT { + t.Run(testName, stest) + } +} + +func testFltrITConnect(t *testing.T) { + dataDBIn, err := NewMigratorDataDB(fltrCfgIn.DataDbCfg().DataDbType, + fltrCfgIn.DataDbCfg().DataDbHost, fltrCfgIn.DataDbCfg().DataDbPort, + fltrCfgIn.DataDbCfg().DataDbName, fltrCfgIn.DataDbCfg().DataDbUser, + fltrCfgIn.DataDbCfg().DataDbPass, fltrCfgIn.GeneralCfg().DBDataEncoding, + config.CgrConfig().CacheCfg(), "") + if err != nil { + log.Fatal(err) + } + dataDBOut, err := NewMigratorDataDB(fltrCfgOut.DataDbCfg().DataDbType, + fltrCfgOut.DataDbCfg().DataDbHost, fltrCfgOut.DataDbCfg().DataDbPort, + fltrCfgOut.DataDbCfg().DataDbName, fltrCfgOut.DataDbCfg().DataDbUser, + fltrCfgOut.DataDbCfg().DataDbPass, fltrCfgOut.GeneralCfg().DBDataEncoding, + config.CgrConfig().CacheCfg(), "") + if err != nil { + log.Fatal(err) + } + fltrMigrator, err = NewMigrator(dataDBIn, dataDBOut, + nil, nil, false, false, false) + if err != nil { + log.Fatal(err) + } +} + +func testFltrITFlush(t *testing.T) { + fltrMigrator.dmOut.DataManager().DataDB().Flush("") + if err := engine.SetDBVersions(fltrMigrator.dmOut.DataManager().DataDB()); err != nil { + t.Error("Error ", err.Error()) + } + fltrMigrator.dmIN.DataManager().DataDB().Flush("") + if err := engine.SetDBVersions(fltrMigrator.dmIN.DataManager().DataDB()); err != nil { + t.Error("Error ", err.Error()) + } +} + +func testFltrITMigrateAndMove(t *testing.T) { + Filters := &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaPrefix, + FieldName: "Account", + Values: []string{"1001"}, + }, + }, + } + expFilters := &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaPrefix, + FieldName: "~Account", + Values: []string{"1001"}, + }, + }, + } + expFilters.Compile() + + attrProf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_1", + Contexts: []string{utils.META_ANY}, + FilterIDs: []string{"*string:Account:1001", "FLTR_2"}, + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Weight: 10, + } + expAttrProf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_1", + Contexts: []string{utils.META_ANY}, + FilterIDs: []string{"*string:~Account:1001", "FLTR_2"}, + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Weight: 10, + } + expAttrProf.Compile() + attrProf.Compile() + switch fltrAction { + case utils.Migrate: + if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil { + t.Error("Error when setting v1 Filters ", err.Error()) + } + if err := fltrMigrator.dmIN.DataManager().SetAttributeProfile(attrProf, false); err != nil { + t.Error("Error when setting attribute profile for v1 Filters ", err.Error()) + } + currentVersion := engine.Versions{utils.RQF: 1} + err := fltrMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for Filters ", err.Error()) + } + //check if version was set correctly + if vrs, err := fltrMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.RQF] != 1 { + t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) + } + //migrate Filters + err, _ = fltrMigrator.Migrate([]string{utils.MetaReqFilters}) + if err != nil { + t.Error("Error when migrating Filters ", err.Error()) + } + //check if version was updated + if vrs, err := fltrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.RQF] != 2 { + t.Errorf("Unexpected version returned: %d", vrs[utils.RQF]) + } + //check if Filters was migrate correctly + result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + if err != nil { + t.Fatalf("Error when getting Attributes %v", err.Error()) + } + result.Compile() + if !reflect.DeepEqual(*expFilters, *result) { + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expFilters), utils.ToJSON(result)) + } + + resultattr, err := fltrMigrator.dmOut.DataManager().DataDB().GetAttributeProfileDrv(attrProf.Tenant, attrProf.ID) + if err != nil { + t.Fatalf("Error when getting Attributes %v", err.Error()) + } + resultattr.Compile() + if !reflect.DeepEqual(*expAttrProf, *resultattr) { + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultattr)) + } + expFltrIdx := map[string]utils.StringMap{ + "*prefix:~Account:1001": utils.StringMap{"ATTR_1": true}, + "*string:~Account:1001": utils.StringMap{"ATTR_1": true}} + + if fltridx, err := fltrMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), utils.MetaString, nil); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expFltrIdx, fltridx) { + t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) + } + case utils.Move: + if err := fltrMigrator.dmIN.DataManager().SetFilter(Filters); err != nil { + t.Error(err) + } + currentVersion := engine.CurrentDataDBVersions() + err := fltrMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for Filters ", err.Error()) + } + //migrate accounts + err, _ = fltrMigrator.Migrate([]string{utils.MetaReqFilters}) + if err != nil { + t.Error("Error when fltrMigratorrating Filters ", err.Error()) + } + //check if account was migrate correctly + result, err := fltrMigrator.dmOut.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + if err != nil { + t.Error(err) + } + result.Compile() + if !reflect.DeepEqual(Filters, result) { + t.Errorf("Expecting: %+v, received: %+v", Filters, result) + } + // check if old account was deleted + result, err = fltrMigrator.dmIN.DataManager().GetFilter(Filters.Tenant, Filters.ID, false, false, utils.NonTransactional) + if err != utils.ErrNotFound { + t.Error(err) + } + } +} diff --git a/migrator/filters_test.go b/migrator/filters_test.go new file mode 100644 index 000000000..8c12e6885 --- /dev/null +++ b/migrator/filters_test.go @@ -0,0 +1,116 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "reflect" + "testing" + + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +func TestFiltersInlineMigrate(t *testing.T) { + data := []struct{ in, exp string }{ + { + in: "*string:Account:1002", + exp: "*string:~Account:1002", + }, + { + in: "*string:~Account:1002", + exp: "*string:~Account:1002", + }, + { + in: "FLTR_1", + exp: "FLTR_1", + }, + { + in: "", + exp: "", + }, + { + in: "*rsr::~Tenant(~^cgr.*\\.org$)", + exp: "*rsr::~Tenant(~^cgr.*\\.org$)", + }, + } + for _, m := range data { + if rply := migrateInlineFilter(m.in); rply != m.exp { + t.Errorf("Expected: %s, recived: %s", m.exp, rply) + } + } + +} + +func TestFiltersMigrate(t *testing.T) { + data := []struct{ in, exp *engine.Filter }{ + { + in: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_1", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaString, + FieldName: "Account", + Values: []string{}, + }, + }, + }, + exp: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_1", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaString, + FieldName: "~Account", + Values: []string{}, + }, + }, + }, + }, + { + in: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaPrefix, + FieldName: "~Account", + Values: []string{}, + }, + }, + }, + exp: &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_2", + Rules: []*engine.FilterRule{ + &engine.FilterRule{ + Type: engine.MetaPrefix, + FieldName: "~Account", + Values: []string{}, + }, + }, + }, + }, + } + for _, m := range data { + if rply := migrateFilterV1(m.in); !reflect.DeepEqual(rply, m.exp) { + t.Errorf("Expected: %s, recived: %s", utils.ToJSON(m.exp), utils.ToJSON(rply)) + } + } +} diff --git a/migrator/thresholds.go b/migrator/thresholds.go index 41aef08f3..3135b11a6 100644 --- a/migrator/thresholds.go +++ b/migrator/thresholds.go @@ -190,14 +190,9 @@ func (m *Migrator) migrateThresholds() (err error) { if m.sameDataDB { return } - if err := m.migrateCurrentThresholds(); err != nil { - return err - } - return - + return m.migrateCurrentThresholds() case 1: return m.migrateV2ActionTriggers() - case 2: return m.migrateV2Thresholds() } diff --git a/migrator/user.go b/migrator/user.go index c4ee51766..a73bc8044 100644 --- a/migrator/user.go +++ b/migrator/user.go @@ -72,7 +72,7 @@ func userProfile2attributeProfile(user *v1UserProfile) (attr *engine.AttributePr fieldName = utils.RequestType } if utils.IsSliceMember(usrFltr, fieldName) { - attr.FilterIDs = append(attr.FilterIDs, fmt.Sprintf("*string:%s:%s", fieldName, substitute)) + attr.FilterIDs = append(attr.FilterIDs, fmt.Sprintf("*string:~%s:%s", fieldName, substitute)) continue } attr.Attributes = append(attr.Attributes, &engine.Attribute{ diff --git a/migrator/user_it_test.go b/migrator/user_it_test.go index a97cf9072..a83ce08df 100644 --- a/migrator/user_it_test.go +++ b/migrator/user_it_test.go @@ -145,7 +145,7 @@ func testUsrITMigrateAndMove(t *testing.T) { Tenant: defaultTenant, ID: "1001", Contexts: []string{utils.META_ANY}, - FilterIDs: []string{"*string:Account:1002"}, + FilterIDs: []string{"*string:~Account:1002"}, ActivationInterval: nil, Attributes: []*engine.Attribute{ { @@ -211,7 +211,7 @@ func testUsrITMigrateAndMove(t *testing.T) { } expUsrIdx := map[string]utils.StringMap{ - "*string:Account:1002": utils.StringMap{ + "*string:~Account:1002": utils.StringMap{ "1001": true, }, } diff --git a/migrator/user_test.go b/migrator/user_test.go index b9f94c778..d0afe6c18 100644 --- a/migrator/user_test.go +++ b/migrator/user_test.go @@ -88,7 +88,7 @@ func TestUserProfile2attributeProfile(t *testing.T) { Tenant: defaultTenant, ID: "1001", Contexts: []string{utils.META_ANY}, - FilterIDs: []string{"*string:Account:1002"}, + FilterIDs: []string{"*string:~Account:1002"}, ActivationInterval: nil, Attributes: []*engine.Attribute{ { @@ -107,7 +107,7 @@ func TestUserProfile2attributeProfile(t *testing.T) { Tenant: defaultTenant, ID: "1001", Contexts: []string{utils.META_ANY}, - FilterIDs: []string{"*string:Account:1002"}, + FilterIDs: []string{"*string:~Account:1002"}, ActivationInterval: nil, Attributes: []*engine.Attribute{ { @@ -126,7 +126,7 @@ func TestUserProfile2attributeProfile(t *testing.T) { Tenant: defaultTenant, ID: "1001", Contexts: []string{utils.META_ANY}, - FilterIDs: []string{"*string:Account:1002"}, + FilterIDs: []string{"*string:~Account:1002"}, ActivationInterval: nil, Attributes: []*engine.Attribute{ {