From 6d2e28edec3e59124bef643d6133c1c7fc2f9c75 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 15 Jan 2019 16:01:16 +0200 Subject: [PATCH] Added integration test for alias migrating to attributes --- engine/version.go | 2 +- migrator/alias.go | 111 +++++++++--- migrator/alias_it_test.go | 260 +++++++++++++++++++++++++++ migrator/alias_test.go | 289 +++++++++++++++++++++++++++++++ migrator/migrator_datadb.go | 4 + migrator/storage_map_datadb.go | 16 ++ migrator/storage_mongo_datadb.go | 76 ++++++++ migrator/storage_redis.go | 88 ++++++++++ 8 files changed, 820 insertions(+), 26 deletions(-) create mode 100644 migrator/alias_it_test.go create mode 100644 migrator/alias_test.go diff --git a/engine/version.go b/engine/version.go index 7b2388909..9ca5cd346 100644 --- a/engine/version.go +++ b/engine/version.go @@ -149,7 +149,7 @@ func CurrentDataDBVersions() Versions { utils.RQF: 1, utils.Resource: 1, utils.ReverseAlias: 1, - utils.Alias: 1, + utils.Alias: 2, utils.User: 1, utils.Subscribers: 1, utils.DerivedChargersV: 1, diff --git a/migrator/alias.go b/migrator/alias.go index c2f267abc..da9d4c49f 100644 --- a/migrator/alias.go +++ b/migrator/alias.go @@ -28,27 +28,48 @@ import ( "github.com/cgrates/cgrates/utils" ) -type Alias struct { +type v1Alias struct { Direction string Tenant string Category string Account string Subject string Context string - Values AliasValues + Values v1AliasValues } -type AliasValues []*AliasValue -type AliasValue struct { +type v1AliasValues []*v1AliasValue + +type v1AliasValue struct { DestinationId string //=Destination - Pairs AliasPairs + Pairs v1AliasPairs Weight float64 } -type AliasPairs map[string]map[string]string -func alias2AtttributeProfile(alias Alias, defaultTenant string) engine.AttributeProfile { - out := engine.AttributeProfile{ +type v1AliasPairs map[string]map[string]string + +func (al *v1Alias) SetId(id string) error { + vals := strings.Split(id, utils.CONCATENATED_KEY_SEP) + if len(vals) != 6 { + return utils.ErrInvalidKey + } + al.Direction = vals[0] + al.Tenant = vals[1] + al.Category = vals[2] + al.Account = vals[3] + al.Subject = vals[4] + al.Context = vals[5] + return nil +} + +func (al *v1Alias) GetId() string { + return utils.ConcatenatedKey(al.Direction, al.Tenant, al.Category, al.Account, al.Subject, al.Context) +} + +func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.AttributeProfile { + out := &engine.AttributeProfile{ Tenant: alias.Tenant, + ID: alias.GetId(), Contexts: []string{"*any"}, FilterIDs: make([]string, 0), ActivationInterval: nil, @@ -57,8 +78,7 @@ func alias2AtttributeProfile(alias Alias, defaultTenant string) engine.Attribute Weight: 10, } if len(out.Tenant) == 0 || out.Tenant == utils.META_ANY { - // cfg, _ := config.NewDefaultCGRConfig() - out.Tenant = defaultTenant //cfg.GeneralCfg().DefaultTenant + out.Tenant = defaultTenant } if len(alias.Category) != 0 && alias.Category != utils.META_ANY { out.FilterIDs = append(out.FilterIDs, "*string:Category:"+alias.Category) @@ -71,7 +91,7 @@ func alias2AtttributeProfile(alias Alias, defaultTenant string) engine.Attribute } var destination string for _, av := range alias.Values { - if len(destination) == 0 || destination != utils.META_ANY { + if len(destination) == 0 || destination == utils.META_ANY { destination = av.DestinationId } for fieldname, vals := range av.Pairs { @@ -85,13 +105,56 @@ func alias2AtttributeProfile(alias Alias, defaultTenant string) engine.Attribute } } } - if len(destination) == 0 || destination != utils.META_ANY { + if len(destination) != 0 && destination != utils.META_ANY { out.FilterIDs = append(out.FilterIDs, "*string:Destination:"+destination) } return out } -func (m *Migrator) migrateCurrentAlias() (err error) { +func (m *Migrator) migrateAlias2Attributes() (err error) { + cfg, err := config.NewDefaultCGRConfig() + if err != nil { + return err + } + defaultTenant := cfg.GeneralCfg().DefaultTenant + for { + alias, err := m.dmIN.getV1Alias() + if err == utils.ErrNoMoreData { + break + } + if err != nil { + return err + } + if alias == nil || m.dryRun { + continue + } + attr := alias2AtttributeProfile(alias, defaultTenant) + if len(attr.Attributes) == 0 { + continue + } + if err := m.dmIN.remV1Alias(alias.GetId()); err != nil { + return err + } + if err := m.dmOut.DataManager().DataDB().SetAttributeProfileDrv(attr); err != nil { + return err + } + m.stats[utils.Alias] += 1 + } + if m.dryRun { + return + } + // All done, update version wtih current one + vrs := engine.Versions{utils.Alias: engine.CurrentDataDBVersions()[utils.Alias]} + if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating Alias version into dataDB", err.Error())) + } + return +} + +func (m *Migrator) migrateV1Alias() (err error) { var ids []string ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.ALIASES_PREFIX) if err != nil { @@ -103,14 +166,13 @@ func (m *Migrator) migrateCurrentAlias() (err error) { if err != nil { return err } - if usr != nil { - if m.dryRun != true { - if err := m.dmOut.DataManager().DataDB().SetAlias(usr, utils.NonTransactional); err != nil { - return err - } - m.stats[utils.Alias] += 1 - } + if usr == nil || m.dryRun { + continue } + if err := m.dmOut.DataManager().DataDB().SetAlias(usr, utils.NonTransactional); err != nil { + return err + } + m.stats[utils.Alias] += 1 } return } @@ -156,7 +218,7 @@ func (m *Migrator) migrateCurrentAlias() (err error) { func (m *Migrator) migrateAlias() (err error) { var vrs engine.Versions current := engine.CurrentDataDBVersions() - vrs, err = m.dmOut.DataManager().DataDB().GetVersions("") + vrs, err = m.dmIN.DataManager().DataDB().GetVersions("") if err != nil { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, @@ -173,10 +235,9 @@ func (m *Migrator) migrateAlias() (err error) { if m.sameDataDB { return } - if err := m.migrateCurrentAlias(); err != nil { - return err - } - return + return m.migrateV1Alias() + case 1: + return m.migrateAlias2Attributes() } return } diff --git a/migrator/alias_it_test.go b/migrator/alias_it_test.go new file mode 100644 index 000000000..c2dfa202f --- /dev/null +++ b/migrator/alias_it_test.go @@ -0,0 +1,260 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "log" + "path" + "reflect" + "sort" + "testing" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + alsCfgIn *config.CGRConfig + alsCfgOut *config.CGRConfig + alsMigrator *Migrator + alsAction string +) + +var sTestsAlsIT = []func(t *testing.T){ + testAlsITConnect, + testAlsITFlush, + testAlsITMigrateAndMove, +} + +func TestAliasMigrateITRedis(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testStart("TestAliasMigrateITRedis", inPath, inPath, utils.Migrate, t) +} + +func TestAliasMigrateITMongo(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + testStart("TestAliasMigrateITMongo", inPath, inPath, utils.Migrate, t) +} + +func TestAliasITMove(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testStart("TestAliasITMove", inPath, outPath, utils.Move, t) +} + +func TestAliasITMigrateMongo2Redis(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + testStart("TestAliasITMigrateMongo2Redis", inPath, outPath, utils.Migrate, t) +} + +func TestAliasITMoveEncoding(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmongo") + outPath := path.Join(*dataDir, "conf", "samples", "tutmongojson") + testStart("TestAliasITMoveEncoding", inPath, outPath, utils.Move, t) +} + +func TestAliasITMoveEncoding2(t *testing.T) { + inPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + outPath := path.Join(*dataDir, "conf", "samples", "tutmysqljson") + testStart("TestAliasITMoveEncoding2", inPath, outPath, utils.Move, t) +} + +func testStart(testName, inPath, outPath, action string, t *testing.T) { + var err error + alsAction = action + if alsCfgIn, err = config.NewCGRConfigFromFolder(inPath); err != nil { + t.Fatal(err) + } + if alsCfgOut, err = config.NewCGRConfigFromFolder(outPath); err != nil { + t.Fatal(err) + } + for _, stest := range sTestsAlsIT { + t.Run(testName, stest) + } +} + +func testAlsITConnect(t *testing.T) { + dataDBIn, err := NewMigratorDataDB(alsCfgIn.DataDbCfg().DataDbType, + alsCfgIn.DataDbCfg().DataDbHost, alsCfgIn.DataDbCfg().DataDbPort, + alsCfgIn.DataDbCfg().DataDbName, alsCfgIn.DataDbCfg().DataDbUser, + alsCfgIn.DataDbCfg().DataDbPass, alsCfgIn.GeneralCfg().DBDataEncoding, + config.CgrConfig().CacheCfg(), "") + if err != nil { + log.Fatal(err) + } + dataDBOut, err := NewMigratorDataDB(alsCfgOut.DataDbCfg().DataDbType, + alsCfgOut.DataDbCfg().DataDbHost, alsCfgOut.DataDbCfg().DataDbPort, + alsCfgOut.DataDbCfg().DataDbName, alsCfgOut.DataDbCfg().DataDbUser, + alsCfgOut.DataDbCfg().DataDbPass, alsCfgOut.GeneralCfg().DBDataEncoding, + config.CgrConfig().CacheCfg(), "") + if err != nil { + log.Fatal(err) + } + alsMigrator, err = NewMigrator(dataDBIn, dataDBOut, + nil, nil, false, false, false) + if err != nil { + log.Fatal(err) + } +} + +func testAlsITFlush(t *testing.T) { + alsMigrator.dmOut.DataManager().DataDB().Flush("") + if err := engine.SetDBVersions(alsMigrator.dmOut.DataManager().DataDB()); err != nil { + t.Error("Error ", err.Error()) + } + alsMigrator.dmIN.DataManager().DataDB().Flush("") + if err := engine.SetDBVersions(alsMigrator.dmIN.DataManager().DataDB()); err != nil { + t.Error("Error ", err.Error()) + } +} + +func testAlsITMigrateAndMove(t *testing.T) { + alias := &v1Alias{ + Tenant: utils.META_ANY, + Direction: "*out", + Category: utils.META_ANY, + Account: "1001", + Subject: "call_1001", + Context: "*rated", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: "1003", + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + }, + "Category": map[string]string{ + "call_1001": "call_1002", + }, + }, + Weight: 10, + }, + }, + } + attrProf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: alias.GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: []string{ + "*string:Account:1001", + "*string:Subject:call_1001", + "*string:Destination:1003", + }, + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + { + FieldName: "Category", + Initial: "call_1001", + Substitute: config.NewRSRParsersMustCompile("call_1002", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + } + attrProf.Compile() + switch alsAction { + case utils.Migrate: + err := alsMigrator.dmIN.setV1Alias(alias) + if err != nil { + t.Error("Error when setting v1 Alias ", err.Error()) + } + currentVersion := engine.Versions{utils.Alias: 1} + err = alsMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for Alias ", err.Error()) + } + //check if version was set correctly + if vrs, err := alsMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.Alias] != 1 { + t.Errorf("Unexpected version returned: %d", vrs[utils.Alias]) + } + //migrate alias + err, _ = alsMigrator.Migrate([]string{utils.MetaAlias}) + if err != nil { + t.Error("Error when migrating Alias ", err.Error()) + } + //check if version was updated + if vrs, err := alsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.Alias] != 2 { + t.Errorf("Unexpected version returned: %d", vrs[utils.Alias]) + } + //check if alias was migrate correctly + result, err := alsMigrator.dmOut.DataManager().DataDB().GetAttributeProfileDrv("cgrates.org", alias.GetId()) + if err != nil { + t.Fatalf("Error when getting Attributes %v", err.Error()) + } + result.Compile() + sort.Slice(result.Attributes, func(i, j int) bool { + if result.Attributes[i].FieldName == result.Attributes[j].FieldName { + return result.Attributes[i].Initial.(string) < result.Attributes[j].Initial.(string) + } + return result.Attributes[i].FieldName < result.Attributes[j].FieldName + }) // only for test; map returns random keys + if !reflect.DeepEqual(*attrProf, *result) { + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(attrProf), utils.ToJSON(result)) + } + //check if old account was deleted + if _, err = alsMigrator.dmIN.getV1Alias(); err != utils.ErrNoMoreData { + t.Error("Error should be not found : ", err) + } + + case utils.Move: + /* // No Move tests + if err := alsMigrator.dmIN.DataManager().DataDB().SetAlias(alias, utils.NonTransactional); err != nil { + t.Error(err) + } + currentVersion := engine.CurrentDataDBVersions() + err := alsMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for Alias ", err.Error()) + } + //migrate accounts + err, _ = alsMigrator.Migrate([]string{utils.MetaAlias}) + if err != nil { + t.Error("Error when alsMigratorrating Alias ", err.Error()) + } + //check if account was migrate correctly + result, err := alsMigrator.dmOut.DataManager().DataDB().GetAlias(alias.GetId(), false) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(alias, result) { + t.Errorf("Expecting: %+v, received: %+v", alias, result) + } + //check if old account was deleted + result, err = alsMigrator.dmIN.DataManager().DataDB().GetAlias(alias.GetId(), false) + if err != utils.ErrNotFound { + t.Error(err) + } + // */ + } +} diff --git a/migrator/alias_test.go b/migrator/alias_test.go new file mode 100644 index 000000000..b3e90988f --- /dev/null +++ b/migrator/alias_test.go @@ -0,0 +1,289 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "reflect" + "sort" + "testing" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var defaultTenant = "cgrates.org" + +func TestAlias2AtttributeProfile(t *testing.T) { + aliases := map[int]*v1Alias{ + 0: { + Tenant: utils.META_ANY, + Direction: utils.META_OUT, + Category: utils.META_ANY, + Account: utils.META_ANY, + Subject: utils.META_ANY, + Context: "*rating", + Values: v1AliasValues{}, + }, + 1: { + Tenant: utils.META_ANY, + Direction: utils.META_OUT, + Category: utils.META_ANY, + Account: utils.META_ANY, + Subject: utils.META_ANY, + Context: "*rating", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: utils.META_ANY, + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + }, + }, + Weight: 10, + }, + }, + }, + 2: { + Tenant: utils.META_ANY, + Direction: utils.META_OUT, + Category: utils.META_ANY, + Account: utils.META_ANY, + Subject: utils.META_ANY, + Context: "*rating", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: utils.META_ANY, + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + "1003": "1004", + }, + }, + Weight: 10, + }, + }, + }, + 3: { + Tenant: "", + Direction: "", + Category: "", + Account: "", + Subject: "", + Context: "", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: utils.META_ANY, + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + "1003": "1004", + }, + }, + Weight: 10, + }, + }, + }, + 4: { + Tenant: "notDefaultTenant", + Direction: "*out", + Category: "*voice", + Account: "1001", + Subject: utils.META_ANY, + Context: "*rated", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: "1003", + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + }, + "Subject": map[string]string{ + "1001": "call_1001", + }, + }, + Weight: 10, + }, + }, + }, + 5: { + Tenant: "notDefaultTenant", + Direction: "*out", + Category: utils.META_ANY, + Account: "1001", + Subject: "call_1001", + Context: "*rated", + Values: v1AliasValues{ + &v1AliasValue{ + DestinationId: "1003", + Pairs: map[string]map[string]string{ + "Account": map[string]string{ + "1001": "1002", + }, + "Category": map[string]string{ + "call_1001": "call_1002", + }, + }, + Weight: 10, + }, + }, + }, + } + expected := map[int]*engine.AttributeProfile{ + 0: { + Tenant: defaultTenant, + ID: aliases[0].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: make([]string, 0), + ActivationInterval: nil, + Attributes: make([]*engine.Attribute, 0), + Blocker: false, + Weight: 10, + }, + 1: { + Tenant: defaultTenant, + ID: aliases[1].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: make([]string, 0), + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + }, + 2: { + Tenant: defaultTenant, + ID: aliases[2].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: make([]string, 0), + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + { + FieldName: "Account", + Initial: "1003", + Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + }, + 3: { + Tenant: defaultTenant, + ID: aliases[3].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: make([]string, 0), + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + { + FieldName: "Account", + Initial: "1003", + Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + }, + 4: { + Tenant: "notDefaultTenant", + ID: aliases[4].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: []string{ + "*string:Category:*voice", + "*string:Account:1001", + "*string:Destination:1003", + }, + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + { + FieldName: "Subject", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("call_1001", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + }, + 5: { + Tenant: "notDefaultTenant", + ID: aliases[5].GetId(), + Contexts: []string{utils.META_ANY}, + FilterIDs: []string{ + "*string:Account:1001", + "*string:Subject:call_1001", + "*string:Destination:1003", + }, + ActivationInterval: nil, + Attributes: []*engine.Attribute{ + { + FieldName: "Account", + Initial: "1001", + Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), + Append: true, + }, + { + FieldName: "Category", + Initial: "call_1001", + Substitute: config.NewRSRParsersMustCompile("call_1002", true, utils.INFIELD_SEP), + Append: true, + }, + }, + Blocker: false, + Weight: 10, + }, + } + for i := range expected { + rply := alias2AtttributeProfile(aliases[i], defaultTenant) + sort.Slice(rply.Attributes, func(i, j int) bool { + if rply.Attributes[i].FieldName == rply.Attributes[j].FieldName { + return rply.Attributes[i].Initial.(string) < rply.Attributes[j].Initial.(string) + } + return rply.Attributes[i].FieldName < rply.Attributes[j].FieldName + }) // only for test; map returns random keys + if !reflect.DeepEqual(expected[i], rply) { + t.Errorf("For %v expected: %s ,recived: %s ", i, utils.ToJSON(expected[i]), utils.ToJSON(rply)) + } + } +} diff --git a/migrator/migrator_datadb.go b/migrator/migrator_datadb.go index c25d1c6a0..664a1ba50 100644 --- a/migrator/migrator_datadb.go +++ b/migrator/migrator_datadb.go @@ -46,5 +46,9 @@ type MigratorDataDB interface { getV2ThresholdProfile() (v2T *v2Threshold, err error) setV2ThresholdProfile(x *v2Threshold) (err error) remV2ThresholdProfile(tenant, id string) (err error) + getV1Alias() (v1a *v1Alias, err error) + setV1Alias(al *v1Alias) (err error) + remV1Alias(key string) (err error) + DataManager() *engine.DataManager } diff --git a/migrator/storage_map_datadb.go b/migrator/storage_map_datadb.go index 2a7791de1..1bc5d1ce5 100755 --- a/migrator/storage_map_datadb.go +++ b/migrator/storage_map_datadb.go @@ -166,3 +166,19 @@ func (mM *mapMigrator) setV2ThresholdProfile(x *v2Threshold) (err error) { func (mM *mapMigrator) remV2ThresholdProfile(tenant, id string) (err error) { return utils.ErrNotImplemented } + +//Alias methods +//get +func (v1ms *mapMigrator) getV1Alias() (v1a *v1Alias, err error) { + return nil, utils.ErrNotImplemented +} + +//set +func (v1ms *mapMigrator) setV1Alias(al *v1Alias) (err error) { + return utils.ErrNotImplemented +} + +//rem +func (v1ms *mapMigrator) remV1Alias(key string) (err error) { + return utils.ErrNotImplemented +} diff --git a/migrator/storage_mongo_datadb.go b/migrator/storage_mongo_datadb.go index 927707b9d..07b489521 100644 --- a/migrator/storage_mongo_datadb.go +++ b/migrator/storage_mongo_datadb.go @@ -23,6 +23,7 @@ import ( "github.com/cgrates/cgrates/utils" "github.com/mongodb/mongo-go-driver/bson" "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/mongo/options" ) const ( @@ -30,6 +31,7 @@ const ( v1ActionTriggersCol = "action_triggers" v1AttributeProfilesCol = "attribute_profiles" v2ThresholdProfileCol = "threshold_profiles" + v1AliasCol = "aliases" ) type mongoMigrator struct { @@ -356,3 +358,77 @@ func (v1ms *mongoMigrator) remV2ThresholdProfile(tenant, id string) (err error) _, err = v1ms.mgoDB.DB().Collection(v2ThresholdProfileCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id}) return } + +//Alias methods +//get +func (v1ms *mongoMigrator) getV1Alias() (v1a *v1Alias, err error) { + if v1ms.cursor == nil { + var cursor mongo.Cursor + cursor, err = v1ms.mgoDB.DB().Collection(v1AliasCol).Find(v1ms.mgoDB.GetContext(), bson.D{}) + if err != nil { + return nil, err + } + v1ms.cursor = &cursor + } + if !(*v1ms.cursor).Next(v1ms.mgoDB.GetContext()) { + (*v1ms.cursor).Close(v1ms.mgoDB.GetContext()) + v1ms.cursor = nil + return nil, utils.ErrNoMoreData + } + v1a = new(v1Alias) + if err := (*v1ms.cursor).Decode(v1a); err != nil { + return nil, err + } + return v1a, nil +} + +//set +func (v1ms *mongoMigrator) setV1Alias(al *v1Alias) (err error) { + _, err = v1ms.mgoDB.DB().Collection(v1AliasCol).UpdateOne(v1ms.mgoDB.GetContext(), bson.M{"key": al.GetId()}, + bson.M{"$set": struct { + Key string + Value v1AliasValues + }{Key: al.GetId(), Value: al.Values}}, + options.Update().SetUpsert(true), + ) + return err +} + +//rem +func (v1ms *mongoMigrator) remV1Alias(key string) (err error) { + al := new(v1Alias) + al.SetId(key) + var kv struct { + Key string + Value v1AliasValues + } + cur := v1ms.mgoDB.DB().Collection(v1AliasCol).FindOne(v1ms.mgoDB.GetContext(), bson.M{"key": key}) + if err := cur.Decode(&kv); err != nil { + if err == mongo.ErrNoDocuments { + return utils.ErrNotFound + } + return err + } + al.Values = kv.Value + dr, err := v1ms.mgoDB.DB().Collection(v1AliasCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"key": key}) + if dr.DeletedCount == 0 { + return utils.ErrNotFound + } + if err != nil { + return err + } + for _, value := range al.Values { + tmpKey := utils.ConcatenatedKey(al.GetId(), value.DestinationId) + for target, pairs := range value.Pairs { + for _, alias := range pairs { + rKey := alias + target + al.Context + _, err = v1ms.mgoDB.DB().Collection(v1AliasCol).UpdateOne(v1ms.mgoDB.GetContext(), bson.M{"key": rKey}, + bson.M{"$pull": bson.M{"value": tmpKey}}) + if err != nil { + return err + } + } + } + } + return +} diff --git a/migrator/storage_redis.go b/migrator/storage_redis.go index 16a0ac84b..426a32e0b 100644 --- a/migrator/storage_redis.go +++ b/migrator/storage_redis.go @@ -19,8 +19,11 @@ along with this program. If not, see package migrator import ( + "strings" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/mediocregopher/radix.v2/redis" ) type redisMigrator struct { @@ -473,3 +476,88 @@ func (v1rs *redisMigrator) remV2ThresholdProfile(tenant, id string) (err error) key := utils.ThresholdProfilePrefix + utils.ConcatenatedKey(tenant, id) return v1rs.rds.Cmd("DEL", key).Err } + +//ThresholdProfile methods +//get +func (v1rs *redisMigrator) getV1Alias() (v1a *v1Alias, err error) { + v1a = &v1Alias{Values: make(v1AliasValues, 0)} + if v1rs.qryIdx == nil { + v1rs.dataKeys, err = v1rs.rds.GetKeysForPrefix(utils.ALIASES_PREFIX) + if err != nil { + return + } else if len(v1rs.dataKeys) == 0 { + return nil, utils.ErrNotFound + } + v1rs.qryIdx = utils.IntPointer(0) + } + if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 { + key := v1rs.dataKeys[*v1rs.qryIdx] + strVal, err := v1rs.rds.Cmd("GET", key).Bytes() + if err != nil { + return nil, err + } + v1a.SetId(strings.TrimPrefix(key, utils.ALIASES_PREFIX)) + if err := v1rs.rds.Marshaler().Unmarshal(strVal, &v1a.Values); err != nil { + return nil, err + } + *v1rs.qryIdx = *v1rs.qryIdx + 1 + } else { + v1rs.qryIdx = nil + return nil, utils.ErrNoMoreData + } + return v1a, nil +} + +//set +func (v1rs *redisMigrator) setV1Alias(al *v1Alias) (err error) { + var result []byte + result, err = v1rs.rds.Marshaler().Marshal(al.Values) + if err != nil { + return + } + key := utils.ALIASES_PREFIX + al.GetId() + if err = v1rs.rds.Cmd("SET", key, result).Err; err != nil { + return + } + return +} + +//rem +func (v1rs *redisMigrator) remV1Alias(key string) (err error) { + + // get alias for values list + + var values []byte + if values, err = v1rs.rds.Cmd("GET", + utils.ALIASES_PREFIX+key).Bytes(); err != nil { + if err == redis.ErrRespNil { // did not find the destination + err = utils.ErrNotFound + } + return + } + al := &v1Alias{Values: make(v1AliasValues, 0)} + al.SetId(key) + if err = v1rs.rds.Marshaler().Unmarshal(values, &al.Values); err != nil { + return err + } + + err = v1rs.rds.Cmd("DEL", utils.ALIASES_PREFIX+key).Err + if err != nil { + return err + } + for _, value := range al.Values { + tmpKey := utils.ConcatenatedKey(al.GetId(), value.DestinationId) + for target, pairs := range value.Pairs { + for _, alias := range pairs { + revID := alias + target + al.Context + err = v1rs.rds.Cmd("SREM", utils.REVERSE_ALIASES_PREFIX+revID, tmpKey).Err + if err != nil { + return err + } + } + } + } + return + + return v1rs.rds.Cmd("DEL", key).Err +}