From 787174d89720c0d243879341a9d5af956bbcd303 Mon Sep 17 00:00:00 2001 From: TeoV Date: Tue, 8 May 2018 07:23:03 -0400 Subject: [PATCH] Update migrator test and add stats_it_test.go --- migrator/accounts_it_test.go | 20 ++- migrator/cdrs_it_test.go | 54 +----- migrator/migrator_it_test.go | 2 +- migrator/stats.go | 17 +- migrator/stats_it_test.go | 320 +++++++++++++++++++++++++++++++++++ 5 files changed, 356 insertions(+), 57 deletions(-) create mode 100755 migrator/stats_it_test.go diff --git a/migrator/accounts_it_test.go b/migrator/accounts_it_test.go index f266b4238..8ed9a86fb 100755 --- a/migrator/accounts_it_test.go +++ b/migrator/accounts_it_test.go @@ -117,9 +117,9 @@ func testAccITConnect(t *testing.T) { if err != nil { log.Fatal(err) } - oldDataDB, err := ConfigureV1DataStorage(accCfgOut.DataDbType, - accCfgOut.DataDbHost, accCfgOut.DataDbPort, accCfgOut.DataDbName, - accCfgOut.DataDbUser, accCfgOut.DataDbPass, accCfgOut.DBDataEncoding) + oldDataDB, err := ConfigureV1DataStorage(accCfgIn.DataDbType, + accCfgIn.DataDbHost, accCfgIn.DataDbPort, accCfgIn.DataDbName, + accCfgIn.DataDbUser, accCfgIn.DataDbPass, accCfgIn.DBDataEncoding) if err != nil { log.Fatal(err) } @@ -230,10 +230,24 @@ func testAccITMigrateAndMove(t *testing.T) { if err != nil { t.Error("Error when setting version for Accounts ", err.Error()) } + + if vrs, err := accMigrator.dmOut.DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.Accounts] != 1 { + t.Errorf("Unexpected version returned: %d", vrs[utils.Accounts]) + } + err, _ = accMigrator.Migrate([]string{utils.MetaAccounts}) if err != nil { t.Error("Error when migrating Accounts ", err.Error()) } + + if vrs, err := accMigrator.dmOut.DataDB().GetVersions(""); err != nil { + t.Error(err) + } else if vrs[utils.Accounts] != 3 { + t.Errorf("Unexpected version returned: %d", vrs[utils.Accounts]) + } + result, err := accMigrator.dmOut.DataDB().GetAccount(testAccount.ID) if err != nil { t.Error("Error when getting Accounts ", err.Error()) diff --git a/migrator/cdrs_it_test.go b/migrator/cdrs_it_test.go index 9abd979d1..cf9208b05 100755 --- a/migrator/cdrs_it_test.go +++ b/migrator/cdrs_it_test.go @@ -41,66 +41,36 @@ var ( ) var sTestsCdrIT = []func(t *testing.T){ + testCdrITConnect, testCdrITFlush, testCdrITMigrateAndMove, } -func TestCdrITMongoConnection(t *testing.T) { +func TestCdrITMongo(t *testing.T) { var err error cdrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo") cdrCfgIn, err = config.NewCGRConfigFromFolder(cdrPathIn) if err != nil { t.Error(err) } - storDBIn, err := engine.ConfigureStorDB(cdrCfgIn.StorDBType, cdrCfgIn.StorDBHost, - cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName, - cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass, - config.CgrConfig().StorDBMaxOpenConns, - config.CgrConfig().StorDBMaxIdleConns, - config.CgrConfig().StorDBConnMaxLifetime, - config.CgrConfig().StorDBCDRSIndexes) - if err != nil { - t.Error(err) - } - storDBOut, err := engine.ConfigureStorDB(cdrCfgIn.StorDBType, - cdrCfgIn.StorDBHost, cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName, - cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass, - config.CgrConfig().StorDBMaxOpenConns, - config.CgrConfig().StorDBMaxIdleConns, - config.CgrConfig().StorDBConnMaxLifetime, - config.CgrConfig().StorDBCDRSIndexes) - if err != nil { - t.Error(err) - } - oldStorDB, err := ConfigureV1StorDB(cdrCfgIn.StorDBType, - cdrCfgIn.StorDBHost, cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName, - cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass) - if err != nil { - log.Fatal(err) - } - - cdrMigrator, err = NewMigrator(nil, nil, cdrCfgIn.DataDbType, - cdrCfgIn.DBDataEncoding, storDBIn, storDBOut, cdrCfgIn.StorDBType, nil, - cdrCfgIn.DataDbType, cdrCfgIn.DBDataEncoding, oldStorDB, cdrCfgIn.StorDBType, - false, false, false, false, false) - if err != nil { - t.Error(err) - } -} - -func TestCdrITMongo(t *testing.T) { for _, stest := range sTestsCdrIT { t.Run("TestCdrITMigrateMongo", stest) } } -func TestCdrITMySqlConnection(t *testing.T) { +func TestCdrITMySql(t *testing.T) { var err error cdrPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql") cdrCfgIn, err = config.NewCGRConfigFromFolder(cdrPathIn) if err != nil { t.Error(err) } + for _, stest := range sTestsCdrIT { + t.Run("TestCdrITMigrateMySql", stest) + } +} + +func testCdrITConnect(t *testing.T) { storDBIn, err := engine.ConfigureStorDB(cdrCfgIn.StorDBType, cdrCfgIn.StorDBHost, cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName, cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass, @@ -137,12 +107,6 @@ func TestCdrITMySqlConnection(t *testing.T) { } } -func TestCdrITMySql(t *testing.T) { - for _, stest := range sTestsCdrIT { - t.Run("TestCdrITMigrateMySql", stest) - } -} - func testCdrITFlush(t *testing.T) { if err := cdrMigrator.storDBOut.Flush( path.Join(cdrCfgIn.DataFolderPath, "storage", cdrCfgIn.StorDBType)); err != nil { diff --git a/migrator/migrator_it_test.go b/migrator/migrator_it_test.go index 2d4dadd4c..c801778c0 100644 --- a/migrator/migrator_it_test.go +++ b/migrator/migrator_it_test.go @@ -52,7 +52,7 @@ var sTestsITMigrator = []func(t *testing.T){ testMigratorActions, testMigratorSharedGroups, testMigratorStats, - testMigratorSessionsCosts, + testMigratorSessionsCosts, // Done testFlush, testMigratorAlias, //FIXME testMigratorReverseAlias, diff --git a/migrator/stats.go b/migrator/stats.go index 94ca5425a..57235ed60 100644 --- a/migrator/stats.go +++ b/migrator/stats.go @@ -144,7 +144,7 @@ func (m *Migrator) migrateV1CDRSTATS() (err error) { } if m.dryRun != true { // All done, update version wtih current one - vrs := engine.Versions{utils.StatS: engine.CurrentStorDBVersions()[utils.StatS]} + vrs := engine.Versions{utils.StatS: engine.CurrentDataDBVersions()[utils.StatS]} if err = m.dmOut.DataDB().SetVersions(vrs, false); err != nil { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, @@ -171,6 +171,10 @@ func (m *Migrator) migrateStats() (err error) { "version number is not defined for ActionTriggers model") } switch vrs[utils.StatS] { + case 1: + if err := m.migrateV1CDRSTATS(); err != nil { + return err + } case current[utils.StatS]: if m.sameDataDB { return @@ -179,11 +183,6 @@ func (m *Migrator) migrateStats() (err error) { return err } return - - case 1: - if err := m.migrateV1CDRSTATS(); err != nil { - return err - } } return } @@ -191,13 +190,15 @@ func (m *Migrator) migrateStats() (err error) { func (v1Sts v1Stat) AsStatQP() (filter *engine.Filter, sq *engine.StatQueue, stq *engine.StatQueueProfile, err error) { var filters []*engine.FilterRule if len(v1Sts.SetupInterval) == 1 { - x, err := engine.NewFilterRule(engine.MetaGreaterOrEqual, "SetupInterval", []string{v1Sts.SetupInterval[0].String()}) + x, err := engine.NewFilterRule(engine.MetaGreaterOrEqual, + "SetupInterval", []string{v1Sts.SetupInterval[0].String()}) if err != nil { return nil, nil, nil, err } filters = append(filters, x) } else if len(v1Sts.SetupInterval) == 2 { - x, err := engine.NewFilterRule(engine.MetaLessThan, "SetupInterval", []string{v1Sts.SetupInterval[1].String()}) + x, err := engine.NewFilterRule(engine.MetaLessThan, + "SetupInterval", []string{v1Sts.SetupInterval[1].String()}) if err != nil { return nil, nil, nil, err } diff --git a/migrator/stats_it_test.go b/migrator/stats_it_test.go new file mode 100755 index 000000000..9d76bb257 --- /dev/null +++ b/migrator/stats_it_test.go @@ -0,0 +1,320 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "log" + "path" + "reflect" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + stsPathIn string + stsPathOut string + stsCfgIn *config.CGRConfig + stsCfgOut *config.CGRConfig + stsMigrator *Migrator + stsAction string + stsSetupTime time.Time +) + +var sTestsStsIT = []func(t *testing.T){ + testStsITConnect, + testStsITFlush, + testStsITMigrateAndMove, +} + +func TestStatsQueueITRedis(t *testing.T) { + var err error + stsPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql") + stsCfgIn, err = config.NewCGRConfigFromFolder(stsPathIn) + if err != nil { + t.Fatal(err) + } + stsCfgOut, err = config.NewCGRConfigFromFolder(stsPathIn) + if err != nil { + t.Fatal(err) + } + stsAction = utils.Migrate + for _, stest := range sTestsStsIT { + t.Run("TestStatsQueueITMigrateRedis", stest) + } +} + +func TestStatsQueueITMongo(t *testing.T) { + var err error + stsPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo") + stsCfgIn, err = config.NewCGRConfigFromFolder(stsPathIn) + if err != nil { + t.Fatal(err) + } + stsCfgOut, err = config.NewCGRConfigFromFolder(stsPathIn) + if err != nil { + t.Fatal(err) + } + stsAction = utils.Migrate + for _, stest := range sTestsStsIT { + t.Run("TestStatsQueueITMigrateMongo", stest) + } +} + +func TestStatsQueueITMove(t *testing.T) { + var err error + stsPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo") + stsCfgIn, err = config.NewCGRConfigFromFolder(stsPathIn) + if err != nil { + t.Fatal(err) + } + stsPathOut = path.Join(*dataDir, "conf", "samples", "tutmysql") + stsCfgOut, err = config.NewCGRConfigFromFolder(stsPathOut) + if err != nil { + t.Fatal(err) + } + stsAction = utils.Move + for _, stest := range sTestsStsIT { + t.Run("TestStatsQueueITMove", stest) + } +} + +func testStsITConnect(t *testing.T) { + dataDBIn, err := engine.ConfigureDataStorage(stsCfgIn.DataDbType, + stsCfgIn.DataDbHost, stsCfgIn.DataDbPort, stsCfgIn.DataDbName, + stsCfgIn.DataDbUser, stsCfgIn.DataDbPass, stsCfgIn.DBDataEncoding, + config.CgrConfig().CacheCfg(), *loadHistorySize) + if err != nil { + log.Fatal(err) + } + dataDBOut, err := engine.ConfigureDataStorage(stsCfgOut.DataDbType, + stsCfgOut.DataDbHost, stsCfgOut.DataDbPort, stsCfgOut.DataDbName, + stsCfgOut.DataDbUser, stsCfgOut.DataDbPass, stsCfgOut.DBDataEncoding, + config.CgrConfig().CacheCfg(), *loadHistorySize) + if err != nil { + log.Fatal(err) + } + oldDataDB, err := ConfigureV1DataStorage(stsCfgIn.DataDbType, + stsCfgIn.DataDbHost, stsCfgIn.DataDbPort, stsCfgIn.DataDbName, + stsCfgIn.DataDbUser, stsCfgIn.DataDbPass, stsCfgIn.DBDataEncoding) + if err != nil { + log.Fatal(err) + } + stsMigrator, err = NewMigrator(dataDBIn, dataDBOut, stsCfgIn.DataDbType, + stsCfgIn.DBDataEncoding, nil, nil, stsCfgIn.StorDBType, oldDataDB, + stsCfgIn.DataDbType, stsCfgIn.DBDataEncoding, nil, stsCfgIn.StorDBType, + false, false, false, false, false) + if err != nil { + log.Fatal(err) + } +} + +func testStsITFlush(t *testing.T) { + stsMigrator.dmOut.DataDB().Flush("") + if err := engine.SetDBVersions(stsMigrator.dmOut.DataDB()); err != nil { + t.Error("Error ", err.Error()) + } +} + +func testStsITMigrateAndMove(t *testing.T) { + tim := time.Date(2012, time.February, 27, 23, 59, 59, 0, time.UTC) + var filters []*engine.FilterRule + v1Sts := &v1Stat{ + Id: "test", // Config id, unique per config instance + QueueLength: 10, // Number of items in the stats buffer + TimeWindow: time.Duration(1) * time.Second, // Will only keep the CDRs who's call setup time is not older than time.Now()-TimeWindow + SaveInterval: time.Duration(1) * time.Second, + Metrics: []string{"ASR", "ACD", "ACC"}, + SetupInterval: []time.Time{time.Now()}, + TOR: []string{}, + CdrHost: []string{}, + CdrSource: []string{}, + ReqType: []string{}, + Direction: []string{}, + Tenant: []string{}, + Category: []string{}, + Account: []string{}, + Subject: []string{}, + DestinationIds: []string{}, + UsageInterval: []time.Duration{1 * time.Second}, + PddInterval: []time.Duration{1 * time.Second}, + Supplier: []string{}, + DisconnectCause: []string{}, + MediationRunIds: []string{}, + RatedAccount: []string{}, + RatedSubject: []string{}, + CostInterval: []float64{}, + Triggers: engine.ActionTriggers{ + &engine.ActionTrigger{ + ID: "Test", + Balance: &engine.BalanceFilter{ + ID: utils.StringPointer("TESTB"), + Timings: []*engine.RITiming{}, + ExpirationDate: utils.TimePointer(tim), + Type: utils.StringPointer(utils.MONETARY), + Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), + }, + ExpirationDate: tim, + LastExecutionTime: tim, + ActivationDate: tim, + ThresholdType: utils.TRIGGER_MAX_BALANCE, + ThresholdValue: 2, + ActionsID: "TEST_ACTIONS", + Executed: true, + }, + }, + } + + // Here remove extra info from SetupInterval + if err := utils.Clone(v1Sts.SetupInterval[0], &stsSetupTime); err != nil { + t.Error(err) + } + + x, _ := engine.NewFilterRule(engine.MetaGreaterOrEqual, + "SetupInterval", []string{stsSetupTime.String()}) + filters = append(filters, x) + x, _ = engine.NewFilterRule(engine.MetaGreaterOrEqual, + "UsageInterval", []string{v1Sts.UsageInterval[0].String()}) + filters = append(filters, x) + x, _ = engine.NewFilterRule(engine.MetaGreaterOrEqual, + "PddInterval", []string{v1Sts.PddInterval[0].String()}) + filters = append(filters, x) + + filter := &engine.Filter{Tenant: config.CgrConfig().DefaultTenant, ID: v1Sts.Id, Rules: filters} + + sqp := &engine.StatQueueProfile{ + Tenant: "cgrates.org", + ID: "test", + FilterIDs: []string{v1Sts.Id}, + QueueLength: 10, + TTL: time.Duration(0) * time.Second, + Metrics: []*utils.MetricWithParams{ + &utils.MetricWithParams{MetricID: "*asr", Parameters: ""}, + &utils.MetricWithParams{MetricID: "*acd", Parameters: ""}, + &utils.MetricWithParams{MetricID: "*acc", Parameters: ""}, + }, + ThresholdIDs: []string{"Test"}, + Blocker: false, + Stored: true, + Weight: float64(0), + MinItems: 0, + } + sq := &engine.StatQueue{Tenant: config.CgrConfig().DefaultTenant, + ID: v1Sts.Id, + SQMetrics: make(map[string]engine.StatMetric), + } + for _, metricwparam := range sqp.Metrics { + if metric, err := engine.NewStatMetric(metricwparam.MetricID, + 0, metricwparam.Parameters); err != nil { + t.Error("Error when creating newstatMETRIc ", err.Error()) + } else { + if _, has := sq.SQMetrics[metricwparam.MetricID]; !has { + sq.SQMetrics[metricwparam.MetricID] = metric + } + } + } + switch stsAction { + case utils.Migrate: + err := stsMigrator.oldDataDB.setV1Stats(v1Sts) + if err != nil { + t.Error("Error when setting v1Stat ", err.Error()) + } + currentVersion := engine.Versions{ + utils.StatS: 1, + utils.Thresholds: 2, + utils.Accounts: 2, + utils.Actions: 2, + utils.ActionTriggers: 2, + utils.ActionPlans: 2, + utils.SharedGroups: 2} + err = stsMigrator.dmOut.DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for stats ", err.Error()) + } + err, _ = stsMigrator.Migrate([]string{utils.MetaStats}) + if err != nil { + t.Error("Error when migrating Stats ", err.Error()) + } + + result, err := stsMigrator.dmOut.GetStatQueueProfile("cgrates.org", + v1Sts.Id, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + if !reflect.DeepEqual(sqp, result) { + t.Errorf("Expecting: %+v, received: %+v", sqp, result) + } + + result1, err := stsMigrator.dmOut.GetFilter("cgrates.org", + v1Sts.Id, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + if !reflect.DeepEqual(filter.ID, result1.ID) { + t.Errorf("Expecting: %+v, received: %+v", filter.ID, result1.ID) + } else if !reflect.DeepEqual(len(filter.Rules), len(result1.Rules)) { + t.Errorf("Expecting: %+v, received: %+v", len(filter.Rules), len(result1.Rules)) + } + + result2, err := stsMigrator.dmOut.GetStatQueue("cgrates.org", sq.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + if !reflect.DeepEqual(sq.ID, result2.ID) { + t.Errorf("Expecting: %+v, received: %+v", sq.ID, result2.ID) + } + + case utils.Move: + if err := stsMigrator.dmIN.SetStatQueueProfile(sqp, true); err != nil { + t.Error("Error when setting Stats ", err.Error()) + } + if err := stsMigrator.dmIN.SetStatQueue(sq); err != nil { + t.Error("Error when setting Stats ", err.Error()) + } + currentVersion := engine.CurrentDataDBVersions() + err := stsMigrator.dmOut.DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for stats ", err.Error()) + } + err, _ = stsMigrator.Migrate([]string{utils.MetaStats}) + if err != nil { + t.Error("Error when migrating Stats ", err.Error()) + } + result, err := stsMigrator.dmOut.GetStatQueueProfile(sqp.Tenant, sqp.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + result1, err := stsMigrator.dmOut.GetStatQueue(sq.Tenant, sq.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + if !reflect.DeepEqual(sqp, result) { + t.Errorf("Expecting: %+v, received: %+v", sqp, result) + } + if !reflect.DeepEqual(sq.ID, result1.ID) { + t.Errorf("Expecting: %+v, received: %+v", sq.ID, result1.ID) + } + } + +}