Migrator test for each tp

This commit is contained in:
TeoV
2018-05-04 11:33:44 -04:00
committed by Dan Christian Bogos
parent 6009be5fc7
commit e7d72c115d
14 changed files with 396 additions and 30 deletions

View File

@@ -80,10 +80,10 @@ func (self *ApierV1) GetTPThresholdIDs(attrs AttrGetTPThresholdIds, reply *[]str
// Removes specific Threshold on Tariff plan
func (self *ApierV1) RemTPThreshold(attrs AttrGetTPThreshold, reply *string) error {
if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ID"}); len(missing) != 0 { //Params missing
if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Tenant", "ID"}); len(missing) != 0 { //Params missing
return utils.NewErrMandatoryIeMissing(missing...)
}
if err := self.StorDb.RemTpData(utils.TBLTPThresholds, attrs.TPid, map[string]string{"id": attrs.ID}); err != nil {
if err := self.StorDb.RemTpData(utils.TBLTPThresholds, attrs.TPid, map[string]string{"tenant": attrs.Tenant, "id": attrs.ID}); err != nil {
return utils.NewErrServerError(err)
} else {
*reply = utils.OK

View File

@@ -52,8 +52,11 @@ func InitDataDb(cfg *config.CGRConfig) error {
func InitStorDb(cfg *config.CGRConfig) error {
x := []string{utils.MYSQL, utils.POSTGRES}
storDb, err := ConfigureLoadStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBConnMaxLifetime, cfg.StorDBCDRSIndexes)
storDb, err := ConfigureLoadStorage(cfg.StorDBType,
cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName,
cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns,
cfg.StorDBConnMaxLifetime, cfg.StorDBCDRSIndexes)
if err != nil {
return err
}

View File

@@ -51,7 +51,7 @@ var sTestsAccIT = []func(t *testing.T){
func TestAccountITRedisConnection(t *testing.T) {
var err error
accPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
accPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql")
accCfgIn, err = config.NewCGRConfigFromFolder(accPathIn)
if err != nil {
t.Fatal(err)
@@ -94,7 +94,7 @@ func TestAccountITRedis(t *testing.T) {
func TestAccountITMongoConnection(t *testing.T) {
var err error
accPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql")
accPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
accCfgIn, err = config.NewCGRConfigFromFolder(accPathIn)
if err != nil {
t.Fatal(err)
@@ -126,7 +126,6 @@ func TestAccountITMongoConnection(t *testing.T) {
if err != nil {
log.Fatal(err)
}
}
func TestAccountITMongo(t *testing.T) {

View File

@@ -59,7 +59,9 @@ func (m *Migrator) migrateCDRs() (err error) {
}
switch vrs[utils.CDRs] {
case current[utils.CDRs]:
if err := m.migrateCurrentCDRs(); err != nil {
return err
}
case 1:
if err := m.migrateV1CDRs(); err != nil {
return err

166
migrator/cdrs_it_test.go Executable file
View File

@@ -0,0 +1,166 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"log"
"path"
"testing"
"time"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
cdrPathIn string
cdrPathOut string
cdrCfgIn *config.CGRConfig
cdrCfgOut *config.CGRConfig
cdrMigrator *Migrator
cdrAction string
)
var sTestsCdrIT = []func(t *testing.T){
testCdrITFlush,
testCdrITMigrateAndMove,
}
func TestCdrITMongoConnection(t *testing.T) {
var err error
cdrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
cdrCfgIn, err = config.NewCGRConfigFromFolder(cdrPathIn)
if err != nil {
t.Error(err)
}
storDBIn, err := engine.ConfigureStorDB(cdrCfgIn.StorDBType, cdrCfgIn.StorDBHost,
cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName,
cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass,
config.CgrConfig().StorDBMaxOpenConns,
config.CgrConfig().StorDBMaxIdleConns,
config.CgrConfig().StorDBConnMaxLifetime,
config.CgrConfig().StorDBCDRSIndexes)
if err != nil {
t.Error(err)
}
storDBOut, err := engine.ConfigureStorDB(cdrCfgIn.StorDBType,
cdrCfgIn.StorDBHost, cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName,
cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass,
config.CgrConfig().StorDBMaxOpenConns,
config.CgrConfig().StorDBMaxIdleConns,
config.CgrConfig().StorDBConnMaxLifetime,
config.CgrConfig().StorDBCDRSIndexes)
if err != nil {
t.Error(err)
}
oldStorDB, err := ConfigureV1StorDB(cdrCfgIn.StorDBType,
cdrCfgIn.StorDBHost, cdrCfgIn.StorDBPort, cdrCfgIn.StorDBName,
cdrCfgIn.StorDBUser, cdrCfgIn.StorDBPass)
if err != nil {
log.Fatal(err)
}
cdrMigrator, err = NewMigrator(nil, nil, cdrCfgIn.DataDbType,
cdrCfgIn.DBDataEncoding, storDBIn, storDBOut, cdrCfgIn.StorDBType, nil,
cdrCfgIn.DataDbType, cdrCfgIn.DBDataEncoding, oldStorDB, cdrCfgIn.StorDBType,
false, false, false, false, false)
if err != nil {
t.Error(err)
}
}
func TestCdrITMongo(t *testing.T) {
for _, stest := range sTestsCdrIT {
t.Run("TestCdrITMigrateMongo", stest)
}
}
func testCdrITFlush(t *testing.T) {
if err := cdrMigrator.storDBOut.Flush(
path.Join(cdrCfgIn.DataFolderPath, "storage", cdrCfgIn.StorDBType)); err != nil {
t.Error(err)
}
}
func testCdrITMigrateAndMove(t *testing.T) {
cc := &engine.CallCost{
Direction: utils.OUT,
Destination: "0723045326",
Timespans: []*engine.TimeSpan{
&engine.TimeSpan{
TimeStart: time.Date(2013, 9, 24, 10, 48, 0, 0, time.UTC),
TimeEnd: time.Date(2013, 9, 24, 10, 48, 10, 0, time.UTC),
DurationIndex: 0,
RateInterval: &engine.RateInterval{
Rating: &engine.RIRate{
Rates: engine.RateGroups{
&engine.Rate{
GroupIntervalStart: 0,
Value: 100,
RateIncrement: 10 * time.Second,
RateUnit: time.Second,
},
},
},
},
},
},
TOR: utils.VOICE,
}
v1Cdr := &v1Cdrs{
CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1",
Source: utils.UNIT_TEST, RequestType: utils.META_RATED, Tenant: "cgrates.org",
Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
RunID: utils.DEFAULT_RUNID, Usage: time.Duration(10),
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
Cost: 1.01, Rated: true,
CostDetails: cc,
}
var err error
if err = cdrMigrator.oldStorDB.setV1CDR(v1Cdr); err != nil {
t.Error(err)
}
currentVersion := engine.Versions{
utils.COST_DETAILS: 2,
utils.SessionSCosts: 3,
utils.CDRs: 1,
}
err = cdrMigrator.storDBOut.SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for CDRs ", err.Error())
}
if rcvCDRs, _, err := cdrMigrator.storDBOut.GetCDRs(new(utils.CDRsFilter), false); err != utils.ErrNotFound {
t.Error(err)
}
err, _ = cdrMigrator.Migrate([]string{utils.MetaCDRs})
if err != nil {
t.Error("Error when migrating CDRs ", err.Error())
}
if rcvCDRs, _, err := cdrMigrator.storDBOut.GetCDRs(new(utils.CDRsFilter), false); err != nil {
t.Error(err)
} else if len(rcvCDRs) != 1 {
t.Errorf("Unexpected number of CDRs returned: %d", len(rcvCDRs))
}
}

View File

@@ -26,9 +26,12 @@ import (
"github.com/cgrates/cgrates/utils"
)
func NewMigrator(dmIN *engine.DataManager, dmOut *engine.DataManager, dataDBType, dataDBEncoding string,
storDBIn engine.StorDB, storDBOut engine.StorDB, storDBType string, oldDataDB MigratorDataDB, oldDataDBType, oldDataDBEncoding string,
oldStorDB MigratorStorDB, oldStorDBType string, dryRun bool, sameDataDB bool, sameStorDB bool,
func NewMigrator(dmIN *engine.DataManager, dmOut *engine.DataManager,
dataDBType, dataDBEncoding string,
storDBIn engine.StorDB, storDBOut engine.StorDB, storDBType string,
oldDataDB MigratorDataDB, oldDataDBType, oldDataDBEncoding string,
oldStorDB MigratorStorDB, oldStorDBType string,
dryRun bool, sameDataDB bool, sameStorDB bool,
datadb_versions bool, stordb_versions bool) (m *Migrator, err error) {
var mrshlr engine.Marshaler
var oldmrshlr engine.Marshaler

View File

@@ -69,7 +69,6 @@ func (m *Migrator) migrateSessionSCosts() (err error) {
if err := m.storDBOut.RemoveSMCost(nil); err != nil {
return err
}
}
m.stats[utils.SessionSCosts] = -1
vrs = engine.Versions{utils.SessionSCosts: engine.CurrentStorDBVersions()[utils.SessionSCosts]}

View File

@@ -31,20 +31,27 @@ func (m *Migrator) migrateCurrentTPfilters() (err error) {
return err
}
for _, tpid := range tpids {
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPFilters, utils.TPDistinctIds{"id"}, map[string]string{}, nil)
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPFilters,
utils.TPDistinctIds{"id"}, map[string]string{}, nil)
if err != nil {
return err
}
for _, id := range ids {
dest, err := m.storDBIn.GetTPFilters(tpid, id)
fltrs, err := m.storDBIn.GetTPFilters(tpid, id)
if err != nil {
return err
}
if dest != nil {
if fltrs != nil {
if m.dryRun != true {
if err := m.storDBOut.SetTPFilters(dest); err != nil {
if err := m.storDBOut.SetTPFilters(fltrs); err != nil {
return err
}
for _, fltr := range fltrs {
if err := m.storDBIn.RemTpData(utils.TBLTPFilters,
fltr.TPid, map[string]string{"tenant": fltr.Tenant, "id": fltr.ID}); err != nil {
return err
}
}
m.stats[utils.TpFilters] += 1
}
}

160
migrator/tp_filters_it_test.go Executable file
View File

@@ -0,0 +1,160 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"log"
"path"
"reflect"
"testing"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
tpFltrPathIn string
tpFltrPathOut string
tpFltrCfgIn *config.CGRConfig
tpFltrCfgOut *config.CGRConfig
tpFltrMigrator *Migrator
tpFilters []*utils.TPFilterProfile
)
var sTestsTpFltrIT = []func(t *testing.T){
testTpFltrITConnect,
testTpFltrITFlush,
testTpFltrITPopulate,
testTpFltrITMove,
testTpFltrITCheckData,
}
func TestTpFltrMove(t *testing.T) {
for _, stest := range sTestsTpFltrIT {
t.Run("TestTpFltrMove", stest)
}
}
func testTpFltrITConnect(t *testing.T) {
var err error
tpFltrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
tpFltrCfgIn, err = config.NewCGRConfigFromFolder(tpFltrPathIn)
if err != nil {
t.Fatal(err)
}
tpFltrPathOut = path.Join(*dataDir, "conf", "samples", "tutmysql")
tpFltrCfgOut, err = config.NewCGRConfigFromFolder(tpFltrPathOut)
if err != nil {
t.Fatal(err)
}
storDBIn, err := engine.ConfigureStorDB(tpFltrCfgIn.StorDBType, tpFltrCfgIn.StorDBHost,
tpFltrCfgIn.StorDBPort, tpFltrCfgIn.StorDBName,
tpFltrCfgIn.StorDBUser, tpFltrCfgIn.StorDBPass,
config.CgrConfig().StorDBMaxOpenConns,
config.CgrConfig().StorDBMaxIdleConns,
config.CgrConfig().StorDBConnMaxLifetime,
config.CgrConfig().StorDBCDRSIndexes)
if err != nil {
log.Fatal(err)
}
storDBOut, err := engine.ConfigureStorDB(tpFltrCfgOut.StorDBType,
tpFltrCfgOut.StorDBHost, tpFltrCfgOut.StorDBPort, tpFltrCfgOut.StorDBName,
tpFltrCfgOut.StorDBUser, tpFltrCfgOut.StorDBPass,
config.CgrConfig().StorDBMaxOpenConns,
config.CgrConfig().StorDBMaxIdleConns,
config.CgrConfig().StorDBConnMaxLifetime,
config.CgrConfig().StorDBCDRSIndexes)
if err != nil {
log.Fatal(err)
}
tpFltrMigrator, err = NewMigrator(nil, nil, tpFltrCfgIn.DataDbType,
tpFltrCfgIn.DBDataEncoding, storDBIn, storDBOut, tpFltrCfgIn.StorDBType, nil,
tpFltrCfgIn.DataDbType, tpFltrCfgIn.DBDataEncoding, nil,
tpFltrCfgIn.StorDBType, false, false, false, false, false)
if err != nil {
log.Fatal(err)
}
}
func testTpFltrITFlush(t *testing.T) {
if err := tpFltrMigrator.storDBIn.Flush(
path.Join(tpFltrCfgIn.DataFolderPath, "storage", tpFltrCfgIn.StorDBType)); err != nil {
t.Error(err)
}
if err := tpFltrMigrator.storDBOut.Flush(
path.Join(tpFltrCfgOut.DataFolderPath, "storage", tpFltrCfgOut.StorDBType)); err != nil {
t.Error(err)
}
}
func testTpFltrITPopulate(t *testing.T) {
tpFilters = []*utils.TPFilterProfile{
&utils.TPFilterProfile{
TPid: "TP1",
Tenant: "cgrates.org",
ID: "Filter",
Filters: []*utils.TPFilter{
&utils.TPFilter{
Type: "*string",
FieldName: "Account",
Values: []string{"1001", "1002"},
},
},
ActivationInterval: &utils.TPActivationInterval{
ActivationTime: "2014-07-29T15:00:00Z",
ExpiryTime: "",
},
},
}
if err := tpFltrMigrator.storDBIn.SetTPFilters(tpFilters); err != nil {
t.Error("Error when setting TpFilter ", err.Error())
}
currentVersion := engine.CurrentStorDBVersions()
err := tpFltrMigrator.storDBOut.SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for TpFilter ", err.Error())
}
}
func testTpFltrITMove(t *testing.T) {
err, _ := tpFltrMigrator.Migrate([]string{utils.MetaTpFilters})
if err != nil {
t.Error("Error when migrating TpFilter ", err.Error())
}
}
func testTpFltrITCheckData(t *testing.T) {
result, err := tpFltrMigrator.storDBOut.GetTPFilters(
tpFilters[0].TPid, tpFilters[0].ID)
if err != nil {
t.Error("Error when getting TpFilter ", err.Error())
}
if !reflect.DeepEqual(tpFilters[0], result[0]) {
t.Errorf("Expecting: %+v, received: %+v", tpFilters[0], result[0])
}
result, err = tpFltrMigrator.storDBIn.GetTPFilters(
tpFilters[0].TPid, tpFilters[0].ID)
if err != utils.ErrNotFound {
t.Error(err)
}
}

View File

@@ -32,21 +32,29 @@ func (m *Migrator) migrateCurrentTPSuppliers() (err error) {
}
for _, tpid := range tpids {
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPSuppliers, utils.TPDistinctIds{"id"}, map[string]string{}, nil)
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPSuppliers,
utils.TPDistinctIds{"id"}, map[string]string{}, nil)
if err != nil {
return err
}
for _, id := range ids {
dest, err := m.storDBIn.GetTPSuppliers(tpid, id)
suppliers, err := m.storDBIn.GetTPSuppliers(tpid, id)
if err != nil {
return err
}
if dest != nil {
if suppliers != nil {
if m.dryRun != true {
if err := m.storDBOut.SetTPSuppliers(dest); err != nil {
if err := m.storDBOut.SetTPSuppliers(suppliers); err != nil {
return err
}
for _, suppliers := range suppliers {
if err := m.storDBIn.RemTpData(utils.TBLTPSuppliers, attrs.TPid,
map[string]string{"tenant": attrs.Tenant, "id": attrs.ID}); err != nil {
return err
}
}
m.stats[utils.TpSuppliers] += 1
}
}

View File

@@ -32,21 +32,28 @@ func (m *Migrator) migrateCurrentTPthresholds() (err error) {
}
for _, tpid := range tpids {
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPThresholds, utils.TPDistinctIds{"id"}, map[string]string{}, nil)
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPThresholds,
utils.TPDistinctIds{"id"}, map[string]string{}, nil)
if err != nil {
return err
}
for _, id := range ids {
dest, err := m.storDBIn.GetTPThresholds(tpid, id)
thresholds, err := m.storDBIn.GetTPThresholds(tpid, id)
if err != nil {
return err
}
if dest != nil {
if thresholds != nil {
if m.dryRun != true {
if err := m.storDBOut.SetTPThresholds(dest); err != nil {
if err := m.storDBOut.SetTPThresholds(thresholds); err != nil {
return err
}
for _, threshold := range thresholds {
if err := m.storDBIn.RemTpData(utils.TBLTPThresholds, threshold.TPid,
map[string]string{"tenant": threshold.Tenant, "id": threshold.ID}); err != nil {
return err
}
}
m.stats[utils.TpThresholds] += 1
}
}

View File

@@ -32,12 +32,12 @@ func (m *Migrator) migrateCurrentTPTiming() (err error) {
}
for _, tpid := range tpids {
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPTimings, utils.TPDistinctIds{"tag"}, map[string]string{}, nil)
ids, err := m.storDBIn.GetTpTableIds(tpid, utils.TBLTPTimings,
utils.TPDistinctIds{"tag"}, map[string]string{}, nil)
if err != nil {
return err
}
for _, id := range ids {
tm, err := m.storDBIn.GetTPTimings(tpid, id)
if err != nil {
return err
@@ -47,6 +47,12 @@ func (m *Migrator) migrateCurrentTPTiming() (err error) {
if err := m.storDBOut.SetTPTimings(tm); err != nil {
return err
}
for _, timing := range tm {
if err := m.storDBIn.RemTpData(utils.TBLTPTimings,
timing.TPid, map[string]string{"tag": timing.ID}); err != nil {
return err
}
}
m.stats[utils.TpTiming] += 1
}
}

View File

@@ -32,15 +32,21 @@ func (m *Migrator) migrateCurrentTPusers() (err error) {
}
for _, tpid := range tpids {
dest, err := m.storDBIn.GetTPUsers(&utils.TPUsers{TPid: tpid})
users, err := m.storDBIn.GetTPUsers(&utils.TPUsers{TPid: tpid})
if err != nil {
return err
}
if dest != nil {
if users != nil {
if m.dryRun != true {
if err := m.storDBOut.SetTPUsers(dest); err != nil {
if err := m.storDBOut.SetTPUsers(users); err != nil {
return err
}
for _, user := range users {
if err := m.storDBIn.RemTpData(utils.TBLTPUsers, user.TPid,
map[string]string{"tenant": user.Tenant, "user_name": user.UserName}); err != nil {
return err
}
}
m.stats[utils.TpUsers] += 1
}
}

View File

@@ -41,7 +41,7 @@ func (v1ms *v1Mongo) getV1CDR() (v1Cdr *v1Cdrs, err error) {
//set
func (v1ms *v1Mongo) setV1CDR(v1Cdr *v1Cdrs) (err error) {
if err := v1ms.session.DB(v1ms.db).C(engine.ColCDRs).Insert(v1Cdr); err != nil {
if err = v1ms.session.DB(v1ms.db).C(engine.ColCDRs).Insert(v1Cdr); err != nil {
return err
}
return