Updated migrator *set_version

This commit is contained in:
Trial97
2019-03-08 18:14:22 +02:00
committed by Dan Christian Bogos
parent a9d64b4c97
commit c780baa47f
37 changed files with 243 additions and 64 deletions

View File

@@ -33,12 +33,16 @@ import (
var (
cgrMigratorFlags = flag.NewFlagSet("cgr-migrator", flag.ContinueOnError)
sameDataDB, sameStorDB bool
dmIN, dmOUT migrator.MigratorDataDB
storDBIn, storDBOut migrator.MigratorStorDB
err error
dfltCfg, _ = config.NewDefaultCGRConfig()
cfgPath = cgrMigratorFlags.String("config_path", "",
sameDataDB bool
sameStorDB bool
sameOutDB bool
dmIN migrator.MigratorDataDB
dmOUT migrator.MigratorDataDB
storDBIn migrator.MigratorStorDB
storDBOut migrator.MigratorStorDB
err error
dfltCfg, _ = config.NewDefaultCGRConfig()
cfgPath = cgrMigratorFlags.String("config_path", "",
"Configuration directory path.")
migrate = cgrMigratorFlags.String("migrate", "", "fire up automatic migration "+
@@ -326,9 +330,14 @@ func main() {
log.Fatal(err)
}
sameOutDB = mgrCfg.MigratorCgrCfg().OutStorDBType == mgrCfg.MigratorCgrCfg().OutDataDBType &&
mgrCfg.MigratorCgrCfg().OutStorDBHost == mgrCfg.MigratorCgrCfg().OutDataDBHost &&
mgrCfg.MigratorCgrCfg().OutStorDBPort == mgrCfg.MigratorCgrCfg().OutDataDBPort &&
mgrCfg.MigratorCgrCfg().OutStorDBName == mgrCfg.MigratorCgrCfg().OutDataDBName
m, err := migrator.NewMigrator(dmIN, dmOUT,
storDBIn, storDBOut,
*dryRun, sameDataDB, sameStorDB)
*dryRun, sameDataDB, sameStorDB, sameOutDB)
if err != nil {
log.Fatal(err)
}

View File

@@ -64,9 +64,6 @@ func (self *SQLStorage) Flush(scriptsPath string) (err error) {
if _, err := self.Db.Query(fmt.Sprintf("SELECT 1 FROM %s", utils.CDRsTBL)); err != nil {
return err
}
if err := SetDBVersions(self); err != nil {
return err
}
return nil
}

View File

@@ -137,6 +137,7 @@ func testVersionsFlush(t *testing.T) {
if err := storageDb.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDbCfg().StorDBType)); err != nil {
t.Error(err)
}
SetDBVersions(storageDb)
}
func testVersion(t *testing.T) {

View File

@@ -103,7 +103,7 @@ func testAcc2ITConnect(t *testing.T) {
}
acc2Migrator, err = NewMigrator(dataDBIn, dataDBOut,
storDBIn, storDBOut,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -174,7 +174,7 @@ func testAccITConnect(t *testing.T) {
}
accMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -171,7 +171,7 @@ func testActITConnect(t *testing.T) {
}
actMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -171,7 +171,7 @@ func testActPlnITConnect(t *testing.T) {
}
actPlnMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -91,7 +91,7 @@ func testAlsITConnect(t *testing.T) {
log.Fatal(err)
}
alsMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil, false, false, false)
nil, nil, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -190,7 +190,7 @@ func testAttrITConnect(t *testing.T) {
}
attrMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -95,7 +95,7 @@ func testCdrITConnect(t *testing.T) {
cdrMigrator, err = NewMigrator(nil, nil,
storDBIn, storDBOut,
false, false, false)
false, false, false, false)
if err != nil {
t.Error(err)
}

View File

@@ -92,7 +92,7 @@ func testDCITConnect(t *testing.T) {
log.Fatal(err)
}
dcMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil, false, false, false)
nil, nil, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -138,7 +138,7 @@ func testDspITConnect(t *testing.T) {
}
dspMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -110,7 +110,7 @@ func testFltrITConnect(t *testing.T) {
log.Fatal(err)
}
fltrMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil, false, false, false)
nil, nil, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -26,14 +26,9 @@ import (
"github.com/cgrates/cgrates/utils"
)
func NewMigrator(
dmIN MigratorDataDB,
dmOut MigratorDataDB,
storDBIn MigratorStorDB,
storDBOut MigratorStorDB,
dryRun bool,
sameDataDB bool,
sameStorDB bool) (m *Migrator, err error) {
func NewMigrator(dmIN, dmOut MigratorDataDB,
storDBIn, storDBOut MigratorStorDB,
dryRun, sameDataDB, sameStorDB, sameOutDB bool) (m *Migrator, err error) {
stats := make(map[string]int)
m = &Migrator{
dmOut: dmOut,
@@ -43,6 +38,7 @@ func NewMigrator(
dryRun: dryRun,
sameDataDB: sameDataDB,
sameStorDB: sameStorDB,
sameOutDB: sameOutDB,
stats: stats,
}
return m, err
@@ -58,6 +54,7 @@ type Migrator struct {
dryRun bool
sameDataDB bool
sameStorDB bool
sameOutDB bool
stats map[string]int
}
@@ -72,21 +69,23 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) {
utils.UnsupportedMigrationTask,
fmt.Sprintf("task <%s> is not a supported migration task", taskID))
case utils.MetaSetVersions:
if m.dryRun != true {
if err := engine.OverwriteDBVersions(m.dmOut.DataManager().DataDB()); err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,
err.Error(),
fmt.Sprintf("error: <%s> when updating CostDetails version into StorDB", err.Error())), nil
}
if err := engine.OverwriteDBVersions(m.storDBOut.StorDB()); err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,
err.Error(),
fmt.Sprintf("error: <%s> when updating CostDetails version into StorDB", err.Error())), nil
}
} else {
if m.dryRun {
log.Print("Cannot dryRun SetVersions!")
return
}
err = engine.OverwriteDBVersions(m.dmOut.DataManager().DataDB())
if err != nil {
return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, err.Error(),
fmt.Sprintf("error: <%s> when seting versions for DataDB", err.Error())), nil
}
if m.sameOutDB {
err = engine.SetDBVersions(m.storDBOut.StorDB())
} else {
err = engine.OverwriteDBVersions(m.storDBOut.StorDB())
}
if err != nil {
return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, err.Error(),
fmt.Sprintf("error: <%s> when seting versions for StorDB", err.Error())), nil
}
case utils.MetaCDRs:
err = m.migrateCDRs()

View File

@@ -104,7 +104,7 @@ func testSessionCostITConnect(t *testing.T) {
}
sCostMigrator, err = NewMigrator(nil, nil,
storDBIn, storDBOut,
false, false, false)
false, false, false, false)
if err != nil {
t.Error(err)
}

View File

@@ -153,7 +153,7 @@ func testShrGrpITConnect(t *testing.T) {
}
shrGrpMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -118,7 +118,7 @@ func testStsITConnect(t *testing.T) {
log.Fatal(err)
}
stsMigrator, err = NewMigrator(dataDBIn, dataDBOut, nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -154,7 +154,7 @@ func testTrsITConnect(t *testing.T) {
}
trsMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpAccActITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpAccActMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpAccActMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpActPlnITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpActPlnMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpActPlnMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpActTrgITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpActTrgMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpActTrgMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpActITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpActMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpActMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpDstRtITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpDstRtMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpDstRtMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpDstITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpDstMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpDstMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpDispITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpDispMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpDispMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpFltrITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpFltrMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpFltrMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpRatesITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpRatesMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpRatesMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpRatPlnITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpRatPlnMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpRatPlnMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpRatPrfITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpRatPrfMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpRatPrfMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -90,7 +90,7 @@ func testTpResITConnect(t *testing.T) {
log.Fatal(err)
}
tpResMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut,
false, false, false)
false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpShrGrITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpShrGrMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpShrGrMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpStatsITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpStatsMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpStatsMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -89,7 +89,7 @@ func testTpSplITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpSplMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpSplMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -89,7 +89,7 @@ func testTpTresITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpTresMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpTresMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -88,7 +88,7 @@ func testTpTimITConnect(t *testing.T) {
if err != nil {
log.Fatal(err)
}
tpTimMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false)
tpTimMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -92,7 +92,7 @@ func testUsrITConnect(t *testing.T) {
log.Fatal(err)
}
usrMigrator, err = NewMigrator(dataDBIn, dataDBOut,
nil, nil, false, false, false)
nil, nil, false, false, false, false)
if err != nil {
log.Fatal(err)
}

View File

@@ -0,0 +1,173 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"path"
"reflect"
"testing"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
vrsPath string
vrsSameOutDB bool
vrsCfg *config.CGRConfig
vrsMigrator *Migrator
)
var sTestsVrsIT = []func(t *testing.T){
testVrsITConnect,
testVrsITFlush,
testVrsITMigrate,
}
func TestVersionITRedis(t *testing.T) {
var err error
vrsPath = path.Join(*dataDir, "conf", "samples", "tutmysql")
vrsCfg, err = config.NewCGRConfigFromFolder(vrsPath)
if err != nil {
t.Fatal(err)
}
vrsSameOutDB = false
for _, stest := range sTestsVrsIT {
t.Run("TestVrsionITMigrateRedis", stest)
}
}
func TestVersionITMongo(t *testing.T) {
var err error
vrsPath = path.Join(*dataDir, "conf", "samples", "tutmongo")
vrsCfg, err = config.NewCGRConfigFromFolder(vrsPath)
if err != nil {
t.Fatal(err)
}
vrsCfg.StorDbCfg().StorDBName = vrsCfg.DataDbCfg().DataDbName
vrsSameOutDB = true
for _, stest := range sTestsVrsIT {
t.Run("TestVrsionITMigrateMongo", stest)
}
}
func testVrsITConnect(t *testing.T) {
dataDBOut, err := NewMigratorDataDB(vrsCfg.DataDbCfg().DataDbType,
vrsCfg.DataDbCfg().DataDbHost, vrsCfg.DataDbCfg().DataDbPort,
vrsCfg.DataDbCfg().DataDbName, vrsCfg.DataDbCfg().DataDbUser,
vrsCfg.DataDbCfg().DataDbPass, vrsCfg.GeneralCfg().DBDataEncoding,
config.CgrConfig().CacheCfg(), "")
if err != nil {
t.Fatal(err)
}
storDBOut, err := NewMigratorStorDB(vrsCfg.StorDbCfg().StorDBType,
vrsCfg.StorDbCfg().StorDBHost, vrsCfg.StorDbCfg().StorDBPort,
vrsCfg.StorDbCfg().StorDBName, vrsCfg.StorDbCfg().StorDBUser,
vrsCfg.StorDbCfg().StorDBPass, vrsCfg.StorDbCfg().StorDBMaxOpenConns,
vrsCfg.StorDbCfg().StorDBMaxIdleConns,
vrsCfg.StorDbCfg().StorDBConnMaxLifetime,
vrsCfg.StorDbCfg().StorDBCDRSIndexes)
if err != nil {
t.Error(err)
}
vrsMigrator, err = NewMigrator(nil, dataDBOut, nil, storDBOut,
false, false, false, vrsSameOutDB)
if err != nil {
t.Fatal(err)
}
}
func testVrsITFlush(t *testing.T) {
vrsMigrator.dmOut.DataManager().DataDB().Flush("")
vrsMigrator.storDBOut.StorDB().Flush((path.Join(vrsCfg.DataFolderPath, "storage",
vrsCfg.StorDbCfg().StorDBType)))
if vrs, err := vrsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err == nil || err.Error() != utils.ErrNotFound.Error() {
t.Errorf("Expected err=%s recived err=%v and rply=%s", utils.ErrNotFound.Error(), err, utils.ToJSON(vrs))
}
if vrs, err := vrsMigrator.storDBOut.StorDB().GetVersions(""); err == nil || err.Error() != utils.ErrNotFound.Error() {
t.Errorf("Expected err=%s recived err=%v and rply=%s", utils.ErrNotFound.Error(), err, utils.ToJSON(vrs))
}
}
func testVrsITMigrate(t *testing.T) {
//check if version was set correctly
// var emptyVers engine.Versions
vrsMigrator.Migrate([]string{utils.MetaSetVersions})
if vrsSameOutDB {
expVrs := engine.CurrentAllDBVersions()
if vrs, err := vrsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
} else {
expVrs := engine.CurrentDataDBVersions()
if vrs, err := vrsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
expVrs = engine.CurrentStorDBVersions()
if vrs, err := vrsMigrator.storDBOut.StorDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
}
currentVersion := engine.Versions{Alias: 0}
err := vrsMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version ", err.Error())
}
err = vrsMigrator.storDBOut.StorDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version ", err.Error())
}
vrsMigrator.Migrate([]string{utils.MetaSetVersions})
if vrsSameOutDB {
expVrs := engine.CurrentAllDBVersions()
if vrs, err := vrsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
} else {
expVrs := engine.CurrentDataDBVersions()
if vrs, err := vrsMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
expVrs = engine.CurrentStorDBVersions()
if vrs, err := vrsMigrator.storDBOut.StorDB().GetVersions(""); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expVrs, vrs) {
t.Errorf("Expected %s recived %s", utils.ToJSON(expVrs), utils.ToJSON(vrs))
}
}
}