Refactored some tests and made some changes

cmd/cgr-engine/cgr-engine.go

@@ -720,7 +720,7 @@ func main() {
 		}
 		defer dataDB.Close()
 		engine.SetDataStorage(dataDB)
-		if err := engine.CheckVersion(nil); err != nil {
+		if err := engine.CheckVersions(nil); err != nil {
 			fmt.Println(err.Error())
 			return
 		}

engine/onstor_it_test.go

@@ -1928,37 +1928,23 @@ func testOnStorITCRUDResource(t *testing.T) {
 }
 
 func testOnStorITCRUDStructVersion(t *testing.T) {
-	cv := &StructVersion{
-		Destinations:    "1",
-		RatingPlans:     "1",
-		RatingProfiles:  "1",
-		Lcrs:            "1",
-		DerivedChargers: "1",
-		Actions:         "1",
-		ActionPlans:     "1",
-		ActionTriggers:  "1",
-		SharedGroups:    "1",
-		Accounts:        "1",
-		CdrStats:        "1",
-		Users:           "1",
-		Alias:           "1",
-		PubSubs:         "1",
-		LoadHistory:     "1",
-		Cdrs:            "1",
-		SMCosts:         "1",
-		ResourceConfigs: "1",
-		Timings:         "1",
-	}
-	if _, rcvErr := onStor.GetStructVersion(); rcvErr != utils.ErrNotFound {
+CurrentVersion:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2}
+
+	if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound {
 		t.Error(rcvErr)
 	}
-	if err := onStor.SetStructVersion(CurrentVersion); err != nil {
+	if err := onStor.SetVersions(CurrentVersion,false); err != nil {
 		t.Error(err)
 	}
-	if rcv, err := onStor.GetStructVersion(); err != nil {
+	if rcv, err := onStor.GetVersions(utils.TBLVersions); err != nil {
 		t.Error(err)
-	} else if !reflect.DeepEqual(cv, rcv) {
-		t.Errorf("Expecting: %v, received: %v", cv, rcv)
+	} else if !reflect.DeepEqual(CurrentVersion, rcv) {
+		t.Errorf("Expecting: %v, received: %v", CurrentVersion, rcv)
+	}else if err = onStor.RemoveVersions(rcv); err != nil {
+		t.Error(err)
+	}
+	if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound {
+		t.Error(rcvErr)
 	}
 }
 

engine/storage_interface.go

@@ -109,8 +109,6 @@ type DataDB interface {
 	RemoveTiming(string, string) error
 	GetLoadHistory(int, bool, string) ([]*utils.LoadInstance, error)
 	AddLoadHistory(*utils.LoadInstance, int, string) error
-	//GetStructVersion() (*StructVersion, error)
-	//SetStructVersion(*StructVersion) error
 	GetReqFilterIndexes(dbKey string) (indexes map[string]map[string]utils.StringMap, err error)
 	SetReqFilterIndexes(dbKey string, indexes map[string]map[string]utils.StringMap) (err error)
 	MatchReqFilterIndex(dbKey, fieldName, fieldVal string) (itemIDs utils.StringMap, err error)

engine/storage_mongo_datadb.go

@@ -1833,33 +1833,6 @@ func (ms *MongoStorage) GetAllCdrStats() (css []*CdrStats, err error) {
 	return
 }
 
-// func (ms *MongoStorage) SetStructVersion(v *StructVersion) (err error) {
-// 	session, col := ms.conn(colVer)
-// 	defer session.Close()
-// 	_, err = col.Upsert(bson.M{"key": utils.VERSION_PREFIX + "struct"}, &struct {
-// 		Key   string
-// 		Value *StructVersion
-// 	}{utils.VERSION_PREFIX + "struct", v})
-// 	return
-// }
-
-// func (ms *MongoStorage) GetStructVersion() (rsv *StructVersion, err error) {
-// 	var result struct {
-// 		Key   string
-// 		Value StructVersion
-// 	}
-// 	session, col := ms.conn(colVer)
-// 	defer session.Close()
-// 	if err = col.Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(&result); err != nil {
-// 		if err == mgo.ErrNotFound {
-// 			err = utils.ErrNotFound
-// 		}
-// 		return nil, err
-// 	}
-// 	rsv = &result.Value
-// 	return
-// }
-
 func (ms *MongoStorage) GetResourceCfg(id string, skipCache bool, transactionID string) (rl *ResourceCfg, err error) {
 	key := utils.ResourceConfigsPrefix + id
 	if !skipCache {

engine/storage_mongo_stordb.go

@@ -1203,7 +1203,7 @@ func (ms *MongoStorage) SetTPThreshold(tpTHs []*utils.TPThreshold) (err error) {
 func (ms *MongoStorage) GetVersions(itm string) (vrs Versions, err error) {
 session, col := ms.conn(colVer)
 	defer session.Close()
-	if err = col.Find(bson.M{"key":colVer}).One(vrs); err != nil {
+	if err = col.Find(bson.M{}).One(&vrs); err != nil {
 		if err == mgo.ErrNotFound {
 			err = utils.ErrNotFound
 		}
@@ -1216,23 +1216,25 @@ func (ms *MongoStorage) SetVersions(vrs Versions, overwrite bool) (err error) {
 	session, col := ms.conn(colVer)
 	defer session.Close()
 	if overwrite {
-if err=ms.RemoveVersions(vrs);err!=nil{
+	if err=ms.RemoveVersions(vrs);err!=nil{
 		return err
 		}
+	}	
+	if _, err = col.Upsert(bson.M{},&vrs);err!=nil{
+		return 
 	}
-	_, err = col.Upsert(bson.M{"key":colVer },vrs)
+	
 	return
 }
 
 func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) {
 	session, col := ms.conn(colVer)
 	defer session.Close()
-	for key,_:=range vrs{
-	if err = col.Remove(bson.M{"id": key}); err != nil {
-		return
-	}
+	err = col.Remove(bson.M{})
+	if err == mgo.ErrNotFound {
+			err = utils.ErrNotFound
+		}else{	
+			return err
+		}
 	return nil
-}
-
-	return
-}
+}

engine/storage_redis.go

@@ -1357,28 +1357,6 @@ func (rs *RedisStorage) GetAllCdrStats() (css []*CdrStats, err error) {
 	return
 }
 
-//Remove?
-// func (rs *RedisStorage) SetStructVersion(v *StructVersion) (err error) {
-// 	var result []byte
-// 	result, err = rs.ms.Marshal(v)
-// 	if err != nil {
-// 		return
-// 	}
-// 	return rs.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err
-// }
-
-// func (rs *RedisStorage) GetStructVersion() (rsv *StructVersion, err error) {
-// 	var values []byte
-// 	if values, err = rs.Cmd("GET", utils.VERSION_PREFIX+"struct").Bytes(); err != nil {
-// 		if err == redis.ErrRespNil { // did not find the destination
-// 			err = utils.ErrNotFound
-// 		}
-// 		return
-// 	}
-// 	err = rs.ms.Unmarshal(values, &rsv)
-// 	return
-// }
-
 func (rs *RedisStorage) GetResourceCfg(id string, skipCache bool, transactionID string) (rl *ResourceCfg, err error) {
 	key := utils.ResourceConfigsPrefix + id
 	if !skipCache {
@@ -1604,7 +1582,7 @@ if overwrite{
 
 func (rs *RedisStorage) RemoveVersions(vrs Versions) (err error) {
 for key,_:=range vrs{
-	err = rs.Cmd("HDEL", key).Err
+	err = rs.Cmd("HDEL",utils.TBLVersions, key).Err
 	if err!=nil{
 		return err
 		}

engine/version.go

@@ -25,7 +25,7 @@ import (
 )
 
 func CheckVersions(storage Storage) error {
-x:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2}
+	x:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2}
 	// get current db version
 	if storage == nil {
 		storage = dataStorage

migrator/migrator_it_test.go

@@ -18,15 +18,15 @@ package migrator
 
 import (
 	"flag"
-	//	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"log"
 	"path"
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 var (
@@ -38,40 +38,7 @@ var (
 	dbtype    string
 	mig       *Migrator
 	dataDir   = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
-
-	dataDBType = flag.String("datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database <redis>")
-	dataDBHost = flag.String("datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.")
-	dataDBPort = flag.String("datadb_port", config.CgrConfig().DataDbPort, "The DataDb port to bind to.")
-	dataDBName = flag.String("datadb_name", config.CgrConfig().DataDbName, "The name/number of the DataDb to connect to.")
-	dataDBUser = flag.String("datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.")
-	dataDBPass = flag.String("datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.")
-
-	storDBType = flag.String("stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database <mysql>")
-	storDBHost = flag.String("stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
-	storDBPort = flag.String("stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
-	storDBName = flag.String("stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
-	storDBUser = flag.String("stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
-	storDBPass = flag.String("stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")
-
-	oldDataDBType = flag.String("old_datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database <redis>")
-	oldDataDBHost = flag.String("old_datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.")
-	oldDataDBPort = flag.String("old_datadb_port", config.CgrConfig().DataDbPort, "The DataDb port to bind to.")
-	oldDataDBName = flag.String("old_datadb_name", "11", "The name/number of the DataDb to connect to.")
-	oldDataDBUser = flag.String("old_datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.")
-	oldDataDBPass = flag.String("old_datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.")
-
-	oldStorDBType = flag.String("old_stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database <mysql>")
-	oldStorDBHost = flag.String("old_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
-	oldStorDBPort = flag.String("old_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
-	oldStorDBName = flag.String("old_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
-	oldStorDBUser = flag.String("old_stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
-	oldStorDBPass = flag.String("old_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")
-
 	loadHistorySize    = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history")
-	oldLoadHistorySize = flag.Int("old_load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history")
-
-	dbDataEncoding    = flag.String("dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings")
-	oldDBDataEncoding = flag.String("old_dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings")
 )
 
 // subtests to be executed for each migrator
@@ -85,26 +52,69 @@ var sTestsITMigrator = []func(t *testing.T){
 	testOnStorITFlush,
 }
 
+func TestOnStorITPostgresConnect(t *testing.T) {
+	cdrsPostgresCfgPath := path.Join(*dataDir, "conf", "samples", "tutpostgres")
+	postgresITCfg, err := config.NewCGRConfigFromFolder(cdrsPostgresCfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dataDB, err := engine.ConfigureDataStorage(postgresITCfg.DataDbType, postgresITCfg.DataDbHost, postgresITCfg.DataDbPort, postgresITCfg.DataDbName, postgresITCfg.DataDbUser, postgresITCfg.DataDbPass, postgresITCfg.DBDataEncoding, postgresITCfg.CacheConfig, *loadHistorySize)
+	if err != nil {
+		log.Fatal(err)
+	}
+	oldDataDB, err := ConfigureV1DataStorage(postgresITCfg.DataDbType, postgresITCfg.DataDbHost, postgresITCfg.DataDbPort, postgresITCfg.DataDbName, postgresITCfg.DataDbUser, postgresITCfg.DataDbPass, postgresITCfg.DBDataEncoding)
+	if err != nil {
+		log.Fatal(err)
+	}
+	storDB, err := engine.ConfigureStorStorage(postgresITCfg.StorDBType, postgresITCfg.StorDBHost, postgresITCfg.StorDBPort, postgresITCfg.StorDBName, postgresITCfg.StorDBUser, postgresITCfg.StorDBPass, postgresITCfg.DBDataEncoding,
+		config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	oldstorDB, err := engine.ConfigureStorStorage(postgresITCfg.StorDBType, postgresITCfg.StorDBHost, postgresITCfg.StorDBPort, postgresITCfg.StorDBName, postgresITCfg.StorDBUser, postgresITCfg.StorDBPass, postgresITCfg.DBDataEncoding,
+		config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	mig, err = NewMigrator(dataDB, postgresITCfg.DataDbType, postgresITCfg.DBDataEncoding, storDB, postgresITCfg.StorDBType, oldDataDB, postgresITCfg.DataDbType, postgresITCfg.DBDataEncoding, oldstorDB, postgresITCfg.StorDBType)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func TestOnStorITPostgres(t *testing.T) {
+	dbtype = utils.REDIS
+	for _, stest := range sTestsITMigrator {
+		t.Run("TestITMigratorOnPostgres", stest)
+	}
+}
+
+
 func TestOnStorITRedisConnect(t *testing.T) {
-	dataDB, err := engine.ConfigureDataStorage(*dataDBType, *dataDBHost, *dataDBPort, *dataDBName, *dataDBUser, *dataDBPass, *dbDataEncoding, config.CgrConfig().CacheConfig, *loadHistorySize)
+	cdrsMysqlCfgPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+	mysqlITCfg, err := config.NewCGRConfigFromFolder(cdrsMysqlCfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dataDB, err := engine.ConfigureDataStorage(mysqlITCfg.DataDbType, mysqlITCfg.DataDbHost, mysqlITCfg.DataDbPort, mysqlITCfg.DataDbName, mysqlITCfg.DataDbUser, mysqlITCfg.DataDbPass, mysqlITCfg.DBDataEncoding, mysqlITCfg.CacheConfig, *loadHistorySize)
 	if err != nil {
 		log.Fatal(err)
 	}
-	oldDataDB, err := ConfigureV1DataStorage(*oldDataDBType, *oldDataDBHost, *oldDataDBPort, *oldDataDBName, *oldDataDBUser, *oldDataDBPass, *oldDBDataEncoding)
+	oldDataDB, err := ConfigureV1DataStorage(mysqlITCfg.DataDbType, mysqlITCfg.DataDbHost, mysqlITCfg.DataDbPort, mysqlITCfg.DataDbName, mysqlITCfg.DataDbUser, mysqlITCfg.DataDbPass, mysqlITCfg.DBDataEncoding)
 	if err != nil {
 		log.Fatal(err)
 	}
-	storDB, err := engine.ConfigureStorStorage(*storDBType, *storDBHost, *storDBPort, *storDBName, *storDBUser, *storDBPass, *dbDataEncoding,
+	storDB, err := engine.ConfigureStorStorage(mysqlITCfg.StorDBType, mysqlITCfg.StorDBHost, mysqlITCfg.StorDBPort, mysqlITCfg.StorDBName, mysqlITCfg.StorDBUser, mysqlITCfg.StorDBPass, mysqlITCfg.DBDataEncoding,
 		config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
 	if err != nil {
 		log.Fatal(err)
 	}
-	oldstorDB, err := engine.ConfigureStorStorage(*oldStorDBType, *oldStorDBHost, *oldStorDBPort, *oldStorDBName, *oldStorDBUser, *oldStorDBPass, *oldDBDataEncoding,
+	oldstorDB, err := engine.ConfigureStorStorage(mysqlITCfg.StorDBType, mysqlITCfg.StorDBHost, mysqlITCfg.StorDBPort, mysqlITCfg.StorDBName, mysqlITCfg.StorDBUser, mysqlITCfg.StorDBPass, mysqlITCfg.DBDataEncoding,
 		config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
 	if err != nil {
 		log.Fatal(err)
 	}
-	mig, err = NewMigrator(dataDB, *dataDBType, *dbDataEncoding, storDB, *storDBType, oldDataDB, *oldDataDBType, *oldDBDataEncoding, oldstorDB, *oldStorDBType)
+	mig, err = NewMigrator(dataDB, mysqlITCfg.DataDbType, mysqlITCfg.DBDataEncoding, storDB, mysqlITCfg.StorDBType, oldDataDB, mysqlITCfg.DataDbType, mysqlITCfg.DBDataEncoding, oldstorDB, mysqlITCfg.StorDBType)
 	if err != nil {
 		log.Fatal(err)
 	}