From 7cb965963b8af18a164550424dd1bb6ba5835f50 Mon Sep 17 00:00:00 2001 From: edwardro22 Date: Wed, 6 Sep 2017 14:56:05 +0000 Subject: [PATCH] Refactored some tests and made some changes --- cmd/cgr-engine/cgr-engine.go | 2 +- engine/onstor_it_test.go | 38 +++++--------- engine/storage_interface.go | 2 - engine/storage_mongo_datadb.go | 27 ---------- engine/storage_mongo_stordb.go | 24 +++++---- engine/storage_redis.go | 24 +-------- engine/version.go | 2 +- migrator/migrator_it_test.go | 94 +++++++++++++++++++--------------- 8 files changed, 80 insertions(+), 133 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 46a11931c..327c33e66 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -720,7 +720,7 @@ func main() { } defer dataDB.Close() engine.SetDataStorage(dataDB) - if err := engine.CheckVersion(nil); err != nil { + if err := engine.CheckVersions(nil); err != nil { fmt.Println(err.Error()) return } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 930cd9f63..211cb2f78 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -1928,37 +1928,23 @@ func testOnStorITCRUDResource(t *testing.T) { } func testOnStorITCRUDStructVersion(t *testing.T) { - cv := &StructVersion{ - Destinations: "1", - RatingPlans: "1", - RatingProfiles: "1", - Lcrs: "1", - DerivedChargers: "1", - Actions: "1", - ActionPlans: "1", - ActionTriggers: "1", - SharedGroups: "1", - Accounts: "1", - CdrStats: "1", - Users: "1", - Alias: "1", - PubSubs: "1", - LoadHistory: "1", - Cdrs: "1", - SMCosts: "1", - ResourceConfigs: "1", - Timings: "1", - } - if _, rcvErr := onStor.GetStructVersion(); rcvErr != utils.ErrNotFound { +CurrentVersion:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2} + + if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetStructVersion(CurrentVersion); err != nil { + if err := onStor.SetVersions(CurrentVersion,false); err != nil { t.Error(err) } - if rcv, err := onStor.GetStructVersion(); err != nil { + if rcv, err := onStor.GetVersions(utils.TBLVersions); err != nil { t.Error(err) - } else if !reflect.DeepEqual(cv, rcv) { - t.Errorf("Expecting: %v, received: %v", cv, rcv) + } else if !reflect.DeepEqual(CurrentVersion, rcv) { + t.Errorf("Expecting: %v, received: %v", CurrentVersion, rcv) + }else if err = onStor.RemoveVersions(rcv); err != nil { + t.Error(err) + } + if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { + t.Error(rcvErr) } } diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 9bf78850d..3222cd27e 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -109,8 +109,6 @@ type DataDB interface { RemoveTiming(string, string) error GetLoadHistory(int, bool, string) ([]*utils.LoadInstance, error) AddLoadHistory(*utils.LoadInstance, int, string) error - //GetStructVersion() (*StructVersion, error) - //SetStructVersion(*StructVersion) error GetReqFilterIndexes(dbKey string) (indexes map[string]map[string]utils.StringMap, err error) SetReqFilterIndexes(dbKey string, indexes map[string]map[string]utils.StringMap) (err error) MatchReqFilterIndex(dbKey, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 0452c1257..bd0f0733c 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1833,33 +1833,6 @@ func (ms *MongoStorage) GetAllCdrStats() (css []*CdrStats, err error) { return } -// func (ms *MongoStorage) SetStructVersion(v *StructVersion) (err error) { -// session, col := ms.conn(colVer) -// defer session.Close() -// _, err = col.Upsert(bson.M{"key": utils.VERSION_PREFIX + "struct"}, &struct { -// Key string -// Value *StructVersion -// }{utils.VERSION_PREFIX + "struct", v}) -// return -// } - -// func (ms *MongoStorage) GetStructVersion() (rsv *StructVersion, err error) { -// var result struct { -// Key string -// Value StructVersion -// } -// session, col := ms.conn(colVer) -// defer session.Close() -// if err = col.Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(&result); err != nil { -// if err == mgo.ErrNotFound { -// err = utils.ErrNotFound -// } -// return nil, err -// } -// rsv = &result.Value -// return -// } - func (ms *MongoStorage) GetResourceCfg(id string, skipCache bool, transactionID string) (rl *ResourceCfg, err error) { key := utils.ResourceConfigsPrefix + id if !skipCache { diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index ecef0d210..78a813c3c 100755 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -1203,7 +1203,7 @@ func (ms *MongoStorage) SetTPThreshold(tpTHs []*utils.TPThreshold) (err error) { func (ms *MongoStorage) GetVersions(itm string) (vrs Versions, err error) { session, col := ms.conn(colVer) defer session.Close() - if err = col.Find(bson.M{"key":colVer}).One(vrs); err != nil { + if err = col.Find(bson.M{}).One(&vrs); err != nil { if err == mgo.ErrNotFound { err = utils.ErrNotFound } @@ -1216,23 +1216,25 @@ func (ms *MongoStorage) SetVersions(vrs Versions, overwrite bool) (err error) { session, col := ms.conn(colVer) defer session.Close() if overwrite { -if err=ms.RemoveVersions(vrs);err!=nil{ + if err=ms.RemoveVersions(vrs);err!=nil{ return err } + } + if _, err = col.Upsert(bson.M{},&vrs);err!=nil{ + return } - _, err = col.Upsert(bson.M{"key":colVer },vrs) + return } func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) { session, col := ms.conn(colVer) defer session.Close() - for key,_:=range vrs{ - if err = col.Remove(bson.M{"id": key}); err != nil { - return - } + err = col.Remove(bson.M{}) + if err == mgo.ErrNotFound { + err = utils.ErrNotFound + }else{ + return err + } return nil -} - - return -} +} \ No newline at end of file diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 7b3649563..f73b1326f 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1357,28 +1357,6 @@ func (rs *RedisStorage) GetAllCdrStats() (css []*CdrStats, err error) { return } -//Remove? -// func (rs *RedisStorage) SetStructVersion(v *StructVersion) (err error) { -// var result []byte -// result, err = rs.ms.Marshal(v) -// if err != nil { -// return -// } -// return rs.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err -// } - -// func (rs *RedisStorage) GetStructVersion() (rsv *StructVersion, err error) { -// var values []byte -// if values, err = rs.Cmd("GET", utils.VERSION_PREFIX+"struct").Bytes(); err != nil { -// if err == redis.ErrRespNil { // did not find the destination -// err = utils.ErrNotFound -// } -// return -// } -// err = rs.ms.Unmarshal(values, &rsv) -// return -// } - func (rs *RedisStorage) GetResourceCfg(id string, skipCache bool, transactionID string) (rl *ResourceCfg, err error) { key := utils.ResourceConfigsPrefix + id if !skipCache { @@ -1604,7 +1582,7 @@ if overwrite{ func (rs *RedisStorage) RemoveVersions(vrs Versions) (err error) { for key,_:=range vrs{ - err = rs.Cmd("HDEL", key).Err + err = rs.Cmd("HDEL",utils.TBLVersions, key).Err if err!=nil{ return err } diff --git a/engine/version.go b/engine/version.go index 660a1171c..4d3154136 100644 --- a/engine/version.go +++ b/engine/version.go @@ -25,7 +25,7 @@ import ( ) func CheckVersions(storage Storage) error { -x:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2} + x:=Versions{utils.Accounts: 2,utils.Actions: 2,utils.ActionTriggers: 2,utils.ActionPlans: 2,utils.SharedGroups: 2,utils.COST_DETAILS: 2} // get current db version if storage == nil { storage = dataStorage diff --git a/migrator/migrator_it_test.go b/migrator/migrator_it_test.go index 5bf01cc11..4711072df 100644 --- a/migrator/migrator_it_test.go +++ b/migrator/migrator_it_test.go @@ -18,15 +18,15 @@ package migrator import ( "flag" - // "fmt" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" "log" "path" "reflect" "testing" "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) var ( @@ -38,40 +38,7 @@ var ( dbtype string mig *Migrator dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") - - dataDBType = flag.String("datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database ") - dataDBHost = flag.String("datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.") - dataDBPort = flag.String("datadb_port", config.CgrConfig().DataDbPort, "The DataDb port to bind to.") - dataDBName = flag.String("datadb_name", config.CgrConfig().DataDbName, "The name/number of the DataDb to connect to.") - dataDBUser = flag.String("datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.") - dataDBPass = flag.String("datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.") - - storDBType = flag.String("stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database ") - storDBHost = flag.String("stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.") - storDBPort = flag.String("stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.") - storDBName = flag.String("stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.") - storDBUser = flag.String("stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.") - storDBPass = flag.String("stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.") - - oldDataDBType = flag.String("old_datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database ") - oldDataDBHost = flag.String("old_datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.") - oldDataDBPort = flag.String("old_datadb_port", config.CgrConfig().DataDbPort, "The DataDb port to bind to.") - oldDataDBName = flag.String("old_datadb_name", "11", "The name/number of the DataDb to connect to.") - oldDataDBUser = flag.String("old_datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.") - oldDataDBPass = flag.String("old_datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.") - - oldStorDBType = flag.String("old_stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database ") - oldStorDBHost = flag.String("old_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.") - oldStorDBPort = flag.String("old_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.") - oldStorDBName = flag.String("old_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.") - oldStorDBUser = flag.String("old_stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.") - oldStorDBPass = flag.String("old_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.") - loadHistorySize = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history") - oldLoadHistorySize = flag.Int("old_load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history") - - dbDataEncoding = flag.String("dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings") - oldDBDataEncoding = flag.String("old_dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings") ) // subtests to be executed for each migrator @@ -85,26 +52,69 @@ var sTestsITMigrator = []func(t *testing.T){ testOnStorITFlush, } +func TestOnStorITPostgresConnect(t *testing.T) { + cdrsPostgresCfgPath := path.Join(*dataDir, "conf", "samples", "tutpostgres") + postgresITCfg, err := config.NewCGRConfigFromFolder(cdrsPostgresCfgPath) + if err != nil { + t.Fatal(err) + } + dataDB, err := engine.ConfigureDataStorage(postgresITCfg.DataDbType, postgresITCfg.DataDbHost, postgresITCfg.DataDbPort, postgresITCfg.DataDbName, postgresITCfg.DataDbUser, postgresITCfg.DataDbPass, postgresITCfg.DBDataEncoding, postgresITCfg.CacheConfig, *loadHistorySize) + if err != nil { + log.Fatal(err) + } + oldDataDB, err := ConfigureV1DataStorage(postgresITCfg.DataDbType, postgresITCfg.DataDbHost, postgresITCfg.DataDbPort, postgresITCfg.DataDbName, postgresITCfg.DataDbUser, postgresITCfg.DataDbPass, postgresITCfg.DBDataEncoding) + if err != nil { + log.Fatal(err) + } + storDB, err := engine.ConfigureStorStorage(postgresITCfg.StorDBType, postgresITCfg.StorDBHost, postgresITCfg.StorDBPort, postgresITCfg.StorDBName, postgresITCfg.StorDBUser, postgresITCfg.StorDBPass, postgresITCfg.DBDataEncoding, + config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) + if err != nil { + log.Fatal(err) + } + oldstorDB, err := engine.ConfigureStorStorage(postgresITCfg.StorDBType, postgresITCfg.StorDBHost, postgresITCfg.StorDBPort, postgresITCfg.StorDBName, postgresITCfg.StorDBUser, postgresITCfg.StorDBPass, postgresITCfg.DBDataEncoding, + config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) + if err != nil { + log.Fatal(err) + } + mig, err = NewMigrator(dataDB, postgresITCfg.DataDbType, postgresITCfg.DBDataEncoding, storDB, postgresITCfg.StorDBType, oldDataDB, postgresITCfg.DataDbType, postgresITCfg.DBDataEncoding, oldstorDB, postgresITCfg.StorDBType) + if err != nil { + log.Fatal(err) + } +} + +func TestOnStorITPostgres(t *testing.T) { + dbtype = utils.REDIS + for _, stest := range sTestsITMigrator { + t.Run("TestITMigratorOnPostgres", stest) + } +} + + func TestOnStorITRedisConnect(t *testing.T) { - dataDB, err := engine.ConfigureDataStorage(*dataDBType, *dataDBHost, *dataDBPort, *dataDBName, *dataDBUser, *dataDBPass, *dbDataEncoding, config.CgrConfig().CacheConfig, *loadHistorySize) + cdrsMysqlCfgPath := path.Join(*dataDir, "conf", "samples", "tutmysql") + mysqlITCfg, err := config.NewCGRConfigFromFolder(cdrsMysqlCfgPath) + if err != nil { + t.Fatal(err) + } + dataDB, err := engine.ConfigureDataStorage(mysqlITCfg.DataDbType, mysqlITCfg.DataDbHost, mysqlITCfg.DataDbPort, mysqlITCfg.DataDbName, mysqlITCfg.DataDbUser, mysqlITCfg.DataDbPass, mysqlITCfg.DBDataEncoding, mysqlITCfg.CacheConfig, *loadHistorySize) if err != nil { log.Fatal(err) } - oldDataDB, err := ConfigureV1DataStorage(*oldDataDBType, *oldDataDBHost, *oldDataDBPort, *oldDataDBName, *oldDataDBUser, *oldDataDBPass, *oldDBDataEncoding) + oldDataDB, err := ConfigureV1DataStorage(mysqlITCfg.DataDbType, mysqlITCfg.DataDbHost, mysqlITCfg.DataDbPort, mysqlITCfg.DataDbName, mysqlITCfg.DataDbUser, mysqlITCfg.DataDbPass, mysqlITCfg.DBDataEncoding) if err != nil { log.Fatal(err) } - storDB, err := engine.ConfigureStorStorage(*storDBType, *storDBHost, *storDBPort, *storDBName, *storDBUser, *storDBPass, *dbDataEncoding, + storDB, err := engine.ConfigureStorStorage(mysqlITCfg.StorDBType, mysqlITCfg.StorDBHost, mysqlITCfg.StorDBPort, mysqlITCfg.StorDBName, mysqlITCfg.StorDBUser, mysqlITCfg.StorDBPass, mysqlITCfg.DBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - oldstorDB, err := engine.ConfigureStorStorage(*oldStorDBType, *oldStorDBHost, *oldStorDBPort, *oldStorDBName, *oldStorDBUser, *oldStorDBPass, *oldDBDataEncoding, + oldstorDB, err := engine.ConfigureStorStorage(mysqlITCfg.StorDBType, mysqlITCfg.StorDBHost, mysqlITCfg.StorDBPort, mysqlITCfg.StorDBName, mysqlITCfg.StorDBUser, mysqlITCfg.StorDBPass, mysqlITCfg.DBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - mig, err = NewMigrator(dataDB, *dataDBType, *dbDataEncoding, storDB, *storDBType, oldDataDB, *oldDataDBType, *oldDBDataEncoding, oldstorDB, *oldStorDBType) + mig, err = NewMigrator(dataDB, mysqlITCfg.DataDbType, mysqlITCfg.DBDataEncoding, storDB, mysqlITCfg.StorDBType, oldDataDB, mysqlITCfg.DataDbType, mysqlITCfg.DBDataEncoding, oldstorDB, mysqlITCfg.StorDBType) if err != nil { log.Fatal(err) }