diff --git a/engine/libtest.go b/engine/libtest.go index 0ede3e3a6..33b7cd245 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -42,15 +42,15 @@ func InitDataDb(cfg *config.CGRConfig) error { return err } dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) - // Write version before starting + // Write version before starting if err := SetDBVersions(dataDB); err != nil { return err } - return nil } func InitStorDb(cfg *config.CGRConfig) error { + x := []string{utils.MYSQL, utils.POSTGRES} storDb, err := ConfigureLoadStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBConnMaxLifetime, cfg.StorDBCDRSIndexes) if err != nil { @@ -59,9 +59,10 @@ func InitStorDb(cfg *config.CGRConfig) error { if err := storDb.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { return err } - // Write version before starting - if err := SetDBVersions(storDb); err != nil { - return err + if utils.IsSliceMember(x, cfg.StorDBType) { + if err := SetDBVersions(storDb); err != nil { + return err + } } return nil } diff --git a/engine/storage_map.go b/engine/storage_map.go index e7047eb72..0c6b6d46c 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -147,6 +147,12 @@ func (ms *MapStorage) RebuildReverseForPrefix(prefix string) error { return nil } +func (ms *MapStorage) IsDBEmpty() (resp bool, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + return len(ms.dict) == 0, nil +} + func (ms *MapStorage) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string) (err error) { if ms.cacheCfg == nil { return @@ -292,12 +298,6 @@ func (ms *MapStorage) CacheDataFromDB(prefix string, IDs []string, mustBeCached return } -func (ms *MapStorage) IsDBEmpty() (resp bool, err error) { - ms.mu.RLock() - defer ms.mu.RUnlock() - return len(ms.dict) == 0, nil -} - func (ms *MapStorage) GetKeysForPrefix(prefix string) ([]string, error) { ms.mu.RLock() defer ms.mu.RUnlock() diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index f6345a748..9d285e92e 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -557,12 +557,11 @@ func (ms *MongoStorage) IsDBEmpty() (resp bool, err error) { session := ms.session.Copy() defer session.Close() db := session.DB(ms.db) - - col, err := db.CollectionNames() + cols, err := db.CollectionNames() if err != nil { return } - return len(col) == 0, nil + return len(cols) == 0 || cols[0] == "cdrs", nil } func (ms *MongoStorage) GetKeysForPrefix(prefix string) (result []string, err error) { diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index c26d5370a..8a509e2d3 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -86,6 +86,6 @@ func (self *MySQLStorage) notExtraFieldsValueQry(field, value string) string { return fmt.Sprintf(" extra_fields NOT LIKE '%%\"%s\":\"%s\"%%'", field, value) } -func (self *SQLStorage) GetStorageType() string { +func (self *MySQLStorage) GetStorageType() string { return utils.MYSQL } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 3955bc962..f995115f3 100755 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -60,11 +60,9 @@ func (self *SQLStorage) Flush(scriptsPath string) (err error) { if _, err := self.Db.Query(fmt.Sprintf("SELECT 1 FROM %s", utils.TBLCDRs)); err != nil { return err } - - if err = SetDBVersions(self); err != nil { + if err := SetDBVersions(self); err != nil { return err } - return nil } @@ -1622,7 +1620,6 @@ func (self *SQLStorage) GetVersions(itm string) (vrs Versions, err error) { } if len(vrs) == 0 { return nil, utils.ErrNotFound - } return } diff --git a/engine/version.go b/engine/version.go index 980b2b27b..18df01307 100644 --- a/engine/version.go +++ b/engine/version.go @@ -29,7 +29,7 @@ func CheckVersions(storage Storage) error { storType := storage.GetStorageType() x := CurrentDBVersions(storType) dbVersion, err := storage.GetVersions(utils.TBLVersions) - if err != nil { + if err == utils.ErrNotFound { empty, err := storage.IsDBEmpty() if err != nil { return err @@ -107,7 +107,11 @@ func (vers Versions) Compare(curent Versions, storType string) string { func CurrentDBVersions(storType string) Versions { dataDbVersions := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} storDbVersions := Versions{utils.COST_DETAILS: 2} - allVersions := dataDbVersions + + allVersions := make(Versions) + for k, v := range dataDbVersions { + allVersions[k] = v + } for k, v := range storDbVersions { allVersions[k] = v } diff --git a/engine/versions_it_test.go b/engine/versions_it_test.go index 6c23fee00..f2fb18360 100644 --- a/engine/versions_it_test.go +++ b/engine/versions_it_test.go @@ -37,81 +37,66 @@ var ( var sTestsITVersions = []func(t *testing.T){ testVersionsFlush, - TestVersion, + testVersion, + testVersionsFlush, } -func TestVersionsITMongoConnect(t *testing.T) { - cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "tutmongo") - cfg, err := config.NewCGRConfigFromFolder(cdrsMongoCfgPath) - if err != nil { +func TestVersionsITMongo(t *testing.T) { + var err error + if cfg, err = config.NewCGRConfigFromFolder(path.Join(*dataDir, "conf", "samples", "tutmongo")); err != nil { t.Fatal(err) } - dataDB, err := ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize) - if err != nil { + if dataDb, err = ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, + cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize); err != nil { log.Fatal(err) } - storDB, err := ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, + storageDb, err = ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - storageDb = storDB - dataDb = dataDB -} - -func TestVersionsITMongo(t *testing.T) { dbtype = utils.MONGO for _, stest := range sTestsITVersions { t.Run("TestVersionsITMongo", stest) } } -func TestVersionsITRedisConnect(t *testing.T) { - cdrsMysqlCfgPath := path.Join(*dataDir, "conf", "samples", "tutmysql") - cfg, err := config.NewCGRConfigFromFolder(cdrsMysqlCfgPath) - if err != nil { +func TestVersionsITRedisMYSQL(t *testing.T) { + var err error + if cfg, err = config.NewCGRConfigFromFolder(path.Join(*dataDir, "conf", "samples", "tutmysql")); err != nil { t.Fatal(err) } - dataDB, err := ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize) + dataDb, err = ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize) if err != nil { log.Fatal(err) } - storDB, err := ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, + + storageDb, err = ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - storageDb = storDB - dataDb = dataDB -} - -func TestVersionsITRedis(t *testing.T) { dbtype = utils.REDIS for _, stest := range sTestsITVersions { t.Run("TestVersionsITRedis", stest) } } -func TestVersionsITPostgresConnect(t *testing.T) { - cdrsPostgresCfgPath := path.Join(*dataDir, "conf", "samples", "tutpostgres") - cfg, err := config.NewCGRConfigFromFolder(cdrsPostgresCfgPath) - if err != nil { +func TestVersionsITRedisPostgres(t *testing.T) { + var err error + if cfg, err = config.NewCGRConfigFromFolder(path.Join(*dataDir, "conf", "samples", "storage", "postgres")); err != nil { t.Fatal(err) } - dataDB, err := ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize) + dataDb, err = ConfigureDataStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, *loadHistorySize) if err != nil { log.Fatal(err) } - storDB, err := ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, + storageDb, err = ConfigureStorStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - storageDb = storDB - dataDb = dataDB -} -func TestMigratorITPostgres(t *testing.T) { dbtype = utils.REDIS for _, stest := range sTestsITVersions { t.Run("TestMigratorITPostgres", stest) @@ -119,35 +104,16 @@ func TestMigratorITPostgres(t *testing.T) { } func testVersionsFlush(t *testing.T) { - switch { - case dbtype == utils.REDIS: - dataDB := dataDb.(*RedisStorage) - err := dataDB.Cmd("FLUSHALL").Err - if err != nil { - t.Error("Error when flushing Redis ", err.Error()) - } - if err := storDB.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { - t.Error(err) - } - case dbtype == utils.MONGO: - err := dataDb.Flush("") - if err != nil { - t.Error("Error when flushing Mongo ", err.Error()) - } - if err := storDB.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { - t.Error(err) - } + err := dataDb.Flush("") + if err != nil { + t.Error("Error when flushing Mongo ", err.Error()) } - if err = SetDBVersions(dataDb); err != nil { + if err := storageDb.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { t.Error(err) } - if err = SetDBVersions(storDB); err != nil { - t.Error(err) - } - } -func TestVersion(t *testing.T) { +func testVersion(t *testing.T) { var test string var currentVersion Versions var testVersion Versions @@ -157,10 +123,6 @@ func TestVersion(t *testing.T) { currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} test = "Migration needed: please backup cgr data and run : " - case utils.POSTGRES, utils.MYSQL: - currentVersion = CurrentStorDBVersions() - testVersion = Versions{utils.COST_DETAILS: 1} - test = "Migration needed: please backup cgr data and run : " case utils.REDIS: currentVersion = CurrentDataDBVersions() testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} @@ -168,8 +130,8 @@ func TestVersion(t *testing.T) { } //dataDB - if err := SetDBVersions(dataDb); err != nil { - t.Error(err) + if _, rcvErr := dataDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { + t.Error(rcvErr) } if err := CheckVersions(dataDb); err != nil { t.Error(err) @@ -194,11 +156,7 @@ func TestVersion(t *testing.T) { if err = dataDb.RemoveVersions(testVersion); err != nil { t.Error(err) } - if err := SetDBVersions(dataDb); err != nil { - t.Error(err) - } - - storType = storDb.GetStorageType() + storType = storageDb.GetStorageType() switch storType { case utils.MONGO, utils.MAPSTOR: currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} @@ -208,40 +166,30 @@ func TestVersion(t *testing.T) { currentVersion = CurrentStorDBVersions() testVersion = Versions{utils.COST_DETAILS: 1} test = "Migration needed: please backup cgr data and run : " - case utils.REDIS: - currentVersion = CurrentDataDBVersions() - testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} - test = "Migration needed: please backup cgr data and run : " - } - //storDB - if err := SetDBVersions(storDb); err != nil { - t.Error(err) - } - if err := CheckVersions(storDb); err != nil { - t.Error(err) } + //storageDb - if rcv, err := storDb.GetVersions(utils.TBLVersions); err != nil { + if err := CheckVersions(storageDb); err != nil { + t.Error(err) + } + if rcv, err := storageDb.GetVersions(utils.TBLVersions); err != nil { t.Error(err) } else if len(currentVersion) != len(rcv) { t.Errorf("Expecting: %v, received: %v", currentVersion, rcv) } - if err = storDb.RemoveVersions(currentVersion); err != nil { + if err = storageDb.RemoveVersions(currentVersion); err != nil { t.Error(err) } - if _, rcvErr := storDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { + if _, rcvErr := storageDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := storDb.SetVersions(testVersion, false); err != nil { + if err := storageDb.SetVersions(testVersion, false); err != nil { t.Error(err) } - if err := CheckVersions(storDb); err.Error() != test { + if err := CheckVersions(storageDb); err.Error() != test { t.Error(err) } - if err = storDb.RemoveVersions(testVersion); err != nil { - t.Error(err) - } - if err := SetDBVersions(storDb); err != nil { + if err = storageDb.RemoveVersions(testVersion); err != nil { t.Error(err) }