diff --git a/engine/libtest.go b/engine/libtest.go index b04ec85e3..0ede3e3a6 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -43,7 +43,7 @@ func InitDataDb(cfg *config.CGRConfig) error { } dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) // Write version before starting - if err := CheckVersions(dataDB); err != nil { + if err := SetDBVersions(dataDB); err != nil { return err } @@ -59,6 +59,10 @@ func InitStorDb(cfg *config.CGRConfig) error { if err := storDb.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { return err } + // Write version before starting + if err := SetDBVersions(storDb); err != nil { + return err + } return nil } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index e6a26183c..0645337d6 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -43,6 +43,7 @@ var ( // subtests to be executed for each confDIR var sTestsOnStorIT = []func(t *testing.T){ testOnStorITFlush, + testOnStorITIsDBEmpty, testOnStorITSetGetDerivedCharges, testOnStorITSetReqFilterIndexes, testOnStorITGetReqFilterIndexes, @@ -135,6 +136,15 @@ func testOnStorITFlush(t *testing.T) { } cache.Flush() } +func testOnStorITIsDBEmpty(t *testing.T) { + test, err := onStor.IsDBEmpty() + if err != nil { + t.Error(err) + } else if test != true { + t.Errorf("\nExpecting: true got :%+v", test) + } + +} func testOnStorITSetGetDerivedCharges(t *testing.T) { keyCharger1 := utils.ConcatenatedKey("*out", "cgrates.org", "call", "dan", "dan") diff --git a/engine/storage_interface.go b/engine/storage_interface.go index d6c459a89..4c4fc8028 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -40,6 +40,7 @@ type Storage interface { RemoveVersions(vrs Versions) (err error) SelectDatabase(dbName string) (err error) GetStorageType() string + IsDBEmpty() (resp bool, err error) } // OnlineStorage contains methods to use for administering online data diff --git a/engine/storage_map.go b/engine/storage_map.go index 20e094e92..0313ded67 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -292,6 +292,43 @@ func (ms *MapStorage) CacheDataFromDB(prefix string, IDs []string, mustBeCached return } +func (ms *MapStorage) IsDBEmpty() (resp bool, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + var keys []string + prefixes := []string{ + utils.DESTINATION_PREFIX, + utils.REVERSE_DESTINATION_PREFIX, + utils.RATING_PLAN_PREFIX, + utils.RATING_PROFILE_PREFIX, + utils.ACTION_PREFIX, + utils.ACTION_PLAN_PREFIX, + utils.ACTION_TRIGGER_PREFIX, + utils.SHARED_GROUP_PREFIX, + utils.DERIVEDCHARGERS_PREFIX, + utils.LCR_PREFIX, + utils.ACCOUNT_PREFIX, + utils.ALIASES_PREFIX, + utils.REVERSE_ALIASES_PREFIX, + utils.ResourceProfilesPrefix, + utils.ResourcesPrefix, + utils.StatQueuePrefix, // used with CDRStatS + utils.StatQueueProfilePrefix, + utils.AccountActionPlansPrefix, + utils.TimingsPrefix, + } + for _, key := range prefixes { + keys, err = ms.GetKeysForPrefix(key) + if err != nil { + return + } + if len(keys) != 0 { + return false, nil + } + } + return true, nil +} + func (ms *MapStorage) GetKeysForPrefix(prefix string) ([]string, error) { ms.mu.RLock() defer ms.mu.RUnlock() diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index f085ed716..f0955a86f 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -22,16 +22,15 @@ import ( "bytes" "compress/zlib" "fmt" - "io/ioutil" - "strings" - "time" - "github.com/cgrates/cgrates/cache" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/guardian" "github.com/cgrates/cgrates/utils" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" + "io/ioutil" + "strings" + "time" ) const ( @@ -546,6 +545,18 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached return } +func (ms *MongoStorage) IsDBEmpty() (resp bool, err error) { + session := ms.session.Copy() + defer session.Close() + db := session.DB(ms.db) + + col, err := db.CollectionNames() + if err != nil { + return + } + return len(col) == 0, nil +} + func (ms *MongoStorage) GetKeysForPrefix(prefix string) (result []string, err error) { var category, subject string keyLen := len(utils.DESTINATION_PREFIX) diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index da90e63b8..287886393 100755 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -1231,7 +1231,7 @@ func (ms *MongoStorage) SetVersions(vrs Versions, overwrite bool) (err error) { } } if _, err = col.Upsert(bson.M{}, &vrs); err != nil { - return + return err } return @@ -1246,7 +1246,7 @@ func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) { } else { return err } - return nil + return } func (ms *MongoStorage) GetStorageType() string { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 0f3923110..ad5193bbc 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -119,6 +119,42 @@ func (rs *RedisStorage) SelectDatabase(dbName string) (err error) { return rs.Cmd("SELECT", dbName).Err } +func (rs *RedisStorage) IsDBEmpty() (resp bool, err error) { + + var keys []string + prefixes := []string{ + utils.DESTINATION_PREFIX, + utils.REVERSE_DESTINATION_PREFIX, + utils.RATING_PLAN_PREFIX, + utils.RATING_PROFILE_PREFIX, + utils.ACTION_PREFIX, + utils.ACTION_PLAN_PREFIX, + utils.ACTION_TRIGGER_PREFIX, + utils.SHARED_GROUP_PREFIX, + utils.DERIVEDCHARGERS_PREFIX, + utils.LCR_PREFIX, + utils.ACCOUNT_PREFIX, + utils.ALIASES_PREFIX, + utils.REVERSE_ALIASES_PREFIX, + utils.ResourceProfilesPrefix, + utils.ResourcesPrefix, + utils.StatQueuePrefix, // used with CDRStatS + utils.StatQueueProfilePrefix, + utils.AccountActionPlansPrefix, + utils.TimingsPrefix, + } + for _, key := range prefixes { + keys, err = rs.GetKeysForPrefix(key) + if err != nil { + return + } + if len(keys) != 0 { + return false, nil + } + } + return true, nil +} + func (rs *RedisStorage) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aaPlIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rpIDs, resIDs []string) (err error) { for key, ids := range map[string][]string{ diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 80398a7f1..ba2fc5b04 100755 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -60,6 +60,11 @@ func (self *SQLStorage) Flush(scriptsPath string) (err error) { if _, err := self.Db.Query(fmt.Sprintf("SELECT 1 FROM %s", utils.TBLCDRs)); err != nil { return err } + + if err = SetDBVersions(self); err != nil { + return err + } + return nil } @@ -97,6 +102,19 @@ func (self *SQLStorage) CreateTablesFromScript(scriptPath string) error { return nil } +func (self *SQLStorage) IsDBEmpty() (resp bool, err error) { + tbls := []string{utils.TBLTPTimings, utils.TBLTPDestinations, utils.TBLTPRates, utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, utils.TBLTPRateProfiles, + utils.TBLTPSharedGroups, utils.TBLTPCdrStats, utils.TBLTPLcrs, utils.TBLTPActions, utils.TBLTPActionPlans, utils.TBLTPActionTriggers, utils.TBLTPAccountActions, + utils.TBLTPDerivedChargers, utils.TBLTPAliases, utils.TBLTPUsers, utils.TBLTPResources, utils.TBLTPStats} + for _, tbl := range tbls { + resp = self.db.HasTable(tbl) + if resp != false { + return false, nil + } + } + return true, nil +} + // Return a list with all TPids defined in the system, even if incomplete, isolated in some table. func (self *SQLStorage) GetTpIds() ([]string, error) { rows, err := self.Db.Query( diff --git a/engine/stordb_it_test.go b/engine/stordb_it_test.go index b2aae4f4b..91fecce2f 100755 --- a/engine/stordb_it_test.go +++ b/engine/stordb_it_test.go @@ -39,6 +39,7 @@ var ( // subtests to be executed for each confDIR var sTestsStorDBit = []func(t *testing.T){ testStorDBitFlush, + testStorDBitIsDBEmpty, testStorDBitCRUDVersions, testStorDBitCRUDTpTimings, testStorDBitCRUDTpDestinations, @@ -115,6 +116,25 @@ func TestStorDBitMongo(t *testing.T) { } } } +func testStorDBitIsDBEmpty(t *testing.T) { + x := storDB.GetStorageType() + switch x { + case utils.MONGO: + test, err := storDB.IsDBEmpty() + if err != nil { + t.Error(err) + } else if test != true { + t.Errorf("\nExpecting: true got :%+v", test) + } + case utils.POSTGRES, utils.MYSQL: + test, err := storDB.IsDBEmpty() + if err != nil { + t.Error(err) + } else if test != false { + t.Errorf("\nExpecting: false got :%+v", test) + } + } +} func testStorDBitCRUDTpTimings(t *testing.T) { // READ @@ -1534,7 +1554,7 @@ func testStorDBitCRUDTpResources(t *testing.T) { func testStorDBitCRUDTpStats(t *testing.T) { // READ - if _, err := storDB.GetTPStats("testTPid", ""); err != utils.ErrNotFound { + if _, err := storDB.GetTPStats("TEST_TPID", ""); err != utils.ErrNotFound { t.Error(err) } //WRITE @@ -1611,11 +1631,11 @@ func testStorDBitCRUDTpStats(t *testing.T) { } // REMOVE - if err := storDB.RemTpData("", "testTPid", nil); err != nil { + if err := storDB.RemTpData(utils.TBLTPStats, "TEST_TPID", nil); err != nil { t.Error(err) } // READ - if _, err := storDB.GetTPStats("testTPid", ""); err != utils.ErrNotFound { + if _, err := storDB.GetTPStats("TEST_TPID", ""); err != utils.ErrNotFound { t.Error(err) } } diff --git a/engine/version.go b/engine/version.go index b5623f60d..09c2f11e6 100644 --- a/engine/version.go +++ b/engine/version.go @@ -26,16 +26,21 @@ import ( func CheckVersions(storage Storage) error { // get current db version - if storage == nil { - storage = dataStorage - } storType := storage.GetStorageType() x := CurrentDBVersions(storType) dbVersion, err := storage.GetVersions(utils.TBLVersions) if err != nil { + empty, err := storage.IsDBEmpty() + if err != nil { + return err + } + if !empty { + msg := "Migration needed: please backup cgr data and run : " + storType + return errors.New(msg) + } // no data, write version - if err := storage.SetVersions(x, false); err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + if err := SetDBVersions(storage); err != nil { + return err } } else { @@ -50,6 +55,17 @@ func CheckVersions(storage Storage) error { return nil } +func SetDBVersions(storage Storage) error { + storType := storage.GetStorageType() + x := CurrentDBVersions(storType) + // no data, write version + if err := storage.SetVersions(x, false); err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + } + return nil + +} + func (vers Versions) Compare(curent Versions, storType string) string { var x map[string]string m := map[string]string{ @@ -73,9 +89,7 @@ func (vers Versions) Compare(curent Versions, storType string) string { switch storType { case utils.MONGO: x = m - case utils.POSTGRES: - x = stor - case utils.MYSQL: + case utils.POSTGRES, utils.MYSQL: x = stor case utils.REDIS: x = data @@ -91,17 +105,16 @@ func (vers Versions) Compare(curent Versions, storType string) string { } func CurrentDBVersions(storType string) Versions { + dataDbVersions := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} + storDbVersions := Versions{utils.COST_DETAILS: 2} + allVersions := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} switch storType { - case utils.MONGO: - return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - case utils.POSTGRES: - return Versions{utils.COST_DETAILS: 2} - case utils.MYSQL: - return Versions{utils.COST_DETAILS: 2} + case utils.MONGO, utils.MAPSTOR: + return allVersions + case utils.POSTGRES, utils.MYSQL: + return storDbVersions case utils.REDIS: - return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} - case utils.MAPSTOR: - return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + return dataDbVersions } return nil } diff --git a/engine/versions_it_test.go b/engine/versions_it_test.go index 603e57280..6c23fee00 100644 --- a/engine/versions_it_test.go +++ b/engine/versions_it_test.go @@ -38,7 +38,6 @@ var ( var sTestsITVersions = []func(t *testing.T){ testVersionsFlush, TestVersion, - testVersionsFlush, } func TestVersionsITMongoConnect(t *testing.T) { @@ -139,6 +138,13 @@ func testVersionsFlush(t *testing.T) { t.Error(err) } } + if err = SetDBVersions(dataDb); err != nil { + t.Error(err) + } + if err = SetDBVersions(storDB); err != nil { + t.Error(err) + } + } func TestVersion(t *testing.T) { @@ -147,15 +153,11 @@ func TestVersion(t *testing.T) { var testVersion Versions storType := dataDb.GetStorageType() switch storType { - case utils.MONGO: + case utils.MONGO, utils.MAPSTOR: currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} test = "Migration needed: please backup cgr data and run : " - case utils.POSTGRES: - currentVersion = CurrentStorDBVersions() - testVersion = Versions{utils.COST_DETAILS: 1} - test = "Migration needed: please backup cgr data and run : " - case utils.MYSQL: + case utils.POSTGRES, utils.MYSQL: currentVersion = CurrentStorDBVersions() testVersion = Versions{utils.COST_DETAILS: 1} test = "Migration needed: please backup cgr data and run : " @@ -163,15 +165,11 @@ func TestVersion(t *testing.T) { currentVersion = CurrentDataDBVersions() testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} test = "Migration needed: please backup cgr data and run : " - case utils.MAPSTOR: - currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - test = "Migration needed: please backup cgr data and run : " } //dataDB - if _, rcvErr := dataDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { - t.Error(rcvErr) + if err := SetDBVersions(dataDb); err != nil { + t.Error(err) } if err := CheckVersions(dataDb); err != nil { t.Error(err) @@ -196,18 +194,17 @@ func TestVersion(t *testing.T) { if err = dataDb.RemoveVersions(testVersion); err != nil { t.Error(err) } + if err := SetDBVersions(dataDb); err != nil { + t.Error(err) + } storType = storDb.GetStorageType() switch storType { - case utils.MONGO: + case utils.MONGO, utils.MAPSTOR: currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} test = "Migration needed: please backup cgr data and run : " - case utils.POSTGRES: - currentVersion = CurrentStorDBVersions() - testVersion = Versions{utils.COST_DETAILS: 1} - test = "Migration needed: please backup cgr data and run : " - case utils.MYSQL: + case utils.POSTGRES, utils.MYSQL: currentVersion = CurrentStorDBVersions() testVersion = Versions{utils.COST_DETAILS: 1} test = "Migration needed: please backup cgr data and run : " @@ -215,18 +212,15 @@ func TestVersion(t *testing.T) { currentVersion = CurrentDataDBVersions() testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} test = "Migration needed: please backup cgr data and run : " - case utils.MAPSTOR: - currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - test = "Migration needed: please backup cgr data and run : " } //storDB - if _, rcvErr := storDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { - t.Error(rcvErr) + if err := SetDBVersions(storDb); err != nil { + t.Error(err) } if err := CheckVersions(storDb); err != nil { t.Error(err) } + if rcv, err := storDb.GetVersions(utils.TBLVersions); err != nil { t.Error(err) } else if len(currentVersion) != len(rcv) { @@ -247,5 +241,8 @@ func TestVersion(t *testing.T) { if err = storDb.RemoveVersions(testVersion); err != nil { t.Error(err) } + if err := SetDBVersions(storDb); err != nil { + t.Error(err) + } } diff --git a/migrator/migrator_it_test.go b/migrator/migrator_it_test.go index 075f635e8..9f4a88445 100644 --- a/migrator/migrator_it_test.go +++ b/migrator/migrator_it_test.go @@ -172,6 +172,10 @@ func testFlush(t *testing.T) { t.Error("Error when flushing Mongo ", err.Error()) } } + if err = SetDBVersions(mig.dataDB); err != nil { + return err + } + } func testMigratorAccounts(t *testing.T) {