From 0bd2c32f5c55a72ed3f25d293dae673efc1b0221 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 6 Dec 2016 19:02:28 +0100 Subject: [PATCH] Get/Set/Remove Versions in StorDB/MySQL; new MySQL table - versions --- .../mysql/create_tariffplan_tables.sql | 10 +++ engine/models.go | 10 +++ engine/storage_interface.go | 9 +++ engine/storage_map.go | 12 +++ engine/storage_mongo_stordb.go | 12 +++ engine/storage_redis.go | 12 +++ engine/storage_sql.go | 47 +++++++++++ engine/stordb_it_test.go | 79 +++++++++++++++++++ engine/version.go | 7 ++ utils/consts.go | 1 + 10 files changed, 199 insertions(+) create mode 100644 engine/stordb_it_test.go diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 6d1f184c5..945ce4106 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -409,4 +409,14 @@ CREATE TABLE tp_resource_limits ( UNIQUE KEY `unique_tp_resource_limits` (`tpid`, `tag`) ); +DROP TABLE IF EXISTS versions; +CREATE TABLE versions ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `item` varchar(64) NOT NULL, + `version` int(11) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `item` (`item`) +); + + diff --git a/engine/models.go b/engine/models.go index 48da27f74..d87e24729 100644 --- a/engine/models.go +++ b/engine/models.go @@ -463,3 +463,13 @@ type TpResourceLimit struct { ActionTriggerIds string `index:"7" re:""` CreatedAt time.Time } + +type TBLVersion struct { + ID uint + Item string + Version int64 +} + +func (t TBLVersion) TableName() string { + return utils.TBLVersions +} diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 86332f6f1..a436cb8e0 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -34,6 +34,9 @@ type Storage interface { GetKeysForPrefix(string) ([]string, error) PreloadCacheForPrefix(string) error RebuildReverseForPrefix(string) error + GetVersions(itm string) (vrs Versions, err error) + SetVersions(vrs Versions) (err error) + RemoveVersions(vrs Versions) (err error) } // Interface for storage providers. @@ -180,6 +183,12 @@ type OnlineStorage interface { CacheDataFromDB(prefix string, IDs []string, mustBeCached bool) error // ToDo: Move this to dataManager } +type StorDB interface { + CdrStorage + LoadReader + LoadWriter +} + type CdrStorage interface { Storage SetCDR(*CDR, bool) error diff --git a/engine/storage_map.go b/engine/storage_map.go index 19390acdf..716954b8d 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1260,3 +1260,15 @@ func (ms *MapStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs ut cache.Set(dbKey+fieldValKey, itemIDs, true, utils.NonTransactional) return } + +func (ms *MapStorage) GetVersions(itm string) (vrs Versions, err error) { + return +} + +func (ms *MapStorage) SetVersions(vrs Versions) (err error) { + return +} + +func (ms *MapStorage) RemoveVersions(vrs Versions) (err error) { + return +} diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index 5e09b748f..cd90eb7b1 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -979,3 +979,15 @@ func (ms *MongoStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, } return cdrs, 0, nil } + +func (ms *MongoStorage) GetVersions(itm string) (vrs Versions, err error) { + return +} + +func (ms *MongoStorage) SetVersions(vrs Versions) (err error) { + return +} + +func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) { + return +} diff --git a/engine/storage_redis.go b/engine/storage_redis.go index da50216fc..7df455003 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1413,3 +1413,15 @@ func (rs *RedisStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs cache.Set(cacheKey, itemIDs, true, utils.NonTransactional) return } + +func (rs *RedisStorage) GetVersions(itm string) (vrs Versions, err error) { + return +} + +func (rs *RedisStorage) SetVersions(vrs Versions) (err error) { + return +} + +func (rs *RedisStorage) RemoveVersions(vrs Versions) (err error) { + return +} diff --git a/engine/storage_sql.go b/engine/storage_sql.go index d1e8c9c7c..518898935 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -1365,3 +1365,50 @@ func (self *SQLStorage) GetTpResourceLimits(tpid, tag string) (TpResourceLimits, } return tpResourceLimits, nil } + +// GetVersions returns slice of all versions or a specific version if tag is specified +func (self *SQLStorage) GetVersions(itm string) (vrs Versions, err error) { + q := self.db.Model(&TBLVersion{}) + if itm != "" { + q = self.db.Where(&TBLVersion{Item: itm}) + } + var verModels []*TBLVersion + if err = q.Find(&verModels).Error; err != nil { + return + } + vrs = make(Versions) + for _, verModel := range verModels { + vrs[verModel.Item] = verModel.Version + } + return +} + +// SetVersions will set a slice of versions, updating existing +func (self *SQLStorage) SetVersions(vrs Versions) (err error) { + tx := self.db.Begin() + for key, val := range vrs { + vrModel := &TBLVersion{Item: key, Version: val} + if err = tx.Save(vrModel).Error; err != nil { + if err = tx.Model(&TBLVersion{}).Where( + TBLVersion{Item: vrModel.Item}).Updates(TBLVersion{Version: val}).Error; err != nil { + tx.Rollback() + return + } + } + } + tx.Commit() + return +} + +// RemoveVersions will remove specific versions out of storage +func (self *SQLStorage) RemoveVersions(vrs Versions) (err error) { + tx := self.db.Begin() + for key := range vrs { + if err = tx.Delete(&TBLVersion{Item: key}).Error; err != nil { + tx.Rollback() + return + } + } + tx.Commit() + return +} diff --git a/engine/stordb_it_test.go b/engine/stordb_it_test.go new file mode 100644 index 000000000..e6ac225e4 --- /dev/null +++ b/engine/stordb_it_test.go @@ -0,0 +1,79 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package engine + +import ( + "path" + "reflect" + "testing" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/utils" +) + +var ( + cfg *config.CGRConfig + storDB StorDB +) + +// subtests to be executed for each confDIR +var sTestsStorDBit = []func(t *testing.T){ + testStorDBitFlush, + testStorDBitCRUDVersions, +} + +func TestStorDBitMySQL(t *testing.T) { + if cfg, err = config.NewCGRConfigFromFolder(path.Join(*dataDir, "conf", "samples", "storage", "mysql")); err != nil { + t.Fatal(err) + } + if storDB, err = NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, + cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { + t.Fatal(err) + } + for _, stest := range sTestsStorDBit { + t.Run("TestStorDBitMySQL", stest) + } +} + +func testStorDBitFlush(t *testing.T) { + if err := storDB.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil { + t.Error(err) + } +} + +func testStorDBitCRUDVersions(t *testing.T) { + vrs := Versions{utils.COST_DETAILS: 1} + if err := storDB.SetVersions(vrs); err != nil { + t.Error(err) + } + if rcv, err := storDB.GetVersions(""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(vrs, rcv) { + t.Errorf("Expecting: %+v, received: %+v", vrs, rcv) + } + if err := storDB.RemoveVersions(vrs); err != nil { + t.Error(err) + } + if rcv, err := storDB.GetVersions(""); err != nil { + t.Error(err) + } else if len(rcv) != 0 { + t.Errorf("Received: %+v", rcv) + } +} diff --git a/engine/version.go b/engine/version.go index 234c8a1eb..57d7a86b4 100644 --- a/engine/version.go +++ b/engine/version.go @@ -238,3 +238,10 @@ func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) []*MigrationInf } return migrationInfoList } + +func CurrentStorDBVersions() Versions { + return Versions{utils.COST_DETAILS: 1} +} + +// Versions will keep trac of various item versions +type Versions map[string]int64 // map[item]versionNr diff --git a/utils/consts.go b/utils/consts.go index 952f90069..19768633e 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -64,6 +64,7 @@ const ( TBLSMCosts = "sm_costs" TBLTPResourceLimits = "tp_resource_limits" TBL_CDRS = "cdrs" + TBLVersions = "versions" TIMINGS_CSV = "Timings.csv" DESTINATIONS_CSV = "Destinations.csv" RATES_CSV = "Rates.csv"