Get/Set/Remove Versions in StorDB/MySQL; new MySQL table - versions

data/storage/mysql/create_tariffplan_tables.sql

@@ -409,4 +409,14 @@ CREATE TABLE tp_resource_limits (
   UNIQUE KEY `unique_tp_resource_limits` (`tpid`, `tag`)
 );
 
+DROP TABLE IF EXISTS versions;
+CREATE TABLE versions (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `item` varchar(64) NOT NULL,
+  `version` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `item` (`item`)
+);
+
+
 

engine/models.go

@@ -463,3 +463,13 @@ type TpResourceLimit struct {
 	ActionTriggerIds string  `index:"7" re:""`
 	CreatedAt        time.Time
 }
+
+type TBLVersion struct {
+	ID      uint
+	Item    string
+	Version int64
+}
+
+func (t TBLVersion) TableName() string {
+	return utils.TBLVersions
+}

engine/storage_interface.go

@@ -34,6 +34,9 @@ type Storage interface {
 	GetKeysForPrefix(string) ([]string, error)
 	PreloadCacheForPrefix(string) error
 	RebuildReverseForPrefix(string) error
+	GetVersions(itm string) (vrs Versions, err error)
+	SetVersions(vrs Versions) (err error)
+	RemoveVersions(vrs Versions) (err error)
 }
 
 // Interface for storage providers.
@@ -180,6 +183,12 @@ type OnlineStorage interface {
 	CacheDataFromDB(prefix string, IDs []string, mustBeCached bool) error // ToDo: Move this to dataManager
 }
 
+type StorDB interface {
+	CdrStorage
+	LoadReader
+	LoadWriter
+}
+
 type CdrStorage interface {
 	Storage
 	SetCDR(*CDR, bool) error

engine/storage_map.go

@@ -1260,3 +1260,15 @@ func (ms *MapStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs ut
 	cache.Set(dbKey+fieldValKey, itemIDs, true, utils.NonTransactional)
 	return
 }
+
+func (ms *MapStorage) GetVersions(itm string) (vrs Versions, err error) {
+	return
+}
+
+func (ms *MapStorage) SetVersions(vrs Versions) (err error) {
+	return
+}
+
+func (ms *MapStorage) RemoveVersions(vrs Versions) (err error) {
+	return
+}

engine/storage_mongo_stordb.go

@@ -979,3 +979,15 @@ func (ms *MongoStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR,
 	}
 	return cdrs, 0, nil
 }
+
+func (ms *MongoStorage) GetVersions(itm string) (vrs Versions, err error) {
+	return
+}
+
+func (ms *MongoStorage) SetVersions(vrs Versions) (err error) {
+	return
+}
+
+func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) {
+	return
+}

engine/storage_redis.go

@@ -1413,3 +1413,15 @@ func (rs *RedisStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs
 	cache.Set(cacheKey, itemIDs, true, utils.NonTransactional)
 	return
 }
+
+func (rs *RedisStorage) GetVersions(itm string) (vrs Versions, err error) {
+	return
+}
+
+func (rs *RedisStorage) SetVersions(vrs Versions) (err error) {
+	return
+}
+
+func (rs *RedisStorage) RemoveVersions(vrs Versions) (err error) {
+	return
+}

engine/storage_sql.go

@@ -1365,3 +1365,50 @@ func (self *SQLStorage) GetTpResourceLimits(tpid, tag string) (TpResourceLimits,
 	}
 	return tpResourceLimits, nil
 }
+
+// GetVersions returns slice of all versions or a specific version if tag is specified
+func (self *SQLStorage) GetVersions(itm string) (vrs Versions, err error) {
+	q := self.db.Model(&TBLVersion{})
+	if itm != "" {
+		q = self.db.Where(&TBLVersion{Item: itm})
+	}
+	var verModels []*TBLVersion
+	if err = q.Find(&verModels).Error; err != nil {
+		return
+	}
+	vrs = make(Versions)
+	for _, verModel := range verModels {
+		vrs[verModel.Item] = verModel.Version
+	}
+	return
+}
+
+// SetVersions will set a slice of versions, updating existing
+func (self *SQLStorage) SetVersions(vrs Versions) (err error) {
+	tx := self.db.Begin()
+	for key, val := range vrs {
+		vrModel := &TBLVersion{Item: key, Version: val}
+		if err = tx.Save(vrModel).Error; err != nil {
+			if err = tx.Model(&TBLVersion{}).Where(
+				TBLVersion{Item: vrModel.Item}).Updates(TBLVersion{Version: val}).Error; err != nil {
+				tx.Rollback()
+				return
+			}
+		}
+	}
+	tx.Commit()
+	return
+}
+
+// RemoveVersions will remove specific versions out of storage
+func (self *SQLStorage) RemoveVersions(vrs Versions) (err error) {
+	tx := self.db.Begin()
+	for key := range vrs {
+		if err = tx.Delete(&TBLVersion{Item: key}).Error; err != nil {
+			tx.Rollback()
+			return
+		}
+	}
+	tx.Commit()
+	return
+}

engine/stordb_it_test.go

@@ -0,0 +1,79 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package engine
+
+import (
+	"path"
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	cfg    *config.CGRConfig
+	storDB StorDB
+)
+
+// subtests to be executed for each confDIR
+var sTestsStorDBit = []func(t *testing.T){
+	testStorDBitFlush,
+	testStorDBitCRUDVersions,
+}
+
+func TestStorDBitMySQL(t *testing.T) {
+	if cfg, err = config.NewCGRConfigFromFolder(path.Join(*dataDir, "conf", "samples", "storage", "mysql")); err != nil {
+		t.Fatal(err)
+	}
+	if storDB, err = NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName,
+		cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil {
+		t.Fatal(err)
+	}
+	for _, stest := range sTestsStorDBit {
+		t.Run("TestStorDBitMySQL", stest)
+	}
+}
+
+func testStorDBitFlush(t *testing.T) {
+	if err := storDB.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDBType)); err != nil {
+		t.Error(err)
+	}
+}
+
+func testStorDBitCRUDVersions(t *testing.T) {
+	vrs := Versions{utils.COST_DETAILS: 1}
+	if err := storDB.SetVersions(vrs); err != nil {
+		t.Error(err)
+	}
+	if rcv, err := storDB.GetVersions(""); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(vrs, rcv) {
+		t.Errorf("Expecting: %+v, received: %+v", vrs, rcv)
+	}
+	if err := storDB.RemoveVersions(vrs); err != nil {
+		t.Error(err)
+	}
+	if rcv, err := storDB.GetVersions(""); err != nil {
+		t.Error(err)
+	} else if len(rcv) != 0 {
+		t.Errorf("Received: %+v", rcv)
+	}
+}

engine/version.go

@@ -238,3 +238,10 @@ func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) []*MigrationInf
 	}
 	return migrationInfoList
 }
+
+func CurrentStorDBVersions() Versions {
+	return Versions{utils.COST_DETAILS: 1}
+}
+
+// Versions will keep trac of various item versions
+type Versions map[string]int64 // map[item]versionNr

utils/consts.go

@@ -64,6 +64,7 @@ const (
 	TBLSMCosts                    = "sm_costs"
 	TBLTPResourceLimits           = "tp_resource_limits"
 	TBL_CDRS                      = "cdrs"
+	TBLVersions                   = "versions"
 	TIMINGS_CSV                   = "Timings.csv"
 	DESTINATIONS_CSV              = "Destinations.csv"
 	RATES_CSV                     = "Rates.csv"