uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-21 07:08:45 +05:00
Code Issues Packages Projects Releases Wiki Activity

Added move feature(to migrate data from one db to another) and updated GetTPIds , IsDBempty and the versioning accordingly

Browse Source
This commit is contained in:
edwardro22
2017-11-14 13:16:02 +00:00
committed by DanB
parent f43dd4d35d
commit c35b2a90bb
56 changed files with 4180 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
engine/loader_it_test.go
View File

@@ -367,7 +367,7 @@ func TestLoaderITImportToStorDb(t *testing.T) {
if err := csvImporter.Run(); err != nil {
t.Error("Error when importing tpdata to storDb: ", err)
}
if tpids, err := storDb.GetTpIds(); err != nil {
if tpids, err := storDb.GetTpIds(""); err != nil {
t.Error("Error when querying storDb for imported data: ", err)
} else if len(tpids) != 1 || tpids[0] != utils.TEST_SQL {
t.Errorf("Data in storDb is different than expected %v", tpids)

2
engine/storage_csv.go
View File

@@ -705,7 +705,7 @@ func (csvs *CSVStorage) GetTPFilters(tpid, id string) ([]*utils.TPFilter, error)
return tpFilter.AsTPFilter(), nil
}
func (csvs *CSVStorage) GetTpIds() ([]string, error) {
func (csvs *CSVStorage) GetTpIds(x string) ([]string, error) {
return nil, utils.ErrNotImplemented
}

2
engine/storage_interface.go
View File

@@ -153,7 +153,7 @@ type LoadStorage interface {
// LoadReader reads from .csv or TP tables and provides the data ready for the tp_db or data_db.
type LoadReader interface {
GetTpIds() ([]string, error)
GetTpIds(string) ([]string, error)
GetTpTableIds(string, string, utils.TPDistinctIds, map[string]string, *utils.Paginator) ([]string, error)
GetTPTimings(string, string) ([]*utils.ApierTPTiming, error)
GetTPDestinations(string, string) ([]*utils.TPDestination, error)

40
engine/storage_mongo_stordb.go
View File

@@ -28,31 +28,47 @@ import (
"time"
)
func (ms *MongoStorage) GetTpIds() ([]string, error) {
func (ms *MongoStorage) GetTpIds(x string) ([]string, error) {
tpidMap := make(map[string]bool)
session := ms.session.Copy()
db := session.DB(ms.db)
defer session.Close()
cols, err := db.CollectionNames()
if err != nil {
return nil, err
}
for _, col := range cols {
if strings.HasPrefix(col, "tp_") {
var tpids []string
if x != "" {
if strings.HasPrefix(x, "tp_") {
tpids := make([]string, 0)
if err := db.C(col).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
if err := db.C(x).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
return nil, err
}
for _, tpid := range tpids {
tpidMap[tpid] = true
}
}
}
var tpids []string
for tpid := range tpidMap {
tpids = append(tpids, tpid)
for tpid := range tpidMap {
tpids = append(tpids, tpid)
}
} else {
cols, err := db.CollectionNames()
if err != nil {
return nil, err
}
for _, col := range cols {
if strings.HasPrefix(col, "tp_") {
tpids := make([]string, 0)
if err := db.C(col).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
return nil, err
}
for _, tpid := range tpids {
tpidMap[tpid] = true
}
}
}
for tpid := range tpidMap {
tpids = append(tpids, tpid)
}
}
return tpids, nil
}
func (ms *MongoStorage) GetTpTableIds(tpid, table string, distinct utils.TPDistinctIds, filter map[string]string, pag *utils.Paginator) ([]string, error) {

99
engine/storage_sql.go
View File

@@ -101,11 +101,14 @@ func (self *SQLStorage) CreateTablesFromScript(scriptPath string) error {
}
func (self *SQLStorage) IsDBEmpty() (resp bool, err error) {
tbls := []string{utils.TBLTPTimings, utils.TBLTPDestinations, utils.TBLTPRates,
tbls := []string{
utils.TBLTPTimings, utils.TBLTPDestinations, utils.TBLTPRates,
utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, utils.TBLTPRateProfiles,
utils.TBLTPSharedGroups, utils.TBLTPCdrStats, utils.TBLTPLcrs, utils.TBLTPActions,
utils.TBLTPActionPlans, utils.TBLTPActionTriggers, utils.TBLTPAccountActions,
utils.TBLTPDerivedChargers, utils.TBLTPAliases, utils.TBLTPUsers, utils.TBLTPResources, utils.TBLTPStats}
utils.TBLTPActionTriggers, utils.TBLTPAccountActions, utils.TBLTPDerivedChargers, utils.TBLTPUsers,
utils.TBLTPAliases, utils.TBLTPResources, utils.TBLTPStats, utils.TBLTPThresholds,
utils.TBLTPFilters, utils.TBLSMCosts, utils.TBLCDRs, utils.TBLTPActionPlans, utils.TBLVersions,
}
for _, tbl := range tbls {
if self.db.HasTable(tbl) {
return false, nil
@@ -115,35 +118,76 @@ func (self *SQLStorage) IsDBEmpty() (resp bool, err error) {
return true, nil
}
// update
// Return a list with all TPids defined in the system, even if incomplete, isolated in some table.
func (self *SQLStorage) GetTpIds() ([]string, error) {
rows, err := self.Db.Query(
fmt.Sprintf("(SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s)",
utils.TBLTPTimings,
utils.TBLTPDestinations,
utils.TBLTPRates,
utils.TBLTPDestinationRates,
utils.TBLTPRatingPlans,
utils.TBLTPRateProfiles))
if err != nil {
return nil, err
}
defer rows.Close()
ids := make([]string, 0)
i := 0
for rows.Next() {
i++ //Keep here a reference so we know we got at least one
var id string
err = rows.Scan(&id)
func (self *SQLStorage) GetTpIds(x string) ([]string, error) {
if x != "" {
rows, err := self.Db.Query(
fmt.Sprintf(" (SELECT tpid FROM %s)", x))
if err != nil {
return nil, err
}
ids = append(ids, id)
defer rows.Close()
ids := make([]string, 0)
i := 0
for rows.Next() {
i++ //Keep here a reference so we know we got at least one
var id string
err = rows.Scan(&id)
if err != nil {
return nil, err
}
ids = append(ids, id)
}
if i == 0 {
return nil, nil
}
return ids, nil
} else {
rows, err := self.Db.Query(
fmt.Sprintf(
"(SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s)",
utils.TBLTPTimings,
utils.TBLTPDestinations,
utils.TBLTPRates,
utils.TBLTPDestinationRates,
utils.TBLTPRatingPlans,
utils.TBLTPRateProfiles,
utils.TBLTPSharedGroups,
utils.TBLTPCdrStats,
utils.TBLTPLcrs,
utils.TBLTPActions,
utils.TBLTPActionTriggers,
utils.TBLTPAccountActions,
utils.TBLTPDerivedChargers,
utils.TBLTPUsers,
utils.TBLTPAliases,
utils.TBLTPResources,
utils.TBLTPStats,
utils.TBLTPThresholds,
utils.TBLTPFilters,
utils.TBLTPActionPlans))
if err != nil {
return nil, err
}
defer rows.Close()
ids := make([]string, 0)
i := 0
for rows.Next() {
i++ //Keep here a reference so we know we got at least one
var id string
err = rows.Scan(&id)
if err != nil {
return nil, err
}
ids = append(ids, id)
}
if i == 0 {
return nil, nil
}
return ids, nil
}
if i == 0 {
return nil, nil
}
return ids, nil
}
// ToDo: TEST
@@ -206,6 +250,7 @@ func (self *SQLStorage) GetTpTableIds(tpid, table string, distinct utils.TPDisti
func (self *SQLStorage) RemTpData(table, tpid string, args map[string]string) error {
tx := self.db.Begin()
if len(table) == 0 { // Remove tpid out of all tables
for _, tblName := range []string{utils.TBLTPTimings, utils.TBLTPDestinations, utils.TBLTPRates, utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, utils.TBLTPRateProfiles,
utils.TBLTPSharedGroups, utils.TBLTPCdrStats, utils.TBLTPLcrs, utils.TBLTPActions, utils.TBLTPActionPlans, utils.TBLTPActionTriggers, utils.TBLTPAccountActions,

53
engine/version.go
View File

@@ -128,11 +128,60 @@ func CurrentDBVersions(storType string) Versions {
}
func CurrentDataDBVersions() Versions {
return Versions{utils.StatS: 2, utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.Thresholds: 2}
return Versions{
utils.StatS: 2,
utils.Accounts: 2,
utils.Actions: 2,
utils.ActionTriggers: 2,
utils.ActionPlans: 2,
utils.SharedGroups: 2,
utils.Thresholds: 2,
utils.Timing: 2,
utils.RQF: 2,
utils.Resource: 2,
utils.ReverseAlias: 2,
utils.Alias: 2,
utils.User: 2,
utils.Subscribers: 2,
utils.DerivedChargersV: 2,
utils.CdrStats: 2,
utils.Destinations: 2,
utils.ReverseDestinations: 2,
utils.LCR: 2,
utils.RatingPlan: 2,
utils.RatingProfile: 2,
}
}
func CurrentStorDBVersions() Versions {
return Versions{utils.COST_DETAILS: 2}
return Versions{
utils.COST_DETAILS: 2,
utils.TpRatingPlans: 2,
utils.TpLcrs: 2,
utils.TpFilters: 2,
utils.TpDestinationRates: 2,
utils.TpActionTriggers: 2,
utils.TpAccountActions: 2,
utils.TpActionPlans: 2,
utils.TpActions: 2,
utils.TpDerivedCharges: 2,
utils.TpThresholds: 2,
utils.TpStats: 2,
utils.TpSharedGroups: 2,
utils.TpRatingProfiles: 2,
utils.TpResources: 2,
utils.TpRates: 2,
utils.TpTiming: 2,
utils.TpResource: 2,
utils.TpAliases: 2,
utils.TpUsers: 2,
utils.TpDerivedChargersV: 2,
utils.TpCdrStats: 2,
utils.TpDestinations: 2,
utils.TpLCR: 2,
utils.TpRatingPlan: 2,
utils.TpRatingProfile: 2,
}
}
// Versions will keep trac of various item versions

34
engine/versions_it_test.go
View File

@@ -126,16 +126,29 @@ func testVersion(t *testing.T) {
var test string
var currentVersion Versions
var testVersion Versions
dataDbVersions := CurrentDataDBVersions()
storDbVersions := CurrentStorDBVersions()
allVersions := make(Versions)
for k, v := range dataDbVersions {
allVersions[k] = v
}
for k, v := range storDbVersions {
allVersions[k] = v
}
storType := dm3.DataDB().GetStorageType()
switch storType {
case utils.MONGO, utils.MAPSTOR:
currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2,
utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2, utils.CDRs: 2}
testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2}
currentVersion = allVersions
testVersion = allVersions
testVersion[utils.Accounts] = 1
test = "Migration needed: please backup cgr data and run : <cgr-migrator -migrate=*accounts>"
case utils.REDIS:
currentVersion = CurrentDataDBVersions()
testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2}
currentVersion = dataDbVersions
testVersion = dataDbVersions
testVersion[utils.Accounts] = 1
test = "Migration needed: please backup cgr data and run : <cgr-migrator -migrate=*accounts>"
}
@@ -169,14 +182,21 @@ func testVersion(t *testing.T) {
storType = storageDb.GetStorageType()
switch storType {
case utils.MONGO, utils.MAPSTOR:
<<<<<<< HEAD
currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2,
utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2, utils.CDRs: 2}
testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2,
utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2, utils.CDRs: 2}
=======
currentVersion = allVersions
testVersion = allVersions
testVersion[utils.Accounts] = 1
>>>>>>> Added move feature(to migrate data from one db to another) and updated GetTPIds , IsDBempty and the versioning accordingly
test = "Migration needed: please backup cgr data and run : <cgr-migrator -migrate=*accounts>"
case utils.POSTGRES, utils.MYSQL:
currentVersion = CurrentStorDBVersions()
testVersion = Versions{utils.COST_DETAILS: 1}
currentVersion = storDbVersions
testVersion = allVersions
testVersion[utils.COST_DETAILS] = 1
test = "Migration needed: please backup cgr data and run : <cgr-migrator -migrate=*cost_details>"
}
//storageDb