Update tests

apier/v1/apier.go

@@ -1312,11 +1312,3 @@ func (v1 *ApierV1) SetCacheSConnection(chS rpcclient.RpcClientConnection) {
 func (v1 *ApierV1) SetSchedulerSConnection(schS rpcclient.RpcClientConnection) {
 	v1.SchedulerS = schS
 }
-
-func (v1 *ApierV1) Preload(items []string, reply *string) (err error) {
-	if err = v1.DataManager.Preload(items); err != nil {
-		return
-	}
-	*reply = utils.OK
-	return
-}

config/config.go

@@ -257,7 +257,6 @@ type CGRConfig struct {
 	CdrcProfiles map[string][]*CdrcCfg // Number of CDRC instances running imports, format map[dirPath][]{Configs}
 	loaderCfg    LoaderSCfgs           // LoaderS configs
 	httpAgentCfg HttpAgentCfgs         // HttpAgent configs
-	rmtDataDB    *DataDbCfg            // Remote DataDB, used for preload
 
 	ConfigReloads map[string]chan struct{} // Signals to specific entities that a config reload should occur
 	rldChans      map[string]chan struct{} // index here the channels used for reloads

config/config_json_test.go

@@ -189,15 +189,16 @@ func TestDfListenJsonCfg(t *testing.T) {
 
 func TestDfDataDbJsonCfg(t *testing.T) {
 	eCfg := &DbJsonCfg{
-		Db_type:        utils.StringPointer("*redis"),
-		Db_host:        utils.StringPointer("127.0.0.1"),
-		Db_port:        utils.IntPointer(6379),
-		Db_name:        utils.StringPointer("10"),
-		Db_user:        utils.StringPointer("cgrates"),
-		Db_password:    utils.StringPointer(""),
-		Redis_sentinel: utils.StringPointer(""),
-		Query_timeout:  utils.StringPointer("10s"),
-		Preload_url:    utils.StringPointer(""),
+		Db_type:           utils.StringPointer("*redis"),
+		Db_host:           utils.StringPointer("127.0.0.1"),
+		Db_port:           utils.IntPointer(6379),
+		Db_name:           utils.StringPointer("10"),
+		Db_user:           utils.StringPointer("cgrates"),
+		Db_password:       utils.StringPointer(""),
+		Redis_sentinel:    utils.StringPointer(""),
+		Query_timeout:     utils.StringPointer("10s"),
+		Replicate_db_urls: &[]string{},
+		Remote_db_urls:    &[]string{},
 	}
 	if cfg, err := dfCgrJsonCfg.DbJsonCfg(DATADB_JSN); err != nil {
 		t.Error(err)

config/config_test.go

@@ -420,8 +420,11 @@ func TestCgrCfgJSONDefaultsjsnDataDb(t *testing.T) {
 	if cgrCfg.DataDbCfg().DataDbPass != "" {
 		t.Errorf("Expecting:  , recived: %+v", cgrCfg.DataDbCfg().DataDbPass)
 	}
-	if cgrCfg.DataDbCfg().PreloadURL != "" {
-		t.Errorf("Expecting:  , recived: %+v", cgrCfg.DataDbCfg().PreloadURL)
+	if len(cgrCfg.DataDbCfg().RmtDataDBCfgs) != 0 {
+		t.Errorf("Expecting:  0, recived: %+v", len(cgrCfg.DataDbCfg().RmtDataDBCfgs))
+	}
+	if len(cgrCfg.DataDbCfg().RplDataDBCfgs) != 0 {
+		t.Errorf("Expecting:  0, recived: %+v", len(cgrCfg.DataDbCfg().RplDataDBCfgs))
 	}
 }
 

engine/datamanager.go

@@ -19,8 +19,6 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/utils"
@@ -102,9 +100,10 @@ func NewDataManager(dataDB DataDB, cacheCfg config.CacheCfg) *DataManager {
 // DataManager is the data storage manager for CGRateS
 // transparently manages data retrieval, further serialization and caching
 type DataManager struct {
-	dataDB    DataDB
-	rmtDataDB DataDB
-	cacheCfg  config.CacheCfg
+	dataDB     DataDB
+	rmtDataDBs []DataDB
+	rplDataDBs []DataDB
+	cacheCfg   config.CacheCfg
 }
 
 // DataDB exports access to dataDB
@@ -1432,69 +1431,3 @@ func (dm *DataManager) Reconnect(marshaler string, newcfg *config.DataDbCfg) (er
 	dm.dataDB = d
 	return
 }
-
-func (dm *DataManager) Preload(items []string) (err error) {
-	// sourceDbCfg will be taken directly from config
-	sourceDbCfg, err := config.NewDataDBCfgFromPreloadUrl("")
-	if err != nil {
-		return err
-	}
-	switch sourceDbCfg.DataDbType {
-	case utils.REDIS:
-		var dbNb int
-		dbNb, err = strconv.Atoi(sourceDbCfg.DataDbName)
-		if err != nil {
-			utils.Logger.Crit("Redis db name must be an integer!")
-			return err
-		}
-		host := sourceDbCfg.DataDbHost
-		if sourceDbCfg.DataDbPort != "" && strings.Index(host, ":") == -1 {
-			host += ":" + sourceDbCfg.DataDbPort
-		}
-		dm.rmtDataDB, err = NewRedisStorage(host, dbNb, sourceDbCfg.DataDbPass,
-			config.CgrConfig().GeneralCfg().DBDataEncoding, utils.REDIS_MAX_CONNS, sourceDbCfg.DataDbSentinelName)
-	case utils.MONGO:
-		dm.rmtDataDB, err = NewMongoStorage(sourceDbCfg.DataDbHost, sourceDbCfg.DataDbPort, sourceDbCfg.DataDbName,
-			sourceDbCfg.DataDbUser, sourceDbCfg.DataDbPass, utils.DataDB, nil, true)
-	default:
-		err = fmt.Errorf("unknown db '%s' valid options are '%s' or '%s or '%s'",
-			sourceDbCfg.DataDbType, utils.REDIS, utils.MONGO)
-	}
-	if err != nil {
-		return
-	}
-
-	// based on items load items from source into current datadb
-	if len(items) == 0 {
-		// items = listOfAllItems
-	}
-	var wg sync.WaitGroup // wait for sync to finish
-	errChan := make(chan error)
-	doneChan := make(chan struct{})
-	for _, item := range items {
-		wg.Add(1)
-		go func(item string) {
-			var errSync error
-			switch item {
-			case utils.MetaAttributes:
-				errSync = dm.SyncAttributes()
-			case utils.MetaThresholds:
-				fmt.Println("test")
-			}
-			if errSync != nil {
-				errChan <- errSync
-			}
-			wg.Done()
-		}(item)
-	}
-	time.Sleep(1) // switch context
-	go func() {   // report wg.Wait on doneChan
-		wg.Wait()
-		close(doneChan)
-	}()
-	select {
-	case err = <-errChan:
-	case <-doneChan:
-	}
-	return
-}

engine/dmpreload.go

@@ -1,37 +0,0 @@
-/*
-Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
-Copyright (C) ITsysCOM GmbH
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package engine
-
-import "github.com/cgrates/cgrates/utils"
-
-func (dm *DataManager) SyncAttributes() (err error) {
-
-	keyIDs, err := dm.rmtDataDB.GetKeysForPrefix(utils.AttributeProfilePrefix)
-	if err != nil {
-		return err
-	}
-	for _, keyID := range keyIDs {
-		tntID := utils.NewTenantID(keyID[len(utils.AttributeProfilePrefix):])
-		attr, err := dm.rmtDataDB.GetAttributeProfileDrv(tntID.Tenant, tntID.ID)
-		if err != nil {
-			return err
-		}
-		if err = dm.SetAttributeProfile(attr, true); err != nil {
-			return
-		}
-	}
-	return
-}