From 455ed9e5474277772f3b6221f2df654503ebb530 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 5 Nov 2013 20:16:39 +0200 Subject: [PATCH] no precaching for loader (precache parameter) --- cmd/cgr-engine/cgr-engine.go | 2 +- cmd/cgr-loader/cgr-loader.go | 4 ++-- engine/loader_csv_test.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_redis.go | 34 ++++++++++++++++++---------------- engine/storage_utils.go | 12 +++++++----- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 4006e438c..37aae9086 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -312,7 +312,7 @@ func main() { var logDb engine.LogStorage var loadDb engine.LoadStorage var cdrDb engine.CdrStorage - dataDb, err = engine.ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, cfg.DataDBName, cfg.DataDBUser, cfg.DataDBPass, cfg.DBDataEncoding) + dataDb, err = engine.ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, cfg.DataDBName, cfg.DataDBUser, cfg.DataDBPass, cfg.DBDataEncoding, true) if err != nil { // Cannot configure getter database, show stopper engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err)) return diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 8e8a35aa8..47e4465dd 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -72,12 +72,12 @@ func main() { var storDb engine.LoadStorage // Init necessary db connections if *fromStorDb { - dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding) + dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding, false) storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) } else if *toStorDb { // Import from csv files to storDb storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) } else { // Default load from csv files to dataDb - dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding) + dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding, false) } // Defer databases opened to be closed when we are done for _, db := range []engine.Storage{dataDb, storDb} { diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index e36592500..563d129e2 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -134,7 +134,7 @@ func init() { csvr.LoadActionTriggers() csvr.LoadAccountActions() csvr.WriteToDatabase(false, false) - storageGetter.PreCache() + storageGetter.PreCache(nil, nil) } func TestLoadDestinations(t *testing.T) { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 95fb2e7dd..a53be1b94 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -62,7 +62,7 @@ Interface for storage providers. */ type DataStorage interface { Storage - PreCache() error + PreCache([]string, []string) error GetRatingPlan(string) (*RatingPlan, error) SetRatingPlan(*RatingPlan) error GetRatingProfile(string) (*RatingProfile, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 169867df2..891bab511 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -44,7 +44,7 @@ func (ms *MapStorage) Flush() error { return nil } -func (ms *MapStorage) PreCache() error { +func (ms *MapStorage) PreCache(dKeys, rppKeys []string) error { for k, _ := range ms.dict { if strings.HasPrefix(k, DESTINATION_PREFIX) { if _, err := ms.GetDestination(k[len(DESTINATION_PREFIX):]); err != nil { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 3ebc43313..d7dfdb897 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -80,26 +80,28 @@ func (rs *RedisStorage) Flush() (err error) { return } -func (rs *RedisStorage) PreCache() error { - if keys, err := rs.db.Keys(DESTINATION_PREFIX + "*"); err == nil { - for _, key := range keys { - if _, err = rs.GetDestination(key[len(DESTINATION_PREFIX):]); err != nil { - return err - } +func (rs *RedisStorage) PreCache(dKeys, rpKeys []string) (err error) { + if dKeys == nil { + if dKeys, err = rs.db.Keys(DESTINATION_PREFIX + "*"); err != nil { + return } - } else { - return err } - if keys, err := rs.db.Keys(RATING_PLAN_PREFIX + "*"); err == nil { - for _, key := range keys { - if _, err = rs.GetRatingPlan(key[len(RATING_PLAN_PREFIX):]); err != nil { - return err - } + for _, key := range dKeys { + if _, err = rs.GetDestination(key[len(DESTINATION_PREFIX):]); err != nil { + return err } - } else { - return err } - return nil + if rpKeys == nil { + if rpKeys, err = rs.db.Keys(RATING_PLAN_PREFIX + "*"); err != nil { + return + } + } + for _, key := range rpKeys { + if _, err = rs.GetRatingPlan(key[len(RATING_PLAN_PREFIX):]); err != nil { + return err + } + } + return } func (rs *RedisStorage) GetRatingPlan(key string) (rp *RatingPlan, err error) { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index ad5ccdfa8..5c3f8dad4 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -27,7 +27,7 @@ import ( // Various helpers to deal with database -func ConfigureDataStorage(db_type, host, port, name, user, pass, marshaler string) (db DataStorage, err error) { +func ConfigureDataStorage(db_type, host, port, name, user, pass, marshaler string, precache bool) (db DataStorage, err error) { var d Storage switch db_type { case utils.REDIS: @@ -42,11 +42,13 @@ func ConfigureDataStorage(db_type, host, port, name, user, pass, marshaler strin } d, err = NewRedisStorage(host, db_nb, pass, marshaler) db = d.(DataStorage) - Logger.Info("Started redis pre-caching...") - if err := db.PreCache(); err != nil { - Logger.Err(fmt.Sprintf("Pre-caching error: %v", err)) + if precache { + Logger.Info("Started redis pre-caching...") + if err := db.PreCache(nil, nil); err != nil { + Logger.Err(fmt.Sprintf("Pre-caching error: %v", err)) + } + Logger.Info("Pre-caching done!") } - Logger.Info("Pre-caching done!") case utils.MONGO: d, err = NewMongoStorage(host, port, name, user, pass) db = d.(DataStorage)