From 747af43f8d32ae60ab2cf5e69e3413803de9e679 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 15 Aug 2016 23:05:55 +0300 Subject: [PATCH] initialize cache with config object --- cache2go/cache.go | 24 ++++---- cache2go/cache_store.go | 89 +++++++++++++++++++++--------- cmd/cgr-engine/cgr-engine.go | 7 ++- config/cacheconfig.go | 70 ++++++++++++++++++++++- config/config.go | 18 ++++-- config/config_json_test.go | 3 +- config/libconfig_json.go | 1 - data/conf/cgrates/cgrates.json | 29 +++++----- engine/calldesc.go | 9 +-- engine/libtest.go | 4 +- engine/loader_local_test.go | 12 ++-- engine/storage_map.go | 89 ++++++++++++++++++++++++++---- engine/storage_mongo_datadb.go | 79 ++++++++++++++++++++++++-- engine/storage_redis.go | 80 ++++++++++++++++++++++++--- engine/storage_redis_local_test.go | 2 +- engine/storage_utils.go | 19 ++++--- 16 files changed, 426 insertions(+), 109 deletions(-) diff --git a/cache2go/cache.go b/cache2go/cache.go index 7389a169b..fd0d332e1 100644 --- a/cache2go/cache.go +++ b/cache2go/cache.go @@ -1,7 +1,11 @@ //Simple caching library with expiration capabilities package cache2go -import "sync" +import ( + "sync" + + "github.com/cgrates/cgrates/config" +) const ( PREFIX_LEN = 4 @@ -14,6 +18,7 @@ const ( var ( mux sync.RWMutex cache cacheStore + cfg *config.CacheConfig // transaction stuff transactionBuffer []*transactionItem transactionMux sync.Mutex @@ -28,11 +33,12 @@ type transactionItem struct { } func init() { - if DOUBLE_CACHE { - cache = newLRUTTLStore() - } else { - cache = newSimpleStore() - } + NewCache(nil) +} + +func NewCache(cacheCfg *config.CacheConfig) { + cfg = cacheCfg + cache = newLRUTTL(cfg) } func BeginTransaction() { @@ -117,11 +123,7 @@ func RemPrefixKey(prefix string) { func Flush() { mux.Lock() defer mux.Unlock() - if DOUBLE_CACHE { - cache = newDoubleStore() - } else { - cache = newSimpleStore() - } + cache = newLRUTTL(cfg) } func CountEntries(prefix string) (result int) { diff --git a/cache2go/cache_store.go b/cache2go/cache_store.go index 72069d3ce..c19b12bd1 100644 --- a/cache2go/cache_store.go +++ b/cache2go/cache_store.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) @@ -92,35 +93,69 @@ func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) { return } -func newLRUTTLStore() cacheLRUTTL { - return newLRUTTL(map[string]*cacheParam{ - utils.DESTINATION_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.RATING_PLAN_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.RATING_PROFILE_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.ACTION_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.ACTION_PLAN_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.ACTION_TRIGGER_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.ALIASES_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.REVERSE_ALIASES_PREFIX: &cacheParam{limit: 10000, expiration: 0}, - utils.REVERSE_DESTINATION_PREFIX: &cacheParam{limit: 100000, expiration: 0}, - }) -} - -type cacheParam struct { - limit int - expiration time.Duration -} - -func (ct *cacheParam) createCache() *Cache { - return NewLRUTTL(ct.limit, ct.expiration) -} - type cacheLRUTTL map[string]*Cache -func newLRUTTL(types map[string]*cacheParam) cacheLRUTTL { - c := make(map[string]*Cache, len(types)) - for prefix, param := range types { - c[prefix] = param.createCache() +func newLRUTTL(cfg *config.CacheConfig) cacheLRUTTL { + c := make(map[string]*Cache) + if cfg != nil && cfg.Destinations != nil { + c[utils.DESTINATION_PREFIX] = NewLRUTTL(cfg.Destinations.Limit, cfg.Destinations.TTL) + } else { + c[utils.DESTINATION_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.ReverseDestinations != nil { + c[utils.REVERSE_DESTINATION_PREFIX] = NewLRUTTL(cfg.ReverseDestinations.Limit, cfg.ReverseDestinations.TTL) + } else { + c[utils.REVERSE_DESTINATION_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.RatingPlans != nil { + c[utils.RATING_PLAN_PREFIX] = NewLRUTTL(cfg.RatingPlans.Limit, cfg.RatingPlans.TTL) + } else { + c[utils.RATING_PLAN_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.RatingProfiles != nil { + c[utils.RATING_PROFILE_PREFIX] = NewLRUTTL(cfg.RatingProfiles.Limit, cfg.RatingProfiles.TTL) + } else { + c[utils.RATING_PROFILE_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.Lcr != nil { + c[utils.LCR_PREFIX] = NewLRUTTL(cfg.Lcr.Limit, cfg.Lcr.TTL) + } else { + c[utils.LCR_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.CdrStats != nil { + c[utils.CDR_STATS_PREFIX] = NewLRUTTL(cfg.CdrStats.Limit, cfg.CdrStats.TTL) + } else { + c[utils.CDR_STATS_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.Actions != nil { + c[utils.ACTION_PREFIX] = NewLRUTTL(cfg.Actions.Limit, cfg.Actions.TTL) + } else { + c[utils.ACTION_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.ActionPlans != nil { + c[utils.ACTION_PLAN_PREFIX] = NewLRUTTL(cfg.ActionPlans.Limit, cfg.ActionPlans.TTL) + } else { + c[utils.ACTION_PLAN_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.ActionTriggers != nil { + c[utils.ACTION_TRIGGER_PREFIX] = NewLRUTTL(cfg.ActionTriggers.Limit, cfg.ActionTriggers.TTL) + } else { + c[utils.ACTION_TRIGGER_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.SharedGroups != nil { + c[utils.SHARED_GROUP_PREFIX] = NewLRUTTL(cfg.SharedGroups.Limit, cfg.SharedGroups.TTL) + } else { + c[utils.SHARED_GROUP_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.Aliases != nil { + c[utils.ALIASES_PREFIX] = NewLRUTTL(cfg.Aliases.Limit, cfg.Aliases.TTL) + } else { + c[utils.ALIASES_PREFIX] = NewLRUTTL(10000, 24*time.Hour) + } + if cfg != nil && cfg.ReverseAliases != nil { + c[utils.REVERSE_ALIASES_PREFIX] = NewLRUTTL(cfg.ReverseAliases.Limit, cfg.ReverseAliases.TTL) + } else { + c[utils.REVERSE_ALIASES_PREFIX] = NewLRUTTL(10000, 24*time.Hour) } return c diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 34624c3a6..0391dea1e 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -33,6 +33,7 @@ import ( "github.com/cgrates/cgrates/apier/v1" "github.com/cgrates/cgrates/apier/v2" "github.com/cgrates/cgrates/balancer2go" + "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/cdrc" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -533,6 +534,8 @@ func main() { return } config.SetCgrConfig(cfg) // Share the config object + cache2go.NewCache(cfg.CacheConfig) + if *raterEnabled { cfg.RALsEnabled = *raterEnabled } @@ -548,7 +551,7 @@ func main() { var cdrDb engine.CdrStorage if cfg.RALsEnabled || cfg.SchedulerEnabled || cfg.CDRStatsEnabled { // Only connect to dataDb if necessary ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, - cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize) + cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize) if err != nil { // Cannot configure getter database, show stopper utils.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err)) return @@ -558,7 +561,7 @@ func main() { } if cfg.RALsEnabled || cfg.CDRStatsEnabled || cfg.PubSubServerEnabled || cfg.AliasesServerEnabled || cfg.UserServerEnabled { accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, - cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize) + cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize) if err != nil { // Cannot configure getter database, show stopper utils.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err)) return diff --git a/config/cacheconfig.go b/config/cacheconfig.go index 69475afb5..443075842 100644 --- a/config/cacheconfig.go +++ b/config/cacheconfig.go @@ -49,7 +49,75 @@ type CacheConfig struct { func (self *CacheConfig) loadFromJsonCfg(jsnCfg *CacheJsonCfg) error { if jsnCfg.Destinations != nil { self.Destinations = &CacheParamConfig{} - self.Destinations.loadFromJsonCfg(jsnCfg.Destinations) + if err := self.Destinations.loadFromJsonCfg(jsnCfg.Destinations); err != nil { + return err + } + } + if jsnCfg.Reverse_destinations != nil { + self.ReverseDestinations = &CacheParamConfig{} + if err := self.ReverseDestinations.loadFromJsonCfg(jsnCfg.Reverse_destinations); err != nil { + return err + } + } + if jsnCfg.Rating_plans != nil { + self.RatingPlans = &CacheParamConfig{} + if err := self.RatingPlans.loadFromJsonCfg(jsnCfg.Rating_plans); err != nil { + return err + } + } + if jsnCfg.Rating_profiles != nil { + self.RatingProfiles = &CacheParamConfig{} + if err := self.RatingProfiles.loadFromJsonCfg(jsnCfg.Rating_profiles); err != nil { + return err + } + } + if jsnCfg.Lcr != nil { + self.Lcr = &CacheParamConfig{} + if err := self.Lcr.loadFromJsonCfg(jsnCfg.Lcr); err != nil { + return err + } + } + if jsnCfg.Cdr_stats != nil { + self.CdrStats = &CacheParamConfig{} + if err := self.CdrStats.loadFromJsonCfg(jsnCfg.Cdr_stats); err != nil { + return err + } + } + if jsnCfg.Actions != nil { + self.Actions = &CacheParamConfig{} + if err := self.Actions.loadFromJsonCfg(jsnCfg.Actions); err != nil { + return err + } + } + if jsnCfg.Action_plans != nil { + self.ActionPlans = &CacheParamConfig{} + if err := self.ActionPlans.loadFromJsonCfg(jsnCfg.Action_plans); err != nil { + return err + } + } + if jsnCfg.Action_triggers != nil { + self.ActionTriggers = &CacheParamConfig{} + if err := self.ActionTriggers.loadFromJsonCfg(jsnCfg.Action_triggers); err != nil { + return err + } + } + if jsnCfg.Shared_groups != nil { + self.SharedGroups = &CacheParamConfig{} + if err := self.SharedGroups.loadFromJsonCfg(jsnCfg.Shared_groups); err != nil { + return err + } + } + if jsnCfg.Aliases != nil { + self.Aliases = &CacheParamConfig{} + if err := self.Aliases.loadFromJsonCfg(jsnCfg.Aliases); err != nil { + return err + } + } + if jsnCfg.Reverse_aliases != nil { + self.ReverseAliases = &CacheParamConfig{} + if err := self.ReverseAliases.loadFromJsonCfg(jsnCfg.Reverse_aliases); err != nil { + return err + } } return nil } diff --git a/config/config.go b/config/config.go index 36d62db9c..5d4dc0cbc 100644 --- a/config/config.go +++ b/config/config.go @@ -63,6 +63,7 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { cfg.InstanceID = utils.GenUUID() cfg.DataFolderPath = "/usr/share/cgrates/" cfg.SmGenericConfig = new(SmGenericConfig) + cfg.CacheConfig = new(CacheConfig) cfg.SmFsConfig = new(SmFsConfig) cfg.SmKamConfig = new(SmKamConfig) cfg.SmOsipsConfig = new(SmOsipsConfig) @@ -191,7 +192,8 @@ type CGRConfig struct { StorDBMaxOpenConns int // Maximum database connections opened StorDBMaxIdleConns int // Maximum idle connections to keep opened StorDBCDRSIndexes []string - DBDataEncoding string // The encoding used to store object data in strings: + DBDataEncoding string // The encoding used to store object data in strings: + CacheConfig *CacheConfig RPCJSONListen string // RPC JSON listening address RPCGOBListen string // RPC GOB listening address HTTPListen string // HTTP listening address @@ -212,7 +214,6 @@ type CGRConfig struct { HttpFailedDir string // Directory path where we store failed http requests MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file LockingTimeout time.Duration // locking mechanism timeout to avoid deadlocks - CacheDumpDir string // cache dump for faster start (leave empty to disable)b RALsEnabled bool // start standalone server (no balancer) RALsBalancer string // balancer address host:port RALsCDRStatSConns []*HaPoolConfig // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> @@ -466,6 +467,12 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { return err } + // Load sections out of JSON config, stop on error + jsnCacheCfg, err := jsnCfg.CacheJsonCfg() + if err != nil { + return err + } + jsnListenCfg, err := jsnCfg.ListenJsonCfg() if err != nil { return err @@ -719,8 +726,11 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { return err } } - if jsnGeneralCfg.Cache_dump_dir != nil { - self.CacheDumpDir = *jsnGeneralCfg.Cache_dump_dir + } + + if jsnCacheCfg != nil { + if err := self.CacheConfig.loadFromJsonCfg(jsnCacheCfg); err != nil { + return err } } diff --git a/config/config_json_test.go b/config/config_json_test.go index 0de8e9454..9b9b7b4c0 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -55,8 +55,7 @@ func TestDfGeneralJsonCfg(t *testing.T) { Reply_timeout: utils.StringPointer("2s"), Response_cache_ttl: utils.StringPointer("0s"), Internal_ttl: utils.StringPointer("2m"), - Locking_timeout: utils.StringPointer("5s"), - Cache_dump_dir: utils.StringPointer("")} + Locking_timeout: utils.StringPointer("5s")} if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, gCfg) { diff --git a/config/libconfig_json.go b/config/libconfig_json.go index a730600ba..0180c3c30 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -37,7 +37,6 @@ type GeneralJsonCfg struct { Response_cache_ttl *string Internal_ttl *string Locking_timeout *string - Cache_dump_dir *string } // Listen config section diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index e31b3f515..e4c905da1 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -24,23 +24,22 @@ // "response_cache_ttl": "0s", // the life span of a cached response // "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up // "locking_timeout": "5s", // timeout internal locks to avoid deadlocks -// "cache_dump_dir": "/var/lib/cgrates/cache_dump" // cache dump for faster start (leave empty to disable) // }, - "cache":{ - "destinations": {"limit": 10000, "ttl":0, "precache": false}, - "reverse_destinations": {"limit": 10000, "ttl":0, "precache": false}, - "rating_plans": {"limit": 10000, "ttl":0,"precache": true}, - "rating_profiles": {"limit": 10000, "ttl":0, "precache": false}, - "lcr": {"limit": 10000, "ttl":0, "precache": false}, - "cdr_stats": {"limit": 10000, "ttl":0, "precache": false}, - "actions": {"limit": 10000, "ttl":0, "precache": false}, - "action_plans": {"limit": 10000, "ttl":0, "precache": false}, - "action_triggers": {"limit": 10000, "ttl":0, "precache": false}, - "shared_groups": {"limit": 10000, "ttl":0, "precache": false}, - "aliases": {"limit": 10000, "ttl":0, "precache": false}, - "reverse_aliases": {"limit": 10000, "ttl":0, "precache": false}, - }, +// "cache":{ +// "destinations": {"limit": 10000, "ttl":0, "precache": false}, +// "reverse_destinations": {"limit": 10000, "ttl":0, "precache": false}, +// "rating_plans": {"limit": 10000, "ttl":0,"precache": true}, +// "rating_profiles": {"limit": 10000, "ttl":0, "precache": false}, +// "lcr": {"limit": 10000, "ttl":0, "precache": false}, +// "cdr_stats": {"limit": 10000, "ttl":0, "precache": false}, +// "actions": {"limit": 10000, "ttl":0, "precache": false}, +// "action_plans": {"limit": 10000, "ttl":0, "precache": false}, +// "action_triggers": {"limit": 10000, "ttl":0, "precache": false}, +// "shared_groups": {"limit": 10000, "ttl":0, "precache": false}, +// "aliases": {"limit": 10000, "ttl":0, "precache": false}, +// "reverse_aliases": {"limit": 10000, "ttl":0, "precache": false}, +// }, // "listen": { // "rpc_json": "127.0.0.1:2012", // RPC JSON listening address diff --git a/engine/calldesc.go b/engine/calldesc.go index 063c0e946..712356aad 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -28,6 +28,7 @@ import ( "time" "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/cgrates/rpcclient" ) @@ -47,20 +48,20 @@ func init() { ratingStorage, _ = NewMapStorage() accountingStorage, _ = NewMapStorage() case "mongo": - ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil, "", 10) + ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10) if err != nil { log.Fatal(err) } - accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil, "", 10) + accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10) if err != nil { log.Fatal(err) } case "redis": - ratingStorage, _ = NewRedisStorage("127.0.0.1:6379", 12, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, "", 10) + ratingStorage, _ = NewRedisStorage("127.0.0.1:6379", 12, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10) if err != nil { log.Fatal(err) } - accountingStorage, _ = NewRedisStorage("127.0.0.1:6379", 13, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, "", 10) + accountingStorage, _ = NewRedisStorage("127.0.0.1:6379", 13, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10) if err != nil { log.Fatal(err) } diff --git a/engine/libtest.go b/engine/libtest.go index 027dfc770..a623e6c31 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -33,12 +33,12 @@ import ( ) func InitDataDb(cfg *config.CGRConfig) error { - ratingDb, err := ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize) + ratingDb, err := ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize) if err != nil { return err } accountDb, err := ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, - cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize) + cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize) if err != nil { return err } diff --git a/engine/loader_local_test.go b/engine/loader_local_test.go index 1626ba107..ac02dd433 100644 --- a/engine/loader_local_test.go +++ b/engine/loader_local_test.go @@ -57,25 +57,25 @@ func TestConnDataDbs(t *testing.T) { } lCfg, _ = config.NewDefaultCGRConfig() var err error - if ratingDbCsv, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "4", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + if ratingDbCsv, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "4", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } - if ratingDbStor, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "5", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + if ratingDbStor, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "5", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } - if ratingDbApier, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "6", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + if ratingDbApier, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "6", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } if accountDbCsv, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "7", - lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } if accountDbStor, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "8", - lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } if accountDbApier, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "9", - lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil { + lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil { t.Fatal("Error on ratingDb connection: ", err.Error()) } for _, db := range []Storage{ratingDbCsv, ratingDbStor, ratingDbApier, accountDbCsv, accountDbStor, accountDbApier} { diff --git a/engine/storage_map.go b/engine/storage_map.go index bae6cd484..c02dee3e1 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -28,15 +28,16 @@ import ( "strings" "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) type MapStorage struct { - dict storage - tasks [][]byte - ms Marshaler - mu sync.RWMutex - cacheDumpDir string + dict storage + tasks [][]byte + ms Marshaler + mu sync.RWMutex + cacheCfg *config.CacheConfig } type storage map[string][]byte @@ -71,11 +72,11 @@ func (s storage) smembers(key string, ms Marshaler) (idMap utils.StringMap, ok b } func NewMapStorage() (*MapStorage, error) { - return &MapStorage{dict: make(map[string][]byte), ms: NewCodecMsgpackMarshaler(), cacheDumpDir: "/tmp/cgrates"}, nil + return &MapStorage{dict: make(map[string][]byte), ms: NewCodecMsgpackMarshaler(), cacheCfg: &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}}, nil } func NewMapStorageJson() (*MapStorage, error) { - return &MapStorage{dict: make(map[string][]byte), ms: new(JSONBufMarshaler), cacheDumpDir: "/tmp/cgrates"}, nil + return &MapStorage{dict: make(map[string][]byte), ms: new(JSONBufMarshaler), cacheCfg: &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}}, nil } func (ms *MapStorage) Close() {} @@ -134,15 +135,81 @@ func (ms *MapStorage) RebuildReverseForPrefix(prefix string) error { } func (ms *MapStorage) PreloadRatingCache() error { - err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX) - if err != nil { - return err + if ms.cacheCfg == nil { + return nil + } + if ms.cacheCfg.Destinations != nil && ms.cacheCfg.Destinations.Precache { + if err := ms.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.ReverseDestinations != nil && ms.cacheCfg.ReverseDestinations.Precache { + if err := ms.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.RatingPlans != nil && ms.cacheCfg.RatingPlans.Precache { + if err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.RatingProfiles != nil && ms.cacheCfg.RatingProfiles.Precache { + if err := ms.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.Lcr != nil && ms.cacheCfg.Lcr.Precache { + if err := ms.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.CdrStats != nil && ms.cacheCfg.CdrStats.Precache { + if err := ms.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.Actions != nil && ms.cacheCfg.Actions.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.ActionPlans != nil && ms.cacheCfg.ActionPlans.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.ActionTriggers != nil && ms.cacheCfg.ActionTriggers.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.SharedGroups != nil && ms.cacheCfg.SharedGroups.Precache { + if err := ms.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil { + return err + } } // add more prefixes if needed return nil } func (ms *MapStorage) PreloadAccountingCache() error { + if ms.cacheCfg == nil { + return nil + } + if ms.cacheCfg.Aliases != nil && ms.cacheCfg.Aliases.Precache { + if err := ms.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.ReverseAliases != nil && ms.cacheCfg.ReverseAliases.Precache { + if err := ms.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil { + return err + } + } return nil } @@ -803,7 +870,7 @@ func (ms *MapStorage) RemoveAlias(key string) error { ms.dict.srem(rKey, tmpKey, ms.ms) cache2go.RemKey(rKey) - /*_, err = rs.GetReverseAlias(rKey, true) // recache + /*_, err = ms.GetReverseAlias(rKey, true) // recache if err != nil { return err }*/ diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 862ae85ca..46b8263bf 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" @@ -88,7 +89,7 @@ type MongoStorage struct { session *mgo.Session db string ms Marshaler - cacheDumpDir string + cacheCfg *config.CacheConfig loadHistorySize int } @@ -97,7 +98,7 @@ func (ms *MongoStorage) conn(col string) (*mgo.Session, *mgo.Collection) { return sessionCopy, sessionCopy.DB(ms.db).C(col) } -func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, cacheDumpDir string, loadHistorySize int) (*MongoStorage, error) { +func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, cacheCfg *config.CacheConfig, loadHistorySize int) (*MongoStorage, error) { // We need this object to establish a session to our MongoDB. /*address := fmt.Sprintf("%s:%s", host, port) @@ -287,7 +288,7 @@ func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, ca if err = ndb.C(utils.TBLSMCosts).EnsureIndex(index); err != nil { return nil, err } - return &MongoStorage{db: db, session: session, ms: NewCodecMsgpackMarshaler(), cacheDumpDir: cacheDumpDir, loadHistorySize: loadHistorySize}, err + return &MongoStorage{db: db, session: session, ms: NewCodecMsgpackMarshaler(), cacheCfg: cacheCfg, loadHistorySize: loadHistorySize}, err } func (ms *MongoStorage) getColNameForPrefix(prefix string) (name string, ok bool) { @@ -386,15 +387,81 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) error { } func (ms *MongoStorage) PreloadRatingCache() error { - err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX) - if err != nil { - return err + if ms.cacheCfg == nil { + return nil + } + if ms.cacheCfg.Destinations != nil && ms.cacheCfg.Destinations.Precache { + if err := ms.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.ReverseDestinations != nil && ms.cacheCfg.ReverseDestinations.Precache { + if err := ms.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.RatingPlans != nil && ms.cacheCfg.RatingPlans.Precache { + if err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.RatingProfiles != nil && ms.cacheCfg.RatingProfiles.Precache { + if err := ms.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.Lcr != nil && ms.cacheCfg.Lcr.Precache { + if err := ms.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.CdrStats != nil && ms.cacheCfg.CdrStats.Precache { + if err := ms.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.Actions != nil && ms.cacheCfg.Actions.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.ActionPlans != nil && ms.cacheCfg.ActionPlans.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.ActionTriggers != nil && ms.cacheCfg.ActionTriggers.Precache { + if err := ms.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil { + return err + } + } + if ms.cacheCfg.SharedGroups != nil && ms.cacheCfg.SharedGroups.Precache { + if err := ms.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil { + return err + } } // add more prefixes if needed return nil } func (ms *MongoStorage) PreloadAccountingCache() error { + if ms.cacheCfg == nil { + return nil + } + if ms.cacheCfg.Aliases != nil && ms.cacheCfg.Aliases.Precache { + if err := ms.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil { + return err + } + } + + if ms.cacheCfg.ReverseAliases != nil && ms.cacheCfg.ReverseAliases.Precache { + if err := ms.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil { + return err + } + } return nil } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 6188d6d23..113d75ec0 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/mediocregopher/radix.v2/pool" "github.com/mediocregopher/radix.v2/redis" @@ -39,11 +40,11 @@ var ( type RedisStorage struct { db *pool.Pool ms Marshaler - cacheDumpDir string + cacheCfg *config.CacheConfig loadHistorySize int } -func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int, cacheDumpDir string, loadHistorySize int) (*RedisStorage, error) { +func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int, cacheCfg *config.CacheConfig, loadHistorySize int) (*RedisStorage, error) { df := func(network, addr string) (*redis.Client, error) { client, err := redis.Dial(network, addr) if err != nil { @@ -75,7 +76,7 @@ func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns i } else { return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr) } - return &RedisStorage{db: p, ms: mrshler, cacheDumpDir: cacheDumpDir, loadHistorySize: loadHistorySize}, nil + return &RedisStorage{db: p, ms: mrshler, cacheCfg: cacheCfg, loadHistorySize: loadHistorySize}, nil } func (rs *RedisStorage) Close() { @@ -87,16 +88,81 @@ func (rs *RedisStorage) Flush(ignore string) error { } func (rs *RedisStorage) PreloadRatingCache() error { - err := rs.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX) - if err != nil { - return err + if rs.cacheCfg == nil { + return nil + } + if rs.cacheCfg.Destinations != nil && rs.cacheCfg.Destinations.Precache { + if err := rs.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil { + return err + } + } + + if rs.cacheCfg.ReverseDestinations != nil && rs.cacheCfg.ReverseDestinations.Precache { + if err := rs.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { + return err + } + } + + if rs.cacheCfg.RatingPlans != nil && rs.cacheCfg.RatingPlans.Precache { + if err := rs.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + return err + } + } + + if rs.cacheCfg.RatingProfiles != nil && rs.cacheCfg.RatingProfiles.Precache { + if err := rs.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.Lcr != nil && rs.cacheCfg.Lcr.Precache { + if err := rs.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.CdrStats != nil && rs.cacheCfg.CdrStats.Precache { + if err := rs.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.Actions != nil && rs.cacheCfg.Actions.Precache { + if err := rs.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.ActionPlans != nil && rs.cacheCfg.ActionPlans.Precache { + if err := rs.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.ActionTriggers != nil && rs.cacheCfg.ActionTriggers.Precache { + if err := rs.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil { + return err + } + } + if rs.cacheCfg.SharedGroups != nil && rs.cacheCfg.SharedGroups.Precache { + if err := rs.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil { + return err + } } // add more prefixes if needed return nil } func (rs *RedisStorage) PreloadAccountingCache() error { - // add more prefixes if needed + if rs.cacheCfg == nil { + return nil + } + if rs.cacheCfg.Aliases != nil && rs.cacheCfg.Aliases.Precache { + if err := rs.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil { + return err + } + } + + if rs.cacheCfg.ReverseAliases != nil && rs.cacheCfg.ReverseAliases.Precache { + if err := rs.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil { + return err + } + } return nil } diff --git a/engine/storage_redis_local_test.go b/engine/storage_redis_local_test.go index e3d16ec6d..e87ad31c9 100644 --- a/engine/storage_redis_local_test.go +++ b/engine/storage_redis_local_test.go @@ -35,7 +35,7 @@ func TestConnectRedis(t *testing.T) { return } cfg, _ := config.NewDefaultCGRConfig() - rds, err = NewRedisStorage(fmt.Sprintf("%s:%s", cfg.TpDbHost, cfg.TpDbPort), 4, cfg.TpDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, "", 1) + rds, err = NewRedisStorage(fmt.Sprintf("%s:%s", cfg.TpDbHost, cfg.TpDbPort), 4, cfg.TpDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, nil, 1) if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } diff --git a/engine/storage_utils.go b/engine/storage_utils.go index d7bc99975..0c9522232 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -23,12 +23,13 @@ import ( "fmt" "strconv" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) // Various helpers to deal with database -func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, cacheDumpDir string, loadHistorySize int) (db RatingStorage, err error) { +func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler string, cacheCfg *config.CacheConfig, loadHistorySize int) (db RatingStorage, err error) { var d Storage switch db_type { case utils.REDIS: @@ -41,9 +42,9 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, ca if port != "" { host += ":" + port } - d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheDumpDir, loadHistorySize) + d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheCfg, loadHistorySize) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheDumpDir, loadHistorySize) + d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheCfg, loadHistorySize) db = d.(RatingStorage) default: err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are '%s' or '%s'", @@ -55,7 +56,7 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, ca return d.(RatingStorage), nil } -func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler, cacheDumpDir string, loadHistorySize int) (db AccountingStorage, err error) { +func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler string, cacheCfg *config.CacheConfig, loadHistorySize int) (db AccountingStorage, err error) { var d AccountingStorage switch db_type { case utils.REDIS: @@ -68,9 +69,9 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler if port != "" { host += ":" + port } - d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheDumpDir, loadHistorySize) + d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheCfg, loadHistorySize) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheDumpDir, loadHistorySize) + d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheCfg, loadHistorySize) db = d.(AccountingStorage) default: err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are '%s' or '%s'", @@ -99,7 +100,7 @@ func ConfigureStorStorage(db_type, host, port, name, user, pass, marshaler strin d, err = NewRedisStorage(host, db_nb, pass, marshaler) */ case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, nil, "", 1) + d, err = NewMongoStorage(host, port, name, user, pass, nil, nil, 1) case utils.POSTGRES: d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MYSQL: @@ -122,7 +123,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, "", 1) + d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, nil, 1) default: err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are [%s, %s, %s]", db_type, utils.MYSQL, utils.MONGO, utils.POSTGRES)) @@ -141,7 +142,7 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, "", 1) + d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, nil, 1) default: err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are [%s, %s, %s]", db_type, utils.MYSQL, utils.MONGO, utils.POSTGRES))