uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-16 13:49:53 +05:00
Code Issues Packages Projects Releases Wiki Activity

initialize cache with config object

Browse Source
This commit is contained in:
Radu Ioan Fericean
2016-08-15 23:05:55 +03:00
parent cac660d1ef
commit 747af43f8d
16 changed files with 426 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cache2go/cache.go
View File

@@ -1,7 +1,11 @@
//Simple caching library with expiration capabilities
package cache2go
import "sync"
import (
"sync"
"github.com/cgrates/cgrates/config"
)
const (
PREFIX_LEN = 4
@@ -14,6 +18,7 @@ const (
var (
mux sync.RWMutex
cache cacheStore
cfg *config.CacheConfig
// transaction stuff
transactionBuffer []*transactionItem
transactionMux sync.Mutex
@@ -28,11 +33,12 @@ type transactionItem struct {
}
func init() {
if DOUBLE_CACHE {
cache = newLRUTTLStore()
} else {
cache = newSimpleStore()
}
NewCache(nil)
}
func NewCache(cacheCfg *config.CacheConfig) {
cfg = cacheCfg
cache = newLRUTTL(cfg)
}
func BeginTransaction() {
@@ -117,11 +123,7 @@ func RemPrefixKey(prefix string) {
func Flush() {
mux.Lock()
defer mux.Unlock()
if DOUBLE_CACHE {
cache = newDoubleStore()
} else {
cache = newSimpleStore()
}
cache = newLRUTTL(cfg)
}
func CountEntries(prefix string) (result int) {

89
cache2go/cache_store.go
View File

@@ -5,6 +5,7 @@ import (
"strings"
"time"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
)
@@ -92,35 +93,69 @@ func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
return
}
func newLRUTTLStore() cacheLRUTTL {
return newLRUTTL(map[string]*cacheParam{
utils.DESTINATION_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.RATING_PLAN_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.RATING_PROFILE_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.ACTION_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.ACTION_PLAN_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.ACTION_TRIGGER_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.ALIASES_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.REVERSE_ALIASES_PREFIX: &cacheParam{limit: 10000, expiration: 0},
utils.REVERSE_DESTINATION_PREFIX: &cacheParam{limit: 100000, expiration: 0},
})
}
type cacheParam struct {
limit int
expiration time.Duration
}
func (ct *cacheParam) createCache() *Cache {
return NewLRUTTL(ct.limit, ct.expiration)
}
type cacheLRUTTL map[string]*Cache
func newLRUTTL(types map[string]*cacheParam) cacheLRUTTL {
c := make(map[string]*Cache, len(types))
for prefix, param := range types {
c[prefix] = param.createCache()
func newLRUTTL(cfg *config.CacheConfig) cacheLRUTTL {
c := make(map[string]*Cache)
if cfg != nil && cfg.Destinations != nil {
c[utils.DESTINATION_PREFIX] = NewLRUTTL(cfg.Destinations.Limit, cfg.Destinations.TTL)
} else {
c[utils.DESTINATION_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.ReverseDestinations != nil {
c[utils.REVERSE_DESTINATION_PREFIX] = NewLRUTTL(cfg.ReverseDestinations.Limit, cfg.ReverseDestinations.TTL)
} else {
c[utils.REVERSE_DESTINATION_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.RatingPlans != nil {
c[utils.RATING_PLAN_PREFIX] = NewLRUTTL(cfg.RatingPlans.Limit, cfg.RatingPlans.TTL)
} else {
c[utils.RATING_PLAN_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.RatingProfiles != nil {
c[utils.RATING_PROFILE_PREFIX] = NewLRUTTL(cfg.RatingProfiles.Limit, cfg.RatingProfiles.TTL)
} else {
c[utils.RATING_PROFILE_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.Lcr != nil {
c[utils.LCR_PREFIX] = NewLRUTTL(cfg.Lcr.Limit, cfg.Lcr.TTL)
} else {
c[utils.LCR_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.CdrStats != nil {
c[utils.CDR_STATS_PREFIX] = NewLRUTTL(cfg.CdrStats.Limit, cfg.CdrStats.TTL)
} else {
c[utils.CDR_STATS_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.Actions != nil {
c[utils.ACTION_PREFIX] = NewLRUTTL(cfg.Actions.Limit, cfg.Actions.TTL)
} else {
c[utils.ACTION_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.ActionPlans != nil {
c[utils.ACTION_PLAN_PREFIX] = NewLRUTTL(cfg.ActionPlans.Limit, cfg.ActionPlans.TTL)
} else {
c[utils.ACTION_PLAN_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.ActionTriggers != nil {
c[utils.ACTION_TRIGGER_PREFIX] = NewLRUTTL(cfg.ActionTriggers.Limit, cfg.ActionTriggers.TTL)
} else {
c[utils.ACTION_TRIGGER_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.SharedGroups != nil {
c[utils.SHARED_GROUP_PREFIX] = NewLRUTTL(cfg.SharedGroups.Limit, cfg.SharedGroups.TTL)
} else {
c[utils.SHARED_GROUP_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.Aliases != nil {
c[utils.ALIASES_PREFIX] = NewLRUTTL(cfg.Aliases.Limit, cfg.Aliases.TTL)
} else {
c[utils.ALIASES_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
if cfg != nil && cfg.ReverseAliases != nil {
c[utils.REVERSE_ALIASES_PREFIX] = NewLRUTTL(cfg.ReverseAliases.Limit, cfg.ReverseAliases.TTL)
} else {
c[utils.REVERSE_ALIASES_PREFIX] = NewLRUTTL(10000, 24*time.Hour)
}
return c

7
cmd/cgr-engine/cgr-engine.go
View File

@@ -33,6 +33,7 @@ import (
"github.com/cgrates/cgrates/apier/v1"
"github.com/cgrates/cgrates/apier/v2"
"github.com/cgrates/cgrates/balancer2go"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/cdrc"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
@@ -533,6 +534,8 @@ func main() {
return
}
config.SetCgrConfig(cfg) // Share the config object
cache2go.NewCache(cfg.CacheConfig)
if *raterEnabled {
cfg.RALsEnabled = *raterEnabled
}
@@ -548,7 +551,7 @@ func main() {
var cdrDb engine.CdrStorage
if cfg.RALsEnabled || cfg.SchedulerEnabled || cfg.CDRStatsEnabled { // Only connect to dataDb if necessary
ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort,
cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize)
cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize)
if err != nil { // Cannot configure getter database, show stopper
utils.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
return
@@ -558,7 +561,7 @@ func main() {
}
if cfg.RALsEnabled || cfg.CDRStatsEnabled || cfg.PubSubServerEnabled || cfg.AliasesServerEnabled || cfg.UserServerEnabled {
accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort,
cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize)
cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize)
if err != nil { // Cannot configure getter database, show stopper
utils.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
return

70
config/cacheconfig.go
View File

@@ -49,7 +49,75 @@ type CacheConfig struct {
func (self *CacheConfig) loadFromJsonCfg(jsnCfg *CacheJsonCfg) error {
if jsnCfg.Destinations != nil {
self.Destinations = &CacheParamConfig{}
self.Destinations.loadFromJsonCfg(jsnCfg.Destinations)
if err := self.Destinations.loadFromJsonCfg(jsnCfg.Destinations); err != nil {
return err
}
}
if jsnCfg.Reverse_destinations != nil {
self.ReverseDestinations = &CacheParamConfig{}
if err := self.ReverseDestinations.loadFromJsonCfg(jsnCfg.Reverse_destinations); err != nil {
return err
}
}
if jsnCfg.Rating_plans != nil {
self.RatingPlans = &CacheParamConfig{}
if err := self.RatingPlans.loadFromJsonCfg(jsnCfg.Rating_plans); err != nil {
return err
}
}
if jsnCfg.Rating_profiles != nil {
self.RatingProfiles = &CacheParamConfig{}
if err := self.RatingProfiles.loadFromJsonCfg(jsnCfg.Rating_profiles); err != nil {
return err
}
}
if jsnCfg.Lcr != nil {
self.Lcr = &CacheParamConfig{}
if err := self.Lcr.loadFromJsonCfg(jsnCfg.Lcr); err != nil {
return err
}
}
if jsnCfg.Cdr_stats != nil {
self.CdrStats = &CacheParamConfig{}
if err := self.CdrStats.loadFromJsonCfg(jsnCfg.Cdr_stats); err != nil {
return err
}
}
if jsnCfg.Actions != nil {
self.Actions = &CacheParamConfig{}
if err := self.Actions.loadFromJsonCfg(jsnCfg.Actions); err != nil {
return err
}
}
if jsnCfg.Action_plans != nil {
self.ActionPlans = &CacheParamConfig{}
if err := self.ActionPlans.loadFromJsonCfg(jsnCfg.Action_plans); err != nil {
return err
}
}
if jsnCfg.Action_triggers != nil {
self.ActionTriggers = &CacheParamConfig{}
if err := self.ActionTriggers.loadFromJsonCfg(jsnCfg.Action_triggers); err != nil {
return err
}
}
if jsnCfg.Shared_groups != nil {
self.SharedGroups = &CacheParamConfig{}
if err := self.SharedGroups.loadFromJsonCfg(jsnCfg.Shared_groups); err != nil {
return err
}
}
if jsnCfg.Aliases != nil {
self.Aliases = &CacheParamConfig{}
if err := self.Aliases.loadFromJsonCfg(jsnCfg.Aliases); err != nil {
return err
}
}
if jsnCfg.Reverse_aliases != nil {
self.ReverseAliases = &CacheParamConfig{}
if err := self.ReverseAliases.loadFromJsonCfg(jsnCfg.Reverse_aliases); err != nil {
return err
}
}
return nil
}

18
config/config.go
View File

@@ -63,6 +63,7 @@ func NewDefaultCGRConfig() (*CGRConfig, error) {
cfg.InstanceID = utils.GenUUID()
cfg.DataFolderPath = "/usr/share/cgrates/"
cfg.SmGenericConfig = new(SmGenericConfig)
cfg.CacheConfig = new(CacheConfig)
cfg.SmFsConfig = new(SmFsConfig)
cfg.SmKamConfig = new(SmKamConfig)
cfg.SmOsipsConfig = new(SmOsipsConfig)
@@ -191,7 +192,8 @@ type CGRConfig struct {
StorDBMaxOpenConns int // Maximum database connections opened
StorDBMaxIdleConns int // Maximum idle connections to keep opened
StorDBCDRSIndexes []string
DBDataEncoding string // The encoding used to store object data in strings: <msgpack|json>
DBDataEncoding string // The encoding used to store object data in strings: <msgpack|json>
CacheConfig *CacheConfig
RPCJSONListen string // RPC JSON listening address
RPCGOBListen string // RPC GOB listening address
HTTPListen string // HTTP listening address
@@ -212,7 +214,6 @@ type CGRConfig struct {
HttpFailedDir string // Directory path where we store failed http requests
MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file
LockingTimeout time.Duration // locking mechanism timeout to avoid deadlocks
CacheDumpDir string // cache dump for faster start (leave empty to disable)b
RALsEnabled bool // start standalone server (no balancer)
RALsBalancer string // balancer address host:port
RALsCDRStatSConns []*HaPoolConfig // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234>
@@ -466,6 +467,12 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error {
return err
}
// Load sections out of JSON config, stop on error
jsnCacheCfg, err := jsnCfg.CacheJsonCfg()
if err != nil {
return err
}
jsnListenCfg, err := jsnCfg.ListenJsonCfg()
if err != nil {
return err
@@ -719,8 +726,11 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error {
return err
}
}
if jsnGeneralCfg.Cache_dump_dir != nil {
self.CacheDumpDir = *jsnGeneralCfg.Cache_dump_dir
}
if jsnCacheCfg != nil {
if err := self.CacheConfig.loadFromJsonCfg(jsnCacheCfg); err != nil {
return err
}
}

3
config/config_json_test.go
View File

@@ -55,8 +55,7 @@ func TestDfGeneralJsonCfg(t *testing.T) {
Reply_timeout: utils.StringPointer("2s"),
Response_cache_ttl: utils.StringPointer("0s"),
Internal_ttl: utils.StringPointer("2m"),
Locking_timeout: utils.StringPointer("5s"),
Cache_dump_dir: utils.StringPointer("")}
Locking_timeout: utils.StringPointer("5s")}
if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eCfg, gCfg) {

1
config/libconfig_json.go
View File

@@ -37,7 +37,6 @@ type GeneralJsonCfg struct {
Response_cache_ttl *string
Internal_ttl *string
Locking_timeout *string
Cache_dump_dir *string
}
// Listen config section

29
data/conf/cgrates/cgrates.json
View File

@@ -24,23 +24,22 @@
// "response_cache_ttl": "0s", // the life span of a cached response
// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up
// "locking_timeout": "5s", // timeout internal locks to avoid deadlocks
// "cache_dump_dir": "/var/lib/cgrates/cache_dump" // cache dump for faster start (leave empty to disable)
// },
"cache":{
"destinations": {"limit": 10000, "ttl":0, "precache": false},
"reverse_destinations": {"limit": 10000, "ttl":0, "precache": false},
"rating_plans": {"limit": 10000, "ttl":0,"precache": true},
"rating_profiles": {"limit": 10000, "ttl":0, "precache": false},
"lcr": {"limit": 10000, "ttl":0, "precache": false},
"cdr_stats": {"limit": 10000, "ttl":0, "precache": false},
"actions": {"limit": 10000, "ttl":0, "precache": false},
"action_plans": {"limit": 10000, "ttl":0, "precache": false},
"action_triggers": {"limit": 10000, "ttl":0, "precache": false},
"shared_groups": {"limit": 10000, "ttl":0, "precache": false},
"aliases": {"limit": 10000, "ttl":0, "precache": false},
"reverse_aliases": {"limit": 10000, "ttl":0, "precache": false},
},
// "cache":{
// "destinations": {"limit": 10000, "ttl":0, "precache": false},
// "reverse_destinations": {"limit": 10000, "ttl":0, "precache": false},
// "rating_plans": {"limit": 10000, "ttl":0,"precache": true},
// "rating_profiles": {"limit": 10000, "ttl":0, "precache": false},
// "lcr": {"limit": 10000, "ttl":0, "precache": false},
// "cdr_stats": {"limit": 10000, "ttl":0, "precache": false},
// "actions": {"limit": 10000, "ttl":0, "precache": false},
// "action_plans": {"limit": 10000, "ttl":0, "precache": false},
// "action_triggers": {"limit": 10000, "ttl":0, "precache": false},
// "shared_groups": {"limit": 10000, "ttl":0, "precache": false},
// "aliases": {"limit": 10000, "ttl":0, "precache": false},
// "reverse_aliases": {"limit": 10000, "ttl":0, "precache": false},
// },
// "listen": {
// "rpc_json": "127.0.0.1:2012", // RPC JSON listening address

9
engine/calldesc.go
View File

@@ -28,6 +28,7 @@ import (
"time"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
"github.com/cgrates/rpcclient"
)
@@ -47,20 +48,20 @@ func init() {
ratingStorage, _ = NewMapStorage()
accountingStorage, _ = NewMapStorage()
case "mongo":
ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil, "", 10)
ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10)
if err != nil {
log.Fatal(err)
}
accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil, "", 10)
accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10)
if err != nil {
log.Fatal(err)
}
case "redis":
ratingStorage, _ = NewRedisStorage("127.0.0.1:6379", 12, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, "", 10)
ratingStorage, _ = NewRedisStorage("127.0.0.1:6379", 12, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10)
if err != nil {
log.Fatal(err)
}
accountingStorage, _ = NewRedisStorage("127.0.0.1:6379", 13, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, "", 10)
accountingStorage, _ = NewRedisStorage("127.0.0.1:6379", 13, "", utils.MSGPACK, utils.REDIS_MAX_CONNS, &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}, 10)
if err != nil {
log.Fatal(err)
}

4
engine/libtest.go
View File

@@ -33,12 +33,12 @@ import (
)
func InitDataDb(cfg *config.CGRConfig) error {
ratingDb, err := ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize)
ratingDb, err := ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize)
if err != nil {
return err
}
accountDb, err := ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName,
cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheDumpDir, cfg.LoadHistorySize)
cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding, cfg.CacheConfig, cfg.LoadHistorySize)
if err != nil {
return err
}

12
engine/loader_local_test.go
View File

@@ -57,25 +57,25 @@ func TestConnDataDbs(t *testing.T) {
}
lCfg, _ = config.NewDefaultCGRConfig()
var err error
if ratingDbCsv, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "4", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
if ratingDbCsv, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "4", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
if ratingDbStor, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "5", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
if ratingDbStor, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "5", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
if ratingDbApier, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "6", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
if ratingDbApier, err = ConfigureRatingStorage(lCfg.TpDbType, lCfg.TpDbHost, lCfg.TpDbPort, "6", lCfg.TpDbUser, lCfg.TpDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
if accountDbCsv, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "7",
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
if accountDbStor, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "8",
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
if accountDbApier, err = ConfigureAccountingStorage(lCfg.DataDbType, lCfg.DataDbHost, lCfg.DataDbPort, "9",
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, "", 1); err != nil {
lCfg.DataDbUser, lCfg.DataDbPass, lCfg.DBDataEncoding, nil, 1); err != nil {
t.Fatal("Error on ratingDb connection: ", err.Error())
}
for _, db := range []Storage{ratingDbCsv, ratingDbStor, ratingDbApier, accountDbCsv, accountDbStor, accountDbApier} {

89
engine/storage_map.go
View File

@@ -28,15 +28,16 @@ import (
"strings"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
)
type MapStorage struct {
dict storage
tasks [][]byte
ms Marshaler
mu sync.RWMutex
cacheDumpDir string
dict storage
tasks [][]byte
ms Marshaler
mu sync.RWMutex
cacheCfg *config.CacheConfig
}
type storage map[string][]byte
@@ -71,11 +72,11 @@ func (s storage) smembers(key string, ms Marshaler) (idMap utils.StringMap, ok b
}
func NewMapStorage() (*MapStorage, error) {
return &MapStorage{dict: make(map[string][]byte), ms: NewCodecMsgpackMarshaler(), cacheDumpDir: "/tmp/cgrates"}, nil
return &MapStorage{dict: make(map[string][]byte), ms: NewCodecMsgpackMarshaler(), cacheCfg: &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}}, nil
}
func NewMapStorageJson() (*MapStorage, error) {
return &MapStorage{dict: make(map[string][]byte), ms: new(JSONBufMarshaler), cacheDumpDir: "/tmp/cgrates"}, nil
return &MapStorage{dict: make(map[string][]byte), ms: new(JSONBufMarshaler), cacheCfg: &config.CacheConfig{RatingPlans: &config.CacheParamConfig{Precache: true}}}, nil
}
func (ms *MapStorage) Close() {}
@@ -134,15 +135,81 @@ func (ms *MapStorage) RebuildReverseForPrefix(prefix string) error {
}
func (ms *MapStorage) PreloadRatingCache() error {
err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX)
if err != nil {
return err
if ms.cacheCfg == nil {
return nil
}
if ms.cacheCfg.Destinations != nil && ms.cacheCfg.Destinations.Precache {
if err := ms.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ReverseDestinations != nil && ms.cacheCfg.ReverseDestinations.Precache {
if err := ms.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.RatingPlans != nil && ms.cacheCfg.RatingPlans.Precache {
if err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.RatingProfiles != nil && ms.cacheCfg.RatingProfiles.Precache {
if err := ms.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.Lcr != nil && ms.cacheCfg.Lcr.Precache {
if err := ms.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.CdrStats != nil && ms.cacheCfg.CdrStats.Precache {
if err := ms.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.Actions != nil && ms.cacheCfg.Actions.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ActionPlans != nil && ms.cacheCfg.ActionPlans.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ActionTriggers != nil && ms.cacheCfg.ActionTriggers.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.SharedGroups != nil && ms.cacheCfg.SharedGroups.Precache {
if err := ms.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil {
return err
}
}
// add more prefixes if needed
return nil
}
func (ms *MapStorage) PreloadAccountingCache() error {
if ms.cacheCfg == nil {
return nil
}
if ms.cacheCfg.Aliases != nil && ms.cacheCfg.Aliases.Precache {
if err := ms.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ReverseAliases != nil && ms.cacheCfg.ReverseAliases.Precache {
if err := ms.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil {
return err
}
}
return nil
}
@@ -803,7 +870,7 @@ func (ms *MapStorage) RemoveAlias(key string) error {
ms.dict.srem(rKey, tmpKey, ms.ms)
cache2go.RemKey(rKey)
/*_, err = rs.GetReverseAlias(rKey, true) // recache
/*_, err = ms.GetReverseAlias(rKey, true) // recache
if err != nil {
return err
}*/

79
engine/storage_mongo_datadb.go
View File

@@ -27,6 +27,7 @@ import (
"strings"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
@@ -88,7 +89,7 @@ type MongoStorage struct {
session *mgo.Session
db string
ms Marshaler
cacheDumpDir string
cacheCfg *config.CacheConfig
loadHistorySize int
}
@@ -97,7 +98,7 @@ func (ms *MongoStorage) conn(col string) (*mgo.Session, *mgo.Collection) {
return sessionCopy, sessionCopy.DB(ms.db).C(col)
}
func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, cacheDumpDir string, loadHistorySize int) (*MongoStorage, error) {
func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, cacheCfg *config.CacheConfig, loadHistorySize int) (*MongoStorage, error) {
// We need this object to establish a session to our MongoDB.
/*address := fmt.Sprintf("%s:%s", host, port)
@@ -287,7 +288,7 @@ func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string, ca
if err = ndb.C(utils.TBLSMCosts).EnsureIndex(index); err != nil {
return nil, err
}
return &MongoStorage{db: db, session: session, ms: NewCodecMsgpackMarshaler(), cacheDumpDir: cacheDumpDir, loadHistorySize: loadHistorySize}, err
return &MongoStorage{db: db, session: session, ms: NewCodecMsgpackMarshaler(), cacheCfg: cacheCfg, loadHistorySize: loadHistorySize}, err
}
func (ms *MongoStorage) getColNameForPrefix(prefix string) (name string, ok bool) {
@@ -386,15 +387,81 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) error {
}
func (ms *MongoStorage) PreloadRatingCache() error {
err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX)
if err != nil {
return err
if ms.cacheCfg == nil {
return nil
}
if ms.cacheCfg.Destinations != nil && ms.cacheCfg.Destinations.Precache {
if err := ms.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ReverseDestinations != nil && ms.cacheCfg.ReverseDestinations.Precache {
if err := ms.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.RatingPlans != nil && ms.cacheCfg.RatingPlans.Precache {
if err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.RatingProfiles != nil && ms.cacheCfg.RatingProfiles.Precache {
if err := ms.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.Lcr != nil && ms.cacheCfg.Lcr.Precache {
if err := ms.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.CdrStats != nil && ms.cacheCfg.CdrStats.Precache {
if err := ms.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.Actions != nil && ms.cacheCfg.Actions.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ActionPlans != nil && ms.cacheCfg.ActionPlans.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ActionTriggers != nil && ms.cacheCfg.ActionTriggers.Precache {
if err := ms.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.SharedGroups != nil && ms.cacheCfg.SharedGroups.Precache {
if err := ms.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil {
return err
}
}
// add more prefixes if needed
return nil
}
func (ms *MongoStorage) PreloadAccountingCache() error {
if ms.cacheCfg == nil {
return nil
}
if ms.cacheCfg.Aliases != nil && ms.cacheCfg.Aliases.Precache {
if err := ms.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil {
return err
}
}
if ms.cacheCfg.ReverseAliases != nil && ms.cacheCfg.ReverseAliases.Precache {
if err := ms.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil {
return err
}
}
return nil
}

80
engine/storage_redis.go
View File

@@ -27,6 +27,7 @@ import (
"strings"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
"github.com/mediocregopher/radix.v2/pool"
"github.com/mediocregopher/radix.v2/redis"
@@ -39,11 +40,11 @@ var (
type RedisStorage struct {
db *pool.Pool
ms Marshaler
cacheDumpDir string
cacheCfg *config.CacheConfig
loadHistorySize int
}
func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int, cacheDumpDir string, loadHistorySize int) (*RedisStorage, error) {
func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int, cacheCfg *config.CacheConfig, loadHistorySize int) (*RedisStorage, error) {
df := func(network, addr string) (*redis.Client, error) {
client, err := redis.Dial(network, addr)
if err != nil {
@@ -75,7 +76,7 @@ func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns i
} else {
return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr)
}
return &RedisStorage{db: p, ms: mrshler, cacheDumpDir: cacheDumpDir, loadHistorySize: loadHistorySize}, nil
return &RedisStorage{db: p, ms: mrshler, cacheCfg: cacheCfg, loadHistorySize: loadHistorySize}, nil
}
func (rs *RedisStorage) Close() {
@@ -87,16 +88,81 @@ func (rs *RedisStorage) Flush(ignore string) error {
}
func (rs *RedisStorage) PreloadRatingCache() error {
err := rs.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX)
if err != nil {
return err
if rs.cacheCfg == nil {
return nil
}
if rs.cacheCfg.Destinations != nil && rs.cacheCfg.Destinations.Precache {
if err := rs.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.ReverseDestinations != nil && rs.cacheCfg.ReverseDestinations.Precache {
if err := rs.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.RatingPlans != nil && rs.cacheCfg.RatingPlans.Precache {
if err := rs.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.RatingProfiles != nil && rs.cacheCfg.RatingProfiles.Precache {
if err := rs.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.Lcr != nil && rs.cacheCfg.Lcr.Precache {
if err := rs.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.CdrStats != nil && rs.cacheCfg.CdrStats.Precache {
if err := rs.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.Actions != nil && rs.cacheCfg.Actions.Precache {
if err := rs.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.ActionPlans != nil && rs.cacheCfg.ActionPlans.Precache {
if err := rs.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.ActionTriggers != nil && rs.cacheCfg.ActionTriggers.Precache {
if err := rs.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.SharedGroups != nil && rs.cacheCfg.SharedGroups.Precache {
if err := rs.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil {
return err
}
}
// add more prefixes if needed
return nil
}
func (rs *RedisStorage) PreloadAccountingCache() error {
// add more prefixes if needed
if rs.cacheCfg == nil {
return nil
}
if rs.cacheCfg.Aliases != nil && rs.cacheCfg.Aliases.Precache {
if err := rs.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil {
return err
}
}
if rs.cacheCfg.ReverseAliases != nil && rs.cacheCfg.ReverseAliases.Precache {
if err := rs.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil {
return err
}
}
return nil
}

2
engine/storage_redis_local_test.go
View File

@@ -35,7 +35,7 @@ func TestConnectRedis(t *testing.T) {
return
}
cfg, _ := config.NewDefaultCGRConfig()
rds, err = NewRedisStorage(fmt.Sprintf("%s:%s", cfg.TpDbHost, cfg.TpDbPort), 4, cfg.TpDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, "", 1)
rds, err = NewRedisStorage(fmt.Sprintf("%s:%s", cfg.TpDbHost, cfg.TpDbPort), 4, cfg.TpDbPass, cfg.DBDataEncoding, utils.REDIS_MAX_CONNS, nil, 1)
if err != nil {
t.Fatal("Could not connect to Redis", err.Error())
}

19
engine/storage_utils.go
View File

@@ -23,12 +23,13 @@ import (
"fmt"
"strconv"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
)
// Various helpers to deal with database
func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, cacheDumpDir string, loadHistorySize int) (db RatingStorage, err error) {
func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler string, cacheCfg *config.CacheConfig, loadHistorySize int) (db RatingStorage, err error) {
var d Storage
switch db_type {
case utils.REDIS:
@@ -41,9 +42,9 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, ca
if port != "" {
host += ":" + port
}
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheDumpDir, loadHistorySize)
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheCfg, loadHistorySize)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheDumpDir, loadHistorySize)
d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheCfg, loadHistorySize)
db = d.(RatingStorage)
default:
err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are '%s' or '%s'",
@@ -55,7 +56,7 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler, ca
return d.(RatingStorage), nil
}
func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler, cacheDumpDir string, loadHistorySize int) (db AccountingStorage, err error) {
func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler string, cacheCfg *config.CacheConfig, loadHistorySize int) (db AccountingStorage, err error) {
var d AccountingStorage
switch db_type {
case utils.REDIS:
@@ -68,9 +69,9 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler
if port != "" {
host += ":" + port
}
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheDumpDir, loadHistorySize)
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS, cacheCfg, loadHistorySize)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheDumpDir, loadHistorySize)
d, err = NewMongoStorage(host, port, name, user, pass, nil, cacheCfg, loadHistorySize)
db = d.(AccountingStorage)
default:
err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are '%s' or '%s'",
@@ -99,7 +100,7 @@ func ConfigureStorStorage(db_type, host, port, name, user, pass, marshaler strin
d, err = NewRedisStorage(host, db_nb, pass, marshaler)
*/
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass, nil, "", 1)
d, err = NewMongoStorage(host, port, name, user, pass, nil, nil, 1)
case utils.POSTGRES:
d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MYSQL:
@@ -122,7 +123,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin
case utils.MYSQL:
d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, "", 1)
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, nil, 1)
default:
err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are [%s, %s, %s]",
db_type, utils.MYSQL, utils.MONGO, utils.POSTGRES))
@@ -141,7 +142,7 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn,
case utils.MYSQL:
d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, "", 1)
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes, nil, 1)
default:
err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are [%s, %s, %s]",
db_type, utils.MYSQL, utils.MONGO, utils.POSTGRES))