From 257ecc51c1ae16ee7f9e080761650ebb69eef6e5 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 5 Jul 2016 18:11:04 +0300 Subject: [PATCH] save only the parts being cached --- engine/cache.go | 2 +- engine/cache_factory.go | 187 ++------------------------------- engine/cache_store.go | 37 ++++--- engine/storage_mongo_datadb.go | 24 ++--- engine/storage_redis.go | 24 ++--- 5 files changed, 61 insertions(+), 213 deletions(-) diff --git a/engine/cache.go b/engine/cache.go index 3c0423b20..46f524a07 100644 --- a/engine/cache.go +++ b/engine/cache.go @@ -78,7 +78,7 @@ func CacheCommitTransaction() { transactionMux.Unlock() } -func CacheSave(path string, keys []string, cfi *utils.CacheFileInfo) error { +func CacheSave(path string, keys map[string][]string, cfi *utils.CacheFileInfo) error { mux.Lock() defer mux.Unlock() return cache.Save(path, keys, cfi) diff --git a/engine/cache_factory.go b/engine/cache_factory.go index 90795a6d6..cb64892d4 100644 --- a/engine/cache_factory.go +++ b/engine/cache_factory.go @@ -2,193 +2,28 @@ package engine import "github.com/cgrates/cgrates/utils" -type cacheKeyValue interface { - Key() string - Value() interface{} -} - -type mapKeyValue struct { - K string - V map[string]struct{} -} - -func (mkv *mapKeyValue) Key() string { - return mkv.K -} - -func (mkv *mapKeyValue) Value() interface{} { - return mkv.V -} - -type rpKeyValue struct { - K string - V *RatingPlan -} - -func (mkv *rpKeyValue) Key() string { - return mkv.K -} - -func (mkv *rpKeyValue) Value() interface{} { - return mkv.V -} - -type rpfKeyValue struct { - K string - V *RatingProfile -} - -func (mkv *rpfKeyValue) Key() string { - return mkv.K -} - -func (mkv *rpfKeyValue) Value() interface{} { - return mkv.V -} - -type lcrKeyValue struct { - K string - V *LCR -} - -func (mkv *lcrKeyValue) Key() string { - return mkv.K -} - -func (mkv *lcrKeyValue) Value() interface{} { - return mkv.V -} - -type dcKeyValue struct { - K string - V *utils.DerivedChargers -} - -func (mkv *dcKeyValue) Key() string { - return mkv.K -} - -func (mkv *dcKeyValue) Value() interface{} { - return mkv.V -} - -type acsKeyValue struct { - K string - V Actions -} - -func (mkv *acsKeyValue) Key() string { - return mkv.K -} - -func (mkv *acsKeyValue) Value() interface{} { - return mkv.V -} - -type aplKeyValue struct { - K string - V *ActionPlan -} - -func (mkv *aplKeyValue) Key() string { - return mkv.K -} - -func (mkv *aplKeyValue) Value() interface{} { - return mkv.V -} - -type sgKeyValue struct { - K string - V *SharedGroup -} - -func (mkv *sgKeyValue) Key() string { - return mkv.K -} - -func (mkv *sgKeyValue) Value() interface{} { - return mkv.V -} - -type alsKeyValue struct { - K string - V AliasValues -} - -func (mkv *alsKeyValue) Key() string { - return mkv.K -} - -func (mkv *alsKeyValue) Value() interface{} { - return mkv.V -} - -type loadKeyValue struct { - K string - V []*utils.LoadInstance -} - -func (mkv *loadKeyValue) Key() string { - return mkv.K -} - -func (mkv *loadKeyValue) Value() interface{} { - return mkv.V -} - -func CacheTypeFactory(prefix string, key string, value interface{}) cacheKeyValue { +func CacheTypeFactory(prefix string) interface{} { switch prefix { case utils.DESTINATION_PREFIX: - if value != nil { - return &mapKeyValue{key, value.(map[string]struct{})} - } - return &mapKeyValue{"", make(map[string]struct{})} + return make(map[string]struct{}) case utils.RATING_PLAN_PREFIX: - if value != nil { - return &rpKeyValue{key, value.(*RatingPlan)} - } - return &rpfKeyValue{"", &RatingProfile{}} + return &RatingPlan{} case utils.RATING_PROFILE_PREFIX: - if value != nil { - return &rpfKeyValue{key, value.(*RatingProfile)} - } - return &rpfKeyValue{"", &RatingProfile{}} + return &RatingProfile{} case utils.LCR_PREFIX: - if value != nil { - return &lcrKeyValue{key, value.(*LCR)} - } - return &lcrKeyValue{"", &LCR{}} + return &LCR{} case utils.DERIVEDCHARGERS_PREFIX: - if value != nil { - return &dcKeyValue{key, value.(*utils.DerivedChargers)} - } - return &dcKeyValue{"", &utils.DerivedChargers{}} + return &utils.DerivedChargers{} case utils.ACTION_PREFIX: - if value != nil { - return &acsKeyValue{key, value.(Actions)} - } - return &acsKeyValue{"", Actions{}} + return Actions{} case utils.ACTION_PLAN_PREFIX: - if value != nil { - return &aplKeyValue{key, value.(*ActionPlan)} - } - return &aplKeyValue{"", &ActionPlan{}} + return &ActionPlan{} case utils.SHARED_GROUP_PREFIX: - if value != nil { - return &sgKeyValue{key, value.(*SharedGroup)} - } - return &sgKeyValue{"", &SharedGroup{}} + return &SharedGroup{} case utils.ALIASES_PREFIX: - if value != nil { - return &alsKeyValue{key, value.(AliasValues)} - } - return &alsKeyValue{"", AliasValues{}} + return AliasValues{} case utils.LOADINST_KEY[:PREFIX_LEN]: - if value != nil { - return &loadKeyValue{key, value.([]*utils.LoadInstance)} - } - return &loadKeyValue{"", make([]*utils.LoadInstance, 0)} + return make([]*utils.LoadInstance, 0) } return nil } diff --git a/engine/cache_store.go b/engine/cache_store.go index f0ac2ed3c..a01ae7480 100644 --- a/engine/cache_store.go +++ b/engine/cache_store.go @@ -27,7 +27,7 @@ type cacheStore interface { CountEntriesForPrefix(string) int GetAllForPrefix(string) (map[string]interface{}, error) GetKeysForPrefix(string) []string - Save(string, []string, *utils.CacheFileInfo) error + Save(string, map[string][]string, *utils.CacheFileInfo) error Load(string, []string) error } @@ -120,9 +120,10 @@ func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) { return } -func (cs cacheDoubleStore) Save(path string, prefixes []string, cfi *utils.CacheFileInfo) error { +func (cs cacheDoubleStore) Save(path string, prefixKeysMap map[string][]string, cfi *utils.CacheFileInfo) error { + start := time.Now() //log.Printf("path: %s prefixes: %v", path, prefixes) - if path == "" || len(prefixes) == 0 { + if path == "" || len(prefixKeysMap) == 0 { return nil } //log.Print("saving cache prefixes: ", prefixes) @@ -133,24 +134,35 @@ func (cs cacheDoubleStore) Save(path string, prefixes []string, cfi *utils.Cache } var wg sync.WaitGroup - for _, prefix := range prefixes { + var prefixSlice []string + for prefix, keys := range prefixKeysMap { prefix = prefix[:PREFIX_LEN] mapValue, found := cs[prefix] if !found { continue } + prefixSlice = append(prefixSlice, prefix) wg.Add(1) - go func(key string, data map[string]interface{}) { + go func(pref string, ks []string, data map[string]interface{}) { defer wg.Done() - dataEncoder := NewCodecMsgpackMarshaler() - db, err := leveldb.OpenFile(filepath.Join(path, key+".cache"), nil) + db, err := leveldb.OpenFile(filepath.Join(path, pref+".cache"), nil) if err != nil { log.Fatal(err) } defer db.Close() - for k, v := range data { + // destinations are reverse mapped + if pref == utils.DESTINATION_PREFIX { + ks = make([]string, len(cs[utils.DESTINATION_PREFIX])) + i := 0 + for dk := range cs[utils.DESTINATION_PREFIX] { + ks[i] = dk + i++ + } + } + for _, k := range ks { + v := data[k] if encData, err := dataEncoder.Marshal(v); err == nil { if len(encData) > 1000 { var buf bytes.Buffer @@ -165,9 +177,11 @@ func (cs cacheDoubleStore) Save(path string, prefixes []string, cfi *utils.Cache break } } - }(prefix, mapValue) + }(prefix, keys, mapValue) } wg.Wait() + + utils.Logger.Info(fmt.Sprintf("Cache %v save time: %v", prefixSlice, time.Since(start))) return utils.SaveCacheFileInfo(path, cfi) } @@ -221,8 +235,7 @@ func (cs cacheDoubleStore) Load(path string, prefixes []string) error { } else { encData = data } - kv := CacheTypeFactory(key, "", nil) - v := kv.Value() + v := CacheTypeFactory(key) if err := dataDecoder.Unmarshal(encData, &v); err != nil { //log.Printf("%s err5", key) utils.Logger.Info(": " + err.Error()) @@ -366,7 +379,7 @@ func (cs cacheSimpleStore) GetKeysForPrefix(prefix string) (keys []string) { return } -func (cs cacheSimpleStore) Save(path string, keys []string, cfi *utils.CacheFileInfo) error { +func (cs cacheSimpleStore) Save(path string, keys map[string][]string, cfi *utils.CacheFileInfo) error { utils.Logger.Info("simplestore save") return nil } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index ab78ff7ec..a708fe965 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -665,30 +665,30 @@ func (ms *MongoStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac utils.Logger.Info(fmt.Sprintf("error saving load history: %v (%v)", loadHist, err)) return err } - var keys []string + keys := make(map[string][]string) if len(dKeys) > 0 { - keys = append(keys, utils.DESTINATION_PREFIX) + keys[utils.DESTINATION_PREFIX] = dKeys } if len(rpKeys) > 0 { - keys = append(keys, utils.RATING_PLAN_PREFIX) + keys[utils.RATING_PLAN_PREFIX] = rpKeys } if len(rpfKeys) > 0 { - keys = append(keys, utils.RATING_PROFILE_PREFIX) + keys[utils.RATING_PROFILE_PREFIX] = rpfKeys } if len(lcrKeys) > 0 { - keys = append(keys, utils.LCR_PREFIX) + keys[utils.LCR_PREFIX] = lcrKeys } if len(actKeys) > 0 { - keys = append(keys, utils.ACTION_PREFIX) + keys[utils.ACTION_PREFIX] = actKeys } - if len(actKeys) > 0 { - keys = append(keys, utils.DERIVEDCHARGERS_PREFIX) + if len(dcsKeys) > 0 { + keys[utils.DERIVEDCHARGERS_PREFIX] = dcsKeys } if len(aplKeys) > 0 { - keys = append(keys, utils.ACTION_PLAN_PREFIX) + keys[utils.ACTION_PLAN_PREFIX] = aplKeys } if len(shgKeys) > 0 { - keys = append(keys, utils.SHARED_GROUP_PREFIX) + keys[utils.SHARED_GROUP_PREFIX] = shgKeys } return CacheSave(ms.cacheDumpDir, keys, &utils.CacheFileInfo{Encoding: utils.GOB, LoadInfo: loadHist}) } @@ -773,9 +773,9 @@ func (ms *MongoStorage) cacheAccounting(alsKeys []string) (err error) { utils.Logger.Info("Finished load history caching.") CacheCommitTransaction() utils.Logger.Info(fmt.Sprintf("Cache accounting creation time: %v", time.Since(start))) - var keys []string + keys := make(map[string][]string) if len(alsKeys) > 0 { - keys = append(keys, utils.ALIASES_PREFIX) + keys[utils.ALIASES_PREFIX] = alsKeys } var loadHist *utils.LoadInstance diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 9338f62e9..a8a85bc81 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -340,30 +340,30 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac return err } - var keys []string + keys := make(map[string][]string) if len(dKeys) > 0 { - keys = append(keys, utils.DESTINATION_PREFIX) + keys[utils.DESTINATION_PREFIX] = dKeys } if len(rpKeys) > 0 { - keys = append(keys, utils.RATING_PLAN_PREFIX) + keys[utils.RATING_PLAN_PREFIX] = rpKeys } if len(rpfKeys) > 0 { - keys = append(keys, utils.RATING_PROFILE_PREFIX) + keys[utils.RATING_PROFILE_PREFIX] = rpfKeys } if len(lcrKeys) > 0 { - keys = append(keys, utils.LCR_PREFIX) + keys[utils.LCR_PREFIX] = lcrKeys } if len(actKeys) > 0 { - keys = append(keys, utils.ACTION_PREFIX) + keys[utils.ACTION_PREFIX] = actKeys } - if len(actKeys) > 0 { - keys = append(keys, utils.DERIVEDCHARGERS_PREFIX) + if len(dcsKeys) > 0 { + keys[utils.DERIVEDCHARGERS_PREFIX] = dcsKeys } if len(aplKeys) > 0 { - keys = append(keys, utils.ACTION_PLAN_PREFIX) + keys[utils.ACTION_PLAN_PREFIX] = aplKeys } if len(shgKeys) > 0 { - keys = append(keys, utils.SHARED_GROUP_PREFIX) + keys[utils.SHARED_GROUP_PREFIX] = shgKeys } return CacheSave(rs.cacheDumpDir, keys, &utils.CacheFileInfo{Encoding: utils.GOB, LoadInfo: loadHist}) } @@ -443,9 +443,9 @@ func (rs *RedisStorage) cacheAccounting(alsKeys []string) (err error) { utils.Logger.Info("Finished load history caching.") CacheCommitTransaction() utils.Logger.Info(fmt.Sprintf("Cache accounting creation time: %v", time.Since(start))) - var keys []string + keys := make(map[string][]string) if len(alsKeys) > 0 { - keys = append(keys, utils.ALIASES_PREFIX) + keys[utils.ALIASES_PREFIX] = alsKeys } loadHistList, err := rs.GetLoadHistory(1, true) if err != nil || len(loadHistList) == 0 {