From b0fe280b34d42f6fba6b09652207aa342b4c33af Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 21 Nov 2015 16:26:27 +0200 Subject: [PATCH 1/3] improve reverse aliases --- cache2go/cache.go | 21 ++++++++++++++++--- cache2go/cache_test.go | 18 ++++++++++++++-- cache2go/store.go | 37 +++++++++++++++++++++++++++++--- engine/aliases.go | 4 ++-- engine/aliases_test.go | 39 +++++++++++++++++++++++++++++++++- engine/loader_csv_test.go | 6 ++++-- engine/storage_map.go | 30 +++++--------------------- engine/storage_mongo.go | 29 +++++--------------------- engine/storage_redis.go | 44 +++++++-------------------------------- engine/storage_test.go | 4 ++-- utils/server.go | 1 + 11 files changed, 133 insertions(+), 100 deletions(-) diff --git a/cache2go/cache.go b/cache2go/cache.go index 7642bb44d..6fcc6d8d1 100644 --- a/cache2go/cache.go +++ b/cache2go/cache.go @@ -8,6 +8,7 @@ const ( KIND_ADD = "ADD" KIND_ADP = "ADP" KIND_REM = "REM" + KIND_POP = "POP" KIND_PRF = "PRF" DOUBLE_CACHE = true ) @@ -62,7 +63,9 @@ func CommitTransaction() { case KIND_ADD: Cache(item.key, item.value) case KIND_ADP: - CachePush(item.key, item.value) + Push(item.key, item.value) + case KIND_POP: + Pop(item.key, item.value) } } mux.Unlock() @@ -79,14 +82,14 @@ func Cache(key string, value interface{}) { } if !transactionON { cache.Put(key, value) - //fmt.Println("ADD: ", key) + //log.Println("ADD: ", key) } else { transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: value, kind: KIND_ADD}) } } // Appends to an existing slice in the cache key -func CachePush(key string, value interface{}) { +func Push(key string, value interface{}) { if !transactionLock { mux.Lock() defer mux.Unlock() @@ -105,6 +108,18 @@ func Get(key string) (v interface{}, err error) { return cache.Get(key) } +func Pop(key string, value interface{}) { + if !transactionLock { + mux.Lock() + defer mux.Unlock() + } + if !transactionON { + cache.Pop(key, value) + } else { + transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: value, kind: KIND_POP}) + } +} + func RemKey(key string) { if !transactionLock { mux.Lock() diff --git a/cache2go/cache_test.go b/cache2go/cache_test.go index 4b062ecf1..ce9f50e92 100644 --- a/cache2go/cache_test.go +++ b/cache2go/cache_test.go @@ -86,14 +86,28 @@ func TestRemPrefixKey(t *testing.T) { } func TestCachePush(t *testing.T) { - CachePush("ccc_t1", "1") - CachePush("ccc_t1", "2") + Push("ccc_t1", "1") + Push("ccc_t1", "2") v, err := Get("ccc_t1") if err != nil || len(v.(map[interface{}]struct{})) != 2 { t.Error("Error in cache push: ", v) } } +func TestCachePop(t *testing.T) { + Push("ccc_t1", "1") + Push("ccc_t1", "2") + v, err := Get("ccc_t1") + if err != nil || len(v.(map[interface{}]struct{})) != 2 { + t.Error("Error in cache push: ", v) + } + Pop("ccc_t1", "1") + v, err = Get("ccc_t1") + if err != nil || len(v.(map[interface{}]struct{})) != 1 { + t.Error("Error in cache pop: ", v) + } +} + func TestCount(t *testing.T) { Cache("dst_A1", "1") Cache("dst_A2", "2") diff --git a/cache2go/store.go b/cache2go/store.go index fbd675775..572d02e4b 100644 --- a/cache2go/store.go +++ b/cache2go/store.go @@ -11,6 +11,7 @@ type cacheStore interface { Put(string, interface{}) Append(string, interface{}) Get(string) (interface{}, error) + Pop(string, interface{}) Delete(string) DeletePrefix(string) CountEntriesForPrefix(string) int @@ -27,10 +28,12 @@ func newDoubleStore() cacheDoubleStore { func (cs cacheDoubleStore) Put(key string, value interface{}) { prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:] - if _, ok := cs[prefix]; !ok { - cs[prefix] = make(map[string]interface{}) + mp, ok := cs[prefix] + if !ok { + mp = make(map[string]interface{}) + cs[prefix] = mp } - cs[prefix][key] = value + mp[key] = value } func (cs cacheDoubleStore) Append(key string, value interface{}) { @@ -54,6 +57,20 @@ func (cs cacheDoubleStore) Get(key string) (interface{}, error) { return nil, utils.ErrNotFound } +func (cs cacheDoubleStore) Pop(key string, value interface{}) { + if v, err := cs.Get(key); err == nil { + elements, ok := v.(map[interface{}]struct{}) + if ok { + delete(elements, value) + if len(elements) > 0 { + cache.Put(key, elements) + } else { + cache.Delete(key) + } + } + } +} + func (cs cacheDoubleStore) Delete(key string) { prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:] if keyMap, ok := cs[prefix]; ok { @@ -130,6 +147,20 @@ func (cs cacheSimpleStore) Get(key string) (interface{}, error) { return nil, utils.ErrNotFound } +func (cs cacheSimpleStore) Pop(key string, value interface{}) { + if v, err := cs.Get(key); err == nil { + elements, ok := v.(map[interface{}]struct{}) + if ok { + delete(elements, value) + if len(elements) > 0 { + cache.Put(key, elements) + } else { + cache.Delete(key) + } + } + } +} + func (cs cacheSimpleStore) Delete(key string) { if _, ok := cs.cache[key]; ok { delete(cs.cache, key) diff --git a/engine/aliases.go b/engine/aliases.go index c6a49520f..bf5086c55 100644 --- a/engine/aliases.go +++ b/engine/aliases.go @@ -222,7 +222,7 @@ func (am *AliasHandler) RemoveReverseAlias(attr AttrReverseAlias, reply *string) defer am.mu.Unlock() rKey := utils.REVERSE_ALIASES_PREFIX + attr.Alias + attr.Target + attr.Context if x, err := cache2go.Get(rKey); err == nil { - existingKeys := x.(map[string]bool) + existingKeys := x.(map[string]struct{}) for key := range existingKeys { // get destination id elems := strings.Split(key, utils.CONCATENATED_KEY_SEP) @@ -258,7 +258,7 @@ func (am *AliasHandler) GetReverseAlias(attr AttrReverseAlias, result *map[strin aliases := make(map[string][]*Alias) rKey := utils.REVERSE_ALIASES_PREFIX + attr.Alias + attr.Target + attr.Context if x, err := cache2go.Get(rKey); err == nil { - existingKeys := x.(map[string]bool) + existingKeys := x.(map[string]struct{}) for key := range existingKeys { // get destination id diff --git a/engine/aliases_test.go b/engine/aliases_test.go index 7d4db2034..67d89dd9a 100644 --- a/engine/aliases_test.go +++ b/engine/aliases_test.go @@ -1,6 +1,11 @@ package engine -import "testing" +import ( + "testing" + + "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/utils" +) func init() { aliasService = NewAliasHandler(accountingStorage) @@ -72,3 +77,35 @@ func TestAliasesLoadAlias(t *testing.T) { t.Errorf("Aliases failed to change interface: %+v", cd) } } + +func TestAliasesCache(t *testing.T) { + key := "*out:cgrates.org:call:remo:remo:*rating" + a, err := cache2go.Get(utils.ALIASES_PREFIX + key) + if err != nil || a == nil { + //log.Printf("Test: %+v", cache2go.GetEntriesKeys(utils.REVERSE_ALIASES_PREFIX)) + t.Error("Error getting alias from cache: ", err, a) + } + rKey1 := "minuAccount*rating" + ra1, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey1) + if err != nil || len(ra1.(map[interface{}]struct{})) != 2 { + t.Error("Error getting reverse alias 1: ", ra1) + } + rKey2 := "minuSubject*rating" + ra2, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey2) + if err != nil || len(ra2.(map[interface{}]struct{})) != 2 { + t.Error("Error getting reverse alias 2: ", ra2) + } + accountingStorage.RemoveAlias(key) + a, err = cache2go.Get(utils.ALIASES_PREFIX + key) + if err == nil { + t.Error("Error getting alias from cache: ", err) + } + ra1, err = cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey1) + if err != nil || len(ra1.(map[interface{}]struct{})) != 1 { + t.Error("Error getting reverse alias 1: ", ra1) + } + ra2, err = cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey2) + if err != nil || len(ra2.(map[interface{}]struct{})) != 1 { + t.Error("Error getting reverse alias 2: ", ra2) + } +} diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 1e5694beb..a7e5baf33 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -231,7 +231,7 @@ cgrates.org,dan,,another,value,10 cgrates.org,mas,true,another,value,10 ` aliases = ` -#Direction[0],Tenant[1],Category[2],Account[3],Subject[4],DestinationId[5],Group[6],Alias[7],Weight[8] +#Direction[0],Tenant[1],Category[2],Account[3],Subject[4],DestinationId[5],Context[6],Target[7],Original[8],Alias[9],Weight[10] *out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Subject,dan,dan1,10 *out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Subject,rif,rif1,10 *out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Cli,0723,0724,10 @@ -240,6 +240,8 @@ cgrates.org,mas,true,another,value,10 *any,*any,*any,*any,*any,*any,*rating,Account,*any,dan1,10 *out,vdf,0,a1,a1,*any,*rating,Subject,a1,minu,10 *out,vdf,0,a1,a1,*any,*rating,Account,a1,minu,10 +*out,cgrates.org,call,remo,remo,*any,*rating,Subject,remo,minu,10 +*out,cgrates.org,call,remo,remo,*any,*rating,Account,remo,minu,10 ` ) @@ -1182,7 +1184,7 @@ func TestLoadUsers(t *testing.T) { } func TestLoadAliases(t *testing.T) { - if len(csvr.aliases) != 3 { + if len(csvr.aliases) != 4 { t.Error("Failed to load aliases: ", len(csvr.aliases)) } alias1 := &Alias{ diff --git a/engine/storage_map.go b/engine/storage_map.go index 7a5937c99..d74b73576 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -385,7 +385,7 @@ func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error) err = ms.ms.Unmarshal(out, dest) // create optimized structure for _, p := range dest.Prefixes { - cache2go.CachePush(utils.DESTINATION_PREFIX+p, dest.Id) + cache2go.Push(utils.DESTINATION_PREFIX+p, dest.Id) } } else { return nil, utils.ErrNotFound @@ -592,18 +592,10 @@ func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error if err == nil { cache2go.Cache(key, al.Values) for _, value := range al.Values { - for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } else { - existingKeys = make(map[string]bool) - } - existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true - cache2go.Cache(rKey, existingKeys) + rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") + cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) } } } @@ -625,23 +617,11 @@ func (ms *MapStorage) RemoveAlias(key string) error { } delete(ms.dict, key) for _, value := range aliasValues { + tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } - for eKey := range existingKeys { - if strings.HasPrefix(eKey, origKey) { - delete(existingKeys, eKey) - } - } - if len(existingKeys) == 0 { - cache2go.RemKey(rKey) - } else { - cache2go.Cache(rKey, existingKeys) - } + cache2go.Pop(rKey, tmpKey) } cache2go.RemKey(key) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index cdde8ee48..19b2e17c3 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -690,7 +690,7 @@ func (ms *MongoStorage) GetDestination(key string) (result *Destination, err err } // create optimized structure for _, p := range result.Prefixes { - cache2go.CachePush(utils.DESTINATION_PREFIX+p, result.Id) + cache2go.Push(utils.DESTINATION_PREFIX+p, result.Id) } return } @@ -900,15 +900,8 @@ func (ms *MongoStorage) GetAlias(key string, skipCache bool) (al *Alias, err err for _, value := range al.Values { for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } else { - existingKeys = make(map[string]bool) - } - existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true - cache2go.Cache(rKey, existingKeys) + rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") + cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) } } } @@ -934,23 +927,11 @@ func (ms *MongoStorage) RemoveAlias(key string) (err error) { err = ms.db.C(colAls).Remove(bson.M{"key": origKey}) if err == nil { for _, value := range aliasValues { + tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } - for eKey := range existingKeys { - if strings.HasPrefix(origKey, eKey) { - delete(existingKeys, eKey) - } - } - if len(existingKeys) == 0 { - cache2go.RemKey(rKey) - } else { - cache2go.Cache(rKey, existingKeys) - } + cache2go.Pop(rKey, tmpKey) } cache2go.RemKey(key) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ee3f74927..47c50656f 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -247,9 +247,6 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac if len(dcsKeys) != 0 { utils.Logger.Info("Finished derived chargers caching.") } - if actKeys == nil { - cache2go.RemPrefixKey(utils.ACTION_PREFIX) - } if actKeys == nil { utils.Logger.Info("Caching all actions") if actKeys, err = conn.Cmd("KEYS", utils.ACTION_PREFIX+"*").List(); err != nil { @@ -271,9 +268,6 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac utils.Logger.Info("Finished actions caching.") } - if aplKeys == nil { - cache2go.RemPrefixKey(utils.ACTION_PLAN_PREFIX) - } if aplKeys == nil { utils.Logger.Info("Caching all action plans") if aplKeys, err = rs.db.Cmd("KEYS", utils.ACTION_PLAN_PREFIX+"*").List(); err != nil { @@ -295,15 +289,13 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac utils.Logger.Info("Finished action plans caching.") } - if shgKeys == nil { - cache2go.RemPrefixKey(utils.SHARED_GROUP_PREFIX) - } if shgKeys == nil { utils.Logger.Info("Caching all shared groups") if shgKeys, err = conn.Cmd("KEYS", utils.SHARED_GROUP_PREFIX+"*").List(); err != nil { cache2go.RollbackTransaction() return err } + cache2go.RemPrefixKey(utils.SHARED_GROUP_PREFIX) } else if len(shgKeys) != 0 { utils.Logger.Info(fmt.Sprintf("Caching shared groups: %v", shgKeys)) } @@ -359,15 +351,14 @@ func (rs *RedisStorage) cacheAccounting(alsKeys []string) (err error) { return err } defer rs.db.Put(conn) - if alsKeys == nil { - cache2go.RemPrefixKey(utils.ALIASES_PREFIX) - } if alsKeys == nil { utils.Logger.Info("Caching all aliases") if alsKeys, err = conn.Cmd("KEYS", utils.ALIASES_PREFIX+"*").List(); err != nil { cache2go.RollbackTransaction() return err } + cache2go.RemPrefixKey(utils.ALIASES_PREFIX) + cache2go.RemPrefixKey(utils.REVERSE_ALIASES_PREFIX) } else if len(alsKeys) != 0 { utils.Logger.Info(fmt.Sprintf("Caching aliases: %v", alsKeys)) } @@ -538,7 +529,7 @@ func (rs *RedisStorage) GetDestination(key string) (dest *Destination, err error err = rs.ms.Unmarshal(out, dest) // create optimized structure for _, p := range dest.Prefixes { - cache2go.CachePush(utils.DESTINATION_PREFIX+p, dest.Id) + cache2go.Push(utils.DESTINATION_PREFIX+p, dest.Id) } } else { return nil, errors.New("not found") @@ -767,15 +758,8 @@ func (rs *RedisStorage) GetAlias(key string, skipCache bool) (al *Alias, err err for _, value := range al.Values { for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } else { - existingKeys = make(map[string]bool) - } - existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true - cache2go.Cache(rKey, existingKeys) + rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") + cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) } } } @@ -801,23 +785,11 @@ func (rs *RedisStorage) RemoveAlias(key string) (err error) { err = conn.Cmd("DEL", key).Err if err == nil { for _, value := range aliasValues { + tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) for target, pairs := range value.Pairs { for _, alias := range pairs { - var existingKeys map[string]bool rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - if x, err := cache2go.Get(rKey); err == nil { - existingKeys = x.(map[string]bool) - } - for eKey := range existingKeys { - if strings.HasPrefix(origKey, eKey) { - delete(existingKeys, eKey) - } - } - if len(existingKeys) == 0 { - cache2go.RemKey(rKey) - } else { - cache2go.Cache(rKey, existingKeys) - } + cache2go.Pop(rKey, tmpKey) } cache2go.RemKey(key) } diff --git a/engine/storage_test.go b/engine/storage_test.go index 306c9289b..ff06d0ac0 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -183,7 +183,7 @@ func TestStorageCacheGetReverseAliases(t *testing.T) { Context: "*other", } if x, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + "aaa" + "Subject" + utils.ALIAS_CONTEXT_RATING); err == nil { - aliasKeys := x.(map[string]bool) + aliasKeys := x.(map[interface{}]struct{}) _, found := aliasKeys[utils.ConcatenatedKey(ala.GetId(), utils.ANY)] if !found { t.Error("Error getting reverse alias: ", aliasKeys, ala.GetId()+utils.ANY) @@ -192,7 +192,7 @@ func TestStorageCacheGetReverseAliases(t *testing.T) { t.Error("Error getting reverse alias: ", err) } if x, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + "aaa" + "Account" + "*other"); err == nil { - aliasKeys := x.(map[string]bool) + aliasKeys := x.(map[interface{}]struct{}) _, found := aliasKeys[utils.ConcatenatedKey(alb.GetId(), utils.ANY)] if !found { t.Error("Error getting reverse alias: ", aliasKeys) diff --git a/utils/server.go b/utils/server.go index 292ae9c9d..5744d796d 100644 --- a/utils/server.go +++ b/utils/server.go @@ -30,6 +30,7 @@ import ( "github.com/cenkalti/rpc2" "golang.org/x/net/websocket" ) +import _ "net/http/pprof" type Server struct { rpcEnabled bool From 86a094b5192775dbe5d6a0c46565b210be4eae5d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 21 Nov 2015 19:48:25 +0200 Subject: [PATCH 2/3] clean reverse aliases on reload --- engine/aliases.go | 23 +++++++++++++++++++++++ engine/storage_map.go | 31 +++++++++++-------------------- engine/storage_mongo.go | 33 +++++++++++---------------------- engine/storage_redis.go | 31 +++++++++++-------------------- 4 files changed, 56 insertions(+), 62 deletions(-) diff --git a/engine/aliases.go b/engine/aliases.go index bf5086c55..419a70970 100644 --- a/engine/aliases.go +++ b/engine/aliases.go @@ -115,6 +115,29 @@ func (al *Alias) SetId(id string) error { return nil } +func (al *Alias) SetReverseCache() { + for _, value := range al.Values { + for target, pairs := range value.Pairs { + for _, alias := range pairs { + rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") + cache2go.Push(rKey, utils.ConcatenatedKey(al.GetId(), value.DestinationId)) + } + } + } +} + +func (al *Alias) RemoveReverseCache() { + for _, value := range al.Values { + tmpKey := utils.ConcatenatedKey(al.GetId(), value.DestinationId) + for target, pairs := range value.Pairs { + for _, alias := range pairs { + rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context + cache2go.Pop(rKey, tmpKey) + } + } + } +} + type AttrMatchingAlias struct { Destination string Direction string diff --git a/engine/storage_map.go b/engine/storage_map.go index d74b73576..5ad4e2b07 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -232,6 +232,13 @@ func (ms *MapStorage) cacheAccounting(alsKeys []string) error { } for k, _ := range ms.dict { if strings.HasPrefix(k, utils.ALIASES_PREFIX) { + // check if it already exists + // to remove reverse cache keys + if avs, err := cache2go.Get(k); err == nil && avs != nil { + al := &Alias{Values: avs.(AliasValues)} + al.SetId(k[len(utils.ALIASES_PREFIX):]) + al.RemoveReverseCache() + } cache2go.RemKey(k) if _, err := ms.GetAlias(k[len(utils.ALIASES_PREFIX):], true); err != nil { cache2go.RollbackTransaction() @@ -574,7 +581,6 @@ func (ms *MapStorage) SetAlias(al *Alias) error { } func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error) { - origKey := key key = utils.ALIASES_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -591,14 +597,7 @@ func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error err = ms.ms.Unmarshal(values, &al.Values) if err == nil { cache2go.Cache(key, al.Values) - for _, value := range al.Values { - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") - cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) - } - } - } + al.SetReverseCache() } } else { return nil, utils.ErrNotFound @@ -609,23 +608,15 @@ func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error func (ms *MapStorage) RemoveAlias(key string) error { al := &Alias{} al.SetId(key) - origKey := key key = utils.ALIASES_PREFIX + key aliasValues := make(AliasValues, 0) if values, ok := ms.dict[key]; ok { ms.ms.Unmarshal(values, &aliasValues) } + al.Values = aliasValues delete(ms.dict, key) - for _, value := range aliasValues { - tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - cache2go.Pop(rKey, tmpKey) - } - cache2go.RemKey(key) - } - } + al.RemoveReverseCache() + cache2go.RemKey(key) return nil } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 19b2e17c3..aa171baa3 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -21,7 +21,6 @@ package engine import ( "errors" "fmt" - "strings" "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" @@ -543,6 +542,13 @@ func (ms *MongoStorage) cacheAccounting(alsKeys []string) (err error) { utils.Logger.Info(fmt.Sprintf("Caching aliases: %v", alsKeys)) } for _, key := range alsKeys { + // check if it already exists + // to remove reverse cache keys + if avs, err := cache2go.Get(key); err == nil && avs != nil { + al := &Alias{Values: avs.(AliasValues)} + al.SetId(key[len(utils.ALIASES_PREFIX):]) + al.RemoveReverseCache() + } cache2go.RemKey(key) if _, err = ms.GetAlias(key[len(utils.ALIASES_PREFIX):], true); err != nil { cache2go.RollbackTransaction() @@ -897,14 +903,7 @@ func (ms *MongoStorage) GetAlias(key string, skipCache bool) (al *Alias, err err if err == nil { cache2go.Cache(key, al.Values) // cache reverse alias - for _, value := range al.Values { - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") - cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) - } - } - } + al.SetReverseCache() } } return @@ -915,27 +914,17 @@ func (ms *MongoStorage) RemoveAlias(key string) (err error) { al.SetId(key) origKey := key key = utils.ALIASES_PREFIX + key - var aliasValues AliasValues - var kv struct { Key string Value AliasValues } if err := ms.db.C(colAls).Find(bson.M{"key": origKey}).One(&kv); err == nil { - aliasValues = kv.Value + al.Values = kv.Value } err = ms.db.C(colAls).Remove(bson.M{"key": origKey}) if err == nil { - for _, value := range aliasValues { - tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - cache2go.Pop(rKey, tmpKey) - } - cache2go.RemKey(key) - } - } + al.RemoveReverseCache() + cache2go.RemKey(key) } return } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 47c50656f..86107e78b 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -23,7 +23,6 @@ import ( "compress/zlib" "errors" "fmt" - "strings" "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" @@ -363,6 +362,13 @@ func (rs *RedisStorage) cacheAccounting(alsKeys []string) (err error) { utils.Logger.Info(fmt.Sprintf("Caching aliases: %v", alsKeys)) } for _, key := range alsKeys { + // check if it already exists + // to remove reverse cache keys + if avs, err := cache2go.Get(key); err == nil && avs != nil { + al := &Alias{Values: avs.(AliasValues)} + al.SetId(key[len(utils.ALIASES_PREFIX):]) + al.RemoveReverseCache() + } cache2go.RemKey(key) if _, err = rs.GetAlias(key[len(utils.ALIASES_PREFIX):], true); err != nil { cache2go.RollbackTransaction() @@ -755,14 +761,7 @@ func (rs *RedisStorage) GetAlias(key string, skipCache bool) (al *Alias, err err if err == nil { cache2go.Cache(key, al.Values) // cache reverse alias - for _, value := range al.Values { - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "") - cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId)) - } - } - } + al.SetReverseCache() } } return @@ -776,24 +775,16 @@ func (rs *RedisStorage) RemoveAlias(key string) (err error) { defer rs.db.Put(conn) al := &Alias{} al.SetId(key) - origKey := key key = utils.ALIASES_PREFIX + key aliasValues := make(AliasValues, 0) if values, err := conn.Cmd("GET", key).Bytes(); err == nil { rs.ms.Unmarshal(values, &aliasValues) } + al.Values = aliasValues err = conn.Cmd("DEL", key).Err if err == nil { - for _, value := range aliasValues { - tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId) - for target, pairs := range value.Pairs { - for _, alias := range pairs { - rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context - cache2go.Pop(rKey, tmpKey) - } - cache2go.RemKey(key) - } - } + al.RemoveReverseCache() + cache2go.RemKey(key) } return } From 53a8b80ed3443c3f5be0055758193f66b8e3c35c Mon Sep 17 00:00:00 2001 From: rinor Date: Mon, 23 Nov 2015 22:50:31 +0100 Subject: [PATCH 3/3] small fix: log and function name - sessionmanager/fssessionmanager.go : fix log - cmd/cgr-engine/cgr-engine.go : rename RegisterHanlersToServer to RegisterHandlersToServer - engine/cdrs.go : rename RegisterHanlersToServer to RegisterHandlersToServer --- cmd/cgr-engine/cgr-engine.go | 2 +- engine/cdrs.go | 2 +- sessionmanager/fssessionmanager.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index a38ddaa28..d519c7c9c 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -472,7 +472,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, raterConn, pubSubConn, usersConn, aliasesConn, statsConn) utils.Logger.Info("Registering CDRS HTTP Handlers.") - cdrServer.RegisterHanlersToServer(server) + cdrServer.RegisterHandlersToServer(server) utils.Logger.Info("Registering CDRS RPC service.") cdrSrv := v1.CdrsV1{CdrSrv: cdrServer} server.RpcRegister(&cdrSrv) diff --git a/engine/cdrs.go b/engine/cdrs.go index ae246729e..99d47226c 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -84,7 +84,7 @@ func (self *CdrServer) Timezone() string { return self.cgrCfg.DefaultTimezone } -func (self *CdrServer) RegisterHanlersToServer(server *utils.Server) { +func (self *CdrServer) RegisterHandlersToServer(server *utils.Server) { cdrServer = self // Share the server object for handlers server.RegisterHttpFunc("/cdr_http", cgrCdrHandler) server.RegisterHttpFunc("/freeswitch_json", fsCdrHandler) diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 0ed305f12..47a728fbd 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -340,7 +340,7 @@ func (sm *FSSessionManager) Shutdown() (err error) { } for i := 0; len(sm.sessions.getSessions()) > 0 && i < 20; i++ { time.Sleep(100 * time.Millisecond) // wait for the hungup event to be fired - utils.Logger.Info(fmt.Sprintf(" Shutdown waiting on sessions: %v", sm.sessions)) + utils.Logger.Info(fmt.Sprintf(" Shutdown waiting on sessions: %v", sm.sessions)) } return nil }