uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-16 13:49:53 +05:00
Code Issues Packages Projects Releases Wiki Activity

improve reverse aliases

Browse Source
This commit is contained in:
Radu Ioan Fericean
2015-11-21 16:26:27 +02:00
parent bc3520bd2b
commit b0fe280b34
11 changed files with 133 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
cache2go/cache.go
View File

@@ -8,6 +8,7 @@ const (
KIND_ADD = "ADD"
KIND_ADP = "ADP"
KIND_REM = "REM"
KIND_POP = "POP"
KIND_PRF = "PRF"
DOUBLE_CACHE = true
)
@@ -62,7 +63,9 @@ func CommitTransaction() {
case KIND_ADD:
Cache(item.key, item.value)
case KIND_ADP:
CachePush(item.key, item.value)
Push(item.key, item.value)
case KIND_POP:
Pop(item.key, item.value)
}
}
mux.Unlock()
@@ -79,14 +82,14 @@ func Cache(key string, value interface{}) {
}
if !transactionON {
cache.Put(key, value)
//fmt.Println("ADD: ", key)
//log.Println("ADD: ", key)
} else {
transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: value, kind: KIND_ADD})
}
}
// Appends to an existing slice in the cache key
func CachePush(key string, value interface{}) {
func Push(key string, value interface{}) {
if !transactionLock {
mux.Lock()
defer mux.Unlock()
@@ -105,6 +108,18 @@ func Get(key string) (v interface{}, err error) {
return cache.Get(key)
}
func Pop(key string, value interface{}) {
if !transactionLock {
mux.Lock()
defer mux.Unlock()
}
if !transactionON {
cache.Pop(key, value)
} else {
transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: value, kind: KIND_POP})
}
}
func RemKey(key string) {
if !transactionLock {
mux.Lock()

18
cache2go/cache_test.go
View File

@@ -86,14 +86,28 @@ func TestRemPrefixKey(t *testing.T) {
}
func TestCachePush(t *testing.T) {
CachePush("ccc_t1", "1")
CachePush("ccc_t1", "2")
Push("ccc_t1", "1")
Push("ccc_t1", "2")
v, err := Get("ccc_t1")
if err != nil || len(v.(map[interface{}]struct{})) != 2 {
t.Error("Error in cache push: ", v)
}
}
func TestCachePop(t *testing.T) {
Push("ccc_t1", "1")
Push("ccc_t1", "2")
v, err := Get("ccc_t1")
if err != nil || len(v.(map[interface{}]struct{})) != 2 {
t.Error("Error in cache push: ", v)
}
Pop("ccc_t1", "1")
v, err = Get("ccc_t1")
if err != nil || len(v.(map[interface{}]struct{})) != 1 {
t.Error("Error in cache pop: ", v)
}
}
func TestCount(t *testing.T) {
Cache("dst_A1", "1")
Cache("dst_A2", "2")

37
cache2go/store.go
View File

@@ -11,6 +11,7 @@ type cacheStore interface {
Put(string, interface{})
Append(string, interface{})
Get(string) (interface{}, error)
Pop(string, interface{})
Delete(string)
DeletePrefix(string)
CountEntriesForPrefix(string) int
@@ -27,10 +28,12 @@ func newDoubleStore() cacheDoubleStore {
func (cs cacheDoubleStore) Put(key string, value interface{}) {
prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
if _, ok := cs[prefix]; !ok {
cs[prefix] = make(map[string]interface{})
mp, ok := cs[prefix]
if !ok {
mp = make(map[string]interface{})
cs[prefix] = mp
}
cs[prefix][key] = value
mp[key] = value
}
func (cs cacheDoubleStore) Append(key string, value interface{}) {
@@ -54,6 +57,20 @@ func (cs cacheDoubleStore) Get(key string) (interface{}, error) {
return nil, utils.ErrNotFound
}
func (cs cacheDoubleStore) Pop(key string, value interface{}) {
if v, err := cs.Get(key); err == nil {
elements, ok := v.(map[interface{}]struct{})
if ok {
delete(elements, value)
if len(elements) > 0 {
cache.Put(key, elements)
} else {
cache.Delete(key)
}
}
}
}
func (cs cacheDoubleStore) Delete(key string) {
prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
if keyMap, ok := cs[prefix]; ok {
@@ -130,6 +147,20 @@ func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
return nil, utils.ErrNotFound
}
func (cs cacheSimpleStore) Pop(key string, value interface{}) {
if v, err := cs.Get(key); err == nil {
elements, ok := v.(map[interface{}]struct{})
if ok {
delete(elements, value)
if len(elements) > 0 {
cache.Put(key, elements)
} else {
cache.Delete(key)
}
}
}
}
func (cs cacheSimpleStore) Delete(key string) {
if _, ok := cs.cache[key]; ok {
delete(cs.cache, key)

4
engine/aliases.go
View File

@@ -222,7 +222,7 @@ func (am *AliasHandler) RemoveReverseAlias(attr AttrReverseAlias, reply *string)
defer am.mu.Unlock()
rKey := utils.REVERSE_ALIASES_PREFIX + attr.Alias + attr.Target + attr.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys := x.(map[string]bool)
existingKeys := x.(map[string]struct{})
for key := range existingKeys {
// get destination id
elems := strings.Split(key, utils.CONCATENATED_KEY_SEP)
@@ -258,7 +258,7 @@ func (am *AliasHandler) GetReverseAlias(attr AttrReverseAlias, result *map[strin
aliases := make(map[string][]*Alias)
rKey := utils.REVERSE_ALIASES_PREFIX + attr.Alias + attr.Target + attr.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys := x.(map[string]bool)
existingKeys := x.(map[string]struct{})
for key := range existingKeys {
// get destination id

39
engine/aliases_test.go
View File

@@ -1,6 +1,11 @@
package engine
import "testing"
import (
"testing"
"github.com/cgrates/cgrates/cache2go"
"github.com/cgrates/cgrates/utils"
)
func init() {
aliasService = NewAliasHandler(accountingStorage)
@@ -72,3 +77,35 @@ func TestAliasesLoadAlias(t *testing.T) {
t.Errorf("Aliases failed to change interface: %+v", cd)
}
}
func TestAliasesCache(t *testing.T) {
key := "*out:cgrates.org:call:remo:remo:*rating"
a, err := cache2go.Get(utils.ALIASES_PREFIX + key)
if err != nil || a == nil {
//log.Printf("Test: %+v", cache2go.GetEntriesKeys(utils.REVERSE_ALIASES_PREFIX))
t.Error("Error getting alias from cache: ", err, a)
}
rKey1 := "minuAccount*rating"
ra1, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey1)
if err != nil || len(ra1.(map[interface{}]struct{})) != 2 {
t.Error("Error getting reverse alias 1: ", ra1)
}
rKey2 := "minuSubject*rating"
ra2, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey2)
if err != nil || len(ra2.(map[interface{}]struct{})) != 2 {
t.Error("Error getting reverse alias 2: ", ra2)
}
accountingStorage.RemoveAlias(key)
a, err = cache2go.Get(utils.ALIASES_PREFIX + key)
if err == nil {
t.Error("Error getting alias from cache: ", err)
}
ra1, err = cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey1)
if err != nil || len(ra1.(map[interface{}]struct{})) != 1 {
t.Error("Error getting reverse alias 1: ", ra1)
}
ra2, err = cache2go.Get(utils.REVERSE_ALIASES_PREFIX + rKey2)
if err != nil || len(ra2.(map[interface{}]struct{})) != 1 {
t.Error("Error getting reverse alias 2: ", ra2)
}
}

6
engine/loader_csv_test.go
View File

@@ -231,7 +231,7 @@ cgrates.org,dan,,another,value,10
cgrates.org,mas,true,another,value,10
`
aliases = `
#Direction[0],Tenant[1],Category[2],Account[3],Subject[4],DestinationId[5],Group[6],Alias[7],Weight[8]
#Direction[0],Tenant[1],Category[2],Account[3],Subject[4],DestinationId[5],Context[6],Target[7],Original[8],Alias[9],Weight[10]
*out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Subject,dan,dan1,10
*out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Subject,rif,rif1,10
*out,cgrates.org,call,dan,dan,EU_LANDLINE,*rating,Cli,0723,0724,10
@@ -240,6 +240,8 @@ cgrates.org,mas,true,another,value,10
*any,*any,*any,*any,*any,*any,*rating,Account,*any,dan1,10
*out,vdf,0,a1,a1,*any,*rating,Subject,a1,minu,10
*out,vdf,0,a1,a1,*any,*rating,Account,a1,minu,10
*out,cgrates.org,call,remo,remo,*any,*rating,Subject,remo,minu,10
*out,cgrates.org,call,remo,remo,*any,*rating,Account,remo,minu,10
`
)
@@ -1182,7 +1184,7 @@ func TestLoadUsers(t *testing.T) {
}
func TestLoadAliases(t *testing.T) {
if len(csvr.aliases) != 3 {
if len(csvr.aliases) != 4 {
t.Error("Failed to load aliases: ", len(csvr.aliases))
}
alias1 := &Alias{

30
engine/storage_map.go
View File

@@ -385,7 +385,7 @@ func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error)
err = ms.ms.Unmarshal(out, dest)
// create optimized structure
for _, p := range dest.Prefixes {
cache2go.CachePush(utils.DESTINATION_PREFIX+p, dest.Id)
cache2go.Push(utils.DESTINATION_PREFIX+p, dest.Id)
}
} else {
return nil, utils.ErrNotFound
@@ -592,18 +592,10 @@ func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error
if err == nil {
cache2go.Cache(key, al.Values)
for _, value := range al.Values {
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
} else {
existingKeys = make(map[string]bool)
}
existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true
cache2go.Cache(rKey, existingKeys)
rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "")
cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId))
}
}
}
@@ -625,23 +617,11 @@ func (ms *MapStorage) RemoveAlias(key string) error {
}
delete(ms.dict, key)
for _, value := range aliasValues {
tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId)
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
}
for eKey := range existingKeys {
if strings.HasPrefix(eKey, origKey) {
delete(existingKeys, eKey)
}
}
if len(existingKeys) == 0 {
cache2go.RemKey(rKey)
} else {
cache2go.Cache(rKey, existingKeys)
}
cache2go.Pop(rKey, tmpKey)
}
cache2go.RemKey(key)
}

29
engine/storage_mongo.go
View File

@@ -690,7 +690,7 @@ func (ms *MongoStorage) GetDestination(key string) (result *Destination, err err
}
// create optimized structure
for _, p := range result.Prefixes {
cache2go.CachePush(utils.DESTINATION_PREFIX+p, result.Id)
cache2go.Push(utils.DESTINATION_PREFIX+p, result.Id)
}
return
}
@@ -900,15 +900,8 @@ func (ms *MongoStorage) GetAlias(key string, skipCache bool) (al *Alias, err err
for _, value := range al.Values {
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
} else {
existingKeys = make(map[string]bool)
}
existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true
cache2go.Cache(rKey, existingKeys)
rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "")
cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId))
}
}
}
@@ -934,23 +927,11 @@ func (ms *MongoStorage) RemoveAlias(key string) (err error) {
err = ms.db.C(colAls).Remove(bson.M{"key": origKey})
if err == nil {
for _, value := range aliasValues {
tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId)
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
}
for eKey := range existingKeys {
if strings.HasPrefix(origKey, eKey) {
delete(existingKeys, eKey)
}
}
if len(existingKeys) == 0 {
cache2go.RemKey(rKey)
} else {
cache2go.Cache(rKey, existingKeys)
}
cache2go.Pop(rKey, tmpKey)
}
cache2go.RemKey(key)
}

44
engine/storage_redis.go
View File

@@ -247,9 +247,6 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac
if len(dcsKeys) != 0 {
utils.Logger.Info("Finished derived chargers caching.")
}
if actKeys == nil {
cache2go.RemPrefixKey(utils.ACTION_PREFIX)
}
if actKeys == nil {
utils.Logger.Info("Caching all actions")
if actKeys, err = conn.Cmd("KEYS", utils.ACTION_PREFIX+"*").List(); err != nil {
@@ -271,9 +268,6 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac
utils.Logger.Info("Finished actions caching.")
}
if aplKeys == nil {
cache2go.RemPrefixKey(utils.ACTION_PLAN_PREFIX)
}
if aplKeys == nil {
utils.Logger.Info("Caching all action plans")
if aplKeys, err = rs.db.Cmd("KEYS", utils.ACTION_PLAN_PREFIX+"*").List(); err != nil {
@@ -295,15 +289,13 @@ func (rs *RedisStorage) cacheRating(dKeys, rpKeys, rpfKeys, lcrKeys, dcsKeys, ac
utils.Logger.Info("Finished action plans caching.")
}
if shgKeys == nil {
cache2go.RemPrefixKey(utils.SHARED_GROUP_PREFIX)
}
if shgKeys == nil {
utils.Logger.Info("Caching all shared groups")
if shgKeys, err = conn.Cmd("KEYS", utils.SHARED_GROUP_PREFIX+"*").List(); err != nil {
cache2go.RollbackTransaction()
return err
}
cache2go.RemPrefixKey(utils.SHARED_GROUP_PREFIX)
} else if len(shgKeys) != 0 {
utils.Logger.Info(fmt.Sprintf("Caching shared groups: %v", shgKeys))
}
@@ -359,15 +351,14 @@ func (rs *RedisStorage) cacheAccounting(alsKeys []string) (err error) {
return err
}
defer rs.db.Put(conn)
if alsKeys == nil {
cache2go.RemPrefixKey(utils.ALIASES_PREFIX)
}
if alsKeys == nil {
utils.Logger.Info("Caching all aliases")
if alsKeys, err = conn.Cmd("KEYS", utils.ALIASES_PREFIX+"*").List(); err != nil {
cache2go.RollbackTransaction()
return err
}
cache2go.RemPrefixKey(utils.ALIASES_PREFIX)
cache2go.RemPrefixKey(utils.REVERSE_ALIASES_PREFIX)
} else if len(alsKeys) != 0 {
utils.Logger.Info(fmt.Sprintf("Caching aliases: %v", alsKeys))
}
@@ -538,7 +529,7 @@ func (rs *RedisStorage) GetDestination(key string) (dest *Destination, err error
err = rs.ms.Unmarshal(out, dest)
// create optimized structure
for _, p := range dest.Prefixes {
cache2go.CachePush(utils.DESTINATION_PREFIX+p, dest.Id)
cache2go.Push(utils.DESTINATION_PREFIX+p, dest.Id)
}
} else {
return nil, errors.New("not found")
@@ -767,15 +758,8 @@ func (rs *RedisStorage) GetAlias(key string, skipCache bool) (al *Alias, err err
for _, value := range al.Values {
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
} else {
existingKeys = make(map[string]bool)
}
existingKeys[utils.ConcatenatedKey(origKey, value.DestinationId)] = true
cache2go.Cache(rKey, existingKeys)
rKey := strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "")
cache2go.Push(rKey, utils.ConcatenatedKey(origKey, value.DestinationId))
}
}
}
@@ -801,23 +785,11 @@ func (rs *RedisStorage) RemoveAlias(key string) (err error) {
err = conn.Cmd("DEL", key).Err
if err == nil {
for _, value := range aliasValues {
tmpKey := utils.ConcatenatedKey(origKey, value.DestinationId)
for target, pairs := range value.Pairs {
for _, alias := range pairs {
var existingKeys map[string]bool
rKey := utils.REVERSE_ALIASES_PREFIX + alias + target + al.Context
if x, err := cache2go.Get(rKey); err == nil {
existingKeys = x.(map[string]bool)
}
for eKey := range existingKeys {
if strings.HasPrefix(origKey, eKey) {
delete(existingKeys, eKey)
}
}
if len(existingKeys) == 0 {
cache2go.RemKey(rKey)
} else {
cache2go.Cache(rKey, existingKeys)
}
cache2go.Pop(rKey, tmpKey)
}
cache2go.RemKey(key)
}

4
engine/storage_test.go
View File

@@ -183,7 +183,7 @@ func TestStorageCacheGetReverseAliases(t *testing.T) {
Context: "*other",
}
if x, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + "aaa" + "Subject" + utils.ALIAS_CONTEXT_RATING); err == nil {
aliasKeys := x.(map[string]bool)
aliasKeys := x.(map[interface{}]struct{})
_, found := aliasKeys[utils.ConcatenatedKey(ala.GetId(), utils.ANY)]
if !found {
t.Error("Error getting reverse alias: ", aliasKeys, ala.GetId()+utils.ANY)
@@ -192,7 +192,7 @@ func TestStorageCacheGetReverseAliases(t *testing.T) {
t.Error("Error getting reverse alias: ", err)
}
if x, err := cache2go.Get(utils.REVERSE_ALIASES_PREFIX + "aaa" + "Account" + "*other"); err == nil {
aliasKeys := x.(map[string]bool)
aliasKeys := x.(map[interface{}]struct{})
_, found := aliasKeys[utils.ConcatenatedKey(alb.GetId(), utils.ANY)]
if !found {
t.Error("Error getting reverse alias: ", aliasKeys)

1
utils/server.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/cenkalti/rpc2"
"golang.org/x/net/websocket"
)
import _ "net/http/pprof"
type Server struct {
rpcEnabled bool