fixes + easy switch between cache stores

cache2go/cache.go

@@ -2,19 +2,17 @@
 package cache2go
 
 import (
-	"errors"
 	"sync"
 	"time"
-
-	"github.com/cgrates/cgrates/utils"
 )
 
 const (
-	PREFIX_LEN = 4
-	KIND_ADD   = "ADD"
-	KIND_ADP   = "ADP"
-	KIND_REM   = "REM"
-	KIND_PRF   = "PRF"
+	PREFIX_LEN   = 4
+	KIND_ADD     = "ADD"
+	KIND_ADP     = "ADP"
+	KIND_REM     = "REM"
+	KIND_PRF     = "PRF"
+	DOUBLE_CACHE = true
 )
 
 type timestampedValue struct {
@@ -32,10 +30,17 @@ type transactionItem struct {
 	kind  string
 }
 
-var (
-	cache = make(cacheStore)
-	mux   sync.RWMutex
+func init() {
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
+}
 
+var (
+	mux   sync.RWMutex
+	cache cacheStore
 	// transaction stuff
 	transactionBuffer []transactionItem
 	transactionMux    sync.Mutex
@@ -146,34 +151,27 @@ func RemPrefixKey(prefix string) {
 func Flush() {
 	mux.Lock()
 	defer mux.Unlock()
-	cache = make(cacheStore)
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
 }
 
 func CountEntries(prefix string) (result int64) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if _, ok := cache[prefix]; ok {
-		return int64(len(cache[prefix]))
-	}
-	return 0
+	return cache.CountEntriesForPrefix(prefix)
 }
 
 func GetAllEntries(prefix string) (map[string]timestampedValue, error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if keyMap, ok := cache[prefix]; ok {
-		return keyMap, nil
-	}
-	return nil, errors.New(utils.ERR_NOT_FOUND)
+	return cache.GetAllForPrefix(prefix)
 }
 
 func GetEntriesKeys(prefix string) (keys []string) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if keyMap, ok := cache[prefix]; ok {
-		for key := range keyMap {
-			keys = append(keys, key)
-		}
-	}
-	return
+	return cache.GetKeysForPrefix(prefix)
 }

cache2go/store.go

@@ -3,14 +3,32 @@ package cache2go
 
 import (
 	"errors"
+	"strings"
 	"time"
 
 	"github.com/cgrates/cgrates/utils"
 )
 
-type cacheStore map[string]map[string]timestampedValue
+type cacheStore interface {
+	Put(string, interface{})
+	Append(string, interface{})
+	Get(string) (interface{}, error)
+	GetAge(string) (time.Duration, error)
+	Delete(string)
+	DeletePrefix(string)
+	CountEntriesForPrefix(string) int64
+	GetAllForPrefix(string) (map[string]timestampedValue, error)
+	GetKeysForPrefix(string) []string
+}
 
-func (cs cacheStore) Put(key string, value interface{}) {
+// easy to be counted exported by prefix
+type cacheDoubleStore map[string]map[string]timestampedValue
+
+func newDoubleStore() cacheDoubleStore {
+	return make(cacheDoubleStore)
+}
+
+func (cs cacheDoubleStore) Put(key string, value interface{}) {
 	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
 	if _, ok := cs[prefix]; !ok {
 		cs[prefix] = make(map[string]timestampedValue)
@@ -18,7 +36,7 @@ func (cs cacheStore) Put(key string, value interface{}) {
 	cs[prefix][key] = timestampedValue{time.Now(), value}
 }
 
-func (cs cacheStore) Append(key string, value interface{}) {
+func (cs cacheDoubleStore) Append(key string, value interface{}) {
 	var elements []interface{}
 	v, err := cs.Get(key)
 	if err == nil {
@@ -38,7 +56,7 @@ func (cs cacheStore) Append(key string, value interface{}) {
 	cache.Put(key, elements)
 }
 
-func (cs cacheStore) Get(key string) (interface{}, error) {
+func (cs cacheDoubleStore) Get(key string) (interface{}, error) {
 	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
 	if keyMap, ok := cs[prefix]; ok {
 		if ti, exists := keyMap[key]; exists {
@@ -48,7 +66,7 @@ func (cs cacheStore) Get(key string) (interface{}, error) {
 	return nil, errors.New(utils.ERR_NOT_FOUND)
 }
 
-func (cs cacheStore) GetAge(key string) (time.Duration, error) {
+func (cs cacheDoubleStore) GetAge(key string) (time.Duration, error) {
 	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
 	if keyMap, ok := cs[prefix]; ok {
 		if ti, exists := keyMap[key]; exists {
@@ -58,17 +76,166 @@ func (cs cacheStore) GetAge(key string) (time.Duration, error) {
 	return -1, errors.New(utils.ERR_NOT_FOUND)
 }
 
-func (cs cacheStore) Delete(key string) {
+func (cs cacheDoubleStore) Delete(key string) {
 	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
 	if keyMap, ok := cs[prefix]; ok {
-		if _, exists := keyMap[key]; exists {
-			delete(keyMap, key)
+		delete(keyMap, key)
+	}
+}
+
+func (cs cacheDoubleStore) DeletePrefix(prefix string) {
+	delete(cs, prefix)
+}
+
+func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int64 {
+	if _, ok := cs[prefix]; ok {
+		return int64(len(cs[prefix]))
+	}
+	return 0
+}
+
+func (cs cacheDoubleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	if keyMap, ok := cs[prefix]; ok {
+		return keyMap, nil
+	}
+	return nil, errors.New(utils.ERR_NOT_FOUND)
+}
+
+func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	prefix, key := prefix[:PREFIX_LEN], prefix[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		for iterKey := range keyMap {
+			if len(key) > 0 && strings.HasPrefix(iterKey, key) {
+				keys = append(keys, prefix+iterKey)
+			}
+		}
+	}
+	return
+}
+
+// faster to access
+type cacheSimpleStore struct {
+	cache    map[string]timestampedValue
+	counters map[string]int64
+}
+
+func newSimpleStore() cacheSimpleStore {
+	return cacheSimpleStore{
+		cache:    make(map[string]timestampedValue),
+		counters: make(map[string]int64),
+	}
+}
+
+func (cs cacheSimpleStore) Put(key string, value interface{}) {
+	if _, ok := cs.cache[key]; !ok {
+		// only count if the key is not already there
+		cs.count(key)
+	}
+	cs.cache[key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheSimpleStore) Append(key string, value interface{}) {
+	var elements []interface{}
+	if ti, exists := cs.cache[key]; exists {
+		elements = ti.value.([]interface{})
+	}
+	// check if the val is already present
+	found := false
+	for _, v := range elements {
+		if value == v {
+			found = true
+			break
+		}
+	}
+	if !found {
+		elements = append(elements, value)
+	}
+	cs.Put(key, elements)
+}
+
+func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return ti.value, nil
+	}
+	return nil, errors.New(utils.ERR_NOT_FOUND)
+}
+
+func (cs cacheSimpleStore) GetAge(key string) (time.Duration, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return time.Since(ti.timestamp), nil
+	}
+
+	return -1, errors.New(utils.ERR_NOT_FOUND)
+}
+
+func (cs cacheSimpleStore) Delete(key string) {
+	if _, ok := cs.cache[key]; ok {
+		delete(cs.cache, key)
+		cs.descount(key)
+	}
+}
+
+func (cs cacheSimpleStore) DeletePrefix(prefix string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			delete(cs.cache, key)
+			cs.descount(key)
 		}
 	}
 }
 
-func (cs cacheStore) DeletePrefix(prefix string) {
-	if _, ok := cs[prefix]; ok {
-		delete(cs, prefix)
+// increments the counter for the specified key prefix
+func (cs cacheSimpleStore) count(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if _, ok := cs.counters[prefix]; ok {
+		// increase the value
+		cs.counters[prefix] += 1
+	} else {
+		cs.counters[prefix] = 1
 	}
 }
+
+// decrements the counter for the specified key prefix
+func (cs cacheSimpleStore) descount(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if value, ok := cs.counters[prefix]; ok && value > 0 {
+		cs.counters[prefix] -= 1
+	}
+}
+
+func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int64 {
+	if _, ok := cs.counters[prefix]; ok {
+		return cs.counters[prefix]
+	}
+	return 0
+}
+
+func (cs cacheSimpleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	result := make(map[string]timestampedValue)
+	found := false
+	for key, ti := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			result[key[PREFIX_LEN:]] = ti
+			found = true
+		}
+	}
+	if !found {
+		return nil, errors.New(utils.ERR_NOT_FOUND)
+	}
+	return result, nil
+}
+
+func (cs cacheSimpleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			keys = append(keys, key)
+		}
+	}
+	return
+}

engine/storage_redis.go

@@ -485,6 +485,7 @@ func (rs *RedisStorage) RemoveAccAliases(tenantAccounts []*TenantAccount) (err e
 			if tntAcnt.Account != alias {
 				continue
 			}
+			cache2go.RemKey(key)
 			if _, err = rs.db.Del(key); err != nil {
 				return err
 			}
@@ -506,7 +507,6 @@ func (rs *RedisStorage) GetAccountAliases(tenant, account string, skipCache bool
 		}
 	}
 	for _, key := range alsKeys {
-		tenantPrfx := ACC_ALIAS_PREFIX + tenant + utils.CONCATENATED_KEY_SEP
 		if alsAcnt, err := rs.GetAccAlias(key[len(ACC_ALIAS_PREFIX):], skipCache); err != nil {
 			return nil, err
 		} else if alsAcnt == account {

engine/storage_test.go

@@ -165,12 +165,12 @@ func TestRemRSubjAliases(t *testing.T) {
 	if err := dataStorage.RemoveRpAliases([]*TenantRatingSubject{&TenantRatingSubject{Tenant: "cgrates.org", Subject: "1001"}}); err != nil {
 		t.Error(err)
 	}
-	if cgrAliases, err := dataStorage.GetRPAliases("cgrates.org", "1001", false); err != nil {
+	if cgrAliases, err := dataStorage.GetRPAliases("cgrates.org", "1001", true); err != nil {
 		t.Error(err)
 	} else if len(cgrAliases) != 0 {
 		t.Error("Subject aliases not removed: ", cgrAliases)
 	}
-	if iscAliases, err := dataStorage.GetRPAliases("itsyscom.com", "1001", false); err != nil { // Make sure the aliases were removed at tenant level
+	if iscAliases, err := dataStorage.GetRPAliases("itsyscom.com", "1001", true); err != nil { // Make sure the aliases were removed at tenant level
 		t.Error(err)
 	} else if !reflect.DeepEqual(iscAliases, []string{"2003"}) {
 		t.Errorf("Unexpected aliases: %v", iscAliases)