From db50767a99b80d7e709fc58a491fef19cf85b073 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Jul 2016 21:51:42 +0300 Subject: [PATCH] destination storage refactoring started --- engine/cache.go | 6 ++--- engine/cache_store.go | 20 +++++++++++------ engine/storage_interface.go | 4 ++-- engine/storage_mongo_datadb.go | 21 +++++++---------- engine/storage_redis.go | 41 ++++++++++------------------------ 5 files changed, 38 insertions(+), 54 deletions(-) diff --git a/engine/cache.go b/engine/cache.go index 1f564e76d..b960fd907 100644 --- a/engine/cache.go +++ b/engine/cache.go @@ -115,15 +115,15 @@ func CacheGet(key string) (v interface{}, err error) { } // Appends to an existing slice in the cache key -func CachePush(key string, value string) { +func CachePush(key string, values ...string) { if !transactionLock { mux.Lock() defer mux.Unlock() } if !transactionON { - cache.Append(key, value) + cache.Append(key, values...) } else { - transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: value, kind: KIND_ADP}) + transactionBuffer = append(transactionBuffer, &transactionItem{key: key, value: values, kind: KIND_ADP}) } } diff --git a/engine/cache_store.go b/engine/cache_store.go index 1f06d59e8..90af3095e 100644 --- a/engine/cache_store.go +++ b/engine/cache_store.go @@ -16,7 +16,7 @@ import ( type cacheStore interface { Put(string, interface{}) Get(string) (interface{}, error) - Append(string, string) + Append(string, ...string) Pop(string, string) Delete(string) DeletePrefix(string) @@ -56,14 +56,16 @@ func (cs cacheDoubleStore) Get(key string) (interface{}, error) { return nil, utils.ErrNotFound } -func (cs cacheDoubleStore) Append(key string, value string) { +func (cs cacheDoubleStore) Append(key string, values ...string) { var elements map[string]struct{} // using map for faster check if element is present if v, err := cs.Get(key); err == nil { elements = v.(map[string]struct{}) } else { elements = make(map[string]struct{}) } - elements[value] = struct{}{} + for _, value := range values { + elements[value] = struct{}{} + } cache.Put(key, elements) } @@ -205,14 +207,16 @@ func (cs cacheLRUTTL) Get(key string) (interface{}, error) { return nil, utils.ErrNotFound } -func (cs cacheLRUTTL) Append(key string, value string) { +func (cs cacheLRUTTL) Append(key string, values ...string) { var elements map[string]struct{} // using map for faster check if element is present if v, err := cs.Get(key); err == nil { elements = v.(map[string]struct{}) } else { elements = make(map[string]struct{}) } - elements[value] = struct{}{} + for _, value := range values { + elements[value] = struct{}{} + } cache.Put(key, elements) } @@ -288,14 +292,16 @@ func (cs cacheSimpleStore) Put(key string, value interface{}) { cs.cache[key] = value } -func (cs cacheSimpleStore) Append(key string, value string) { +func (cs cacheSimpleStore) Append(key string, values ...string) { var elements map[string]struct{} if v, err := cs.Get(key); err == nil { elements = v.(map[string]struct{}) } else { elements = make(map[string]struct{}) } - elements[value] = struct{}{} + for _, value := range values { + elements[value] = struct{}{} + } cache.Put(key, elements) } diff --git a/engine/storage_interface.go b/engine/storage_interface.go index b200f726c..a9cd1aa91 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -47,8 +47,8 @@ type RatingStorage interface { GetRatingProfile(string, bool) (*RatingProfile, error) SetRatingProfile(*RatingProfile) error RemoveRatingProfile(string) error - GetDestination(string) (*Destination, error) - SetDestination(*Destination) error + GetDestinationIDs(string) ([]string, error) + SetDestinationIDs(*Destination) error RemoveDestination(string) error GetLCR(string, bool) (*LCR, error) SetLCR(*LCR) error diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 4ea52887b..569d7bca1 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -979,20 +979,15 @@ func (ms *MongoStorage) GetDestination(key string) (result *Destination, err err } func (ms *MongoStorage) SetDestination(dest *Destination) (err error) { - result, err := ms.ms.Marshal(dest) - if err != nil { - return err + for _, p := range dest.Prefixes { + session, col := ms.conn(colDst) + if _, err = col.Upsert(bson.M{"key": p}, &struct { + Key string + Value []string + }, bson.M{"key": p}, bson.M{"$addToSet": bson.M{Value: dest.Id}}); err != nil { + break + } } - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write(result) - w.Close() - session, col := ms.conn(colDst) - defer session.Close() - _, err = col.Upsert(bson.M{"key": dest.Id}, &struct { - Key string - Value []byte - }{Key: dest.Id, Value: b.Bytes()}) if err == nil && historyScribe != nil { var response int historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(false), &response) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index c989301c4..8870bd5e6 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -582,42 +582,25 @@ func (rs *RedisStorage) SetLCR(lcr *LCR) (err error) { return } -func (rs *RedisStorage) GetDestination(key string) (dest *Destination, err error) { - key = utils.DESTINATION_PREFIX + key - var values []byte - if values, err = rs.db.Cmd("GET", key).Bytes(); len(values) > 0 && err == nil { - b := bytes.NewBuffer(values) - r, err := zlib.NewReader(b) - if err != nil { - return nil, err - } - out, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - r.Close() - dest = new(Destination) - err = rs.ms.Unmarshal(out, dest) - // create optimized structure - for _, p := range dest.Prefixes { - CachePush(utils.DESTINATION_PREFIX+p, dest.Id) - } +func (rs *RedisStorage) GetDestinationIDs(prefix string) (ids []string, err error) { + prefix = utils.DESTINATION_PREFIX + prefix + var values []string + if values, err = rs.db.Cmd("SMEMBERS", prefix).List(); len(values) > 0 && err == nil { + CachePush(utils.DESTINATION_PREFIX+prefix, values...) } else { return nil, utils.ErrNotFound } return } -func (rs *RedisStorage) SetDestination(dest *Destination) (err error) { - result, err := rs.ms.Marshal(dest) - if err != nil { - return err +func (rs *RedisStorage) SetDestinationIDs(dest *Destination) (err error) { + for _, p := range dest.Prefixes { + err = rs.db.Cmd("SADD", utils.DESTINATION_PREFIX+p, dest.Id).Err + if err != nil { + break + } } - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write(result) - w.Close() - err = rs.db.Cmd("SET", utils.DESTINATION_PREFIX+dest.Id, b.Bytes()).Err + if err == nil && historyScribe != nil { response := 0 go historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(false), &response)