From 7721a99edd5efa501ca1f186e7e3ebce7292ef4a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 25 Mar 2016 15:32:21 +0200 Subject: [PATCH] fix GetAccounts with mongo --- apier/v1/accounts.go | 2 +- apier/v2/accounts.go | 2 +- data/conf/samples/cgradmin/cgradmin.json | 26 +++++------ engine/storage_interface.go | 2 +- engine/storage_map.go | 15 +++--- engine/storage_mongo_datadb.go | 58 ++++++++++++++++++++++-- engine/storage_redis.go | 13 ++++-- engine/storage_sql.go | 2 +- engine/tp_reader.go | 2 +- 9 files changed, 90 insertions(+), 32 deletions(-) diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index 6d006da25..c6c161331 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -318,7 +318,7 @@ func (self *ApierV1) GetAccounts(attr utils.AttrGetAccounts, reply *[]interface{ var accountKeys []string var err error if len(attr.AccountIds) == 0 { - if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + attr.Tenant); err != nil { + if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+attr.Tenant, true); err != nil { return err } } else { diff --git a/apier/v2/accounts.go b/apier/v2/accounts.go index c3c495edd..08f6832dd 100644 --- a/apier/v2/accounts.go +++ b/apier/v2/accounts.go @@ -33,7 +33,7 @@ func (self *ApierV2) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Ac var accountKeys []string var err error if len(attr.AccountIds) == 0 { - if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + utils.ConcatenatedKey(attr.Tenant)); err != nil { + if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Tenant), true); err != nil { return err } } else { diff --git a/data/conf/samples/cgradmin/cgradmin.json b/data/conf/samples/cgradmin/cgradmin.json index 2f6f9e4a1..9dd30abcf 100644 --- a/data/conf/samples/cgradmin/cgradmin.json +++ b/data/conf/samples/cgradmin/cgradmin.json @@ -10,19 +10,19 @@ "http": ":2080", // HTTP listening address }, -//"tariffplan_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mongo", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 27017, // the port to reach the stordb -// "db_name": "tpdb", -//}, -// -//"data_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mongo", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 27017, // the port to reach the stordb -// "db_name": "datadb", -//}, +"tariffplan_db": { // database used to store offline tariff plans and CDRs + "db_type": "mongo", // stor database type to use: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 27017, // the port to reach the stordb + "db_name": "tpdb", +}, + +"data_db": { // database used to store offline tariff plans and CDRs + "db_type": "mongo", // stor database type to use: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 27017, // the port to reach the stordb + "db_name": "datadb", +}, "stor_db": { // database used to store offline tariff plans and CDRs "db_type": "mongo", // stor database type to use: diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 95d2f8d0e..dbf8a8632 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -33,7 +33,7 @@ import ( type Storage interface { Close() Flush(string) error - GetKeysForPrefix(string) ([]string, error) + GetKeysForPrefix(string, bool) ([]string, error) } // Interface for storage providers. diff --git a/engine/storage_map.go b/engine/storage_map.go index 0e77e3ae1..5a898fa32 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -53,14 +53,17 @@ func (ms *MapStorage) Flush(ignore string) error { return nil } -func (ms *MapStorage) GetKeysForPrefix(prefix string) ([]string, error) { - keysForPrefix := make([]string, 0) - for key := range ms.dict { - if strings.HasPrefix(key, prefix) { - keysForPrefix = append(keysForPrefix, key) +func (ms *MapStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + if skipCache { + keysForPrefix := make([]string, 0) + for key := range ms.dict { + if strings.HasPrefix(key, prefix) { + keysForPrefix = append(keysForPrefix, key) + } } + return keysForPrefix, nil } - return keysForPrefix, nil + return cache2go.GetEntriesKeys(prefix), nil } func (ms *MapStorage) CacheRatingAll() error { diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 4fb29e84b..a208ce6cb 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -244,8 +244,60 @@ func (ms *MongoStorage) Close() { ms.session.Close() } -func (ms *MongoStorage) GetKeysForPrefix(prefix string) ([]string, error) { - return nil, nil +func (ms *MongoStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + var category, subject string + length := len(utils.DESTINATION_PREFIX) + if len(prefix) >= length { + category = prefix[:length] // prefix lenght + subject = fmt.Sprintf("^%s", prefix[length:]) + } else { + return nil, fmt.Errorf("unsupported prefix in GetKeysForPrefix: %s", prefix) + } + var result []string + if skipCache { + keyResult := struct{ Key string }{} + idResult := struct{ Id string }{} + switch category { + case utils.DESTINATION_PREFIX: + iter := ms.db.C(colDst).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.DESTINATION_PREFIX+keyResult.Key) + } + return result, nil + case utils.RATING_PLAN_PREFIX: + iter := ms.db.C(colRpl).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.RATING_PLAN_PREFIX+keyResult.Key) + } + return result, nil + case utils.RATING_PROFILE_PREFIX: + iter := ms.db.C(colRpf).Find(bson.M{"id": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"id": 1}).Iter() + for iter.Next(&idResult) { + result = append(result, utils.RATING_PROFILE_PREFIX+idResult.Id) + } + return result, nil + case utils.ACTION_PREFIX: + iter := ms.db.C(colAct).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.ACTION_PREFIX+keyResult.Key) + } + return result, nil + case utils.ACTION_PLAN_PREFIX: + iter := ms.db.C(colApl).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.ACTION_PLAN_PREFIX+keyResult.Key) + } + return result, nil + case utils.ACCOUNT_PREFIX: + iter := ms.db.C(colAcc).Find(bson.M{"id": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"id": 1}).Iter() + for iter.Next(&idResult) { + result = append(result, utils.ACCOUNT_PREFIX+idResult.Id) + } + return result, nil + } + return result, fmt.Errorf("unsupported prefix in GetKeysForPrefix: %s", prefix) + } + return cache2go.GetEntriesKeys(prefix), nil } func (ms *MongoStorage) Flush(ignore string) (err error) { @@ -627,7 +679,7 @@ func (ms *MongoStorage) HasData(category, subject string) (bool, error) { count, err := ms.db.C(colAcc).Find(bson.M{"id": subject}).Count() return count > 0, err } - return false, errors.New("Unsupported category in HasData") + return false, errors.New("unsupported category in HasData") } func (ms *MongoStorage) GetRatingPlan(key string, skipCache bool) (rp *RatingPlan, err error) { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 52d4bb3a7..8d8b1660a 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -84,12 +84,15 @@ func (rs *RedisStorage) Flush(ignore string) error { return rs.db.Cmd("FLUSHDB").Err } -func (rs *RedisStorage) GetKeysForPrefix(prefix string) ([]string, error) { - r := rs.db.Cmd("KEYS", prefix+"*") - if r.Err != nil { - return nil, r.Err +func (rs *RedisStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + if skipCache { + r := rs.db.Cmd("KEYS", prefix+"*") + if r.Err != nil { + return nil, r.Err + } + return r.List() } - return r.List() + return cache2go.GetEntriesKeys(prefix), nil } func (rs *RedisStorage) CacheRatingAll() error { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index cac8c86fc..1e0f5368a 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -55,7 +55,7 @@ func (self *SQLStorage) Flush(scriptsPath string) (err error) { return nil } -func (self *SQLStorage) GetKeysForPrefix(prefix string) ([]string, error) { +func (self *SQLStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { return nil, utils.ErrNotImplemented } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index bf662a335..e458e130e 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -433,7 +433,7 @@ func (tpr *TpReader) LoadLCRs() (err error) { } } if !found && tpr.ratingStorage != nil { - if keys, err := tpr.ratingStorage.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX + ratingProfileSearchKey); err != nil { + if keys, err := tpr.ratingStorage.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX+ratingProfileSearchKey, true); err != nil { return fmt.Errorf("[LCR] error querying ratingDb %s", err.Error()) } else if len(keys) != 0 { found = true