From 4f25ef16c9392a16e5fd5895f02fba6527408055 Mon Sep 17 00:00:00 2001 From: TeoV Date: Tue, 17 Oct 2017 16:13:46 +0300 Subject: [PATCH] Add LoadDataDBCache in DataManager --- apier/v1/apier.go | 2 +- cmd/cgr-engine/rater.go | 2 +- cmd/cgr-tester/cgr-tester.go | 2 +- engine/datamanager.go | 25 ++++++ engine/libtest.go | 2 +- engine/loader_csv_test.go | 2 +- engine/storage_interface.go | 4 +- engine/storage_map.go | 103 +----------------------- engine/storage_mongo_datadb.go | 135 -------------------------------- engine/storage_redis.go | 127 ------------------------------ engine/storage_test.go | 2 +- general_tests/acntacts_test.go | 2 +- general_tests/auth_test.go | 2 +- general_tests/costs1_test.go | 2 +- general_tests/datachrg1_test.go | 2 +- general_tests/ddazmbl1_test.go | 2 +- general_tests/ddazmbl2_test.go | 2 +- general_tests/ddazmbl3_test.go | 2 +- general_tests/smschrg1_test.go | 2 +- 19 files changed, 42 insertions(+), 380 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 7fec905f0..8b6d9342c 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1038,7 +1038,7 @@ func (self *ApierV1) LoadCache(args utils.AttrReloadCache, reply *string) (err e resIDs = *args.ResourceIDs } - if err := self.DataManager.DataDB().LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rspIDs, resIDs); err != nil { + if err := self.DataManager.LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rspIDs, resIDs); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index ecd3f3f8b..eaa5802e6 100755 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -91,7 +91,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } // ToDo: Add here timings - if err := dm.DataDB().LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rspIDs, resIDs); err != nil { + if err := dm.LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rspIDs, resIDs); err != nil { utils.Logger.Crit(fmt.Sprintf(" Cache rating error: %s", err.Error())) exitChan <- true return diff --git a/cmd/cgr-tester/cgr-tester.go b/cmd/cgr-tester/cgr-tester.go index 92a19a6d0..2cad57083 100644 --- a/cmd/cgr-tester/cgr-tester.go +++ b/cmd/cgr-tester/cgr-tester.go @@ -67,7 +67,7 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) { } defer dm.DataDB().Close() engine.SetDataStorage(dm) - if err := dm.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil { + if err := dm.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil { return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error()) } log.Printf("Runnning %d cycles...", *runs) diff --git a/engine/datamanager.go b/engine/datamanager.go index 4f63158a3..66fd90231 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -38,6 +38,31 @@ func (dm *DataManager) DataDB() DataDB { return dm.dataDB } +func (dm *DataManager) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aaPlIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rpIDs, resIDs []string) (err error) { + for key, ids := range map[string][]string{ + utils.DESTINATION_PREFIX: dstIDs, + utils.REVERSE_DESTINATION_PREFIX: rvDstIDs, + utils.RATING_PLAN_PREFIX: rplIDs, + utils.RATING_PROFILE_PREFIX: rpfIDs, + utils.ACTION_PREFIX: actIDs, + utils.ACTION_PLAN_PREFIX: aplIDs, + utils.AccountActionPlansPrefix: aaPlIDs, + utils.ACTION_TRIGGER_PREFIX: atrgIDs, + utils.SHARED_GROUP_PREFIX: sgIDs, + utils.LCR_PREFIX: lcrIDs, + utils.DERIVEDCHARGERS_PREFIX: dcIDs, + utils.ALIASES_PREFIX: alsIDs, + utils.REVERSE_ALIASES_PREFIX: rvAlsIDs, + utils.ResourceProfilesPrefix: rpIDs, + utils.ResourcesPrefix: resIDs, + } { + if err = dm.CacheDataFromDB(key, ids, false); err != nil { + return + } + } + return +} + func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached bool) (err error) { if !utils.IsSliceMember([]string{utils.DESTINATION_PREFIX, utils.REVERSE_DESTINATION_PREFIX, diff --git a/engine/libtest.go b/engine/libtest.go index 8e0093248..993e9faec 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -41,7 +41,7 @@ func InitDataDb(cfg *config.CGRConfig) error { if err := dm.DataDB().Flush(""); err != nil { return err } - dm.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dm.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) // Write version before starting if err := SetDBVersions(dm.dataDB); err != nil { return err diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index b205e9914..c8afc485f 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -360,7 +360,7 @@ func init() { } csvr.WriteToDatabase(false, false, false) cache.Flush() - dm.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dm.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) } func TestLoadDestinations(t *testing.T) { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 1a05ad01b..f4beadac9 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -48,7 +48,7 @@ type DataDB interface { Storage Marshaler() Marshaler HasData(string, string) (bool, error) - LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string) error + //LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string) error GetRatingPlan(string, bool, string) (*RatingPlan, error) SetRatingPlan(*RatingPlan, string) error GetRatingProfile(string, bool, string) (*RatingProfile, error) @@ -129,8 +129,6 @@ type DataDB interface { GetFilterDrv(string, string) (*Filter, error) SetFilterDrv(*Filter) error RemoveFilterDrv(string, string) error - // CacheDataFromDB loads data to cache, prefix represents the cache prefix, IDs should be nil if all available data should be loaded - CacheDataFromDB2(prefix string, IDs []string, mustBeCached bool) error // ToDo: Move this to dataManager } type StorDB interface { diff --git a/engine/storage_map.go b/engine/storage_map.go index 055a040ef..b9f2036da 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -22,7 +22,6 @@ import ( "bytes" "compress/zlib" "errors" - "fmt" "io/ioutil" "strings" "sync" @@ -153,6 +152,7 @@ func (ms *MapStorage) IsDBEmpty() (resp bool, err error) { return len(ms.dict) == 0, nil } +/* func (ms *MapStorage) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string) (err error) { if ms.cacheCfg == nil { return @@ -197,106 +197,7 @@ func (ms *MapStorage) PreloadCacheForPrefix(prefix string) error { return nil } -// CacheDataFromDB loads data to cache, -// prefix represents the cache prefix, IDs should be nil if all available data should be loaded -// ToDo: convert IDs into []*utils.TenantIDs when infrastructure will be ready -func (ms *MapStorage) CacheDataFromDB2(prefix string, IDs []string, mustBeCached bool) (err error) { - if !utils.IsSliceMember([]string{utils.DESTINATION_PREFIX, - utils.REVERSE_DESTINATION_PREFIX, - utils.RATING_PLAN_PREFIX, - utils.RATING_PROFILE_PREFIX, - utils.ACTION_PREFIX, - utils.ACTION_PLAN_PREFIX, - utils.AccountActionPlansPrefix, - utils.ACTION_TRIGGER_PREFIX, - utils.SHARED_GROUP_PREFIX, - utils.DERIVEDCHARGERS_PREFIX, - utils.LCR_PREFIX, - utils.ALIASES_PREFIX, - utils.REVERSE_ALIASES_PREFIX, - utils.ResourceProfilesPrefix, - utils.ResourcesPrefix, - utils.TimingsPrefix}, prefix) { - return utils.NewCGRError(utils.REDIS, - utils.MandatoryIEMissingCaps, - utils.UnsupportedCachePrefix, - fmt.Sprintf("prefix <%s> is not a supported cache prefix", prefix)) - } - if IDs == nil { - keyIDs, err := ms.GetKeysForPrefix(prefix) - if err != nil { - return utils.NewCGRError(utils.REDIS, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("MapStorage error <%s> querying keys for prefix: <%s>", prefix)) - } - for _, keyID := range keyIDs { - if mustBeCached { // Only consider loading ids which are already in cache - if _, hasIt := cache.Get(keyID); !hasIt { - continue - } - } - IDs = append(IDs, keyID[len(prefix):]) - } - var nrItems int - if cCfg, has := ms.cacheCfg[utils.CachePrefixToInstance[prefix]]; has { - nrItems = cCfg.Limit - } - if nrItems > 0 && nrItems < len(IDs) { - IDs = IDs[:nrItems] - } - } - for _, dataID := range IDs { - if mustBeCached { - if _, hasIt := cache.Get(prefix + dataID); !hasIt { // only cache if previously there - continue - } - } - switch prefix { - case utils.DESTINATION_PREFIX: - _, err = ms.GetDestination(dataID, true, utils.NonTransactional) - case utils.REVERSE_DESTINATION_PREFIX: - _, err = ms.GetReverseDestination(dataID, true, utils.NonTransactional) - case utils.RATING_PLAN_PREFIX: - _, err = ms.GetRatingPlan(dataID, true, utils.NonTransactional) - case utils.RATING_PROFILE_PREFIX: - _, err = ms.GetRatingProfile(dataID, true, utils.NonTransactional) - case utils.ACTION_PREFIX: - _, err = ms.GetActions(dataID, true, utils.NonTransactional) - case utils.ACTION_PLAN_PREFIX: - _, err = ms.GetActionPlan(dataID, true, utils.NonTransactional) - case utils.AccountActionPlansPrefix: - _, err = ms.GetAccountActionPlans(dataID, true, utils.NonTransactional) - case utils.ACTION_TRIGGER_PREFIX: - _, err = ms.GetActionTriggers(dataID, true, utils.NonTransactional) - case utils.SHARED_GROUP_PREFIX: - _, err = ms.GetSharedGroup(dataID, true, utils.NonTransactional) - case utils.DERIVEDCHARGERS_PREFIX: - _, err = ms.GetDerivedChargers(dataID, true, utils.NonTransactional) - case utils.LCR_PREFIX: - _, err = ms.GetLCR(dataID, true, utils.NonTransactional) - case utils.ALIASES_PREFIX: - _, err = ms.GetAlias(dataID, true, utils.NonTransactional) - case utils.REVERSE_ALIASES_PREFIX: - _, err = ms.GetReverseAlias(dataID, true, utils.NonTransactional) - case utils.ResourceProfilesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetResourceProfile(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.ResourcesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetResource(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.TimingsPrefix: - _, err = ms.GetTiming(dataID, true, utils.NonTransactional) - } - if err != nil { - return utils.NewCGRError(utils.REDIS, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("error <%s> querying MapStorage for category: <%s>, dataID: <%s>", prefix, dataID)) - } - } - return -} +*/ func (ms *MapStorage) GetKeysForPrefix(prefix string) ([]string, error) { ms.mu.RLock() diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index f2dc2e51d..d41e7e038 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -423,141 +423,6 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) (err error) { return nil } -func (ms *MongoStorage) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aaPlIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rpIDs, resIDs []string) (err error) { - for key, ids := range map[string][]string{ - utils.DESTINATION_PREFIX: dstIDs, - utils.REVERSE_DESTINATION_PREFIX: rvDstIDs, - utils.RATING_PLAN_PREFIX: rplIDs, - utils.RATING_PROFILE_PREFIX: rpfIDs, - utils.ACTION_PREFIX: actIDs, - utils.ACTION_PLAN_PREFIX: aplIDs, - utils.AccountActionPlansPrefix: aaPlIDs, - utils.ACTION_TRIGGER_PREFIX: atrgIDs, - utils.SHARED_GROUP_PREFIX: sgIDs, - utils.LCR_PREFIX: lcrIDs, - utils.DERIVEDCHARGERS_PREFIX: dcIDs, - utils.ALIASES_PREFIX: alsIDs, - utils.REVERSE_ALIASES_PREFIX: rvAlsIDs, - utils.ResourceProfilesPrefix: rpIDs, - utils.ResourcesPrefix: resIDs, - } { - if err = ms.CacheDataFromDB2(key, ids, false); err != nil { - return - } - } - return -} - -// CacheDataFromDB loads data to cache -// prfx represents the cache prefix, ids should be nil if all available data should be loaded -// mustBeCached specifies that data needs to be cached in order to be retrieved from db -func (ms *MongoStorage) CacheDataFromDB2(prfx string, ids []string, mustBeCached bool) (err error) { - if !utils.IsSliceMember([]string{utils.DESTINATION_PREFIX, - utils.REVERSE_DESTINATION_PREFIX, - utils.RATING_PLAN_PREFIX, - utils.RATING_PROFILE_PREFIX, - utils.ACTION_PREFIX, - utils.ACTION_PLAN_PREFIX, - utils.AccountActionPlansPrefix, - utils.ACTION_TRIGGER_PREFIX, - utils.SHARED_GROUP_PREFIX, - utils.DERIVEDCHARGERS_PREFIX, - utils.LCR_PREFIX, - utils.ALIASES_PREFIX, - utils.REVERSE_ALIASES_PREFIX, - utils.ResourceProfilesPrefix, - utils.TimingsPrefix, - utils.ResourcesPrefix}, prfx) { - return utils.NewCGRError(utils.MONGO, - utils.MandatoryIEMissingCaps, - utils.UnsupportedCachePrefix, - fmt.Sprintf("prefix <%s> is not a supported cache prefix", prfx)) - } - if ids == nil { - keyIDs, err := ms.GetKeysForPrefix(prfx) - if err != nil { - return utils.NewCGRError(utils.MONGO, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("mongo error <%s> querying keys for prefix: <%s>", prfx)) - } - for _, keyID := range keyIDs { - if mustBeCached { // Only consider loading ids which are already in cache - if _, hasIt := cache.Get(keyID); !hasIt { - continue - } - } - ids = append(ids, keyID[len(prfx):]) - } - var nrItems int - if cCfg, has := ms.cacheCfg[utils.CachePrefixToInstance[prfx]]; has { - nrItems = cCfg.Limit - } - if nrItems > 0 && nrItems < len(ids) { // More ids than cache config allows it, limit here - ids = ids[:nrItems] - } - } - for _, dataID := range ids { - if mustBeCached { - if _, hasIt := cache.Get(prfx + dataID); !hasIt { // only cache if previously there - continue - } - } - switch prfx { - case utils.DESTINATION_PREFIX: - _, err = ms.GetDestination(dataID, true, utils.NonTransactional) - case utils.REVERSE_DESTINATION_PREFIX: - _, err = ms.GetReverseDestination(dataID, true, utils.NonTransactional) - case utils.RATING_PLAN_PREFIX: - _, err = ms.GetRatingPlan(dataID, true, utils.NonTransactional) - case utils.RATING_PROFILE_PREFIX: - _, err = ms.GetRatingProfile(dataID, true, utils.NonTransactional) - case utils.ACTION_PREFIX: - _, err = ms.GetActions(dataID, true, utils.NonTransactional) - case utils.ACTION_PLAN_PREFIX: - _, err = ms.GetActionPlan(dataID, true, utils.NonTransactional) - case utils.AccountActionPlansPrefix: - _, err = ms.GetAccountActionPlans(dataID, true, utils.NonTransactional) - case utils.ACTION_TRIGGER_PREFIX: - _, err = ms.GetActionTriggers(dataID, true, utils.NonTransactional) - case utils.SHARED_GROUP_PREFIX: - _, err = ms.GetSharedGroup(dataID, true, utils.NonTransactional) - case utils.DERIVEDCHARGERS_PREFIX: - _, err = ms.GetDerivedChargers(dataID, true, utils.NonTransactional) - case utils.LCR_PREFIX: - _, err = ms.GetLCR(dataID, true, utils.NonTransactional) - case utils.ALIASES_PREFIX: - _, err = ms.GetAlias(dataID, true, utils.NonTransactional) - case utils.REVERSE_ALIASES_PREFIX: - _, err = ms.GetReverseAlias(dataID, true, utils.NonTransactional) - case utils.ResourceProfilesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetResourceProfile(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.ResourcesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetResource(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.TimingsPrefix: - _, err = ms.GetTiming(dataID, true, utils.NonTransactional) - case utils.ThresholdProfilePrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetThresholdProfileDrv(tntID.Tenant, tntID.ID) - case utils.ThresholdPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetThresholdDrv(tntID.Tenant, tntID.ID) - case utils.FilterPrefix: - tntID := utils.NewTenantID(dataID) - _, err = ms.GetFilterDrv(tntID.Tenant, tntID.ID) - } - if err != nil { - return utils.NewCGRError(utils.MONGO, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("error <%s> querying mongo for category: <%s>, dataID: <%s>", prfx, dataID)) - } - } - return -} - func (ms *MongoStorage) IsDBEmpty() (resp bool, err error) { session := ms.session.Copy() defer session.Close() diff --git a/engine/storage_redis.go b/engine/storage_redis.go index b537837a3..5f3c5174b 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -131,32 +131,6 @@ func (rs *RedisStorage) IsDBEmpty() (resp bool, err error) { return true, nil } -func (rs *RedisStorage) LoadDataDBCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, - aplIDs, aaPlIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rpIDs, resIDs []string) (err error) { - for key, ids := range map[string][]string{ - utils.DESTINATION_PREFIX: dstIDs, - utils.REVERSE_DESTINATION_PREFIX: rvDstIDs, - utils.RATING_PLAN_PREFIX: rplIDs, - utils.RATING_PROFILE_PREFIX: rpfIDs, - utils.ACTION_PREFIX: actIDs, - utils.ACTION_PLAN_PREFIX: aplIDs, - utils.AccountActionPlansPrefix: aaPlIDs, - utils.ACTION_TRIGGER_PREFIX: atrgIDs, - utils.SHARED_GROUP_PREFIX: sgIDs, - utils.LCR_PREFIX: lcrIDs, - utils.DERIVEDCHARGERS_PREFIX: dcIDs, - utils.ALIASES_PREFIX: alsIDs, - utils.REVERSE_ALIASES_PREFIX: rvAlsIDs, - utils.ResourceProfilesPrefix: rpIDs, - utils.ResourcesPrefix: resIDs, - } { - if err = rs.CacheDataFromDB2(key, ids, false); err != nil { - return - } - } - return -} - func (rs *RedisStorage) RebuildReverseForPrefix(prefix string) (err error) { if !utils.IsSliceMember([]string{utils.REVERSE_DESTINATION_PREFIX, utils.REVERSE_ALIASES_PREFIX, utils.AccountActionPlansPrefix}, prefix) { return utils.ErrInvalidKey @@ -217,107 +191,6 @@ func (rs *RedisStorage) RebuildReverseForPrefix(prefix string) (err error) { return nil } -// CacheDataFromDB loads data to cache -// prfx represents the cache prefix, ids should be nil if all available data should be loaded -// mustBeCached specifies that data needs to be cached in order to be retrieved from db -func (rs *RedisStorage) CacheDataFromDB2(prfx string, ids []string, mustBeCached bool) (err error) { - if !utils.IsSliceMember([]string{utils.DESTINATION_PREFIX, - utils.REVERSE_DESTINATION_PREFIX, - utils.RATING_PLAN_PREFIX, - utils.RATING_PROFILE_PREFIX, - utils.ACTION_PREFIX, - utils.ACTION_PLAN_PREFIX, - utils.AccountActionPlansPrefix, - utils.ACTION_TRIGGER_PREFIX, - utils.SHARED_GROUP_PREFIX, - utils.DERIVEDCHARGERS_PREFIX, - utils.LCR_PREFIX, - utils.ALIASES_PREFIX, - utils.REVERSE_ALIASES_PREFIX, - utils.ResourceProfilesPrefix, - utils.ResourcesPrefix, - utils.TimingsPrefix}, prfx) { - return utils.NewCGRError(utils.REDIS, - utils.MandatoryIEMissingCaps, - utils.UnsupportedCachePrefix, - fmt.Sprintf("prefix <%s> is not a supported cache prefix", prfx)) - } - if ids == nil { - keyIDs, err := rs.GetKeysForPrefix(prfx) - if err != nil { - return utils.NewCGRError(utils.REDIS, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx)) - } - for _, keyID := range keyIDs { - if mustBeCached { // Only consider loading ids which are already in cache - if _, hasIt := cache.Get(keyID); !hasIt { - continue - } - } - ids = append(ids, keyID[len(prfx):]) - } - var nrItems int - if cCfg, has := rs.cacheCfg[utils.CachePrefixToInstance[prfx]]; has { - nrItems = cCfg.Limit - } - if nrItems > 0 && nrItems < len(ids) { - ids = ids[:nrItems] - } - } - for _, dataID := range ids { - if mustBeCached { - if _, hasIt := cache.Get(prfx + dataID); !hasIt { // only cache if previously there - continue - } - } - switch prfx { - case utils.DESTINATION_PREFIX: - _, err = rs.GetDestination(dataID, true, utils.NonTransactional) - case utils.REVERSE_DESTINATION_PREFIX: - _, err = rs.GetReverseDestination(dataID, true, utils.NonTransactional) - case utils.RATING_PLAN_PREFIX: - _, err = rs.GetRatingPlan(dataID, true, utils.NonTransactional) - case utils.RATING_PROFILE_PREFIX: - _, err = rs.GetRatingProfile(dataID, true, utils.NonTransactional) - case utils.ACTION_PREFIX: - _, err = rs.GetActions(dataID, true, utils.NonTransactional) - case utils.ACTION_PLAN_PREFIX: - _, err = rs.GetActionPlan(dataID, true, utils.NonTransactional) - case utils.AccountActionPlansPrefix: - _, err = rs.GetAccountActionPlans(dataID, true, utils.NonTransactional) - case utils.ACTION_TRIGGER_PREFIX: - _, err = rs.GetActionTriggers(dataID, true, utils.NonTransactional) - case utils.SHARED_GROUP_PREFIX: - _, err = rs.GetSharedGroup(dataID, true, utils.NonTransactional) - case utils.DERIVEDCHARGERS_PREFIX: - _, err = rs.GetDerivedChargers(dataID, true, utils.NonTransactional) - case utils.LCR_PREFIX: - _, err = rs.GetLCR(dataID, true, utils.NonTransactional) - case utils.ALIASES_PREFIX: - _, err = rs.GetAlias(dataID, true, utils.NonTransactional) - case utils.REVERSE_ALIASES_PREFIX: - _, err = rs.GetReverseAlias(dataID, true, utils.NonTransactional) - case utils.ResourceProfilesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = rs.GetResourceProfile(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.ResourcesPrefix: - tntID := utils.NewTenantID(dataID) - _, err = rs.GetResource(tntID.Tenant, tntID.ID, true, utils.NonTransactional) - case utils.TimingsPrefix: - _, err = rs.GetTiming(dataID, true, utils.NonTransactional) - } - if err != nil { - return utils.NewCGRError(utils.REDIS, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("error <%s> querying redis for category: <%s>, dataID: <%s>", prfx, dataID)) - } - } - return -} - func (rs *RedisStorage) GetKeysForPrefix(prefix string) ([]string, error) { r := rs.Cmd("KEYS", prefix+"*") if r.Err != nil { diff --git a/engine/storage_test.go b/engine/storage_test.go index 3377f4f93..300bba8f0 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -102,7 +102,7 @@ func TestStorageCacheRefresh(t *testing.T) { dm.DataDB().GetDestination("T11", false, utils.NonTransactional) dm.DataDB().SetDestination(&Destination{"T11", []string{"1"}}, utils.NonTransactional) t.Log("Test cache refresh") - err := dm.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + err := dm.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if err != nil { t.Error("Error cache rating: ", err) } diff --git a/general_tests/acntacts_test.go b/general_tests/acntacts_test.go index e4ddbedc6..1a3f265bc 100644 --- a/general_tests/acntacts_test.go +++ b/general_tests/acntacts_test.go @@ -65,7 +65,7 @@ ENABLE_ACNT,*enable_account,,,,,,,,,,,,,,false,false,10` csvr.WriteToDatabase(false, false, false) cache.Flush() - dbAcntActs.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dbAcntActs.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) expectAcnt := &engine.Account{ID: "cgrates.org:1"} if acnt, err := dbAcntActs.DataDB().GetAccount("cgrates.org:1"); err != nil { diff --git a/general_tests/auth_test.go b/general_tests/auth_test.go index a7518ccb5..72fe62d60 100644 --- a/general_tests/auth_test.go +++ b/general_tests/auth_test.go @@ -77,7 +77,7 @@ RP_ANY,DR_ANY_1CNT,*any,10` } cache.Flush() - dbAuth.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dbAuth.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 { t.Error("Wrong number of cached destinations found", cachedDests) diff --git a/general_tests/costs1_test.go b/general_tests/costs1_test.go index f49c81438..ab48ab23a 100644 --- a/general_tests/costs1_test.go +++ b/general_tests/costs1_test.go @@ -74,7 +74,7 @@ RP_SMS1,DR_SMS_1,ALWAYS,10` } csvr.WriteToDatabase(false, false, false) cache.Flush() - dataDB.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 3 { t.Error("Wrong number of cached rating plans found", cachedRPlans) diff --git a/general_tests/datachrg1_test.go b/general_tests/datachrg1_test.go index 35c9b3218..3886726ae 100644 --- a/general_tests/datachrg1_test.go +++ b/general_tests/datachrg1_test.go @@ -61,7 +61,7 @@ RP_DATA1,DR_DATA_2,TM2,10` } csvr.WriteToDatabase(false, false, false) cache.Flush() - dataDB.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 1 { t.Error("Wrong number of cached rating plans found", cachedRPlans) diff --git a/general_tests/ddazmbl1_test.go b/general_tests/ddazmbl1_test.go index b6194f8a0..2e028b352 100644 --- a/general_tests/ddazmbl1_test.go +++ b/general_tests/ddazmbl1_test.go @@ -112,7 +112,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` t.Error("No account saved") } cache.Flush() - dataDB.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 { t.Error("Wrong number of cached destinations found", cachedDests) diff --git a/general_tests/ddazmbl2_test.go b/general_tests/ddazmbl2_test.go index d21a8d477..724e2a841 100644 --- a/general_tests/ddazmbl2_test.go +++ b/general_tests/ddazmbl2_test.go @@ -112,7 +112,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` t.Error("No account saved") } cache.Flush() - dataDB2.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB2.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 { t.Error("Wrong number of cached destinations found", cachedDests) diff --git a/general_tests/ddazmbl3_test.go b/general_tests/ddazmbl3_test.go index 13c4ee40d..2027dc8ae 100644 --- a/general_tests/ddazmbl3_test.go +++ b/general_tests/ddazmbl3_test.go @@ -110,7 +110,7 @@ RP_UK,DR_UK_Mobile_BIG5,ALWAYS,10` t.Error("No account saved") } cache.Flush() - dataDB3.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB3.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 { t.Error("Wrong number of cached destinations found", cachedDests) diff --git a/general_tests/smschrg1_test.go b/general_tests/smschrg1_test.go index f9fcef038..900cd4ec2 100644 --- a/general_tests/smschrg1_test.go +++ b/general_tests/smschrg1_test.go @@ -59,7 +59,7 @@ func TestSMSLoadCsvTpSmsChrg1(t *testing.T) { } csvr.WriteToDatabase(false, false, false) cache.Flush() - dataDB.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + dataDB.LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 1 { t.Error("Wrong number of cached rating plans found", cachedRPlans)