Preload data on start, properly handle reloads when loadFromFolder or stor, ApierV1.PreloadCache method

2026-02-14 20:59:53 +05:00 · 2016-12-21 16:01:10 +01:00
parent da7e75e013
commit 03a9569190
12 changed files with 277 additions and 235 deletions
--- a/apier/v1/apier.go
+++ b/apier/v1/apier.go
@@ -166,6 +166,9 @@ func (self *ApierV1) LoadDestination(attrs AttrLoadDestination, reply *string) e
 	} else if !loaded {
 		return utils.ErrNotFound
 	}
+	if err := self.RatingDb.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{attrs.DestinationId}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = OK
 	return nil
 }
@@ -180,6 +183,9 @@ func (self *ApierV1) LoadDerivedChargers(attrs utils.TPDerivedChargers, reply *s
 	if err := dbReader.LoadDerivedChargersFiltered(&dc[0], true); err != nil {
 		return utils.NewErrServerError(err)
 	}
+	if err := self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{attrs.GetDerivedChargersKey()}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = OK
 	return nil
 }
@@ -200,7 +206,9 @@ func (self *ApierV1) LoadRatingPlan(attrs AttrLoadRatingPlan, reply *string) err
 	} else if !loaded {
 		return utils.ErrNotFound
 	}
-	self.RatingDb.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX)
+	if err := self.RatingDb.CacheDataFromDB(utils.RATING_PLAN_PREFIX, []string{attrs.RatingPlanId}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = OK
 	return nil
 }
@@ -215,6 +223,9 @@ func (self *ApierV1) LoadRatingProfile(attrs utils.TPRatingProfile, reply *strin
 	if err := dbReader.LoadRatingProfilesFiltered(&rp[0]); err != nil {
 		return utils.NewErrServerError(err)
 	}
+	if err := self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, []string{attrs.KeyId()}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = OK
 	return nil
 }
@@ -233,7 +244,9 @@ func (self *ApierV1) LoadSharedGroup(attrs AttrLoadSharedGroup, reply *string) e
 	if err := dbReader.LoadSharedGroupsFiltered(attrs.SharedGroupId, true); err != nil {
 		return utils.NewErrServerError(err)
 	}
-
+	if err := self.RatingDb.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, []string{attrs.SharedGroupId}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = OK
 	return nil
 }
@@ -285,7 +298,7 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply
 	if err := dbReader.WriteToDatabase(attrs.FlushDb, false, false); err != nil {
 		return utils.NewErrServerError(err)
 	}
-	utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.")
+	utils.Logger.Info("ApierV1.LoadTariffPlanFromStorDb, reloading cache.")
 	for _, prfx := range []string{utils.DESTINATION_PREFIX,
 		utils.RATING_PLAN_PREFIX,
 		utils.RATING_PROFILE_PREFIX,
@@ -293,14 +306,19 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply
 		utils.ACTION_PLAN_PREFIX,
 		utils.SHARED_GROUP_PREFIX,
 		utils.DERIVEDCHARGERS_PREFIX,
-		utils.LCR_PREFIX,
-		utils.ALIASES_PREFIX,
-		utils.ResourceLimitsPrefix} {
+		utils.LCR_PREFIX} {
 		loadedIDs, _ := dbReader.GetLoadedIds(prfx)
 		if err := self.RatingDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
 			return utils.NewErrServerError(err)
 		}
 	}
+	for _, prfx := range []string{utils.ALIASES_PREFIX,
+		utils.ResourceLimitsPrefix} {
+		loadedIDs, _ := dbReader.GetLoadedIds(prfx)
+		if err := self.AccountDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
 	aps, _ := dbReader.GetLoadedIds(utils.ACTION_PLAN_PREFIX)
 	cstKeys, _ := dbReader.GetLoadedIds(utils.CDR_STATS_PREFIX)
 	userKeys, _ := dbReader.GetLoadedIds(utils.USERS_PREFIX)
@@ -827,6 +845,17 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	return nil
 }

+func (self *ApierV1) PreloadCache(ignr string, reply *string) error {
+	if err := self.RatingDb.PreloadRatingCache(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := self.AccountDb.PreloadAccountingCache(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
 func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.CacheStats) error {
 	cs := new(utils.CacheStats)
 	cs.Destinations = cache.CountEntries(utils.DESTINATION_PREFIX)
@@ -916,19 +945,24 @@ func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder,
 		utils.ACTION_PLAN_PREFIX,
 		utils.SHARED_GROUP_PREFIX,
 		utils.DERIVEDCHARGERS_PREFIX,
-		utils.LCR_PREFIX,
-		utils.ALIASES_PREFIX,
-		utils.ResourceLimitsPrefix} {
+		utils.LCR_PREFIX} {
 		loadedIDs, _ := loader.GetLoadedIds(prfx)
 		if err := self.RatingDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
 			return utils.NewErrServerError(err)
 		}
 	}
+	for _, prfx := range []string{utils.ALIASES_PREFIX,
+		utils.ResourceLimitsPrefix} {
+		loadedIDs, _ := loader.GetLoadedIds(prfx)
+		if err := self.AccountDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
 	aps, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX)
 	cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX)
 	userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX)

-	// relase the tp data
+	// relase tp data
 	loader.Init()

 	if len(aps) != 0 {
--- a/apier/v1/apier_it_test.go
+++ b/apier/v1/apier_it_test.go
@@ -107,7 +107,6 @@ func TestApierStartEngine(t *testing.T) {

 // Connect rpc client to rater
 func TestApierRpcConn(t *testing.T) {
-
 	var err error
 	rater, err = jsonrpc.Dial("tcp", cfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
 	if err != nil {
@@ -738,6 +737,14 @@ func TestApierReloadScheduler(t *testing.T) {

 // Test here ReloadCache
 func TestApierReloadCache(t *testing.T) {
+	var rcvStats *utils.CacheStats
+	var args utils.AttrCacheStats
+	expectedStats := &utils.CacheStats{Destinations: 0, RatingPlans: 1, RatingProfiles: 0, Actions: 0, ActionPlans: 0}
+	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
+		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
+	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
+		t.Errorf("Calling ApierV1.GetCacheStats expected: %+v, received: %+v", expectedStats, rcvStats)
+	}
 	reply := ""
 	arc := new(utils.AttrReloadCache)
 	// Simple test that command is executed without errors
@@ -746,14 +753,8 @@ func TestApierReloadCache(t *testing.T) {
 	} else if reply != "OK" {
 		t.Error("Calling ApierV1.ReloadCache got reply: ", reply)
 	}
-}
-
-func TestApierGetCacheStats(t *testing.T) {
-	var rcvStats *utils.CacheStats
-	var args utils.AttrCacheStats
-	err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats)
-	expectedStats := &utils.CacheStats{Destinations: 0, RatingPlans: 1, RatingProfiles: 0, Actions: 0, ActionPlans: 0}
-	if err != nil {
+	expectedStats = &utils.CacheStats{Destinations: 0, RatingPlans: 1, RatingProfiles: 0, Actions: 0, ActionPlans: 0}
+	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
 		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
 	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
 		t.Errorf("Calling ApierV1.GetCacheStats expected: %+v, received: %+v", expectedStats, rcvStats)
--- a/apier/v2/apier.go
+++ b/apier/v2/apier.go
@@ -27,7 +27,6 @@ import (
 	"strings"

 	"github.com/cgrates/cgrates/apier/v1"
-	"github.com/cgrates/cgrates/cache"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -53,6 +52,9 @@ func (self *ApierV2) LoadRatingProfile(attrs AttrLoadRatingProfile, reply *strin
 	if err := dbReader.LoadRatingProfilesFiltered(&rpf[0]); err != nil {
 		return utils.NewErrServerError(err)
 	}
+	if err := self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, []string{attrs.RatingProfileId}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = v1.OK
 	return nil
 }
@@ -104,6 +106,9 @@ func (self *ApierV2) LoadDerivedChargers(attrs AttrLoadDerivedChargers, reply *s
 	if err := dbReader.LoadDerivedChargersFiltered(&dc[0], true); err != nil {
 		return utils.NewErrServerError(err)
 	}
+	if err := self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{attrs.DerivedChargersId}, true); err != nil {
+		return utils.NewErrServerError(err)
+	}
 	*reply = v1.OK
 	return nil
 }
@@ -156,61 +161,34 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder,
 	if err := loader.WriteToDatabase(attrs.FlushDb, false, false); err != nil {
 		return utils.NewErrServerError(err)
 	}
-	// Make sure the items are in the cache
-	dstIds, _ := loader.GetLoadedIds(utils.DESTINATION_PREFIX)
-	dstKeys := make([]string, len(dstIds))
-	for idx, dId := range dstIds {
-		dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys
+
+	utils.Logger.Info("ApierV2.LoadTariffPlanFromFolder, reloading cache.")
+	for _, prfx := range []string{utils.DESTINATION_PREFIX,
+		utils.RATING_PLAN_PREFIX,
+		utils.RATING_PROFILE_PREFIX,
+		utils.ACTION_PREFIX,
+		utils.ACTION_PLAN_PREFIX,
+		utils.SHARED_GROUP_PREFIX,
+		utils.DERIVEDCHARGERS_PREFIX,
+		utils.LCR_PREFIX} {
+		loadedIDs, _ := loader.GetLoadedIds(prfx)
+		if err := self.RatingDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
+			return utils.NewErrServerError(err)
+		}
 	}
-	rplIds, _ := loader.GetLoadedIds(utils.RATING_PLAN_PREFIX)
-	rpKeys := make([]string, len(rplIds))
-	for idx, rpId := range rplIds {
-		rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId
-	}
-	rpfIds, _ := loader.GetLoadedIds(utils.RATING_PROFILE_PREFIX)
-	rpfKeys := make([]string, len(rpfIds))
-	for idx, rpfId := range rpfIds {
-		rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId
-	}
-	actIds, _ := loader.GetLoadedIds(utils.ACTION_PREFIX)
-	actKeys := make([]string, len(actIds))
-	for idx, actId := range actIds {
-		actKeys[idx] = utils.ACTION_PREFIX + actId
-	}
-	aplIds, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX)
-	aplKeys := make([]string, len(aplIds))
-	for idx, aplId := range aplIds {
-		aplKeys[idx] = utils.ACTION_PLAN_PREFIX + aplId
-	}
-	shgIds, _ := loader.GetLoadedIds(utils.SHARED_GROUP_PREFIX)
-	shgKeys := make([]string, len(shgIds))
-	for idx, shgId := range shgIds {
-		shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId
-	}
-	aliases, _ := loader.GetLoadedIds(utils.ALIASES_PREFIX)
-	alsKeys := make([]string, len(aliases))
-	for idx, alias := range aliases {
-		alsKeys[idx] = utils.ALIASES_PREFIX + alias
-	}
-	lcrIds, _ := loader.GetLoadedIds(utils.LCR_PREFIX)
-	lcrKeys := make([]string, len(lcrIds))
-	for idx, lcrId := range lcrIds {
-		lcrKeys[idx] = utils.LCR_PREFIX + lcrId
-	}
-	dcs, _ := loader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX)
-	dcsKeys := make([]string, len(dcs))
-	for idx, dc := range dcs {
-		dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc
+	for _, prfx := range []string{utils.ALIASES_PREFIX,
+		utils.ResourceLimitsPrefix} {
+		loadedIDs, _ := loader.GetLoadedIds(prfx)
+		if err := self.AccountDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil {
+			return utils.NewErrServerError(err)
+		}
 	}
 	aps, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX)
-	utils.Logger.Info("ApierV2.LoadTariffPlanFromFolder, reloading cache.")
-
 	cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX)
 	userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX)
-	loader.Init() // release the tp data
-	cache.Flush()
-	self.RatingDb.PreloadRatingCache()
-	self.AccountDb.PreloadAccountingCache()
+
+	// relase tp data
+	loader.Init()

 	if len(aps) != 0 {
 		sched := self.ServManager.GetScheduler()
--- a/cmd/cgr-engine/cgr-engine.go
+++ b/cmd/cgr-engine/cgr-engine.go
@@ -458,13 +458,6 @@ func startAliasesServer(internalAliaseSChan chan rpcclient.RpcClientConnection,
 		internalAliaseSChan <- aliasesServer
 		return
 	}
-
-	if err := accountDb.PreloadAccountingCache(); err != nil {
-		utils.Logger.Crit(fmt.Sprintf("<Aliases> Could not start, error: %s", err.Error()))
-		exitChan <- true
-		return
-	}
-
 	internalAliaseSChan <- aliasesServer
 }

--- a/cmd/cgr-engine/rater.go
+++ b/cmd/cgr-engine/rater.go
@@ -71,7 +71,12 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC
 		}

 		if err := ratingDb.PreloadRatingCache(); err != nil {
-			utils.Logger.Crit(fmt.Sprintf("Cache rating error: %s", err.Error()))
+			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache rating error: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+		if err := accountDb.PreloadAccountingCache(); err != nil {
+			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache accounting error: %s", err.Error()))
 			exitChan <- true
 			return
 		}
--- a/config/cacheconfig.go
+++ b/config/cacheconfig.go
@@ -61,6 +61,8 @@ type CacheConfig struct {
 	SharedGroups        *CacheParamConfig
 	Aliases             *CacheParamConfig
 	ReverseAliases      *CacheParamConfig
+	DerivedChargers     *CacheParamConfig
+	ResourceLimits      *CacheParamConfig
 }

 func (self *CacheConfig) loadFromJsonCfg(jsnCfg *CacheJsonCfg) error {
@@ -136,5 +138,17 @@ func (self *CacheConfig) loadFromJsonCfg(jsnCfg *CacheJsonCfg) error {
 			return err
 		}
 	}
+	if jsnCfg.Derived_chargers != nil {
+		self.DerivedChargers = &CacheParamConfig{}
+		if err := self.DerivedChargers.loadFromJsonCfg(jsnCfg.Derived_chargers); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Resource_limits != nil {
+		self.ResourceLimits = &CacheParamConfig{}
+		if err := self.ResourceLimits.loadFromJsonCfg(jsnCfg.Resource_limits); err != nil {
+			return err
+		}
+	}
 	return nil
 }
--- a/config/config_defaults.go
+++ b/config/config_defaults.go
@@ -62,6 +62,8 @@ const CGRATES_CFG_JSON = `
 	"shared_groups": {"limit": 10000, "ttl":"0s", "precache": false},
 	"aliases": {"limit": 10000, "ttl":"0s", "precache": false},
 	"reverse_aliases": {"limit": 10000, "ttl":"0s", "precache": false},
+	"derived_chargers": {"limit": 10000, "ttl":"0s", "precache": false},
+	"resource_limits": {"limit": 10000, "ttl":"0s", "precache": false},
 },


--- a/config/config_json_test.go
+++ b/config/config_json_test.go
@@ -91,6 +91,10 @@ func TestCacheJsonCfg(t *testing.T) {
 			Ttl: utils.StringPointer("0s"), Precache: utils.BoolPointer(false)},
 		Reverse_aliases: &CacheParamJsonCfg{Limit: utils.IntPointer(10000),
 			Ttl: utils.StringPointer("0s"), Precache: utils.BoolPointer(false)},
+		Derived_chargers: &CacheParamJsonCfg{Limit: utils.IntPointer(10000),
+			Ttl: utils.StringPointer("0s"), Precache: utils.BoolPointer(false)},
+		Resource_limits: &CacheParamJsonCfg{Limit: utils.IntPointer(10000),
+			Ttl: utils.StringPointer("0s"), Precache: utils.BoolPointer(false)},
 	}
 	if gCfg, err := dfCgrJsonCfg.CacheJsonCfg(); err != nil {
 		t.Error(err)
--- a/config/libconfig_json.go
+++ b/config/libconfig_json.go
@@ -263,6 +263,8 @@ type CacheJsonCfg struct {
 	Shared_groups        *CacheParamJsonCfg
 	Aliases              *CacheParamJsonCfg
 	Reverse_aliases      *CacheParamJsonCfg
+	Derived_chargers     *CacheParamJsonCfg
+	Resource_limits      *CacheParamJsonCfg
 }

 // Represents one connection instance towards FreeSWITCH
--- a/engine/storage_interface.go
+++ b/engine/storage_interface.go
@@ -32,7 +32,6 @@ type Storage interface {
 	Close()
 	Flush(string) error
 	GetKeysForPrefix(string) ([]string, error)
-	PreloadCacheForPrefix(string) error
 	RebuildReverseForPrefix(string) error
 	GetVersions(itm string) (vrs Versions, err error)
 	SetVersions(vrs Versions) (err error)
--- a/engine/storage_mongo_datadb.go
+++ b/engine/storage_mongo_datadb.go
@@ -366,107 +366,75 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) error {
 	return nil
 }

-func (ms *MongoStorage) PreloadRatingCache() error {
+func (ms *MongoStorage) PreloadRatingCache() (err error) {
 	if ms.cacheCfg == nil {
-		return nil
+		return
 	}
-	if ms.cacheCfg.Destinations != nil && ms.cacheCfg.Destinations.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.Destinations.Precache {
+		if err = ms.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if ms.cacheCfg.ReverseDestinations != nil && ms.cacheCfg.ReverseDestinations.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.ReverseDestinations.Precache {
+		if err = ms.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if ms.cacheCfg.RatingPlans != nil && ms.cacheCfg.RatingPlans.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.RatingPlans.Precache {
+		if err = ms.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if ms.cacheCfg.RatingProfiles != nil && ms.cacheCfg.RatingProfiles.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.RatingProfiles.Precache {
+		if err = ms.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.Lcr != nil && ms.cacheCfg.Lcr.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.Actions.Precache {
+		if err = ms.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.CdrStats != nil && ms.cacheCfg.CdrStats.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.ActionPlans.Precache {
+		if err = ms.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.Actions != nil && ms.cacheCfg.Actions.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.SharedGroups.Precache {
+		if err = ms.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.ActionPlans != nil && ms.cacheCfg.ActionPlans.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.DerivedChargers.Precache {
+		if err = ms.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.ActionTriggers != nil && ms.cacheCfg.ActionTriggers.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.Lcr.Precache {
+		if err = ms.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if ms.cacheCfg.SharedGroups != nil && ms.cacheCfg.SharedGroups.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil {
-			return err
-		}
-	}
-	// add more prefixes if needed
-	return nil
+	return
 }

-func (ms *MongoStorage) PreloadAccountingCache() error {
-	if ms.cacheCfg == nil {
-		return nil
-	}
-	if ms.cacheCfg.Aliases != nil && ms.cacheCfg.Aliases.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil {
-			return err
+func (ms *MongoStorage) PreloadAccountingCache() (err error) {
+	if ms.cacheCfg.Aliases.Precache {
+		if err = ms.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if ms.cacheCfg.ReverseAliases != nil && ms.cacheCfg.ReverseAliases.Precache {
-		if err := ms.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil {
-			return err
+	if ms.cacheCfg.ReverseAliases.Precache {
+		if err = ms.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	return nil
-}
-
-func (ms *MongoStorage) PreloadCacheForPrefix(prefix string) error {
-	transID := cache.BeginTransaction()
-	cache.RemPrefixKey(prefix, false, transID)
-	keyList, err := ms.GetKeysForPrefix(prefix)
-	if err != nil {
-		cache.RollbackTransaction(transID)
-		return err
-	}
-	switch prefix {
-	case utils.RATING_PLAN_PREFIX:
-		for _, key := range keyList {
-			_, err := ms.GetRatingPlan(key[len(utils.RATING_PLAN_PREFIX):], true, transID)
-			if err != nil {
-				cache.RollbackTransaction(transID)
-				return err
-			}
+	if ms.cacheCfg.ResourceLimits.Precache {
+		if err = ms.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil {
+			return
 		}
-	default:
-		return utils.ErrInvalidKey
 	}
-	cache.CommitTransaction(transID)
-	return nil
+	return
 }

 // CacheDataFromDB loads data to cache
@@ -490,6 +458,7 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 			utils.UnsupportedCachePrefix,
 			fmt.Sprintf("prefix <%s> is not a supported cache prefix", prfx))
 	}
+	var prefixRemoved bool // Mark prefix reset
 	if ids == nil {
 		if ids, err = ms.GetKeysForPrefix(prfx); err != nil {
 			return utils.NewCGRError(utils.MONGO,
@@ -497,10 +466,50 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 				err.Error(),
 				fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx))
 		}
+		if mustBeCached { // Only consider loading ids which are already in cache
+			for i := 0; i < len(ids); {
+				if _, hasIt := cache.Get(prfx + ids[i]); !hasIt {
+					ids = append(ids[:i], ids[i+1:]...)
+					continue
+				}
+				i++
+			}
+		}
 		cache.RemPrefixKey(prfx, true, utils.NonTransactional)
+		prefixRemoved = true
+		var nrItems int
+		switch prfx {
+		case utils.DESTINATION_PREFIX:
+			nrItems = ms.cacheCfg.Destinations.Limit
+		case utils.REVERSE_DESTINATION_PREFIX:
+			nrItems = ms.cacheCfg.ReverseDestinations.Limit
+		case utils.RATING_PLAN_PREFIX:
+			nrItems = ms.cacheCfg.RatingPlans.Limit
+		case utils.RATING_PROFILE_PREFIX:
+			nrItems = ms.cacheCfg.RatingProfiles.Limit
+		case utils.ACTION_PREFIX:
+			nrItems = ms.cacheCfg.Actions.Limit
+		case utils.ACTION_PLAN_PREFIX:
+			nrItems = ms.cacheCfg.ActionPlans.Limit
+		case utils.SHARED_GROUP_PREFIX:
+			nrItems = ms.cacheCfg.SharedGroups.Limit
+		case utils.DERIVEDCHARGERS_PREFIX:
+			nrItems = ms.cacheCfg.DerivedChargers.Limit
+		case utils.LCR_PREFIX:
+			nrItems = ms.cacheCfg.Lcr.Limit
+		case utils.ALIASES_PREFIX:
+			nrItems = ms.cacheCfg.Aliases.Limit
+		case utils.REVERSE_ALIASES_PREFIX:
+			nrItems = ms.cacheCfg.ReverseAliases.Limit
+		case utils.ResourceLimitsPrefix:
+			nrItems = ms.cacheCfg.ResourceLimits.Limit
+		}
+		if nrItems != 0 && nrItems < len(ids) { // More ids than cache config allows it, limit here
+			ids = ids[:nrItems]
+		}
 	}
 	for _, dataID := range ids {
-		if mustBeCached {
+		if mustBeCached && !prefixRemoved {
 			if _, hasIt := cache.Get(prfx + dataID); !hasIt { // only cache if previously there
 				continue
 			}
--- a/engine/storage_redis.go
+++ b/engine/storage_redis.go
@@ -112,115 +112,75 @@ func (rs *RedisStorage) Flush(ignore string) error {
 	return rs.Cmd("FLUSHDB").Err
 }

-func (rs *RedisStorage) PreloadRatingCache() error {
+func (rs *RedisStorage) PreloadRatingCache() (err error) {
 	if rs.cacheCfg == nil {
-		return nil
+		return
 	}
-	if rs.cacheCfg.Destinations != nil && rs.cacheCfg.Destinations.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.DESTINATION_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.Destinations.Precache {
+		if err = rs.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if rs.cacheCfg.ReverseDestinations != nil && rs.cacheCfg.ReverseDestinations.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.ReverseDestinations.Precache {
+		if err = rs.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if rs.cacheCfg.RatingPlans != nil && rs.cacheCfg.RatingPlans.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.RATING_PLAN_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.RatingPlans.Precache {
+		if err = rs.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if rs.cacheCfg.RatingProfiles != nil && rs.cacheCfg.RatingProfiles.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.RATING_PROFILE_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.RatingProfiles.Precache {
+		if err = rs.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.Lcr != nil && rs.cacheCfg.Lcr.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.LCR_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.Actions.Precache {
+		if err = rs.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.CdrStats != nil && rs.cacheCfg.CdrStats.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.CDR_STATS_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.ActionPlans.Precache {
+		if err = rs.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.Actions != nil && rs.cacheCfg.Actions.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.ACTION_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.SharedGroups.Precache {
+		if err = rs.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.ActionPlans != nil && rs.cacheCfg.ActionPlans.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.ACTION_PLAN_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.DerivedChargers.Precache {
+		if err = rs.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.ActionTriggers != nil && rs.cacheCfg.ActionTriggers.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.Lcr.Precache {
+		if err = rs.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	if rs.cacheCfg.SharedGroups != nil && rs.cacheCfg.SharedGroups.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.SHARED_GROUP_PREFIX); err != nil {
-			return err
-		}
-	}
-	// add more prefixes if needed
-	return nil
+	return
 }

-func (rs *RedisStorage) PreloadAccountingCache() error {
-	if rs.cacheCfg == nil {
-		return nil
-	}
-	if rs.cacheCfg.Aliases != nil && rs.cacheCfg.Aliases.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.ALIASES_PREFIX); err != nil {
-			return err
+func (rs *RedisStorage) PreloadAccountingCache() (err error) {
+	if rs.cacheCfg.Aliases.Precache {
+		if err = rs.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-
-	if rs.cacheCfg.ReverseAliases != nil && rs.cacheCfg.ReverseAliases.Precache {
-		if err := rs.PreloadCacheForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil {
-			return err
+	if rs.cacheCfg.ReverseAliases.Precache {
+		if err = rs.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil {
+			return
 		}
 	}
-	return nil
-}
-
-func (rs *RedisStorage) PreloadCacheForPrefix(prefix string) error {
-	transID := cache.BeginTransaction()
-	cache.RemPrefixKey(prefix, false, transID)
-	keyList, err := rs.GetKeysForPrefix(prefix)
-	if err != nil {
-		cache.RollbackTransaction(transID)
-		return err
-	}
-	switch prefix {
-	case utils.RATING_PLAN_PREFIX:
-		for _, key := range keyList {
-			_, err := rs.GetRatingPlan(key[len(utils.RATING_PLAN_PREFIX):], true, transID)
-			if err != nil {
-				cache.RollbackTransaction(transID)
-				return err
-			}
+	if rs.cacheCfg.ResourceLimits.Precache {
+		if err = rs.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil {
+			return
 		}
-	case utils.ResourceLimitsPrefix:
-		for _, key := range keyList {
-			_, err = rs.GetResourceLimit(key[len(utils.ResourceLimitsPrefix):], true, transID)
-			if err != nil {
-				cache.RollbackTransaction(transID)
-				return err
-			}
-		}
-	default:
-		return utils.ErrInvalidKey
 	}
-	cache.CommitTransaction(transID)
-	return nil
+	return
 }

 func (rs *RedisStorage) RebuildReverseForPrefix(prefix string) error {
@@ -290,6 +250,7 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 			utils.UnsupportedCachePrefix,
 			fmt.Sprintf("prefix <%s> is not a supported cache prefix", prfx))
 	}
+	var prefixRemoved bool // Mark prefix reset
 	if ids == nil {
 		if ids, err = rs.GetKeysForPrefix(prfx); err != nil {
 			return utils.NewCGRError(utils.REDIS,
@@ -297,10 +258,50 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 				err.Error(),
 				fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx))
 		}
+		if mustBeCached { // Only consider loading ids which are already in cache
+			for i := 0; i < len(ids); {
+				if _, hasIt := cache.Get(prfx + ids[i]); !hasIt {
+					ids = append(ids[:i], ids[i+1:]...)
+					continue
+				}
+				i++
+			}
+		}
 		cache.RemPrefixKey(prfx, true, utils.NonTransactional)
+		prefixRemoved = true
+		var nrItems int
+		switch prfx {
+		case utils.DESTINATION_PREFIX:
+			nrItems = rs.cacheCfg.Destinations.Limit
+		case utils.REVERSE_DESTINATION_PREFIX:
+			nrItems = rs.cacheCfg.ReverseDestinations.Limit
+		case utils.RATING_PLAN_PREFIX:
+			nrItems = rs.cacheCfg.RatingPlans.Limit
+		case utils.RATING_PROFILE_PREFIX:
+			nrItems = rs.cacheCfg.RatingProfiles.Limit
+		case utils.ACTION_PREFIX:
+			nrItems = rs.cacheCfg.Actions.Limit
+		case utils.ACTION_PLAN_PREFIX:
+			nrItems = rs.cacheCfg.ActionPlans.Limit
+		case utils.SHARED_GROUP_PREFIX:
+			nrItems = rs.cacheCfg.SharedGroups.Limit
+		case utils.DERIVEDCHARGERS_PREFIX:
+			nrItems = rs.cacheCfg.DerivedChargers.Limit
+		case utils.LCR_PREFIX:
+			nrItems = rs.cacheCfg.Lcr.Limit
+		case utils.ALIASES_PREFIX:
+			nrItems = rs.cacheCfg.Aliases.Limit
+		case utils.REVERSE_ALIASES_PREFIX:
+			nrItems = rs.cacheCfg.ReverseAliases.Limit
+		case utils.ResourceLimitsPrefix:
+			nrItems = rs.cacheCfg.ResourceLimits.Limit
+		}
+		if nrItems != 0 && nrItems < len(ids) {
+			ids = ids[:nrItems]
+		}
 	}
 	for _, dataID := range ids {
-		if mustBeCached {
+		if mustBeCached && !prefixRemoved {
 			if _, hasIt := cache.Get(prfx + dataID); !hasIt { // only cache if previously there
 				continue
 			}