Apierv1.PreloadCache, ApierV1.FlushCache with options, remove Responder.FlushCache, fixes at CacheDataFromDB for all keys

apier/v1/apier.go

@@ -692,6 +692,9 @@ func (self *ApierV1) ReloadScheduler(ignore string, reply *string) error {
 }
 
 func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (err error) {
+	if attrs.FlushAll {
+		cache.Flush()
+	}
 	var dataIDs []string
 	// Reload Destinations
 	if attrs.DestinationIDs == nil {
@@ -699,7 +702,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.DestinationIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.DestinationIDs))
 		for idx, dId := range *attrs.DestinationIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.DESTINATION_PREFIX, dataIDs, true); err != nil {
@@ -711,7 +714,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.ReverseDestinationIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.ReverseDestinationIDs))
 		for idx, dId := range *attrs.ReverseDestinationIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dataIDs, true); err != nil {
@@ -723,7 +726,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.RatingPlanIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.RatingPlanIDs))
 		for idx, dId := range *attrs.RatingPlanIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.RATING_PLAN_PREFIX, dataIDs, true); err != nil {
@@ -735,7 +738,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.RatingProfileIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.RatingProfileIDs))
 		for idx, dId := range *attrs.RatingProfileIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, dataIDs, true); err != nil {
@@ -747,7 +750,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.ActionIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.ActionIDs))
 		for idx, dId := range *attrs.ActionIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PREFIX, dataIDs, true); err != nil {
@@ -759,7 +762,19 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.ActionPlanIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.ActionPlanIDs))
 		for idx, dId := range *attrs.ActionPlanIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
+		}
+	}
+	if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, dataIDs, true); err != nil {
+		return
+	}
+	// ActionTriggers
+	if attrs.ActionTriggerIDs == nil {
+		dataIDs = nil // Reload all
+	} else if len(*attrs.ActionTriggerIDs) > 0 {
+		dataIDs = make([]string, len(*attrs.ActionTriggerIDs))
+		for idx, dId := range *attrs.ActionTriggerIDs {
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, dataIDs, true); err != nil {
@@ -771,7 +786,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.SharedGroupIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.SharedGroupIDs))
 		for idx, dId := range *attrs.SharedGroupIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, dataIDs, true); err != nil {
@@ -783,7 +798,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.LCRids) > 0 {
 		dataIDs = make([]string, len(*attrs.LCRids))
 		for idx, dId := range *attrs.LCRids {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.LCR_PREFIX, dataIDs, true); err != nil {
@@ -795,7 +810,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.DerivedChargerIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.DerivedChargerIDs))
 		for idx, dId := range *attrs.DerivedChargerIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dataIDs, true); err != nil {
@@ -807,7 +822,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.AliasIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.AliasIDs))
 		for idx, dId := range *attrs.AliasIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.AccountDb.CacheDataFromDB(utils.ALIASES_PREFIX, dataIDs, true); err != nil {
@@ -819,7 +834,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.ReverseAliasIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.ReverseAliasIDs))
 		for idx, dId := range *attrs.ReverseAliasIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.AccountDb.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, dataIDs, true); err != nil {
@@ -831,7 +846,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	} else if len(*attrs.ResourceLimitIDs) > 0 {
 		dataIDs = make([]string, len(*attrs.ResourceLimitIDs))
 		for idx, dId := range *attrs.ResourceLimitIDs {
-			dataIDs[idx] = dId // Cache expects them as redis keys
+			dataIDs[idx] = dId
 		}
 	}
 	if err = self.AccountDb.CacheDataFromDB(utils.ResourceLimitsPrefix, dataIDs, true); err != nil {
@@ -841,17 +856,188 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	return nil
 }
 
-func (self *ApierV1) PreloadCache(ignr string, reply *string) error {
-	if err := self.RatingDb.PreloadRatingCache(); err != nil {
+func (self *ApierV1) PreloadCache(args utils.AttrReloadCache, reply *string) (err error) {
+	if args.FlushAll {
+		cache.Flush()
+	}
+	var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs []string
+	if args.DestinationIDs == nil {
+		dstIDs = nil
+	} else {
+		dstIDs = *args.DestinationIDs
+	}
+	if args.ReverseDestinationIDs == nil {
+		rvDstIDs = nil
+	} else {
+		rvDstIDs = *args.ReverseDestinationIDs
+	}
+	if args.RatingPlanIDs == nil {
+		rplIDs = nil
+	} else {
+		rplIDs = *args.RatingPlanIDs
+	}
+	if args.RatingProfileIDs == nil {
+		rpfIDs = nil
+	} else {
+		rpfIDs = *args.RatingProfileIDs
+	}
+	if args.ActionIDs == nil {
+		actIDs = nil
+	} else {
+		actIDs = *args.ActionIDs
+	}
+	if args.ActionPlanIDs == nil {
+		aplIDs = nil
+	} else {
+		aplIDs = *args.ActionPlanIDs
+	}
+	if args.ActionTriggerIDs == nil {
+		atrgIDs = nil
+	} else {
+		atrgIDs = *args.ActionTriggerIDs
+	}
+	if args.SharedGroupIDs == nil {
+		sgIDs = nil
+	} else {
+		sgIDs = *args.SharedGroupIDs
+	}
+	if args.LCRids == nil {
+		lcrIDs = nil
+	} else {
+		lcrIDs = *args.LCRids
+	}
+	if args.DerivedChargerIDs == nil {
+		dcIDs = nil
+	} else {
+		dcIDs = *args.DerivedChargerIDs
+	}
+	if args.AliasIDs == nil {
+		alsIDs = nil
+	} else {
+		alsIDs = *args.AliasIDs
+	}
+	if args.ReverseAliasIDs == nil {
+		rvAlsIDs = nil
+	} else {
+		rvAlsIDs = *args.ReverseAliasIDs
+	}
+	if args.ResourceLimitIDs == nil {
+		rlIDs = nil
+	} else {
+		rlIDs = *args.ResourceLimitIDs
+	}
+	if err := self.RatingDb.PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs); err != nil {
 		return utils.NewErrServerError(err)
 	}
-	if err := self.AccountDb.PreloadAccountingCache(); err != nil {
+	if err := self.AccountDb.PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs); err != nil {
 		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK
 	return nil
 }
 
+// FlushCache wipes out cache for a prefix or completely
+func (self *ApierV1) FlushCache(args utils.AttrReloadCache, reply *string) (err error) {
+	if args.FlushAll {
+		cache.Flush()
+		*reply = utils.OK
+		return
+	}
+	if args.DestinationIDs == nil {
+		cache.RemPrefixKey(utils.DESTINATION_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.DestinationIDs) != 0 {
+		for _, key := range *args.DestinationIDs {
+			cache.RemKey(utils.DESTINATION_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ReverseDestinationIDs == nil {
+		cache.RemPrefixKey(utils.REVERSE_DESTINATION_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.ReverseDestinationIDs) != 0 {
+		for _, key := range *args.ReverseDestinationIDs {
+			cache.RemKey(utils.REVERSE_DESTINATION_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.RatingPlanIDs == nil {
+		cache.RemPrefixKey(utils.RATING_PLAN_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.RatingPlanIDs) != 0 {
+		for _, key := range *args.RatingPlanIDs {
+			cache.RemKey(utils.RATING_PLAN_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.RatingProfileIDs == nil {
+		cache.RemPrefixKey(utils.RATING_PROFILE_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.RatingProfileIDs) != 0 {
+		for _, key := range *args.RatingProfileIDs {
+			cache.RemKey(utils.RATING_PROFILE_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ActionIDs == nil {
+		cache.RemPrefixKey(utils.ACTION_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.ActionIDs) != 0 {
+		for _, key := range *args.ActionIDs {
+			cache.RemKey(utils.ACTION_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ActionPlanIDs == nil {
+		cache.RemPrefixKey(utils.ACTION_PLAN_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.ActionPlanIDs) != 0 {
+		for _, key := range *args.ActionPlanIDs {
+			cache.RemKey(utils.ACTION_PLAN_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ActionTriggerIDs == nil {
+		cache.RemPrefixKey(utils.ACTION_TRIGGER_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.ActionTriggerIDs) != 0 {
+		for _, key := range *args.ActionTriggerIDs {
+			cache.RemKey(utils.ACTION_TRIGGER_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.SharedGroupIDs == nil {
+		cache.RemPrefixKey(utils.SHARED_GROUP_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.SharedGroupIDs) != 0 {
+		for _, key := range *args.SharedGroupIDs {
+			cache.RemKey(utils.SHARED_GROUP_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.LCRids == nil {
+		cache.RemPrefixKey(utils.LCR_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.LCRids) != 0 {
+		for _, key := range *args.LCRids {
+			cache.RemKey(utils.LCR_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.DerivedChargerIDs == nil {
+		cache.RemPrefixKey(utils.DERIVEDCHARGERS_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.DerivedChargerIDs) != 0 {
+		for _, key := range *args.DerivedChargerIDs {
+			cache.RemKey(utils.DERIVEDCHARGERS_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.AliasIDs == nil {
+		cache.RemPrefixKey(utils.ALIASES_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.AliasIDs) != 0 {
+		for _, key := range *args.AliasIDs {
+			cache.RemKey(utils.ALIASES_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ReverseAliasIDs == nil {
+		cache.RemPrefixKey(utils.REVERSE_ALIASES_PREFIX, true, utils.NonTransactional)
+	} else if len(*args.ReverseAliasIDs) != 0 {
+		for _, key := range *args.ReverseAliasIDs {
+			cache.RemKey(utils.REVERSE_ALIASES_PREFIX+key, true, utils.NonTransactional)
+		}
+	}
+	if args.ResourceLimitIDs == nil {
+		cache.RemPrefixKey(utils.ResourceLimitsPrefix, true, utils.NonTransactional)
+	} else if len(*args.ResourceLimitIDs) != 0 {
+		for _, key := range *args.ResourceLimitIDs {
+			cache.RemKey(utils.ResourceLimitsPrefix+key, true, utils.NonTransactional)
+		}
+	}
+	*reply = utils.OK
+	return
+}
+
 func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.CacheStats) error {
 	cs := new(utils.CacheStats)
 	cs.Destinations = cache.CountEntries(utils.DESTINATION_PREFIX)

apier/v1/apier_it_test.go

@@ -692,6 +692,7 @@ func TestApierLoadAccountActions(t *testing.T) {
 	} else if reply != "OK" {
 		t.Error("Calling ApierV1.LoadAccountActions got reply: ", reply)
 	}
+	time.Sleep(10 * time.Millisecond)
 	expectedStats = &utils.CacheStats{Actions: 1, ActionPlans: 1}
 	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
 		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
@@ -1169,13 +1170,12 @@ func TestApierTriggersExecute(t *testing.T) {
 // Start fresh before loading from folder
 func TestApierResetDataBeforeLoadFromFolder(t *testing.T) {
 	TestApierInitDataDb(t)
-	reply := ""
-	arc := new(utils.AttrReloadCache)
+	var reply string
 	// Simple test that command is executed without errors
-	if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil {
-		t.Error("Got error on ApierV1.ReloadCache: ", err.Error())
+	if err := rater.Call("ApierV1.FlushCache", utils.AttrReloadCache{FlushAll: true}, &reply); err != nil {
+		t.Error(err)
 	} else if reply != "OK" {
-		t.Error("Calling ApierV1.ReloadCache got reply: ", reply)
+		t.Error("Reply: ", reply)
 	}
 	var rcvStats *utils.CacheStats
 	var args utils.AttrCacheStats
@@ -1206,29 +1206,42 @@ func TestApierLoadTariffPlanFromFolder(t *testing.T) {
 	} else if reply != "OK" {
 		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
 	}
-	time.Sleep(time.Duration(1 * time.Second))
+	time.Sleep(time.Duration(2 * time.Second))
 }
 
 func TestApierResetDataAfterLoadFromFolder(t *testing.T) {
+	expStats := &utils.CacheStats{Destinations: 3, Actions: 6, ActionPlans: 7, Aliases: 1} // We get partial cache info during load, maybe fix this in the future
+	var rcvStats *utils.CacheStats
+	if err := rater.Call("ApierV1.GetCacheStats", utils.AttrCacheStats{}, &rcvStats); err != nil {
+		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
+	} else if !reflect.DeepEqual(expStats, rcvStats) {
+		t.Errorf("Expecting: %+v, received: %+v", expStats, rcvStats)
+	}
 	reply := ""
-	arc := new(utils.AttrReloadCache)
 	// Simple test that command is executed without errors
-	if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil {
-		t.Error("Got error on ApierV1.ReloadCache: ", err.Error())
+	if err := rater.Call("ApierV1.PreloadCache", utils.AttrReloadCache{}, &reply); err != nil {
+		t.Error(err)
 	} else if reply != "OK" {
 		t.Error("Calling ApierV1.ReloadCache got reply: ", reply)
 	}
-	var rcvStats *utils.CacheStats
-	var args utils.AttrCacheStats
-	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
-		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
+	//expStats = &utils.CacheStats{Destinations: 3, ReverseDestinations: 5}
+	if err := rater.Call("ApierV1.GetCacheStats", utils.AttrCacheStats{}, &rcvStats); err != nil {
+		t.Error(err)
+		//} else if !reflect.DeepEqual(expStats, rcvStats) {
 	} else {
-		if rcvStats.Destinations != 0 ||
+		if rcvStats.Destinations != 3 ||
+			rcvStats.ReverseDestinations != 5 ||
 			rcvStats.RatingPlans != 5 ||
-			rcvStats.RatingProfiles != 0 ||
-			rcvStats.Actions != 0 ||
-			rcvStats.DerivedChargers != 0 {
-			t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats)
+			rcvStats.RatingProfiles != 5 ||
+			rcvStats.Actions != 13 ||
+			rcvStats.ActionPlans != 7 ||
+			rcvStats.SharedGroups != 0 ||
+			rcvStats.DerivedChargers != 3 ||
+			rcvStats.LcrProfiles != 0 ||
+			rcvStats.Aliases != 1 ||
+			rcvStats.ReverseAliases != 2 ||
+			rcvStats.ResourceLimits != 0 {
+			t.Errorf("Expecting: %+v, received: %+v", expStats, rcvStats)
 		}
 	}
 }
@@ -1325,41 +1338,6 @@ func TestApierCdrServer(t *testing.T) {
 	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
 }
 
-/*
-func TestApierExportCdrsToFile(t *testing.T) {
-    var reply *utils.ExportedFileCdrs
-    req := utils.AttrExpFileCdrs{}
-    //if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) {
-    //  t.Error("Failed to detect missing parameter")
-    //}
-    dryRun := utils.CDRE_DRYRUN
-    req.CdrFormat = &dryRun
-    tm1, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:22Z")
-    tm2, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:23Z")
-    expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCGRIDs: []string{utils.Sha1("dsafdsaf", tm1.String()),
-        utils.Sha1("adsafdsaf", tm2.String())}}
-    if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil {
-        t.Error(err.Error())
-    } else if !reflect.DeepEqual(reply, expectReply) {
-        t.Errorf("Unexpected reply: %v", reply)
-    }
-    Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN
-    req.RemoveFromDb = true
-    if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil {
-        t.Error(err.Error())
-    } else if !reflect.DeepEqual(reply, expectReply) {
-        t.Errorf("Unexpected reply: %v", reply)
-    }
-    expectReply.NumberOfRecords = 0 // We should have deleted previously
-    if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil {
-        t.Error(err.Error())
-    } else if !reflect.DeepEqual(reply, expectReply) {
-        t.Errorf("Unexpected reply: %v", reply)
-    }
-
-}
-*/
-
 func TestApierITGetCdrs(t *testing.T) {
 	var reply []*engine.ExternalCDR
 	req := utils.AttrGetCdrs{MediationRunIds: []string{utils.MetaRaw}}
@@ -1486,7 +1464,8 @@ func TestApierITAddRatingSubjectAliases(t *testing.T) {
 	}
 	var alias engine.Alias
 	for _, als := range addRtSubjAliases.Aliases {
-		if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", Tenant: addRtSubjAliases.Tenant, Category: addRtSubjAliases.Category,
+		if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out",
+			Tenant: addRtSubjAliases.Tenant, Category: addRtSubjAliases.Category,
 			Account: als, Subject: als}, &alias); err != nil {
 			t.Error("Unexpected error", err.Error())
 		}
@@ -1503,8 +1482,9 @@ func TestApierITRemRatingSubjectAliases(t *testing.T) {
 	}
 	var alias engine.Alias
 	//al.Direction, al.Tenant, al.Category, al.Account, al.Subject, al.Group
-	if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "2001", Subject: "2001"}, &alias); err == nil || err.Error() != utils.ErrNotFound.Error() {
-		t.Errorf("Unexpected error %v, alias: %+v", err, alias)
+	if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out",
+		Tenant: "cgrates.org", Category: "call", Account: "2001", Subject: "2001"}, &alias); err == nil || err.Error() != utils.ErrNotFound.Error() {
+		t.Errorf("Unexpected error %v, alias: %+v, values: %+v", err, alias, alias.Values[0])
 	}
 }
 
@@ -1572,9 +1552,8 @@ func TestApierInitStorDb2(t *testing.T) {
 
 func TestApierReloadCache2(t *testing.T) {
 	reply := ""
-	arc := new(utils.AttrReloadCache)
 	// Simple test that command is executed without errors
-	if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil {
+	if err := rater.Call("ApierV1.FlushCache", utils.AttrReloadCache{FlushAll: true}, &reply); err != nil {
 		t.Error("Got error on ApierV1.ReloadCache: ", err.Error())
 	} else if reply != utils.OK {
 		t.Error("Calling ApierV1.ReloadCache got reply: ", reply)

cmd/cgr-engine/rater.go

@@ -70,12 +70,12 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC
 			return
 		}
 
-		if err := ratingDb.PreloadRatingCache(); err != nil {
+		if err := ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil {
 			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache rating error: %s", err.Error()))
 			exitChan <- true
 			return
 		}
-		if err := accountDb.PreloadAccountingCache(); err != nil {
+		if err := accountDb.PreloadAccountingCache(nil, nil, nil); err != nil {
 			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache accounting error: %s", err.Error()))
 			exitChan <- true
 			return

cmd/cgr-tester/cgr-tester.go

@@ -76,10 +76,10 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	}
 	defer accountDb.Close()
 	engine.SetAccountingStorage(accountDb)
-	if err := ratingDb.PreloadRatingCache(); err != nil {
+	if err := ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil {
 		return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
 	}
-	if err := accountDb.PreloadAccountingCache(); err != nil {
+	if err := accountDb.PreloadAccountingCache(nil, nil, nil); err != nil {
 		return nilDuration, fmt.Errorf("Cache accounting error: %s", err.Error())
 	}
 	log.Printf("Runnning %d cycles...", *runs)

data/conf/samples/apier/apier.json

@@ -8,6 +8,24 @@
     "log_level": 7,
 },
 
+
+"cache":{
+	"destinations": {"limit": 10000, "ttl":"0s", "precache": true},
+	"reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true},
+	"rating_plans": {"limit": 10000, "ttl":"0s","precache": true},
+	"rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"lcr": {"limit": 10000, "ttl":"0s", "precache": true},
+	"cdr_stats": {"limit": 10000, "ttl":"0s", "precache": true},
+	"actions": {"limit": 10000, "ttl":"0s", "precache": true},
+	"action_plans": {"limit": 10000, "ttl":"0s", "precache": true},
+	"action_triggers": {"limit": 10000, "ttl":"0s", "precache": true},
+	"shared_groups": {"limit": 10000, "ttl":"0s", "precache": true},
+	"aliases": {"limit": 10000, "ttl":"0s", "precache": true},
+	"reverse_aliases": {"limit": 10000, "ttl":"0s", "precache": true},
+	"derived_chargers": {"limit": 10000, "ttl":"0s", "precache": true},
+	"resource_limits": {"limit": 10000, "ttl":"0s", "precache": true},
+},
+
 "listen": {
 	"rpc_json": ":2012",			// RPC JSON listening address
 	"rpc_gob": ":2013",			// RPC GOB listening address

engine/calldesc.go

@@ -863,14 +863,6 @@ func (cd *CallDescriptor) RefundRounding() error {
 	return err
 }
 
-func (cd *CallDescriptor) FlushCache() (err error) {
-	cache.Flush()
-	ratingStorage.PreloadRatingCache()
-	accountingStorage.PreloadAccountingCache()
-	return nil
-
-}
-
 // Creates a CallCost structure copying related data from CallDescriptor
 func (cd *CallDescriptor) CreateCallCost() *CallCost {
 	return &CallCost{

engine/libtest.go

@@ -47,7 +47,7 @@ func InitDataDb(cfg *config.CGRConfig) error {
 			return err
 		}
 	}
-	ratingDb.PreloadRatingCache()
+	ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 	CheckVersion(accountDb) // Write version before starting
 	return nil
 }

engine/loader_csv_test.go

@@ -335,8 +335,8 @@ func init() {
 	}
 	csvr.WriteToDatabase(false, false, false)
 	cache.Flush()
-	ratingStorage.PreloadRatingCache()
-	accountingStorage.PreloadAccountingCache()
+	ratingStorage.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+	accountingStorage.PreloadAccountingCache(nil, nil, nil)
 }
 
 func TestLoadDestinations(t *testing.T) {

engine/responder.go

@@ -573,18 +573,6 @@ func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error {
 	return nil
 }
 
-func (rs *Responder) FlushCache(arg *CallDescriptor, reply *float64) (err error) {
-	if rs.Bal != nil {
-		*reply, err = rs.callMethod(arg, "Responder.FlushCache")
-	} else {
-		r, e := Guardian.Guard(func() (interface{}, error) {
-			return 0, arg.FlushCache()
-		}, 0, arg.GetAccountKey())
-		*reply, err = r.(float64), e
-	}
-	return
-}
-
 func (rs *Responder) Status(arg string, reply *map[string]interface{}) (err error) {
 	if arg != "" { // Introduce  delay in answer, used in some automated tests
 		if delay, err := utils.ParseDurationWithSecs(arg); err == nil {

engine/storage_interface.go

@@ -42,7 +42,7 @@ type Storage interface {
 type RatingStorage interface {
 	Storage
 	HasData(string, string) (bool, error)
-	PreloadRatingCache() error
+	PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error
 	GetRatingPlan(string, bool, string) (*RatingPlan, error)
 	SetRatingPlan(*RatingPlan, string) error
 	GetRatingProfile(string, bool, string) (*RatingProfile, error)
@@ -80,7 +80,7 @@ type RatingStorage interface {
 
 type AccountingStorage interface {
 	Storage
-	PreloadAccountingCache() error
+	PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error
 	GetAccount(string) (*Account, error)
 	SetAccount(*Account) error
 	RemoveAccount(string) error
@@ -117,7 +117,7 @@ type AccountingStorage interface {
 type OnlineStorage interface {
 	Storage
 	HasData(string, string) (bool, error)
-	PreloadRatingCache() error
+	PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error
 	GetRatingPlan(string, bool, string) (*RatingPlan, error)
 	SetRatingPlan(*RatingPlan, string) error
 	GetRatingProfile(string, bool, string) (*RatingProfile, error)
@@ -149,7 +149,7 @@ type OnlineStorage interface {
 	GetAllActionPlans() (map[string]*ActionPlan, error)
 	PushTask(*Task) error
 	PopTask() (*Task, error)
-	PreloadAccountingCache() error
+	PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error
 	GetAccount(string) (*Account, error)
 	SetAccount(*Account) error
 	RemoveAccount(string) error

engine/storage_map.go

@@ -132,7 +132,8 @@ func (ms *MapStorage) RebuildReverseForPrefix(prefix string) error {
 	return nil
 }
 
-func (ms *MapStorage) PreloadRatingCache() error {
+// FixMe
+func (ms *MapStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error {
 	if ms.cacheCfg == nil {
 		return nil
 	}
@@ -193,7 +194,7 @@ func (ms *MapStorage) PreloadRatingCache() error {
 	return nil
 }
 
-func (ms *MapStorage) PreloadAccountingCache() error {
+func (ms *MapStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error {
 	if ms.cacheCfg == nil {
 		return nil
 	}

engine/storage_mongo_datadb.go

@@ -366,76 +366,76 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) error {
 	return nil
 }
 
-func (ms *MongoStorage) PreloadRatingCache() (err error) {
-	if ms.cacheCfg == nil {
-		return
-	}
+func (ms *MongoStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) (err error) {
+	//if ms.cacheCfg == nil {
+	//	return
+	//}
 	if ms.cacheCfg.Destinations.Precache {
-		if err = ms.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.DESTINATION_PREFIX, dstIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.ReverseDestinations.Precache {
-		if err = ms.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, rvDstIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.RatingPlans.Precache {
-		if err = ms.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.RATING_PLAN_PREFIX, rplIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.RatingProfiles.Precache {
-		if err = ms.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, rpfIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.Actions.Precache {
-		if err = ms.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.ACTION_PREFIX, actIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.ActionPlans.Precache {
-		if err = ms.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, aplIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.ActionTriggers.Precache {
-		if err = ms.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, atrgIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.SharedGroups.Precache {
-		if err = ms.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil {
-			return
-		}
-	}
-	if ms.cacheCfg.DerivedChargers.Precache {
-		if err = ms.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.Lcr.Precache {
-		if err = ms.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.LCR_PREFIX, lcrIDs, false); err != nil {
+			return
+		}
+	}
+	if ms.cacheCfg.DerivedChargers.Precache {
+		if err = ms.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dcIDs, false); err != nil {
 			return
 		}
 	}
 	return
 }
 
-func (ms *MongoStorage) PreloadAccountingCache() (err error) {
+func (ms *MongoStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) {
 	if ms.cacheCfg.Aliases.Precache {
-		if err = ms.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.ALIASES_PREFIX, alsIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.ReverseAliases.Precache {
-		if err = ms.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, rvAlsIDs, false); err != nil {
 			return
 		}
 	}
 	if ms.cacheCfg.ResourceLimits.Precache {
-		if err = ms.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil {
+		if err = ms.CacheDataFromDB(utils.ResourceLimitsPrefix, rlIDs, false); err != nil {
 			return
 		}
 	}
@@ -472,12 +472,13 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 				err.Error(),
 				fmt.Sprintf("mongo error <%s> querying keys for prefix: <%s>", prfx))
 		}
-		if mustBeCached { // Only consider loading ids which are already in cache
-			for _, keyID := range keyIDs {
-				if _, hasIt := cache.Get(keyID); hasIt {
-					ids = append(ids, keyID[len(prfx):])
+		for _, keyID := range keyIDs {
+			if mustBeCached { // Only consider loading ids which are already in cache
+				if _, hasIt := cache.Get(keyID); !hasIt {
+					continue
 				}
 			}
+			ids = append(ids, keyID[len(prfx):])
 		}
 		var nrItems int
 		switch prfx {

engine/storage_redis.go

@@ -112,76 +112,76 @@ func (rs *RedisStorage) Flush(ignore string) error {
 	return rs.Cmd("FLUSHDB").Err
 }
 
-func (rs *RedisStorage) PreloadRatingCache() (err error) {
-	if rs.cacheCfg == nil {
-		return
-	}
+func (rs *RedisStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) (err error) {
+	//if rs.cacheCfg == nil {
+	//	return
+	//}
 	if rs.cacheCfg.Destinations.Precache {
-		if err = rs.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.DESTINATION_PREFIX, dstIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.ReverseDestinations.Precache {
-		if err = rs.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, rvDstIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.RatingPlans.Precache {
-		if err = rs.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.RATING_PLAN_PREFIX, rplIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.RatingProfiles.Precache {
-		if err = rs.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, rpfIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.Actions.Precache {
-		if err = rs.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.ACTION_PREFIX, actIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.ActionPlans.Precache {
-		if err = rs.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, aplIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.ActionTriggers.Precache {
-		if err = rs.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, atrgIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.SharedGroups.Precache {
-		if err = rs.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil {
-			return
-		}
-	}
-	if rs.cacheCfg.DerivedChargers.Precache {
-		if err = rs.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.Lcr.Precache {
-		if err = rs.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.LCR_PREFIX, lcrIDs, false); err != nil {
+			return
+		}
+	}
+	if rs.cacheCfg.DerivedChargers.Precache {
+		if err = rs.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dcIDs, false); err != nil {
 			return
 		}
 	}
 	return
 }
 
-func (rs *RedisStorage) PreloadAccountingCache() (err error) {
+func (rs *RedisStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) {
 	if rs.cacheCfg.Aliases.Precache {
-		if err = rs.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.ALIASES_PREFIX, alsIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.ReverseAliases.Precache {
-		if err = rs.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, rvAlsIDs, false); err != nil {
 			return
 		}
 	}
 	if rs.cacheCfg.ResourceLimits.Precache {
-		if err = rs.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil {
+		if err = rs.CacheDataFromDB(utils.ResourceLimitsPrefix, rlIDs, false); err != nil {
 			return
 		}
 	}
@@ -264,12 +264,13 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached
 				err.Error(),
 				fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx))
 		}
-		if mustBeCached { // Only consider loading ids which are already in cache
-			for _, keyID := range keyIDs {
-				if _, hasIt := cache.Get(keyID); hasIt {
-					ids = append(ids, keyID[len(prfx):])
+		for _, keyID := range keyIDs {
+			if mustBeCached { // Only consider loading ids which are already in cache
+				if _, hasIt := cache.Get(keyID); !hasIt {
+					continue
 				}
 			}
+			ids = append(ids, keyID[len(prfx):])
 		}
 		var nrItems int
 		switch prfx {

engine/storage_test.go

@@ -102,7 +102,7 @@ func TestStorageCacheRefresh(t *testing.T) {
 	ratingStorage.GetDestination("T11", false, utils.NonTransactional)
 	ratingStorage.SetDestination(&Destination{"T11", []string{"1"}}, utils.NonTransactional)
 	t.Log("Test cache refresh")
-	err := ratingStorage.PreloadRatingCache()
+	err := ratingStorage.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 	if err != nil {
 		t.Error("Error cache rating: ", err)
 	}

utils/apitpdata.go

@@ -567,12 +567,14 @@ type AttrReloadCache struct {
 	RatingProfileIDs      *[]string
 	ActionIDs             *[]string
 	ActionPlanIDs         *[]string
+	ActionTriggerIDs      *[]string
 	SharedGroupIDs        *[]string
 	LCRids                *[]string
 	DerivedChargerIDs     *[]string
 	AliasIDs              *[]string
 	ReverseAliasIDs       *[]string
 	ResourceLimitIDs      *[]string
+	FlushAll              bool // If provided, cache flush will be executed before any action
 }
 
 type AttrCacheStats struct { // Add in the future filters here maybe so we avoid counting complete cache