From 7111bf239c5687342664c5d88f8d6e164b6cf1c1 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 23 Dec 2016 19:30:14 +0100 Subject: [PATCH] Apierv1.PreloadCache, ApierV1.FlushCache with options, remove Responder.FlushCache, fixes at CacheDataFromDB for all keys --- apier/v1/apier.go | 216 +++++++++++++++++++++++++++-- apier/v1/apier_it_test.go | 95 +++++-------- cmd/cgr-engine/rater.go | 4 +- cmd/cgr-tester/cgr-tester.go | 4 +- data/conf/samples/apier/apier.json | 18 +++ engine/calldesc.go | 8 -- engine/libtest.go | 2 +- engine/loader_csv_test.go | 4 +- engine/responder.go | 12 -- engine/storage_interface.go | 8 +- engine/storage_map.go | 5 +- engine/storage_mongo_datadb.go | 53 +++---- engine/storage_redis.go | 53 +++---- engine/storage_test.go | 2 +- utils/apitpdata.go | 2 + 15 files changed, 327 insertions(+), 159 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 0eb77ac47..7bf6b097e 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -692,6 +692,9 @@ func (self *ApierV1) ReloadScheduler(ignore string, reply *string) error { } func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (err error) { + if attrs.FlushAll { + cache.Flush() + } var dataIDs []string // Reload Destinations if attrs.DestinationIDs == nil { @@ -699,7 +702,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.DestinationIDs) > 0 { dataIDs = make([]string, len(*attrs.DestinationIDs)) for idx, dId := range *attrs.DestinationIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.DESTINATION_PREFIX, dataIDs, true); err != nil { @@ -711,7 +714,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.ReverseDestinationIDs) > 0 { dataIDs = make([]string, len(*attrs.ReverseDestinationIDs)) for idx, dId := range *attrs.ReverseDestinationIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dataIDs, true); err != nil { @@ -723,7 +726,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.RatingPlanIDs) > 0 { dataIDs = make([]string, len(*attrs.RatingPlanIDs)) for idx, dId := range *attrs.RatingPlanIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.RATING_PLAN_PREFIX, dataIDs, true); err != nil { @@ -735,7 +738,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.RatingProfileIDs) > 0 { dataIDs = make([]string, len(*attrs.RatingProfileIDs)) for idx, dId := range *attrs.RatingProfileIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, dataIDs, true); err != nil { @@ -747,7 +750,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.ActionIDs) > 0 { dataIDs = make([]string, len(*attrs.ActionIDs)) for idx, dId := range *attrs.ActionIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PREFIX, dataIDs, true); err != nil { @@ -759,7 +762,19 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.ActionPlanIDs) > 0 { dataIDs = make([]string, len(*attrs.ActionPlanIDs)) for idx, dId := range *attrs.ActionPlanIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId + } + } + if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, dataIDs, true); err != nil { + return + } + // ActionTriggers + if attrs.ActionTriggerIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ActionTriggerIDs) > 0 { + dataIDs = make([]string, len(*attrs.ActionTriggerIDs)) + for idx, dId := range *attrs.ActionTriggerIDs { + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, dataIDs, true); err != nil { @@ -771,7 +786,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.SharedGroupIDs) > 0 { dataIDs = make([]string, len(*attrs.SharedGroupIDs)) for idx, dId := range *attrs.SharedGroupIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, dataIDs, true); err != nil { @@ -783,7 +798,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.LCRids) > 0 { dataIDs = make([]string, len(*attrs.LCRids)) for idx, dId := range *attrs.LCRids { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.LCR_PREFIX, dataIDs, true); err != nil { @@ -795,7 +810,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.DerivedChargerIDs) > 0 { dataIDs = make([]string, len(*attrs.DerivedChargerIDs)) for idx, dId := range *attrs.DerivedChargerIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dataIDs, true); err != nil { @@ -807,7 +822,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.AliasIDs) > 0 { dataIDs = make([]string, len(*attrs.AliasIDs)) for idx, dId := range *attrs.AliasIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.AccountDb.CacheDataFromDB(utils.ALIASES_PREFIX, dataIDs, true); err != nil { @@ -819,7 +834,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.ReverseAliasIDs) > 0 { dataIDs = make([]string, len(*attrs.ReverseAliasIDs)) for idx, dId := range *attrs.ReverseAliasIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.AccountDb.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, dataIDs, true); err != nil { @@ -831,7 +846,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er } else if len(*attrs.ResourceLimitIDs) > 0 { dataIDs = make([]string, len(*attrs.ResourceLimitIDs)) for idx, dId := range *attrs.ResourceLimitIDs { - dataIDs[idx] = dId // Cache expects them as redis keys + dataIDs[idx] = dId } } if err = self.AccountDb.CacheDataFromDB(utils.ResourceLimitsPrefix, dataIDs, true); err != nil { @@ -841,17 +856,188 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er return nil } -func (self *ApierV1) PreloadCache(ignr string, reply *string) error { - if err := self.RatingDb.PreloadRatingCache(); err != nil { +func (self *ApierV1) PreloadCache(args utils.AttrReloadCache, reply *string) (err error) { + if args.FlushAll { + cache.Flush() + } + var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs []string + if args.DestinationIDs == nil { + dstIDs = nil + } else { + dstIDs = *args.DestinationIDs + } + if args.ReverseDestinationIDs == nil { + rvDstIDs = nil + } else { + rvDstIDs = *args.ReverseDestinationIDs + } + if args.RatingPlanIDs == nil { + rplIDs = nil + } else { + rplIDs = *args.RatingPlanIDs + } + if args.RatingProfileIDs == nil { + rpfIDs = nil + } else { + rpfIDs = *args.RatingProfileIDs + } + if args.ActionIDs == nil { + actIDs = nil + } else { + actIDs = *args.ActionIDs + } + if args.ActionPlanIDs == nil { + aplIDs = nil + } else { + aplIDs = *args.ActionPlanIDs + } + if args.ActionTriggerIDs == nil { + atrgIDs = nil + } else { + atrgIDs = *args.ActionTriggerIDs + } + if args.SharedGroupIDs == nil { + sgIDs = nil + } else { + sgIDs = *args.SharedGroupIDs + } + if args.LCRids == nil { + lcrIDs = nil + } else { + lcrIDs = *args.LCRids + } + if args.DerivedChargerIDs == nil { + dcIDs = nil + } else { + dcIDs = *args.DerivedChargerIDs + } + if args.AliasIDs == nil { + alsIDs = nil + } else { + alsIDs = *args.AliasIDs + } + if args.ReverseAliasIDs == nil { + rvAlsIDs = nil + } else { + rvAlsIDs = *args.ReverseAliasIDs + } + if args.ResourceLimitIDs == nil { + rlIDs = nil + } else { + rlIDs = *args.ResourceLimitIDs + } + if err := self.RatingDb.PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs); err != nil { return utils.NewErrServerError(err) } - if err := self.AccountDb.PreloadAccountingCache(); err != nil { + if err := self.AccountDb.PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK return nil } +// FlushCache wipes out cache for a prefix or completely +func (self *ApierV1) FlushCache(args utils.AttrReloadCache, reply *string) (err error) { + if args.FlushAll { + cache.Flush() + *reply = utils.OK + return + } + if args.DestinationIDs == nil { + cache.RemPrefixKey(utils.DESTINATION_PREFIX, true, utils.NonTransactional) + } else if len(*args.DestinationIDs) != 0 { + for _, key := range *args.DestinationIDs { + cache.RemKey(utils.DESTINATION_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ReverseDestinationIDs == nil { + cache.RemPrefixKey(utils.REVERSE_DESTINATION_PREFIX, true, utils.NonTransactional) + } else if len(*args.ReverseDestinationIDs) != 0 { + for _, key := range *args.ReverseDestinationIDs { + cache.RemKey(utils.REVERSE_DESTINATION_PREFIX+key, true, utils.NonTransactional) + } + } + if args.RatingPlanIDs == nil { + cache.RemPrefixKey(utils.RATING_PLAN_PREFIX, true, utils.NonTransactional) + } else if len(*args.RatingPlanIDs) != 0 { + for _, key := range *args.RatingPlanIDs { + cache.RemKey(utils.RATING_PLAN_PREFIX+key, true, utils.NonTransactional) + } + } + if args.RatingProfileIDs == nil { + cache.RemPrefixKey(utils.RATING_PROFILE_PREFIX, true, utils.NonTransactional) + } else if len(*args.RatingProfileIDs) != 0 { + for _, key := range *args.RatingProfileIDs { + cache.RemKey(utils.RATING_PROFILE_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ActionIDs == nil { + cache.RemPrefixKey(utils.ACTION_PREFIX, true, utils.NonTransactional) + } else if len(*args.ActionIDs) != 0 { + for _, key := range *args.ActionIDs { + cache.RemKey(utils.ACTION_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ActionPlanIDs == nil { + cache.RemPrefixKey(utils.ACTION_PLAN_PREFIX, true, utils.NonTransactional) + } else if len(*args.ActionPlanIDs) != 0 { + for _, key := range *args.ActionPlanIDs { + cache.RemKey(utils.ACTION_PLAN_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ActionTriggerIDs == nil { + cache.RemPrefixKey(utils.ACTION_TRIGGER_PREFIX, true, utils.NonTransactional) + } else if len(*args.ActionTriggerIDs) != 0 { + for _, key := range *args.ActionTriggerIDs { + cache.RemKey(utils.ACTION_TRIGGER_PREFIX+key, true, utils.NonTransactional) + } + } + if args.SharedGroupIDs == nil { + cache.RemPrefixKey(utils.SHARED_GROUP_PREFIX, true, utils.NonTransactional) + } else if len(*args.SharedGroupIDs) != 0 { + for _, key := range *args.SharedGroupIDs { + cache.RemKey(utils.SHARED_GROUP_PREFIX+key, true, utils.NonTransactional) + } + } + if args.LCRids == nil { + cache.RemPrefixKey(utils.LCR_PREFIX, true, utils.NonTransactional) + } else if len(*args.LCRids) != 0 { + for _, key := range *args.LCRids { + cache.RemKey(utils.LCR_PREFIX+key, true, utils.NonTransactional) + } + } + if args.DerivedChargerIDs == nil { + cache.RemPrefixKey(utils.DERIVEDCHARGERS_PREFIX, true, utils.NonTransactional) + } else if len(*args.DerivedChargerIDs) != 0 { + for _, key := range *args.DerivedChargerIDs { + cache.RemKey(utils.DERIVEDCHARGERS_PREFIX+key, true, utils.NonTransactional) + } + } + if args.AliasIDs == nil { + cache.RemPrefixKey(utils.ALIASES_PREFIX, true, utils.NonTransactional) + } else if len(*args.AliasIDs) != 0 { + for _, key := range *args.AliasIDs { + cache.RemKey(utils.ALIASES_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ReverseAliasIDs == nil { + cache.RemPrefixKey(utils.REVERSE_ALIASES_PREFIX, true, utils.NonTransactional) + } else if len(*args.ReverseAliasIDs) != 0 { + for _, key := range *args.ReverseAliasIDs { + cache.RemKey(utils.REVERSE_ALIASES_PREFIX+key, true, utils.NonTransactional) + } + } + if args.ResourceLimitIDs == nil { + cache.RemPrefixKey(utils.ResourceLimitsPrefix, true, utils.NonTransactional) + } else if len(*args.ResourceLimitIDs) != 0 { + for _, key := range *args.ResourceLimitIDs { + cache.RemKey(utils.ResourceLimitsPrefix+key, true, utils.NonTransactional) + } + } + *reply = utils.OK + return +} + func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.CacheStats) error { cs := new(utils.CacheStats) cs.Destinations = cache.CountEntries(utils.DESTINATION_PREFIX) diff --git a/apier/v1/apier_it_test.go b/apier/v1/apier_it_test.go index fbe06b425..882fde8a4 100644 --- a/apier/v1/apier_it_test.go +++ b/apier/v1/apier_it_test.go @@ -692,6 +692,7 @@ func TestApierLoadAccountActions(t *testing.T) { } else if reply != "OK" { t.Error("Calling ApierV1.LoadAccountActions got reply: ", reply) } + time.Sleep(10 * time.Millisecond) expectedStats = &utils.CacheStats{Actions: 1, ActionPlans: 1} if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) @@ -1169,13 +1170,12 @@ func TestApierTriggersExecute(t *testing.T) { // Start fresh before loading from folder func TestApierResetDataBeforeLoadFromFolder(t *testing.T) { TestApierInitDataDb(t) - reply := "" - arc := new(utils.AttrReloadCache) + var reply string // Simple test that command is executed without errors - if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil { - t.Error("Got error on ApierV1.ReloadCache: ", err.Error()) + if err := rater.Call("ApierV1.FlushCache", utils.AttrReloadCache{FlushAll: true}, &reply); err != nil { + t.Error(err) } else if reply != "OK" { - t.Error("Calling ApierV1.ReloadCache got reply: ", reply) + t.Error("Reply: ", reply) } var rcvStats *utils.CacheStats var args utils.AttrCacheStats @@ -1206,29 +1206,42 @@ func TestApierLoadTariffPlanFromFolder(t *testing.T) { } else if reply != "OK" { t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply) } - time.Sleep(time.Duration(1 * time.Second)) + time.Sleep(time.Duration(2 * time.Second)) } func TestApierResetDataAfterLoadFromFolder(t *testing.T) { + expStats := &utils.CacheStats{Destinations: 3, Actions: 6, ActionPlans: 7, Aliases: 1} // We get partial cache info during load, maybe fix this in the future + var rcvStats *utils.CacheStats + if err := rater.Call("ApierV1.GetCacheStats", utils.AttrCacheStats{}, &rcvStats); err != nil { + t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) + } else if !reflect.DeepEqual(expStats, rcvStats) { + t.Errorf("Expecting: %+v, received: %+v", expStats, rcvStats) + } reply := "" - arc := new(utils.AttrReloadCache) // Simple test that command is executed without errors - if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil { - t.Error("Got error on ApierV1.ReloadCache: ", err.Error()) + if err := rater.Call("ApierV1.PreloadCache", utils.AttrReloadCache{}, &reply); err != nil { + t.Error(err) } else if reply != "OK" { t.Error("Calling ApierV1.ReloadCache got reply: ", reply) } - var rcvStats *utils.CacheStats - var args utils.AttrCacheStats - if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { - t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) + //expStats = &utils.CacheStats{Destinations: 3, ReverseDestinations: 5} + if err := rater.Call("ApierV1.GetCacheStats", utils.AttrCacheStats{}, &rcvStats); err != nil { + t.Error(err) + //} else if !reflect.DeepEqual(expStats, rcvStats) { } else { - if rcvStats.Destinations != 0 || + if rcvStats.Destinations != 3 || + rcvStats.ReverseDestinations != 5 || rcvStats.RatingPlans != 5 || - rcvStats.RatingProfiles != 0 || - rcvStats.Actions != 0 || - rcvStats.DerivedChargers != 0 { - t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) + rcvStats.RatingProfiles != 5 || + rcvStats.Actions != 13 || + rcvStats.ActionPlans != 7 || + rcvStats.SharedGroups != 0 || + rcvStats.DerivedChargers != 3 || + rcvStats.LcrProfiles != 0 || + rcvStats.Aliases != 1 || + rcvStats.ReverseAliases != 2 || + rcvStats.ResourceLimits != 0 { + t.Errorf("Expecting: %+v, received: %+v", expStats, rcvStats) } } } @@ -1325,41 +1338,6 @@ func TestApierCdrServer(t *testing.T) { time.Sleep(time.Duration(*waitRater) * time.Millisecond) } -/* -func TestApierExportCdrsToFile(t *testing.T) { - var reply *utils.ExportedFileCdrs - req := utils.AttrExpFileCdrs{} - //if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) { - // t.Error("Failed to detect missing parameter") - //} - dryRun := utils.CDRE_DRYRUN - req.CdrFormat = &dryRun - tm1, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:22Z") - tm2, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:23Z") - expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCGRIDs: []string{utils.Sha1("dsafdsaf", tm1.String()), - utils.Sha1("adsafdsaf", tm2.String())}} - if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil { - t.Error(err.Error()) - } else if !reflect.DeepEqual(reply, expectReply) { - t.Errorf("Unexpected reply: %v", reply) - } - Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN - req.RemoveFromDb = true - if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil { - t.Error(err.Error()) - } else if !reflect.DeepEqual(reply, expectReply) { - t.Errorf("Unexpected reply: %v", reply) - } - expectReply.NumberOfRecords = 0 // We should have deleted previously - if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil { - t.Error(err.Error()) - } else if !reflect.DeepEqual(reply, expectReply) { - t.Errorf("Unexpected reply: %v", reply) - } - -} -*/ - func TestApierITGetCdrs(t *testing.T) { var reply []*engine.ExternalCDR req := utils.AttrGetCdrs{MediationRunIds: []string{utils.MetaRaw}} @@ -1486,7 +1464,8 @@ func TestApierITAddRatingSubjectAliases(t *testing.T) { } var alias engine.Alias for _, als := range addRtSubjAliases.Aliases { - if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", Tenant: addRtSubjAliases.Tenant, Category: addRtSubjAliases.Category, + if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", + Tenant: addRtSubjAliases.Tenant, Category: addRtSubjAliases.Category, Account: als, Subject: als}, &alias); err != nil { t.Error("Unexpected error", err.Error()) } @@ -1503,8 +1482,9 @@ func TestApierITRemRatingSubjectAliases(t *testing.T) { } var alias engine.Alias //al.Direction, al.Tenant, al.Category, al.Account, al.Subject, al.Group - if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "2001", Subject: "2001"}, &alias); err == nil || err.Error() != utils.ErrNotFound.Error() { - t.Errorf("Unexpected error %v, alias: %+v", err, alias) + if err := rater.Call("AliasesV1.GetAlias", engine.Alias{Context: utils.ALIAS_CONTEXT_RATING, Direction: "*out", + Tenant: "cgrates.org", Category: "call", Account: "2001", Subject: "2001"}, &alias); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Unexpected error %v, alias: %+v, values: %+v", err, alias, alias.Values[0]) } } @@ -1572,9 +1552,8 @@ func TestApierInitStorDb2(t *testing.T) { func TestApierReloadCache2(t *testing.T) { reply := "" - arc := new(utils.AttrReloadCache) // Simple test that command is executed without errors - if err := rater.Call("ApierV1.ReloadCache", arc, &reply); err != nil { + if err := rater.Call("ApierV1.FlushCache", utils.AttrReloadCache{FlushAll: true}, &reply); err != nil { t.Error("Got error on ApierV1.ReloadCache: ", err.Error()) } else if reply != utils.OK { t.Error("Calling ApierV1.ReloadCache got reply: ", reply) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 8e4608e14..a12b16e02 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -70,12 +70,12 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC return } - if err := ratingDb.PreloadRatingCache(); err != nil { + if err := ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Cache rating error: %s", err.Error())) exitChan <- true return } - if err := accountDb.PreloadAccountingCache(); err != nil { + if err := accountDb.PreloadAccountingCache(nil, nil, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Cache accounting error: %s", err.Error())) exitChan <- true return diff --git a/cmd/cgr-tester/cgr-tester.go b/cmd/cgr-tester/cgr-tester.go index a125a99e3..c97c7d66a 100644 --- a/cmd/cgr-tester/cgr-tester.go +++ b/cmd/cgr-tester/cgr-tester.go @@ -76,10 +76,10 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) { } defer accountDb.Close() engine.SetAccountingStorage(accountDb) - if err := ratingDb.PreloadRatingCache(); err != nil { + if err := ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil { return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error()) } - if err := accountDb.PreloadAccountingCache(); err != nil { + if err := accountDb.PreloadAccountingCache(nil, nil, nil); err != nil { return nilDuration, fmt.Errorf("Cache accounting error: %s", err.Error()) } log.Printf("Runnning %d cycles...", *runs) diff --git a/data/conf/samples/apier/apier.json b/data/conf/samples/apier/apier.json index 9d8b97d99..b88c9a2b9 100644 --- a/data/conf/samples/apier/apier.json +++ b/data/conf/samples/apier/apier.json @@ -8,6 +8,24 @@ "log_level": 7, }, + +"cache":{ + "destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "rating_plans": {"limit": 10000, "ttl":"0s","precache": true}, + "rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "lcr": {"limit": 10000, "ttl":"0s", "precache": true}, + "cdr_stats": {"limit": 10000, "ttl":"0s", "precache": true}, + "actions": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_triggers": {"limit": 10000, "ttl":"0s", "precache": true}, + "shared_groups": {"limit": 10000, "ttl":"0s", "precache": true}, + "aliases": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_aliases": {"limit": 10000, "ttl":"0s", "precache": true}, + "derived_chargers": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_limits": {"limit": 10000, "ttl":"0s", "precache": true}, +}, + "listen": { "rpc_json": ":2012", // RPC JSON listening address "rpc_gob": ":2013", // RPC GOB listening address diff --git a/engine/calldesc.go b/engine/calldesc.go index 66ea9d1f0..610f1260d 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -863,14 +863,6 @@ func (cd *CallDescriptor) RefundRounding() error { return err } -func (cd *CallDescriptor) FlushCache() (err error) { - cache.Flush() - ratingStorage.PreloadRatingCache() - accountingStorage.PreloadAccountingCache() - return nil - -} - // Creates a CallCost structure copying related data from CallDescriptor func (cd *CallDescriptor) CreateCallCost() *CallCost { return &CallCost{ diff --git a/engine/libtest.go b/engine/libtest.go index 8cb8495e2..6e5ba62ae 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -47,7 +47,7 @@ func InitDataDb(cfg *config.CGRConfig) error { return err } } - ratingDb.PreloadRatingCache() + ratingDb.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) CheckVersion(accountDb) // Write version before starting return nil } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 0b5fce18e..7835c4b91 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -335,8 +335,8 @@ func init() { } csvr.WriteToDatabase(false, false, false) cache.Flush() - ratingStorage.PreloadRatingCache() - accountingStorage.PreloadAccountingCache() + ratingStorage.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + accountingStorage.PreloadAccountingCache(nil, nil, nil) } func TestLoadDestinations(t *testing.T) { diff --git a/engine/responder.go b/engine/responder.go index e686d718b..9f00290fa 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -573,18 +573,6 @@ func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { return nil } -func (rs *Responder) FlushCache(arg *CallDescriptor, reply *float64) (err error) { - if rs.Bal != nil { - *reply, err = rs.callMethod(arg, "Responder.FlushCache") - } else { - r, e := Guardian.Guard(func() (interface{}, error) { - return 0, arg.FlushCache() - }, 0, arg.GetAccountKey()) - *reply, err = r.(float64), e - } - return -} - func (rs *Responder) Status(arg string, reply *map[string]interface{}) (err error) { if arg != "" { // Introduce delay in answer, used in some automated tests if delay, err := utils.ParseDurationWithSecs(arg); err == nil { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index b80e840e8..6df26293d 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -42,7 +42,7 @@ type Storage interface { type RatingStorage interface { Storage HasData(string, string) (bool, error) - PreloadRatingCache() error + PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error GetRatingPlan(string, bool, string) (*RatingPlan, error) SetRatingPlan(*RatingPlan, string) error GetRatingProfile(string, bool, string) (*RatingProfile, error) @@ -80,7 +80,7 @@ type RatingStorage interface { type AccountingStorage interface { Storage - PreloadAccountingCache() error + PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error GetAccount(string) (*Account, error) SetAccount(*Account) error RemoveAccount(string) error @@ -117,7 +117,7 @@ type AccountingStorage interface { type OnlineStorage interface { Storage HasData(string, string) (bool, error) - PreloadRatingCache() error + PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error GetRatingPlan(string, bool, string) (*RatingPlan, error) SetRatingPlan(*RatingPlan, string) error GetRatingProfile(string, bool, string) (*RatingProfile, error) @@ -149,7 +149,7 @@ type OnlineStorage interface { GetAllActionPlans() (map[string]*ActionPlan, error) PushTask(*Task) error PopTask() (*Task, error) - PreloadAccountingCache() error + PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error GetAccount(string) (*Account, error) SetAccount(*Account) error RemoveAccount(string) error diff --git a/engine/storage_map.go b/engine/storage_map.go index 716954b8d..070231169 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -132,7 +132,8 @@ func (ms *MapStorage) RebuildReverseForPrefix(prefix string) error { return nil } -func (ms *MapStorage) PreloadRatingCache() error { +// FixMe +func (ms *MapStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) error { if ms.cacheCfg == nil { return nil } @@ -193,7 +194,7 @@ func (ms *MapStorage) PreloadRatingCache() error { return nil } -func (ms *MapStorage) PreloadAccountingCache() error { +func (ms *MapStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error { if ms.cacheCfg == nil { return nil } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index fa6ae5188..4586663ee 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -366,76 +366,76 @@ func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) error { return nil } -func (ms *MongoStorage) PreloadRatingCache() (err error) { - if ms.cacheCfg == nil { - return - } +func (ms *MongoStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) (err error) { + //if ms.cacheCfg == nil { + // return + //} if ms.cacheCfg.Destinations.Precache { - if err = ms.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.DESTINATION_PREFIX, dstIDs, false); err != nil { return } } if ms.cacheCfg.ReverseDestinations.Precache { - if err = ms.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, rvDstIDs, false); err != nil { return } } if ms.cacheCfg.RatingPlans.Precache { - if err = ms.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.RATING_PLAN_PREFIX, rplIDs, false); err != nil { return } } if ms.cacheCfg.RatingProfiles.Precache { - if err = ms.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, rpfIDs, false); err != nil { return } } if ms.cacheCfg.Actions.Precache { - if err = ms.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.ACTION_PREFIX, actIDs, false); err != nil { return } } if ms.cacheCfg.ActionPlans.Precache { - if err = ms.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, aplIDs, false); err != nil { return } } if ms.cacheCfg.ActionTriggers.Precache { - if err = ms.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, atrgIDs, false); err != nil { return } } if ms.cacheCfg.SharedGroups.Precache { - if err = ms.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil { - return - } - } - if ms.cacheCfg.DerivedChargers.Precache { - if err = ms.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgIDs, false); err != nil { return } } if ms.cacheCfg.Lcr.Precache { - if err = ms.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.LCR_PREFIX, lcrIDs, false); err != nil { + return + } + } + if ms.cacheCfg.DerivedChargers.Precache { + if err = ms.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dcIDs, false); err != nil { return } } return } -func (ms *MongoStorage) PreloadAccountingCache() (err error) { +func (ms *MongoStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) { if ms.cacheCfg.Aliases.Precache { - if err = ms.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.ALIASES_PREFIX, alsIDs, false); err != nil { return } } if ms.cacheCfg.ReverseAliases.Precache { - if err = ms.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, rvAlsIDs, false); err != nil { return } } if ms.cacheCfg.ResourceLimits.Precache { - if err = ms.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil { + if err = ms.CacheDataFromDB(utils.ResourceLimitsPrefix, rlIDs, false); err != nil { return } } @@ -472,12 +472,13 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached err.Error(), fmt.Sprintf("mongo error <%s> querying keys for prefix: <%s>", prfx)) } - if mustBeCached { // Only consider loading ids which are already in cache - for _, keyID := range keyIDs { - if _, hasIt := cache.Get(keyID); hasIt { - ids = append(ids, keyID[len(prfx):]) + for _, keyID := range keyIDs { + if mustBeCached { // Only consider loading ids which are already in cache + if _, hasIt := cache.Get(keyID); !hasIt { + continue } } + ids = append(ids, keyID[len(prfx):]) } var nrItems int switch prfx { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 66d62a0c2..c6e0cf997 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -112,76 +112,76 @@ func (rs *RedisStorage) Flush(ignore string) error { return rs.Cmd("FLUSHDB").Err } -func (rs *RedisStorage) PreloadRatingCache() (err error) { - if rs.cacheCfg == nil { - return - } +func (rs *RedisStorage) PreloadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, atrgIDs, sgIDs, lcrIDs, dcIDs []string) (err error) { + //if rs.cacheCfg == nil { + // return + //} if rs.cacheCfg.Destinations.Precache { - if err = rs.CacheDataFromDB(utils.DESTINATION_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.DESTINATION_PREFIX, dstIDs, false); err != nil { return } } if rs.cacheCfg.ReverseDestinations.Precache { - if err = rs.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, rvDstIDs, false); err != nil { return } } if rs.cacheCfg.RatingPlans.Precache { - if err = rs.CacheDataFromDB(utils.RATING_PLAN_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.RATING_PLAN_PREFIX, rplIDs, false); err != nil { return } } if rs.cacheCfg.RatingProfiles.Precache { - if err = rs.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, rpfIDs, false); err != nil { return } } if rs.cacheCfg.Actions.Precache { - if err = rs.CacheDataFromDB(utils.ACTION_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.ACTION_PREFIX, actIDs, false); err != nil { return } } if rs.cacheCfg.ActionPlans.Precache { - if err = rs.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, aplIDs, false); err != nil { return } } if rs.cacheCfg.ActionTriggers.Precache { - if err = rs.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, atrgIDs, false); err != nil { return } } if rs.cacheCfg.SharedGroups.Precache { - if err = rs.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, nil, false); err != nil { - return - } - } - if rs.cacheCfg.DerivedChargers.Precache { - if err = rs.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgIDs, false); err != nil { return } } if rs.cacheCfg.Lcr.Precache { - if err = rs.CacheDataFromDB(utils.LCR_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.LCR_PREFIX, lcrIDs, false); err != nil { + return + } + } + if rs.cacheCfg.DerivedChargers.Precache { + if err = rs.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dcIDs, false); err != nil { return } } return } -func (rs *RedisStorage) PreloadAccountingCache() (err error) { +func (rs *RedisStorage) PreloadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) { if rs.cacheCfg.Aliases.Precache { - if err = rs.CacheDataFromDB(utils.ALIASES_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.ALIASES_PREFIX, alsIDs, false); err != nil { return } } if rs.cacheCfg.ReverseAliases.Precache { - if err = rs.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, rvAlsIDs, false); err != nil { return } } if rs.cacheCfg.ResourceLimits.Precache { - if err = rs.CacheDataFromDB(utils.ResourceLimitsPrefix, nil, false); err != nil { + if err = rs.CacheDataFromDB(utils.ResourceLimitsPrefix, rlIDs, false); err != nil { return } } @@ -264,12 +264,13 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached err.Error(), fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx)) } - if mustBeCached { // Only consider loading ids which are already in cache - for _, keyID := range keyIDs { - if _, hasIt := cache.Get(keyID); hasIt { - ids = append(ids, keyID[len(prfx):]) + for _, keyID := range keyIDs { + if mustBeCached { // Only consider loading ids which are already in cache + if _, hasIt := cache.Get(keyID); !hasIt { + continue } } + ids = append(ids, keyID[len(prfx):]) } var nrItems int switch prfx { diff --git a/engine/storage_test.go b/engine/storage_test.go index ad78f50a6..10e48ce94 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -102,7 +102,7 @@ func TestStorageCacheRefresh(t *testing.T) { ratingStorage.GetDestination("T11", false, utils.NonTransactional) ratingStorage.SetDestination(&Destination{"T11", []string{"1"}}, utils.NonTransactional) t.Log("Test cache refresh") - err := ratingStorage.PreloadRatingCache() + err := ratingStorage.PreloadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if err != nil { t.Error("Error cache rating: ", err) } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 7e29088cc..19f7131b7 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -567,12 +567,14 @@ type AttrReloadCache struct { RatingProfileIDs *[]string ActionIDs *[]string ActionPlanIDs *[]string + ActionTriggerIDs *[]string SharedGroupIDs *[]string LCRids *[]string DerivedChargerIDs *[]string AliasIDs *[]string ReverseAliasIDs *[]string ResourceLimitIDs *[]string + FlushAll bool // If provided, cache flush will be executed before any action } type AttrCacheStats struct { // Add in the future filters here maybe so we avoid counting complete cache