From 3ef60c98d4a4626f84f731418728adf0e183e3f5 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 29 Apr 2021 16:29:00 +0300 Subject: [PATCH] Updated cache handling on tariffplan load --- apier/v1/apier_it_test.go | 11 +-- apier/v1/caches_it_test.go | 2 +- dispatchers/caches_it_test.go | 2 +- engine/tpreader.go | 136 +++++++++++++++++++--------------- loaders/loader.go | 74 +----------------- 5 files changed, 85 insertions(+), 140 deletions(-) diff --git a/apier/v1/apier_it_test.go b/apier/v1/apier_it_test.go index 32b97b4d3..0d7e6b688 100644 --- a/apier/v1/apier_it_test.go +++ b/apier/v1/apier_it_test.go @@ -826,7 +826,8 @@ func testApierLoadRatingProfileWithoutTenant(t *testing.T) { func testApierLoadAccountActions(t *testing.T) { var rcvStats map[string]*ltcache.CacheStats expectedStats := engine.GetDefaultEmptyCacheStats() // Make sure nothing in cache so far - expectedStats[utils.CacheLoadIDs].Items = 4 // we loaded the ratingprofiles + expectedStats[utils.CacheLoadIDs].Items = 1 // we loaded the ratingprofiles + expectedStats[utils.CacheRPCConnections].Items = 1 if err := rater.Call(utils.CacheSv1GetCacheStats, new(utils.AttrCacheIDsWithAPIOpts), &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) } else if !reflect.DeepEqual(expectedStats, rcvStats) { @@ -843,7 +844,7 @@ func testApierLoadAccountActions(t *testing.T) { expectedStats[utils.CacheAccountActionPlans].Items = 1 expectedStats[utils.CacheActionPlans].Items = 1 expectedStats[utils.CacheActions].Items = 1 - expectedStats[utils.CacheLoadIDs].Items = 8 + expectedStats[utils.CacheLoadIDs].Items = 5 expectedStats[utils.CacheRPCConnections].Items = 1 if err := rater.Call(utils.CacheSv1GetCacheStats, new(utils.AttrCacheIDsWithAPIOpts), &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) @@ -881,7 +882,7 @@ func testApierSetRatingProfile(t *testing.T) { expectedStats[utils.CacheActions].Items = 1 expectedStats[utils.CacheRatingProfiles].Items = 1 expectedStats[utils.CacheRPCConnections].Items = 1 - expectedStats[utils.CacheLoadIDs].Items = 8 + expectedStats[utils.CacheLoadIDs].Items = 5 if err := rater.Call(utils.CacheSv1GetCacheStats, new(utils.AttrCacheIDsWithAPIOpts), &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) } else if !reflect.DeepEqual(expectedStats, rcvStats) { @@ -1080,7 +1081,7 @@ func testApierReloadCache(t *testing.T) { expectedStats[utils.CacheRatingProfiles].Items = 3 expectedStats[utils.CacheRatingPlans].Items = 1 expectedStats[utils.CacheReverseDestinations].Items = 10 - expectedStats[utils.CacheLoadIDs].Items = 8 + expectedStats[utils.CacheLoadIDs].Items = 5 expectedStats[utils.CacheRPCConnections].Items = 1 if err := rater.Call(utils.CacheSv1GetCacheStats, new(utils.AttrCacheIDsWithAPIOpts), &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) @@ -1629,7 +1630,7 @@ func testApierResetDataAfterLoadFromFolder(t *testing.T) { expStats[utils.CacheActionPlans].Items = 7 expStats[utils.CacheActions].Items = 5 expStats[utils.CacheDestinations].Items = 3 - expStats[utils.CacheLoadIDs].Items = 17 + expStats[utils.CacheLoadIDs].Items = 14 expStats[utils.CacheRPCConnections].Items = 2 if err := rater.Call(utils.CacheSv1GetCacheStats, new(utils.AttrCacheIDsWithAPIOpts), &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) diff --git a/apier/v1/caches_it_test.go b/apier/v1/caches_it_test.go index f63e2a762..884d24608 100644 --- a/apier/v1/caches_it_test.go +++ b/apier/v1/caches_it_test.go @@ -133,7 +133,7 @@ func testCacheSAfterLoadFromFolder(t *testing.T) { expStats[utils.CacheActionPlans].Items = 7 expStats[utils.CacheActions].Items = 5 expStats[utils.CacheDestinations].Items = 3 - expStats[utils.CacheLoadIDs].Items = 17 + expStats[utils.CacheLoadIDs].Items = 14 expStats[utils.CacheRPCConnections].Items = 2 if err := chcRPC.Call(utils.CacheSv1GetCacheStats, &utils.AttrCacheIDsWithAPIOpts{}, &rcvStats); err != nil { t.Error("Got error on CacheSv1.GetCacheStats: ", err.Error()) diff --git a/dispatchers/caches_it_test.go b/dispatchers/caches_it_test.go index 68abe5607..cdffcfdcf 100644 --- a/dispatchers/caches_it_test.go +++ b/dispatchers/caches_it_test.go @@ -100,7 +100,7 @@ func testDspChcLoadAfterFolder(t *testing.T) { expStats[utils.CacheActionPlans].Items = 1 expStats[utils.CacheActions].Items = 1 expStats[utils.CacheDestinations].Items = 4 - expStats[utils.CacheLoadIDs].Items = 17 + expStats[utils.CacheLoadIDs].Items = 14 expStats[utils.CacheRPCConnections].Items = 2 args := utils.AttrCacheIDsWithAPIOpts{ APIOpts: map[string]interface{}{ diff --git a/engine/tpreader.go b/engine/tpreader.go index 437f4478f..c2c2026a8 100644 --- a/engine/tpreader.go +++ b/engine/tpreader.go @@ -2398,56 +2398,28 @@ func (tpr *TpReader) ReloadCache(caching string, verbose bool, opts map[string]i aps, _ := tpr.GetLoadedIds(utils.ActionPlanPrefix) //compose Reload Cache argument - cacheArgs := utils.AttrReloadCacheWithAPIOpts{ - APIOpts: opts, - ArgsCache: map[string][]string{ - utils.DestinationIDs: dstIds, - utils.ReverseDestinationIDs: revDstIDs, - utils.RatingPlanIDs: rplIds, - utils.RatingProfileIDs: rpfIds, - utils.ActionIDs: actIds, - utils.ActionPlanIDs: aps, - utils.AccountActionPlanIDs: aapIDs, - utils.SharedGroupIDs: shgIds, - utils.ResourceProfileIDs: rspIDs, - utils.ResourceIDs: resIDs, - utils.ActionTriggerIDs: aatIDs, - utils.StatsQueueIDs: stqIDs, - utils.StatsQueueProfileIDs: stqpIDs, - utils.ThresholdIDs: trsIDs, - utils.ThresholdProfileIDs: trspfIDs, - utils.FilterIDs: flrIDs, - utils.RouteProfileIDs: routeIDs, - utils.AttributeProfileIDs: apfIDs, - utils.ChargerProfileIDs: chargerIDs, - utils.DispatcherProfileIDs: dppIDs, - utils.DispatcherHostIDs: dphIDs, - }, - } - - if verbose { - log.Print("Reloading cache") - } - var reply string - switch caching { - case utils.MetaNone: - return - case utils.MetaReload: - if err = connMgr.Call(tpr.cacheConns, nil, utils.CacheSv1ReloadCache, cacheArgs, &reply); err != nil { - return - } - case utils.MetaLoad: - if err = connMgr.Call(tpr.cacheConns, nil, utils.CacheSv1LoadCache, cacheArgs, &reply); err != nil { - return - } - case utils.MetaRemove: - if err = connMgr.Call(tpr.cacheConns, nil, utils.CacheSv1RemoveItems, cacheArgs, &reply); err != nil { - return - } - case utils.MetaClear: - if err = connMgr.Call(tpr.cacheConns, nil, utils.CacheSv1Clear, new(utils.AttrCacheIDsWithAPIOpts), &reply); err != nil { - return - } + cacheArgs := map[string][]string{ + utils.DestinationIDs: dstIds, + utils.ReverseDestinationIDs: revDstIDs, + utils.RatingPlanIDs: rplIds, + utils.RatingProfileIDs: rpfIds, + utils.ActionIDs: actIds, + utils.ActionPlanIDs: aps, + utils.AccountActionPlanIDs: aapIDs, + utils.SharedGroupIDs: shgIds, + utils.ResourceProfileIDs: rspIDs, + utils.ResourceIDs: resIDs, + utils.ActionTriggerIDs: aatIDs, + utils.StatsQueueIDs: stqIDs, + utils.StatsQueueProfileIDs: stqpIDs, + utils.ThresholdIDs: trsIDs, + utils.ThresholdProfileIDs: trspfIDs, + utils.FilterIDs: flrIDs, + utils.RouteProfileIDs: routeIDs, + utils.AttributeProfileIDs: apfIDs, + utils.ChargerProfileIDs: chargerIDs, + utils.DispatcherProfileIDs: dppIDs, + utils.DispatcherHostIDs: dphIDs, } // verify if we need to clear indexes @@ -2476,23 +2448,16 @@ func (tpr *TpReader) ReloadCache(caching string, verbose bool, opts map[string]i if len(flrIDs) != 0 { cacheIDs = append(cacheIDs, utils.CacheReverseFilterIndexes) } - if verbose { - log.Print("Clearing indexes") - } - clearArgs := &utils.AttrCacheIDsWithAPIOpts{ - APIOpts: opts, - CacheIDs: cacheIDs, - } - if err = connMgr.Call(tpr.cacheConns, nil, utils.CacheSv1Clear, clearArgs, &reply); err != nil { - log.Printf("WARNING: Got error on cache clear: %s\n", err.Error()) - } + if err = CallCache(connMgr, tpr.cacheConns, caching, cacheArgs, cacheIDs, opts, verbose); err != nil { + return + } //get loadIDs for all types var loadIDs map[string]int64 if loadIDs, err = tpr.dm.GetItemLoadIDs(utils.EmptyString, false); err != nil { return } - cacheLoadIDs := populateCacheLoadIDs(loadIDs, cacheArgs.ArgsCache) + cacheLoadIDs := populateCacheLoadIDs(loadIDs, cacheArgs) for key, val := range cacheLoadIDs { if err = Cache.Set(utils.CacheLoadIDs, key, val, nil, cacheCommit(utils.NonTransactional), utils.NonTransactional); err != nil { @@ -2502,6 +2467,55 @@ func (tpr *TpReader) ReloadCache(caching string, verbose bool, opts map[string]i return } +// CallCache call the cache reload after data load +func CallCache(connMgr *ConnManager, cacheConns []string, caching string, args map[string][]string, cacheIDs []string, opts map[string]interface{}, verbose bool) (err error) { + for k, v := range args { + if len(v) == 0 { + delete(args, k) + } + } + var method, reply string + var cacheArgs interface{} = utils.AttrReloadCacheWithAPIOpts{ + APIOpts: opts, + ArgsCache: args, + } + switch caching { + case utils.MetaNone: + return + case utils.MetaReload: + method = utils.CacheSv1ReloadCache + case utils.MetaLoad: + method = utils.CacheSv1LoadCache + case utils.MetaRemove: + method = utils.CacheSv1RemoveItems + case utils.MetaClear: + method = utils.CacheSv1Clear + cacheArgs = &utils.AttrCacheIDsWithAPIOpts{APIOpts: opts} + } + if verbose { + log.Print("Reloading cache") + } + + if err = connMgr.Call(cacheConns, nil, method, cacheArgs, &reply); err != nil { + return + } + + if len(cacheIDs) != 0 { + if verbose { + log.Print("Clearing indexes") + } + if err = connMgr.Call(cacheConns, nil, utils.CacheSv1Clear, &utils.AttrCacheIDsWithAPIOpts{ + APIOpts: opts, + CacheIDs: cacheIDs, + }, &reply); err != nil { + if verbose { + log.Printf("WARNING: Got error on cache clear: %s\n", err.Error()) + } + } + } + return +} + func (tpr *TpReader) ReloadScheduler(verbose bool) (err error) { var reply string aps, _ := tpr.GetLoadedIds(utils.ActionPlanPrefix) diff --git a/loaders/loader.go b/loaders/loader.go index 3e9372e37..b0e8751a6 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -576,42 +576,7 @@ func (ldr *Loader) storeLoadedData(loaderType string, } if len(ldr.cacheConns) != 0 { - var reply string - switch caching { - case utils.MetaNone: - return - case utils.MetaReload: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1ReloadCache, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaLoad: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1LoadCache, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaRemove: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1RemoveItems, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaClear: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1Clear, new(utils.AttrCacheIDsWithAPIOpts), &reply); err != nil { - return - } - } - if len(cacheIDs) != 0 { - if err = ldr.connMgr.Call(ldr.cacheConns, nil, utils.CacheSv1Clear, &utils.AttrCacheIDsWithAPIOpts{ - CacheIDs: cacheIDs, - }, &reply); err != nil { - return - } - } - + return engine.CallCache(ldr.connMgr, ldr.cacheConns, caching, cacheArgs, cacheIDs, nil, false) } return } @@ -867,42 +832,7 @@ func (ldr *Loader) removeLoadedData(loaderType string, lds map[string][]LoaderDa } if len(ldr.cacheConns) != 0 { - var reply string - switch caching { - case utils.MetaNone: - return - case utils.MetaReload: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1ReloadCache, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaLoad: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1LoadCache, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaRemove: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1RemoveItems, utils.AttrReloadCacheWithAPIOpts{ - ArgsCache: cacheArgs}, &reply); err != nil { - return - } - case utils.MetaClear: - if err = ldr.connMgr.Call(ldr.cacheConns, nil, - utils.CacheSv1Clear, new(utils.AttrCacheIDsWithAPIOpts), &reply); err != nil { - return - } - } - if len(cacheIDs) != 0 { - if err = ldr.connMgr.Call(ldr.cacheConns, nil, utils.CacheSv1Clear, &utils.AttrCacheIDsWithAPIOpts{ - CacheIDs: cacheIDs, - }, &reply); err != nil { - return - } - } - + return engine.CallCache(ldr.connMgr, ldr.cacheConns, caching, cacheArgs, cacheIDs, nil, false) } return }