From da7e75e01398fb2073363412a02713b79ecfd973 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 19 Dec 2016 19:07:59 +0100 Subject: [PATCH] Cache load management improved, fixes #615, fixes #619 --- apier/v1/accounts.go | 102 +-------- apier/v1/apier.go | 374 ++++++++++++++++----------------- apier/v1/derivedcharging.go | 10 +- apier/v2/accounts.go | 7 + apier/v2/apier.go | 3 + build.sh | 2 +- cache/cache.go | 2 + cmd/cgr-loader/cgr-loader.go | 25 +-- engine/account.go | 35 +-- engine/action.go | 27 ++- engine/aliases.go | 43 ++-- engine/onstor_it_test.go | 3 +- engine/storage_mongo_datadb.go | 25 +-- engine/storage_redis.go | 22 +- utils/apitpdata.go | 24 +-- 15 files changed, 307 insertions(+), 397 deletions(-) diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index 148194a2b..bb7a08165 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -80,7 +80,7 @@ type AttrRemActionTiming struct { } // Removes an ActionTimings or parts of it depending on filters being set -func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) error { +func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) (err error) { if missing := utils.MissingStructFields(&attrs, []string{"ActionPlanId"}); len(missing) != 0 { // Only mandatory ActionPlanId return utils.NewErrMandatoryIeMissing(missing...) } @@ -89,7 +89,7 @@ func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) e return utils.NewErrMandatoryIeMissing(missing...) } } - _, err := engine.Guardian.Guard(func() (interface{}, error) { + _, err = engine.Guardian.Guard(func() (interface{}, error) { ap, err := self.RatingDb.GetActionPlan(attrs.ActionPlanId, false, utils.NonTransactional) if err != nil { return 0, err @@ -100,7 +100,10 @@ func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) e if attrs.Tenant != "" && attrs.Account != "" { accID := utils.AccountKey(attrs.Tenant, attrs.Account) delete(ap.AccountIDs, accID) - err = self.RatingDb.SetActionPlan(ap.Id, ap, true, utils.NonTransactional) + if err = self.RatingDb.SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { + return 0, err + } + err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, []string{ap.Id}, true) goto UPDATE } @@ -575,96 +578,3 @@ func (self *ApierV1) RemoveBalances(attr *utils.AttrSetBalance, reply *string) e *reply = OK return nil } - -/* To be removed after the above one proves reliable -func (self *ApierV1) SetBalance(attr *AttrSetBalance, reply *string) error { - if missing := utils.MissingStructFields(attr, []string{"Tenant", "Account", "BalanceType"}); len(missing) != 0 { - return utils.NewErrMandatoryIeMissing(missing...) - } - - var err error - if attr.ExpiryTime != nil { - attr.expTime, err = utils.ParseTimeDetectLayout(*attr.ExpiryTime, self.Config.DefaultTimezone) - if err != nil { - *reply = err.Error() - return err - } - } - accID := utils.ConcatenatedKey(attr.Tenant, attr.Account) - _, err = engine.Guardian.Guard(func() (interface{}, error) { - account, err := self.AccountDb.GetAccount(accID) - if err != nil { - return 0, utils.ErrNotFound - } - - if account.BalanceMap == nil { - account.BalanceMap = make(map[string]engine.Balances, 1) - } - var previousSharedGroups utils.StringMap // kept for comparison - var balance *engine.Balance - var found bool - for _, b := range account.BalanceMap[attr.BalanceType] { - if b.IsExpired() { - continue - } - if (attr.BalanceUUID != nil && b.Uuid == *attr.BalanceUUID) || - (attr.BalanceID != nil && b.Id == *attr.BalanceID) { - previousSharedGroups = b.SharedGroups - balance = b - found = true - break // only set one balance - } - } - - // if it is not found then we add it to the list - if balance == nil { - balance = &engine.Balance{} - balance.Uuid = utils.GenUUID() // alway overwrite the uuid for consistency - account.BalanceMap[attr.BalanceType] = append(account.BalanceMap[attr.BalanceType], balance) - } - - if attr.BalanceID != nil && *attr.BalanceID == utils.META_DEFAULT { - balance.Id = utils.META_DEFAULT - if attr.Value != nil { - balance.Value = *attr.Value - } - } else { - attr.SetBalance(balance) - } - - if !found || !previousSharedGroups.Equal(balance.SharedGroups) { - _, err = engine.Guardian.Guard(func() (interface{}, error) { - for sgID := range balance.SharedGroups { - // add shared group member - sg, err := self.RatingDb.GetSharedGroup(sgID, false) - if err != nil || sg == nil { - //than is problem - utils.Logger.Warning(fmt.Sprintf("Could not get shared group: %v", sgID)) - } else { - if _, found := sg.MemberIds[account.Id]; !found { - // add member and save - if sg.MemberIds == nil { - sg.MemberIds = make(utils.StringMap) - } - sg.MemberIds[account.Id] = true - self.RatingDb.SetSharedGroup(sg) - } - } - } - return 0, nil - }, 0, balance.SharedGroups.Slice()...) - } - - account.InitCounters() - account.ExecuteActionTriggers(nil) - self.AccountDb.SetAccount(account) - return 0, nil - }, 0, accID) - if err != nil { - *reply = err.Error() - return err - } - *reply = utils.OK - return nil -} -*/ diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 3e93c2177..2d51e4386 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -97,22 +97,31 @@ func (apier *ApierV1) GetSharedGroup(sgId string, reply *engine.SharedGroup) err return nil } -func (self *ApierV1) SetDestination(attrs utils.AttrSetDestination, reply *string) error { +func (self *ApierV1) SetDestination(attrs utils.AttrSetDestination, reply *string) (err error) { if missing := utils.MissingStructFields(&attrs, []string{"Id", "Prefixes"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } - - if !attrs.Overwrite { - if exists, err := self.RatingDb.HasData(utils.DESTINATION_PREFIX, attrs.Id); err != nil { - return utils.NewErrServerError(err) - } else if exists { - return utils.ErrExists - } - } dest := &engine.Destination{Id: attrs.Id, Prefixes: attrs.Prefixes} + var oldDest *engine.Destination + if oldDest, err = self.RatingDb.GetDestination(attrs.Id, false, utils.NonTransactional); err != nil { + if err != utils.ErrNotFound { + return utils.NewErrServerError(err) + } + } else if !attrs.Overwrite { + return utils.ErrExists + } if err := self.RatingDb.SetDestination(dest, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err = self.RatingDb.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{attrs.Id}, true); err != nil { + return + } + if err = self.RatingDb.UpdateReverseDestination(oldDest, dest, utils.NonTransactional); err != nil { + return + } + if err = self.RatingDb.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dest.Prefixes, true); err != nil { + return + } *reply = OK return nil } @@ -276,51 +285,21 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply if err := dbReader.WriteToDatabase(attrs.FlushDb, false, false); err != nil { return utils.NewErrServerError(err) } - // Make sure the items are in the cache - dstIds, _ := dbReader.GetLoadedIds(utils.DESTINATION_PREFIX) - dstKeys := make([]string, len(dstIds)) - for idx, dId := range dstIds { - dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys - } - rplIds, _ := dbReader.GetLoadedIds(utils.RATING_PLAN_PREFIX) - rpKeys := make([]string, len(rplIds)) - for idx, rpId := range rplIds { - rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId - } - rpfIds, _ := dbReader.GetLoadedIds(utils.RATING_PROFILE_PREFIX) - rpfKeys := make([]string, len(rpfIds)) - for idx, rpfId := range rpfIds { - rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId - } - actIds, _ := dbReader.GetLoadedIds(utils.ACTION_PREFIX) - actKeys := make([]string, len(actIds)) - for idx, actId := range actIds { - actKeys[idx] = utils.ACTION_PREFIX + actId - } - aplIds, _ := dbReader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) - aplKeys := make([]string, len(aplIds)) - for idx, aplId := range aplIds { - aplKeys[idx] = utils.ACTION_PLAN_PREFIX + aplId - } - shgIds, _ := dbReader.GetLoadedIds(utils.SHARED_GROUP_PREFIX) - shgKeys := make([]string, len(shgIds)) - for idx, shgId := range shgIds { - shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId - } - aliases, _ := dbReader.GetLoadedIds(utils.ALIASES_PREFIX) - alsKeys := make([]string, len(aliases)) - for idx, alias := range aliases { - alsKeys[idx] = utils.ALIASES_PREFIX + alias - } - lcrIds, _ := dbReader.GetLoadedIds(utils.LCR_PREFIX) - lcrKeys := make([]string, len(lcrIds)) - for idx, lcrId := range lcrIds { - lcrKeys[idx] = utils.LCR_PREFIX + lcrId - } - dcs, _ := dbReader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX) - dcsKeys := make([]string, len(dcs)) - for idx, dc := range dcs { - dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc + utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.") + for _, prfx := range []string{utils.DESTINATION_PREFIX, + utils.RATING_PLAN_PREFIX, + utils.RATING_PROFILE_PREFIX, + utils.ACTION_PREFIX, + utils.ACTION_PLAN_PREFIX, + utils.SHARED_GROUP_PREFIX, + utils.DERIVEDCHARGERS_PREFIX, + utils.LCR_PREFIX, + utils.ALIASES_PREFIX, + utils.ResourceLimitsPrefix} { + loadedIDs, _ := dbReader.GetLoadedIds(prfx) + if err := self.RatingDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil { + return utils.NewErrServerError(err) + } } aps, _ := dbReader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) cstKeys, _ := dbReader.GetLoadedIds(utils.CDR_STATS_PREFIX) @@ -329,10 +308,6 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply // relase tp data dbReader.Init() - cache.Flush() - self.RatingDb.PreloadRatingCache() - self.AccountDb.PreloadAccountingCache() - if len(aps) != 0 { sched := self.ServManager.GetScheduler() if sched != nil { @@ -396,7 +371,7 @@ type AttrSetRatingProfile struct { } // Sets a specific rating profile working with data directly in the RatingDb without involving storDb -func (self *ApierV1) SetRatingProfile(attrs AttrSetRatingProfile, reply *string) error { +func (self *ApierV1) SetRatingProfile(attrs AttrSetRatingProfile, reply *string) (err error) { if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "TOR", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } @@ -437,6 +412,9 @@ func (self *ApierV1) SetRatingProfile(attrs AttrSetRatingProfile, reply *string) if err := self.RatingDb.SetRatingProfile(rpfl, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err = self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, []string{keyId}, true); err != nil { + return + } *reply = OK return nil } @@ -474,7 +452,7 @@ type V1TPAction struct { Weight float64 // Action's weight } -func (self *ApierV1) SetActions(attrs V1AttrSetActions, reply *string) error { +func (self *ApierV1) SetActions(attrs V1AttrSetActions, reply *string) (err error) { if missing := utils.MissingStructFields(&attrs, []string{"ActionsId", "Actions"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } @@ -520,6 +498,9 @@ func (self *ApierV1) SetActions(attrs V1AttrSetActions, reply *string) error { if err := self.RatingDb.SetActions(attrs.ActionsId, storeActions, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PREFIX, []string{attrs.ActionsId}, true); err != nil { + utils.NewErrServerError(err) + } *reply = OK return nil } @@ -580,7 +561,7 @@ type AttrActionPlan struct { Weight float64 // Binding's weight } -func (self *ApierV1) SetActionPlan(attrs AttrSetActionPlan, reply *string) error { +func (self *ApierV1) SetActionPlan(attrs AttrSetActionPlan, reply *string) (err error) { if missing := utils.MissingStructFields(&attrs, []string{"Id", "ActionPlan"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } @@ -622,6 +603,9 @@ func (self *ApierV1) SetActionPlan(attrs AttrSetActionPlan, reply *string) error if err := self.RatingDb.SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, []string{ap.Id}, true); err != nil { + return utils.NewErrServerError(err) + } if attrs.ReloadScheduler { sched := self.ServManager.GetScheduler() if sched == nil { @@ -693,104 +677,152 @@ func (self *ApierV1) ReloadScheduler(ignore string, reply *string) error { return nil } -func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) error { - var dstKeys, rpKeys, rpfKeys, actKeys, aplKeys, shgKeys, lcrKeys, dcsKeys, alsKeys, rlKeys []string - - if attrs.DestinationIds == nil { - dstKeys = nil // Reload all - } else if len(*attrs.DestinationIds) > 0 { - dstKeys = make([]string, len(*attrs.DestinationIds)) - for idx, dId := range *attrs.DestinationIds { - dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys +func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (err error) { + var dataIDs []string + // Reload Destinations + if attrs.ReverseDestinationIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.DestinationIDs) > 0 { + dataIDs = make([]string, len(*attrs.DestinationIDs)) + for idx, dId := range *attrs.DestinationIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.RatingPlanIds == nil { - rpKeys = nil - } else if len(*attrs.RatingPlanIds) > 0 { - rpKeys = make([]string, len(*attrs.RatingPlanIds)) - for idx, rpId := range *attrs.RatingPlanIds { - rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId + if err = self.RatingDb.CacheDataFromDB(utils.DESTINATION_PREFIX, dataIDs, true); err != nil { + return + } + // Reload ReverseDestinations + if attrs.ReverseDestinationIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ReverseDestinationIDs) > 0 { + dataIDs = make([]string, len(*attrs.ReverseDestinationIDs)) + for idx, dId := range *attrs.ReverseDestinationIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.RatingProfileIds == nil { - rpfKeys = nil - } else if len(*attrs.RatingProfileIds) > 0 { - rpfKeys = make([]string, len(*attrs.RatingProfileIds)) - for idx, rpfId := range *attrs.RatingProfileIds { - rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId + if err = self.RatingDb.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dataIDs, true); err != nil { + return + } + // RatingPlans + if attrs.RatingPlanIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.RatingPlanIDs) > 0 { + dataIDs = make([]string, len(*attrs.RatingPlanIDs)) + for idx, dId := range *attrs.RatingPlanIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.ActionIds == nil { - actKeys = nil - } else if len(*attrs.ActionIds) > 0 { - actKeys = make([]string, len(*attrs.ActionIds)) - for idx, actId := range *attrs.ActionIds { - actKeys[idx] = utils.ACTION_PREFIX + actId + if err = self.RatingDb.CacheDataFromDB(utils.RATING_PLAN_PREFIX, dataIDs, true); err != nil { + return + } + // RatingProfiles + if attrs.RatingProfileIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.RatingProfileIDs) > 0 { + dataIDs = make([]string, len(*attrs.RatingProfileIDs)) + for idx, dId := range *attrs.RatingProfileIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.ActionPlanIds == nil { - aplKeys = nil - } else if len(*attrs.ActionPlanIds) > 0 { - aplKeys = make([]string, len(*attrs.ActionPlanIds)) - for idx, aplId := range *attrs.ActionPlanIds { - aplKeys[idx] = utils.ACTION_PLAN_PREFIX + aplId + if err = self.RatingDb.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, dataIDs, true); err != nil { + return + } + // Actions + if attrs.ActionIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ActionIDs) > 0 { + dataIDs = make([]string, len(*attrs.ActionIDs)) + for idx, dId := range *attrs.ActionIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.SharedGroupIds == nil { - shgKeys = nil - } else if len(*attrs.SharedGroupIds) > 0 { - shgKeys = make([]string, len(*attrs.SharedGroupIds)) - for idx, shgId := range *attrs.SharedGroupIds { - shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId + if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PREFIX, dataIDs, true); err != nil { + return + } + // ActionPlans + if attrs.ActionPlanIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ActionPlanIDs) > 0 { + dataIDs = make([]string, len(*attrs.ActionPlanIDs)) + for idx, dId := range *attrs.ActionPlanIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.LCRIds == nil { - lcrKeys = nil - } else if len(*attrs.LCRIds) > 0 { - lcrKeys = make([]string, len(*attrs.LCRIds)) - for idx, lcrId := range *attrs.LCRIds { - lcrKeys[idx] = utils.LCR_PREFIX + lcrId + if err = self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, dataIDs, true); err != nil { + return + } + // SharedGroups + if attrs.SharedGroupIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.SharedGroupIDs) > 0 { + dataIDs = make([]string, len(*attrs.SharedGroupIDs)) + for idx, dId := range *attrs.SharedGroupIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.DerivedChargers == nil { - dcsKeys = nil - } else if len(*attrs.DerivedChargers) > 0 { - dcsKeys = make([]string, len(*attrs.DerivedChargers)) - for idx, dc := range *attrs.DerivedChargers { - dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc + if err = self.RatingDb.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, dataIDs, true); err != nil { + return + } + // LCR Profiles + if attrs.LCRids == nil { + dataIDs = nil // Reload all + } else if len(*attrs.LCRids) > 0 { + dataIDs = make([]string, len(*attrs.LCRids)) + for idx, dId := range *attrs.LCRids { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.Aliases == nil { - alsKeys = nil - } else if len(*attrs.Aliases) > 0 { - alsKeys = make([]string, len(*attrs.Aliases)) - for idx, alias := range *attrs.Aliases { - alsKeys[idx] = utils.ALIASES_PREFIX + alias + if err = self.RatingDb.CacheDataFromDB(utils.LCR_PREFIX, dataIDs, true); err != nil { + return + } + // DerivedChargers + if attrs.DerivedChargerIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.DerivedChargerIDs) > 0 { + dataIDs = make([]string, len(*attrs.DerivedChargerIDs)) + for idx, dId := range *attrs.DerivedChargerIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - if attrs.ResourceLimits == nil { - rlKeys = nil - } else if len(*attrs.ResourceLimits) > 0 { - rlKeys = make([]string, len(*attrs.ResourceLimits)) - for idx, rlID := range *attrs.ResourceLimits { - rlKeys[idx] = utils.ResourceLimitsPrefix + rlID + if err = self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, dataIDs, true); err != nil { + return + } + // Aliases + if attrs.AliasIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.AliasIDs) > 0 { + dataIDs = make([]string, len(*attrs.AliasIDs)) + for idx, dId := range *attrs.AliasIDs { + dataIDs[idx] = dId // Cache expects them as redis keys } } - - // FixMe with CacheS - cache.Flush() - self.RatingDb.PreloadRatingCache() - self.AccountDb.PreloadAccountingCache() - + if err = self.AccountDb.CacheDataFromDB(utils.ALIASES_PREFIX, dataIDs, true); err != nil { + return + } + // ReverseAliases + if attrs.ReverseAliasIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ReverseAliasIDs) > 0 { + dataIDs = make([]string, len(*attrs.ReverseAliasIDs)) + for idx, dId := range *attrs.ReverseAliasIDs { + dataIDs[idx] = dId // Cache expects them as redis keys + } + } + if err = self.AccountDb.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, dataIDs, true); err != nil { + return + } + // ResourceLimits + if attrs.ResourceLimitIDs == nil { + dataIDs = nil // Reload all + } else if len(*attrs.ResourceLimitIDs) > 0 { + dataIDs = make([]string, len(*attrs.ResourceLimitIDs)) + for idx, dId := range *attrs.ResourceLimitIDs { + dataIDs[idx] = dId // Cache expects them as redis keys + } + } + if err = self.AccountDb.CacheDataFromDB(utils.ResourceLimitsPrefix, dataIDs, true); err != nil { + return + } *reply = utils.OK return nil } @@ -876,63 +908,29 @@ func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, if err := loader.WriteToDatabase(attrs.FlushDb, false, false); err != nil { return utils.NewErrServerError(err) } - // Make sure the items are in the cache - dstIds, _ := loader.GetLoadedIds(utils.DESTINATION_PREFIX) - dstKeys := make([]string, len(dstIds)) - for idx, dId := range dstIds { - dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys - } - rplIds, _ := loader.GetLoadedIds(utils.RATING_PLAN_PREFIX) - rpKeys := make([]string, len(rplIds)) - for idx, rpId := range rplIds { - rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId - } - rpfIds, _ := loader.GetLoadedIds(utils.RATING_PROFILE_PREFIX) - rpfKeys := make([]string, len(rpfIds)) - for idx, rpfId := range rpfIds { - rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId - } - actIds, _ := loader.GetLoadedIds(utils.ACTION_PREFIX) - actKeys := make([]string, len(actIds)) - for idx, actId := range actIds { - actKeys[idx] = utils.ACTION_PREFIX + actId - } - aplIds, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) - aplKeys := make([]string, len(aplIds)) - for idx, aplId := range aplIds { - aplKeys[idx] = utils.ACTION_PLAN_PREFIX + aplId - } - shgIds, _ := loader.GetLoadedIds(utils.SHARED_GROUP_PREFIX) - shgKeys := make([]string, len(shgIds)) - for idx, shgId := range shgIds { - shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId - } - aliases, _ := loader.GetLoadedIds(utils.ALIASES_PREFIX) - alsKeys := make([]string, len(aliases)) - for idx, alias := range aliases { - alsKeys[idx] = utils.ALIASES_PREFIX + alias - } - lcrIds, _ := loader.GetLoadedIds(utils.LCR_PREFIX) - lcrKeys := make([]string, len(lcrIds)) - for idx, lcrId := range lcrIds { - lcrKeys[idx] = utils.LCR_PREFIX + lcrId - } - dcs, _ := loader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX) - dcsKeys := make([]string, len(dcs)) - for idx, dc := range dcs { - dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc + utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.") + for _, prfx := range []string{utils.DESTINATION_PREFIX, + utils.RATING_PLAN_PREFIX, + utils.RATING_PROFILE_PREFIX, + utils.ACTION_PREFIX, + utils.ACTION_PLAN_PREFIX, + utils.SHARED_GROUP_PREFIX, + utils.DERIVEDCHARGERS_PREFIX, + utils.LCR_PREFIX, + utils.ALIASES_PREFIX, + utils.ResourceLimitsPrefix} { + loadedIDs, _ := loader.GetLoadedIds(prfx) + if err := self.RatingDb.CacheDataFromDB(prfx, loadedIDs, true); err != nil { + return utils.NewErrServerError(err) + } } aps, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) - utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.") cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX) userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX) // relase the tp data loader.Init() - cache.Flush() - self.RatingDb.PreloadRatingCache() - self.AccountDb.PreloadAccountingCache() if len(aps) != 0 { sched := self.ServManager.GetScheduler() if sched != nil { diff --git a/apier/v1/derivedcharging.go b/apier/v1/derivedcharging.go index fb7bc8343..d53b8eed9 100644 --- a/apier/v1/derivedcharging.go +++ b/apier/v1/derivedcharging.go @@ -81,6 +81,9 @@ func (self *ApierV1) SetDerivedChargers(attrs AttrSetDerivedChargers, reply *str if err := self.RatingDb.SetDerivedChargers(dcKey, dcs, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err := self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{dcKey}, true); err != nil { + return utils.NewErrServerError(err) + } *reply = utils.OK return nil } @@ -107,8 +110,11 @@ func (self *ApierV1) RemDerivedChargers(attrs AttrRemDerivedChargers, reply *str } if err := self.RatingDb.SetDerivedChargers(utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject), nil, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) - } else { - *reply = "OK" } + if err := self.RatingDb.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, + []string{utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject)}, true); err != nil { + return utils.NewErrServerError(err) + } + *reply = "OK" return nil } diff --git a/apier/v2/accounts.go b/apier/v2/accounts.go index 1fbbe0931..069c34a1f 100644 --- a/apier/v2/accounts.go +++ b/apier/v2/accounts.go @@ -156,10 +156,17 @@ func (self *ApierV2) SetAccount(attr AttrSetAccount, reply *string) error { } } } + apIDs := make([]string, len(dirtyActionPlans)) + i := 0 for actionPlanID, ap := range dirtyActionPlans { if err := self.RatingDb.SetActionPlan(actionPlanID, ap, true, utils.NonTransactional); err != nil { return 0, err } + apIDs[i] = actionPlanID + i++ + } + if err := self.RatingDb.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, apIDs, true); err != nil { + return 0, err } return 0, nil }, 0, utils.ACTION_PLAN_PREFIX) diff --git a/apier/v2/apier.go b/apier/v2/apier.go index 100d924af..10f036416 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -391,6 +391,9 @@ func (self *ApierV2) SetActions(attrs utils.AttrSetActions, reply *string) error if err := self.RatingDb.SetActions(attrs.ActionsId, storeActions, utils.NonTransactional); err != nil { return utils.NewErrServerError(err) } + if err := self.RatingDb.CacheDataFromDB(utils.ACTION_PREFIX, []string{attrs.ActionsId}, true); err != nil { + utils.NewErrServerError(err) + } *reply = utils.OK return nil } diff --git a/build.sh b/build.sh index b71a8ba77..366723f10 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #! /usr/bin/env sh -echo "Building CGRateS..." +echo "Building CGRateS ..." go install github.com/cgrates/cgrates/cmd/cgr-engine cr=$? diff --git a/cache/cache.go b/cache/cache.go index dc7ceb38e..181315361 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -111,6 +111,7 @@ func Set(key string, value interface{}, commit bool, transID string) { } } +// RemKey removes a specific item from cache func RemKey(key string, commit bool, transID string) { if commit { if transID == "" { // Lock per operation not transaction @@ -125,6 +126,7 @@ func RemKey(key string, commit bool, transID string) { } } +// RemPrefixKey removes a complete category of data out of cache func RemPrefixKey(prefix string, commit bool, transID string) { if commit { if transID == "" { // Lock locally diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 7d1533b30..43abc5993 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -379,7 +379,7 @@ func main() { dcsIds, _ = tpReader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX) rlIDs, _ = tpReader.GetLoadedIds(utils.ResourceLimitsPrefix) } - actTmgIds, _ := tpReader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) + aps, _ := tpReader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) var statsQueueIds []string if cdrstats != nil { statsQueueIds, _ = tpReader.GetLoadedIds(utils.CDR_STATS_PREFIX) @@ -400,23 +400,24 @@ func main() { log.Print("Reloading cache") } if *flush { - dstIds, rplIds, rpfIds, lcrIds = nil, nil, nil, nil // Should reload all these on flush + dstIds, rplIds, rpfIds, actIds, shgIds, alsIds, lcrIds, dcsIds, rlIDs, aps = nil, nil, nil, nil, nil, nil, nil, nil, nil, nil // Should reload all these on flush } if err = rater.Call("ApierV1.ReloadCache", utils.AttrReloadCache{ - DestinationIds: &dstIds, - RatingPlanIds: &rplIds, - RatingProfileIds: &rpfIds, - ActionIds: &actIds, - SharedGroupIds: &shgIds, - Aliases: &alsIds, - LCRIds: &lcrIds, - DerivedChargers: &dcsIds, - ResourceLimits: &rlIDs, + DestinationIDs: &dstIds, + RatingPlanIDs: &rplIds, + RatingProfileIDs: &rpfIds, + ActionIDs: &actIds, + ActionPlanIDs: &aps, + SharedGroupIDs: &shgIds, + AliasIDs: &alsIds, + LCRids: &lcrIds, + DerivedChargerIDs: &dcsIds, + ResourceLimitIDs: &rlIDs, }, &reply); err != nil { log.Printf("WARNING: Got error on cache reload: %s\n", err.Error()) } - if len(actTmgIds) != 0 { + if len(aps) != 0 { if *verbose { log.Print("Reloading scheduler") } diff --git a/engine/account.go b/engine/account.go index 386bfd32d..776933300 100644 --- a/engine/account.go +++ b/engine/account.go @@ -152,6 +152,8 @@ func (acc *Account) setBalanceAction(a *Action) error { // modify if necessary the shared groups here if !found || !previousSharedGroups.Equal(balance.SharedGroups) { _, err := Guardian.Guard(func() (interface{}, error) { + sgs := make([]string, len(balance.SharedGroups)) + i := 0 for sgID := range balance.SharedGroups { // add shared group member sg, err := ratingStorage.GetSharedGroup(sgID, false, utils.NonTransactional) @@ -168,7 +170,9 @@ func (acc *Account) setBalanceAction(a *Action) error { ratingStorage.SetSharedGroup(sg, utils.NonTransactional) } } + i++ } + ratingStorage.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgs, true) return 0, nil }, 0, balance.SharedGroups.Slice()...) if err != nil { @@ -240,6 +244,8 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { } ub.BalanceMap[balanceType] = append(ub.BalanceMap[balanceType], bClone) _, err := Guardian.Guard(func() (interface{}, error) { + sgs := make([]string, len(bClone.SharedGroups)) + i := 0 for sgId := range bClone.SharedGroups { // add shared group member sg, err := ratingStorage.GetSharedGroup(sgId, false, utils.NonTransactional) @@ -256,7 +262,9 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { ratingStorage.SetSharedGroup(sg, utils.NonTransactional) } } + i++ } + ratingStorage.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, sgs, true) return 0, nil }, 0, bClone.SharedGroups.Slice()...) if err != nil { @@ -268,33 +276,6 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { return nil } -/* -func (ub *Account) enableDisableBalanceAction(a *Action) error { - if a == nil { - return errors.New("nil action") - } - - if ub.BalanceMap == nil { - ub.BalanceMap = make(map[string]Balances) - } - found := false - id := a.BalanceType - disabled := a.Balance.Disabled - a.Balance.Disabled = !disabled // match for the opposite - for _, b := range ub.BalanceMap[id] { - if b.MatchFilter(a.Balance, false) { - b.Disabled = disabled - b.dirty = true - found = true - } - } - a.Balance.Disabled = disabled // restore balance aaction as it is cached - if !found { - return utils.ErrNotFound - } - return nil -} -*/ func (ub *Account) getBalancesForPrefix(prefix, category, direction, tor string, sharedGroup string) Balances { var balances Balances balances = append(balances, ub.BalanceMap[tor]...) diff --git a/engine/action.go b/engine/action.go index ab4de4bb7..8067aa141 100644 --- a/engine/action.go +++ b/engine/action.go @@ -483,7 +483,7 @@ func mailAsync(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) err return nil } -func setddestinations(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error { +func setddestinations(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) (err error) { var ddcDestId string for _, bchain := range ub.BalanceMap { for _, b := range bchain { @@ -511,12 +511,19 @@ func setddestinations(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actio } newDest := &Destination{Id: ddcDestId, Prefixes: prefixes} oldDest, err := ratingStorage.GetDestination(ddcDestId, false, utils.NonTransactional) + if err != nil { + return err + } // update destid in storage - ratingStorage.SetDestination(newDest, utils.NonTransactional) + if err = ratingStorage.SetDestination(newDest, utils.NonTransactional); err != nil { + return err + } + if err = ratingStorage.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{ddcDestId}, true); err != nil { + return err + } if err == nil && oldDest != nil { - err = ratingStorage.UpdateReverseDestination(oldDest, newDest, utils.NonTransactional) - if err != nil { + if err = ratingStorage.UpdateReverseDestination(oldDest, newDest, utils.NonTransactional); err != nil { return err } } @@ -560,6 +567,8 @@ func removeAccountAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Ac return 0, err } //var dirtyAps []string + aps := make([]string, len(allAPs)) + i := 0 for key, ap := range allAPs { if _, exists := ap.AccountIDs[accID]; exists { @@ -569,12 +578,12 @@ func removeAccountAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Ac ratingStorage.SetActionPlan(key, ap, true, utils.NonTransactional) //dirtyAps = append(dirtyAps, utils.ACTION_PLAN_PREFIX+key) } + aps[i] = key + i++ + } + if err = ratingStorage.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, aps, true); err != nil { + return 0, err } - /*if len(dirtyAps) > 0 { - // cache - ratingStorage.CacheRatingPrefixValues("RemoveAccountAction", map[string][]string{ - utils.ACTION_PLAN_PREFIX: dirtyAps}) - }*/ return 0, nil }, 0, utils.ACTION_PLAN_PREFIX) diff --git a/engine/aliases.go b/engine/aliases.go index 60911cd1a..a3e8f4cb1 100644 --- a/engine/aliases.go +++ b/engine/aliases.go @@ -42,6 +42,17 @@ type Alias struct { Values AliasValues } +func (al *Alias) ReverseAliasIDs() (rvAl []string) { + for _, value := range al.Values { + for target, pairs := range value.Pairs { + for _, alias := range pairs { + rvAl = append(rvAl, strings.Join([]string{utils.REVERSE_ALIASES_PREFIX, alias, target, al.Context}, "")) + } + } + } + return +} + type AliasValue struct { DestinationId string Pairs AliasPairs @@ -176,7 +187,7 @@ type AttrAddAlias struct { } // SetAlias will set/overwrite specified alias -func (am *AliasHandler) SetAlias(attr *AttrAddAlias, reply *string) error { +func (am *AliasHandler) SetAlias(attr *AttrAddAlias, reply *string) (err error) { am.mu.Lock() defer am.mu.Unlock() @@ -186,13 +197,17 @@ func (am *AliasHandler) SetAlias(attr *AttrAddAlias, reply *string) error { } if attr.Overwrite || oldAlias == nil { - if err := am.accountingDb.SetAlias(attr.Alias, utils.NonTransactional); err != nil { - *reply = err.Error() + if err = am.accountingDb.SetAlias(attr.Alias, utils.NonTransactional); err != nil { return err } - if err := am.accountingDb.SetReverseAlias(attr.Alias, utils.NonTransactional); err != nil { - *reply = err.Error() - return err + if err = am.accountingDb.CacheDataFromDB(utils.ALIASES_PREFIX, []string{attr.Alias.GetId()}, true); err != nil { + return + } + if err = am.accountingDb.SetReverseAlias(attr.Alias, utils.NonTransactional); err != nil { + return + } + if err = am.accountingDb.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, attr.Alias.ReverseAliasIDs(), true); err != nil { + return } } else { for _, value := range attr.Alias.Values { @@ -219,13 +234,17 @@ func (am *AliasHandler) SetAlias(attr *AttrAddAlias, reply *string) error { oldAlias.Values = append(oldAlias.Values, value) } } - if err := am.accountingDb.SetAlias(oldAlias, utils.NonTransactional); err != nil { - *reply = err.Error() - return err + if err = am.accountingDb.SetAlias(oldAlias, utils.NonTransactional); err != nil { + return } - if err := am.accountingDb.SetReverseAlias(oldAlias, utils.NonTransactional); err != nil { - *reply = err.Error() - return err + if err = am.accountingDb.CacheDataFromDB(utils.ALIASES_PREFIX, []string{oldAlias.GetId()}, true); err != nil { + return + } + if err = am.accountingDb.SetReverseAlias(oldAlias, utils.NonTransactional); err != nil { + return + } + if err = am.accountingDb.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, attr.Alias.ReverseAliasIDs(), true); err != nil { + return } //FIXME: optimize by creating better update reverse alias /*err := am.accountingDb.UpdateReverseAlias(oldAlias, oldAlias) diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index f45caa33a..187f7ae65 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -57,6 +57,7 @@ var sTestsOnStorIT = []func(t *testing.T){ testOnStorITCacheAlias, testOnStorITCacheReverseAlias, testOnStorITCacheResourceLimit, + // ToDo: test cache flush for a prefix } func TestOnStorITRedisConnect(t *testing.T) { @@ -493,7 +494,7 @@ func testOnStorITCacheDerivedChargers(t *testing.T) { SetupTimeField: utils.META_DEFAULT, PDDField: utils.META_DEFAULT, AnswerTimeField: utils.META_DEFAULT, UsageField: utils.META_DEFAULT, SupplierField: utils.META_DEFAULT, DisconnectCauseField: utils.META_DEFAULT, CostField: utils.META_DEFAULT, RatedField: utils.META_DEFAULT}, }} - keyDCS := utils.ConcatenatedKey("*out", "cgrates.org", "call", "dan", "dan") + keyDCS := utils.ConcatenatedKey("*out", "itsyscom.com", "call", "dan", "dan") if err := onStor.SetDerivedChargers(keyDCS, dcs, utils.NonTransactional); err != nil { t.Error(err) } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 85b821dc1..6613b8a6c 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -483,8 +483,8 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached utils.DERIVEDCHARGERS_PREFIX, utils.LCR_PREFIX, utils.ALIASES_PREFIX, - utils.ResourceLimitsPrefix, - utils.REVERSE_ALIASES_PREFIX}, prfx) { + utils.REVERSE_ALIASES_PREFIX, + utils.ResourceLimitsPrefix}, prfx) { return utils.NewCGRError(utils.REDIS, utils.MandatoryIEMissingCaps, utils.UnsupportedCachePrefix, @@ -497,6 +497,7 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached err.Error(), fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx)) } + cache.RemPrefixKey(prfx, true, utils.NonTransactional) } for _, dataID := range ids { if mustBeCached { @@ -525,10 +526,10 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached _, err = ms.GetLCR(dataID, false, utils.NonTransactional) case utils.ALIASES_PREFIX: _, err = ms.GetAlias(dataID, false, utils.NonTransactional) - case utils.ResourceLimitsPrefix: - _, err = ms.GetResourceLimit(dataID, false, utils.NonTransactional) case utils.REVERSE_ALIASES_PREFIX: _, err = ms.GetReverseAlias(dataID, false, utils.NonTransactional) + case utils.ResourceLimitsPrefix: + _, err = ms.GetResourceLimit(dataID, false, utils.NonTransactional) } if err != nil { return utils.NewCGRError(utils.MONGO, @@ -695,8 +696,6 @@ func (ms *MongoStorage) SetRatingPlan(rp *RatingPlan, transactionID string) erro var response int historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(), &response) } - cache.RemKey(utils.RATING_PLAN_PREFIX+rp.Id, - cacheCommit(transactionID), transactionID) return err } @@ -734,8 +733,6 @@ func (ms *MongoStorage) SetRatingProfile(rp *RatingProfile, transactionID string var response int historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(false), &response) } - cache.RemKey(utils.RATING_PROFILE_PREFIX+rp.Id, - cacheCommit(transactionID), transactionID) return } @@ -795,7 +792,6 @@ func (ms *MongoStorage) SetLCR(lcr *LCR, transactionID string) (err error) { }{lcr.GetId(), lcr}); err != nil { return } - cache.RemKey(utils.LCR_PREFIX+lcr.GetId(), cacheCommit(transactionID), transactionID) return } @@ -861,7 +857,6 @@ func (ms *MongoStorage) SetDestination(dest *Destination, transactionID string) var response int historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(false), &response) } - cache.RemKey(utils.DESTINATION_PREFIX+dest.Id, cacheCommit(transactionID), transactionID) return } @@ -896,12 +891,10 @@ func (ms *MongoStorage) GetReverseDestination(prefix string, skipCache bool, tra func (ms *MongoStorage) SetReverseDestination(dest *Destination, transactionID string) (err error) { session, col := ms.conn(colRds) defer session.Close() - cCommit := cacheCommit(transactionID) for _, p := range dest.Prefixes { if _, err = col.Upsert(bson.M{"key": p}, bson.M{"$addToSet": bson.M{"value": dest.Id}}); err != nil { break } - cache.RemKey(utils.REVERSE_DESTINATION_PREFIX+p, cCommit, transactionID) } return } @@ -984,7 +977,6 @@ func (ms *MongoStorage) UpdateReverseDestination(oldDest, newDest *Destination, if err != nil { return err } - cache.RemKey(utils.REVERSE_DESTINATION_PREFIX+addedPrefix, cCommit, transactionID) } return nil } @@ -1027,7 +1019,6 @@ func (ms *MongoStorage) SetActions(key string, as Actions, transactionID string) Key string Value Actions }{Key: key, Value: as}) - cache.RemKey(utils.ACTION_PREFIX+key, cacheCommit(transactionID), transactionID) return err } @@ -1069,7 +1060,6 @@ func (ms *MongoStorage) SetSharedGroup(sg *SharedGroup, transactionID string) (e if _, err = col.Upsert(bson.M{"id": sg.Id}, sg); err != nil { return } - cache.RemKey(utils.SHARED_GROUP_PREFIX+sg.Id, cacheCommit(transactionID), transactionID) return err } @@ -1251,7 +1241,6 @@ func (ms *MongoStorage) SetAlias(al *Alias, transactionID string) (err error) { }{Key: al.GetId(), Value: al.Values}); err != nil { return } - cache.RemKey(utils.ALIASES_PREFIX+al.GetId(), cacheCommit(transactionID), transactionID) return err } @@ -1286,7 +1275,6 @@ func (ms *MongoStorage) GetReverseAlias(reverseID string, skipCache bool, transa func (ms *MongoStorage) SetReverseAlias(al *Alias, transactionID string) (err error) { session, col := ms.conn(colRls) defer session.Close() - cCommit := cacheCommit(transactionID) for _, value := range al.Values { for target, pairs := range value.Pairs { for _, alias := range pairs { @@ -1295,7 +1283,6 @@ func (ms *MongoStorage) SetReverseAlias(al *Alias, transactionID string) (err er if _, err = col.Upsert(bson.M{"key": rKey}, bson.M{"$addToSet": bson.M{"value": id}}); err != nil { break } - cache.RemKey(rKey, cCommit, transactionID) } } } @@ -1561,7 +1548,6 @@ func (ms *MongoStorage) SetActionPlan(key string, ats *ActionPlan, overwrite boo }{Key: key, Value: b.Bytes()}); err != nil { return } - cache.RemKey(dbKey, cCommit, transactionID) return err } @@ -1656,7 +1642,6 @@ func (ms *MongoStorage) SetDerivedChargers(key string, dcs *utils.DerivedCharger }{Key: key, Value: dcs}); err != nil { return } - cache.RemKey(cacheKey, cCommit, transactionID) return err } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 4d0cae1a1..2b39d5f90 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -283,8 +283,8 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached utils.DERIVEDCHARGERS_PREFIX, utils.LCR_PREFIX, utils.ALIASES_PREFIX, - utils.ResourceLimitsPrefix, - utils.REVERSE_ALIASES_PREFIX}, prfx) { + utils.REVERSE_ALIASES_PREFIX, + utils.ResourceLimitsPrefix}, prfx) { return utils.NewCGRError(utils.REDIS, utils.MandatoryIEMissingCaps, utils.UnsupportedCachePrefix, @@ -297,6 +297,7 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached err.Error(), fmt.Sprintf("redis error <%s> querying keys for prefix: <%s>", prfx)) } + cache.RemPrefixKey(prfx, true, utils.NonTransactional) } for _, dataID := range ids { if mustBeCached { @@ -325,10 +326,10 @@ func (rs *RedisStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached _, err = rs.GetLCR(dataID, false, utils.NonTransactional) case utils.ALIASES_PREFIX: _, err = rs.GetAlias(dataID, false, utils.NonTransactional) - case utils.ResourceLimitsPrefix: - _, err = rs.GetResourceLimit(dataID, false, utils.NonTransactional) case utils.REVERSE_ALIASES_PREFIX: _, err = rs.GetReverseAlias(dataID, false, utils.NonTransactional) + case utils.ResourceLimitsPrefix: + _, err = rs.GetResourceLimit(dataID, false, utils.NonTransactional) } if err != nil { return utils.NewCGRError(utils.REDIS, @@ -406,7 +407,6 @@ func (rs *RedisStorage) SetRatingPlan(rp *RatingPlan, transactionID string) (err response := 0 go historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(), &response) } - //cache.Set(utils.RATING_PLAN_PREFIX+rp.Id, rp, cacheCommit(transactionID), transactionID) return } @@ -448,7 +448,6 @@ func (rs *RedisStorage) SetRatingProfile(rpf *RatingProfile, transactionID strin response := 0 go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(false), &response) } - cache.RemKey(key, cacheCommit(transactionID), transactionID) return } @@ -505,7 +504,6 @@ func (rs *RedisStorage) SetLCR(lcr *LCR, transactionID string) (err error) { if err = rs.Cmd("SET", key, result).Err; err != nil { return } - cache.RemKey(key, cacheCommit(transactionID), transactionID) return } @@ -563,7 +561,6 @@ func (rs *RedisStorage) SetDestination(dest *Destination, transactionID string) response := 0 go historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(false), &response) } - cache.RemKey(key, cacheCommit(transactionID), transactionID) return } @@ -594,7 +591,6 @@ func (rs *RedisStorage) SetReverseDestination(dest *Destination, transactionID s if err = rs.Cmd("SADD", key, dest.Id).Err; err != nil { break } - cache.RemKey(key, cacheCommit(transactionID), transactionID) } return } @@ -670,7 +666,6 @@ func (rs *RedisStorage) UpdateReverseDestination(oldDest, newDest *Destination, if err != nil { return err } - cache.RemKey(utils.REVERSE_DESTINATION_PREFIX+addedPrefix, cCommit, transactionID) } return nil } @@ -706,7 +701,6 @@ func (rs *RedisStorage) GetActions(key string, skipCache bool, transactionID str func (rs *RedisStorage) SetActions(key string, as Actions, transactionID string) (err error) { result, err := rs.ms.Marshal(&as) err = rs.Cmd("SET", utils.ACTION_PREFIX+key, result).Err - cache.RemKey(utils.ACTION_PREFIX+key, cacheCommit(transactionID), transactionID) return } @@ -747,7 +741,6 @@ func (rs *RedisStorage) SetSharedGroup(sg *SharedGroup, transactionID string) (e return } err = rs.Cmd("SET", utils.SHARED_GROUP_PREFIX+sg.Id, result).Err - cache.RemKey(utils.SHARED_GROUP_PREFIX+sg.Id, cacheCommit(transactionID), transactionID) return } @@ -915,7 +908,6 @@ func (rs *RedisStorage) SetAlias(al *Alias, transactionID string) (err error) { if err = rs.Cmd("SET", key, result).Err; err != nil { return } - cache.RemKey(key, cacheCommit(transactionID), transactionID) return } @@ -946,7 +938,6 @@ func (rs *RedisStorage) GetReverseAlias(reverseID string, skipCache bool, transa } func (rs *RedisStorage) SetReverseAlias(al *Alias, transactionID string) (err error) { - cCommit := cacheCommit(transactionID) for _, value := range al.Values { for target, pairs := range value.Pairs { for _, alias := range pairs { @@ -955,7 +946,6 @@ func (rs *RedisStorage) SetReverseAlias(al *Alias, transactionID string) (err er if err = rs.Cmd("SADD", rKey, id).Err; err != nil { break } - cache.RemKey(rKey, cCommit, transactionID) } } } @@ -1183,7 +1173,6 @@ func (rs *RedisStorage) SetActionPlan(key string, ats *ActionPlan, overwrite boo if err = rs.Cmd("SET", dbKey, b.Bytes()).Err; err != nil { return } - cache.RemKey(dbKey, cCommit, transactionID) return } @@ -1265,7 +1254,6 @@ func (rs *RedisStorage) SetDerivedChargers(key string, dcs *utils.DerivedCharger if err = rs.Cmd("SET", key, marshaled).Err; err != nil { return } - cache.RemKey(key, cCommit, transactionID) return } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index b4cd51e74..7e29088cc 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -561,18 +561,18 @@ type AttrGetAccounts struct { // Data used to do remote cache reloads via api type AttrReloadCache struct { - DestinationIds *[]string - ReverseDestinationIds *[]string - RatingPlanIds *[]string - RatingProfileIds *[]string - ActionIds *[]string - ActionPlanIds *[]string - SharedGroupIds *[]string - LCRIds *[]string - DerivedChargers *[]string - Aliases *[]string - ReverseAliases *[]string - ResourceLimits *[]string + DestinationIDs *[]string + ReverseDestinationIDs *[]string + RatingPlanIDs *[]string + RatingProfileIDs *[]string + ActionIDs *[]string + ActionPlanIDs *[]string + SharedGroupIDs *[]string + LCRids *[]string + DerivedChargerIDs *[]string + AliasIDs *[]string + ReverseAliasIDs *[]string + ResourceLimitIDs *[]string } type AttrCacheStats struct { // Add in the future filters here maybe so we avoid counting complete cache