mirror of
https://github.com/cgrates/cgrates.git
synced 2026-02-12 18:46:24 +05:00
bd79a0f9e7
Previously made two trips (get and set) for each group of items from an index key being removed. Now, we fetch indexes once at the beginning and store the updated indexes once at the end.
805 lines
30 KiB
Go
805 lines
30 KiB
Go
/*
|
|
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
|
|
Copyright (C) ITsysCOM GmbH
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>
|
|
*/
|
|
|
|
package engine
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/cgrates/birpc/context"
|
|
"github.com/cgrates/cgrates/config"
|
|
"github.com/cgrates/cgrates/guardian"
|
|
"github.com/cgrates/cgrates/utils"
|
|
)
|
|
|
|
var (
|
|
FilterIndexTypes = utils.NewStringSet([]string{utils.MetaPrefix, utils.MetaString, utils.MetaSuffix, utils.MetaExists, utils.MetaNotExists})
|
|
// Element or values of a filter that starts with one of this should not be indexed
|
|
ToNotBeIndexed = []string{utils.DynamicDataPrefix + utils.MetaAccounts,
|
|
utils.DynamicDataPrefix + utils.MetaStats,
|
|
utils.DynamicDataPrefix + utils.MetaResources,
|
|
utils.DynamicDataPrefix + utils.MetaLibPhoneNumber}
|
|
)
|
|
|
|
// newFilterIndex will get the index from DataManager if is not found it will create it
|
|
// is used to update the mentioned index
|
|
func newFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID, transactionID string, filterIDs []string, newFlt *Filter) (indexes map[string]utils.StringSet, err error) {
|
|
tntGrp := tnt
|
|
if grp != utils.EmptyString {
|
|
tntGrp = utils.ConcatenatedKey(tnt, grp)
|
|
}
|
|
indexes = make(map[string]utils.StringSet)
|
|
if len(filterIDs) == 0 { // in case of None
|
|
idxKey := utils.ConcatenatedKey(utils.MetaNone, utils.MetaAny, utils.MetaAny)
|
|
var rcvIndx map[string]utils.StringSet
|
|
if rcvIndx, err = dm.GetIndexes(ctx, idxItmType, tntGrp,
|
|
idxKey, transactionID,
|
|
true, false); err != nil {
|
|
if err != utils.ErrNotFound {
|
|
return
|
|
}
|
|
err = nil
|
|
indexes[idxKey] = make(utils.StringSet) // create an empty index if is not found in DB in case we add them later
|
|
return
|
|
}
|
|
for idxKey, idx := range rcvIndx { // parse the received indexes
|
|
indexes[idxKey] = idx
|
|
}
|
|
return
|
|
}
|
|
// in case of more filters we parse each filter rule and only for supported index types
|
|
// we try to get them from Cache/DataDB or if not found in this location we create them here
|
|
for _, fltrID := range filterIDs {
|
|
var fltr *Filter
|
|
if newFlt != nil && newFlt.Tenant == tnt && newFlt.ID == fltrID {
|
|
fltr = newFlt
|
|
} else if fltr, err = dm.GetFilter(ctx, tnt, fltrID,
|
|
true, false, utils.NonTransactional); err != nil {
|
|
if err == utils.ErrNotFound {
|
|
err = fmt.Errorf("broken reference to filter: %+v for itemType: %+v and ID: %+v",
|
|
fltrID, idxItmType, itemID)
|
|
}
|
|
return
|
|
}
|
|
for _, flt := range fltr.Rules {
|
|
if !FilterIndexTypes.Has(flt.Type) ||
|
|
IsDynamicDPPath(flt.Element) {
|
|
continue
|
|
}
|
|
isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix)
|
|
for _, fldVal := range flt.Values {
|
|
if IsDynamicDPPath(fldVal) {
|
|
continue
|
|
}
|
|
var idxKey string
|
|
if isDyn {
|
|
if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic
|
|
continue
|
|
}
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal)
|
|
} else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) {
|
|
idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element)
|
|
} else {
|
|
// do not index not dynamic filters
|
|
continue
|
|
}
|
|
var rcvIndx map[string]utils.StringSet
|
|
// only read from cache in case if we do not find the index to not cache the negative response
|
|
if rcvIndx, err = dm.GetIndexes(ctx, idxItmType, tntGrp,
|
|
idxKey, transactionID, true, false); err != nil {
|
|
if err != utils.ErrNotFound {
|
|
return
|
|
}
|
|
err = nil
|
|
indexes[idxKey] = make(utils.StringSet) // create an empty index if is not found in DB in case we add them later
|
|
continue
|
|
}
|
|
for idxKey, idx := range rcvIndx { // parse the received indexes
|
|
indexes[idxKey] = idx
|
|
}
|
|
}
|
|
// this case is only for "*exists" and "*notexists" type. These 2 types does not have values, so the key will be just type and element.
|
|
if len(flt.Values) == 0 {
|
|
var rcvIndx map[string]utils.StringSet
|
|
idxKey := utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaAny)
|
|
if flt.Type == utils.MetaNotExists {
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaNone)
|
|
}
|
|
// only read from cache in case if we do not find the index to not cache the negative response
|
|
if rcvIndx, err = dm.GetIndexes(ctx, idxItmType, tntGrp,
|
|
idxKey, transactionID, true, false); err != nil {
|
|
if err != utils.ErrNotFound {
|
|
return
|
|
}
|
|
err = nil
|
|
indexes[idxKey] = make(utils.StringSet) // create an empty index if is not found in DB in case we add them later
|
|
continue
|
|
}
|
|
for idxKey, idx := range rcvIndx { // parse the received indexes
|
|
indexes[idxKey] = idx
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// addItemToFilterIndex will add the itemID to the existing/created index and set it in the DataDB
|
|
func addItemToFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID string, filterIDs []string) (err error) {
|
|
tntGrp := tnt
|
|
if grp != utils.EmptyString {
|
|
tntGrp = utils.ConcatenatedKey(tnt, grp)
|
|
}
|
|
// early lock to be sure that until we do not write back the indexes
|
|
// another goroutine can't create new indexes
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tntGrp)
|
|
defer guardian.Guardian.UnguardIDs(refID)
|
|
|
|
var indexes map[string]utils.StringSet
|
|
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, utils.NonTransactional, filterIDs, nil); err != nil {
|
|
return
|
|
}
|
|
// in case we have a profile with only non indexable filters(e.g. only *gt)
|
|
if len(indexes) == 0 {
|
|
return
|
|
}
|
|
|
|
for indxKey, index := range indexes {
|
|
index.Add(itemID)
|
|
// remove from cache in order to corectly update the index
|
|
if err = Cache.Remove(ctx, idxItmType, utils.ConcatenatedKey(tntGrp, indxKey), true, utils.NonTransactional); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return dm.SetIndexes(ctx, idxItmType, tntGrp, indexes, true, utils.NonTransactional)
|
|
}
|
|
|
|
// removeItemFromFilterIndex will remove the itemID from the existing/created index and set it in the DataDB
|
|
func removeItemFromFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID string, filterIDs []string) (err error) {
|
|
tntGrp := tnt
|
|
if grp != utils.EmptyString {
|
|
tntGrp = utils.ConcatenatedKey(tnt, grp)
|
|
}
|
|
// early lock to be sure that until we do not write back the indexes
|
|
// another goroutine can't create new indexes
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tntGrp)
|
|
defer guardian.Guardian.UnguardIDs(refID)
|
|
|
|
var indexes map[string]utils.StringSet
|
|
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, utils.NonTransactional, filterIDs, nil); err != nil {
|
|
return
|
|
}
|
|
if len(indexes) == 0 { // in case we have a profile with only non indexable filters(e.g. only *gt)
|
|
return
|
|
}
|
|
for idxKey, index := range indexes {
|
|
index.Remove(itemID)
|
|
if index.Size() == 0 { // empty index set it with nil for cache
|
|
indexes[idxKey] = nil // this will not be set in DB(handled by driver)
|
|
}
|
|
// remove from cache in order to corectly update the index
|
|
if err = Cache.Remove(ctx, idxItmType, utils.ConcatenatedKey(tntGrp, idxKey), true, utils.NonTransactional); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return dm.SetIndexes(ctx, idxItmType, tntGrp, indexes, true, utils.NonTransactional)
|
|
}
|
|
|
|
// updatedIndexes will compare the old filtersIDs with the new ones and only update the filters indexes that are added/removed
|
|
// idxItmType - the index object type(e.g.*attribute_filter_indexes, *rate_filter_indexes, *threshold_filter_indexes)
|
|
// tnt - the tenant of the object
|
|
// grp - the rate profile id for rate from RateProfile(sub indexes); for all the rest the grp is "" (DispatcherPrf have a separate function)
|
|
// itemID - the object id
|
|
// oldFilterIds - the filtersIDs that the old object had; this is optional if the object did not exist
|
|
// newFilterIDs - the filtersIDs for the object that will be set
|
|
// useGrp - in case of subindexes(e.g. Rate from RateProfiles) need to add the grp to the itemID when reverse filter indexes are set
|
|
//
|
|
// used when updating the filters
|
|
func updatedIndexes(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID string, oldFilterIds *[]string, newFilterIDs []string, useGrp bool) (err error) {
|
|
itmGrp := itemID
|
|
if useGrp {
|
|
itmGrp = utils.ConcatenatedKey(itemID, grp)
|
|
}
|
|
if oldFilterIds == nil { // nothing to remove so just create the new indexes
|
|
if err = addIndexFiltersItem(ctx, dm, idxItmType, tnt, itmGrp, newFilterIDs); err != nil {
|
|
return
|
|
}
|
|
return addItemToFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, newFilterIDs)
|
|
}
|
|
if len(*oldFilterIds) == 0 && len(newFilterIDs) == 0 { // nothing to update
|
|
return
|
|
}
|
|
|
|
// check what indexes needs to be updated
|
|
oldFltrs := utils.NewStringSet(*oldFilterIds)
|
|
newFltrs := utils.NewStringSet(newFilterIDs)
|
|
|
|
oldFilterIDs := make([]string, 0, len(*oldFilterIds))
|
|
newFilterIDs = make([]string, 0, len(newFilterIDs))
|
|
|
|
for fltrID := range oldFltrs {
|
|
if !newFltrs.Has(fltrID) { // append only if the index needs to be removed
|
|
oldFilterIDs = append(oldFilterIDs, fltrID)
|
|
}
|
|
}
|
|
|
|
for fltrID := range newFltrs {
|
|
if !oldFltrs.Has(fltrID) { // append only if the index needs to be added
|
|
newFilterIDs = append(newFilterIDs, fltrID)
|
|
}
|
|
}
|
|
|
|
if len(oldFilterIDs) != 0 || oldFltrs.Size() == 0 {
|
|
// has some indexes to remove or
|
|
// the old profile doesn't have filters but the new one has so remove the *none index
|
|
if err = removeIndexFiltersItem(ctx, dm, idxItmType, tnt, itmGrp, oldFilterIDs); err != nil {
|
|
return
|
|
}
|
|
if err = removeItemFromFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, oldFilterIDs); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(newFilterIDs) != 0 || newFltrs.Size() == 0 {
|
|
// has some indexes to add or
|
|
// the old profile has filters but the new one does not so add the *none index
|
|
if err = addIndexFiltersItem(ctx, dm, idxItmType, tnt, itmGrp, newFilterIDs); err != nil {
|
|
return
|
|
}
|
|
if err = addItemToFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, newFilterIDs); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// splitFilterIndex splits the cache key so it can be used to recache the indexes
|
|
func splitFilterIndex(tntGrpIdxKey string) (tntGrp, idxKey string, err error) {
|
|
splt := utils.SplitConcatenatedKey(tntGrpIdxKey) // tntCtx:filterType:fieldName:fieldVal
|
|
lsplt := len(splt)
|
|
if lsplt < 4 {
|
|
err = fmt.Errorf("WRONG_IDX_KEY_FORMAT<%s>", tntGrpIdxKey)
|
|
return
|
|
}
|
|
tntGrp = utils.ConcatenatedKey(splt[:lsplt-3]...) // prefix may contain context/subsystems
|
|
idxKey = utils.ConcatenatedKey(splt[lsplt-3:]...)
|
|
return
|
|
}
|
|
|
|
// ComputeIndexes gets the indexes from the DB and ensure that the items are indexed
|
|
// getFilters returns a list of filters IDs for the given profile id
|
|
func ComputeIndexes(ctx *context.Context, dm *DataManager, tnt, grp, idxItmType string, IDs *[]string,
|
|
transactionID string, getFilters func(tnt, id, grp string) (*[]string, error), newFltr *Filter) (indexes utils.StringSet, err error) {
|
|
indexes = make(utils.StringSet)
|
|
var profilesIDs []string
|
|
if IDs == nil { // get all items
|
|
Cache.Clear([]string{idxItmType})
|
|
var ids []string
|
|
if ids, err = dm.DataDB().GetKeysForPrefix(ctx, utils.CacheIndexesToPrefix[idxItmType]); err != nil {
|
|
return
|
|
}
|
|
for _, id := range ids {
|
|
profilesIDs = append(profilesIDs, utils.SplitConcatenatedKey(id)[1])
|
|
}
|
|
} else {
|
|
profilesIDs = *IDs
|
|
}
|
|
tntGrp := tnt
|
|
if grp != utils.EmptyString {
|
|
tntGrp = utils.ConcatenatedKey(tnt, grp)
|
|
}
|
|
// early lock to be sure that until we do not write back the indexes
|
|
// another goroutine can't create new indexes
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tntGrp)
|
|
defer guardian.Guardian.UnguardIDs(refID)
|
|
for _, id := range profilesIDs {
|
|
var filterIDs *[]string
|
|
if filterIDs, err = getFilters(tnt, id, grp); err != nil {
|
|
return
|
|
}
|
|
if filterIDs == nil {
|
|
continue
|
|
}
|
|
var index map[string]utils.StringSet
|
|
if index, err = newFilterIndex(ctx, dm, idxItmType,
|
|
tnt, grp, id, transactionID, *filterIDs, newFltr); err != nil {
|
|
return
|
|
}
|
|
// ensure that the item is in the index set
|
|
for key, idx := range index {
|
|
idx.Add(id)
|
|
indexes.Add(utils.ConcatenatedKey(tntGrp, key))
|
|
}
|
|
if err = dm.SetIndexes(ctx, idxItmType, tntGrp, index, cacheCommit(transactionID), transactionID); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// addIndexFiltersItem will add a reference for the items in the reverse filter index
|
|
func addIndexFiltersItem(ctx *context.Context, dm *DataManager, idxItmType, tnt, itemID string, filterIDs []string) (err error) {
|
|
for _, ID := range filterIDs {
|
|
if strings.HasPrefix(ID, utils.Meta) { // skip inline
|
|
continue
|
|
}
|
|
tntGrp := utils.ConcatenatedKey(tnt, ID)
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntGrp)
|
|
var indexes map[string]utils.StringSet
|
|
if indexes, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp,
|
|
idxItmType, utils.NonTransactional, true, false); err != nil {
|
|
if err != utils.ErrNotFound {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
err = nil
|
|
indexes = map[string]utils.StringSet{
|
|
idxItmType: make(utils.StringSet), // create an empty index if is not found in DB in case we add them later
|
|
}
|
|
}
|
|
indexes[idxItmType].Add(itemID)
|
|
for indxKey := range indexes {
|
|
if err = Cache.Remove(ctx, utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntGrp, indxKey), true, utils.NonTransactional); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
}
|
|
if err = dm.SetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp, indexes, true, utils.NonTransactional); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
}
|
|
return
|
|
}
|
|
|
|
// removeIndexFiltersItem will removes a reference for the items in the reverse filter index
|
|
func removeIndexFiltersItem(ctx *context.Context, dm *DataManager, idxItmType, tnt, itemID string, filterIDs []string) (err error) {
|
|
for _, ID := range filterIDs {
|
|
if strings.HasPrefix(ID, utils.Meta) { // skip inline
|
|
continue
|
|
}
|
|
tntGrp := utils.ConcatenatedKey(tnt, ID)
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntGrp)
|
|
var indexes map[string]utils.StringSet
|
|
if indexes, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp,
|
|
idxItmType, utils.NonTransactional, true, false); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
if err != utils.ErrNotFound {
|
|
return
|
|
}
|
|
err = nil
|
|
continue // it is already removed
|
|
}
|
|
indexes[idxItmType].Remove(itemID)
|
|
|
|
for indxKey := range indexes {
|
|
if err = Cache.Remove(ctx, utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntGrp, indxKey), true, utils.NonTransactional); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
}
|
|
if err = dm.SetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp, indexes, true, utils.NonTransactional); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
}
|
|
return
|
|
}
|
|
|
|
// UpdateFilterIndex will update the indexes for the new Filter
|
|
// we do not care what is added
|
|
// exported for the migrator
|
|
func UpdateFilterIndex(ctx *context.Context, dm *DataManager, oldFlt, newFlt *Filter) (err error) {
|
|
if oldFlt == nil { // no filter before so no index to update
|
|
return // nothing to update
|
|
}
|
|
|
|
// split the rules so we can determine if we need to update the indexes
|
|
oldRules := utils.StringSet{}
|
|
newRules := utils.StringSet{} // we only need to determine if we added new rules to rebuild
|
|
removeRules := utils.StringSet{} // but we need to know what indexes to remove
|
|
for _, flt := range newFlt.Rules {
|
|
if !FilterIndexTypes.Has(flt.Type) ||
|
|
IsDynamicDPPath(flt.Element) {
|
|
continue
|
|
}
|
|
isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix)
|
|
// tthis case is specially for *exists/*notexists filter types
|
|
if len(flt.Values) == 0 {
|
|
switch flt.Type {
|
|
case utils.MetaExists:
|
|
newRules.Add(utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaAny)) // asume it is *exists type
|
|
case utils.MetaNotExists:
|
|
newRules.Add(utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaNone)) // asume it is *notexists type
|
|
}
|
|
}
|
|
for _, fldVal := range flt.Values {
|
|
if IsDynamicDPPath(fldVal) {
|
|
continue
|
|
}
|
|
var idxKey string
|
|
if isDyn {
|
|
if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic
|
|
continue
|
|
}
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal)
|
|
} else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) {
|
|
idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element)
|
|
} else {
|
|
// do not index not dynamic filters
|
|
continue
|
|
}
|
|
newRules.Add(idxKey)
|
|
}
|
|
}
|
|
for _, flt := range oldFlt.Rules {
|
|
if !FilterIndexTypes.Has(flt.Type) ||
|
|
IsDynamicDPPath(flt.Element) {
|
|
continue
|
|
}
|
|
isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix)
|
|
// this case is only for "*exists" and "*notexists" type. These 2 types does not have values, so the key will be just type and element.
|
|
if len(flt.Values) == 0 {
|
|
var idxKey string
|
|
switch flt.Type {
|
|
case utils.MetaExists:
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaAny) // asume it is *exists type
|
|
case utils.MetaNotExists:
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], utils.MetaNone) // asume it is *notexists type
|
|
}
|
|
if !newRules.Has(idxKey) {
|
|
removeRules.Add(idxKey)
|
|
} else {
|
|
oldRules.Add(idxKey)
|
|
}
|
|
}
|
|
for _, fldVal := range flt.Values {
|
|
if IsDynamicDPPath(fldVal) {
|
|
continue
|
|
}
|
|
var idxKey string
|
|
if isDyn {
|
|
if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic
|
|
continue
|
|
}
|
|
idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal)
|
|
} else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) {
|
|
idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element)
|
|
} else {
|
|
// do not index not dynamic filters
|
|
continue
|
|
}
|
|
if !newRules.Has(idxKey) {
|
|
removeRules.Add(idxKey)
|
|
} else {
|
|
oldRules.Add(idxKey)
|
|
}
|
|
}
|
|
}
|
|
needsRebuild := removeRules.Size() != 0 // nothing to remove means nothing to rebuild
|
|
if !needsRebuild { // so check if we added somrthing
|
|
for key := range newRules {
|
|
if needsRebuild = !oldRules.Has(key); needsRebuild {
|
|
break
|
|
}
|
|
}
|
|
if !needsRebuild { // if we did not remove or add we do not need to rebuild the indexes
|
|
return
|
|
}
|
|
}
|
|
|
|
tntID := newFlt.TenantID()
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntID)
|
|
defer guardian.Guardian.UnguardIDs(refID)
|
|
var rcvIndx map[string]utils.StringSet
|
|
// get all reverse indexes from DB
|
|
if rcvIndx, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntID,
|
|
utils.EmptyString, utils.NonTransactional, true, false); err != nil {
|
|
if err != utils.ErrNotFound {
|
|
return
|
|
}
|
|
err = nil // no index for this filter so no update needed
|
|
return
|
|
}
|
|
removeIndexKeys := removeRules.AsSlice()
|
|
|
|
// remove the old indexes and compute the new ones
|
|
for idxItmType, indx := range rcvIndx {
|
|
switch idxItmType {
|
|
case utils.CacheThresholdFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
th, e := dm.GetThresholdProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(th.FilterIDs))
|
|
copy(fltrIDs, th.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheStatFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
sq, e := dm.GetStatQueueProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(sq.FilterIDs))
|
|
copy(fltrIDs, sq.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheResourceFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
rs, e := dm.GetResourceProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(rs.FilterIDs))
|
|
copy(fltrIDs, rs.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheRouteFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
rt, e := dm.GetRouteProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(rt.FilterIDs))
|
|
copy(fltrIDs, rt.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheChargerFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
ch, e := dm.GetChargerProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(ch.FilterIDs))
|
|
copy(fltrIDs, ch.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheAccountsFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, //remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
ap, e := dm.GetAccount(context.Background(), tnt, id)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(ap.FilterIDs))
|
|
copy(fltrIDs, ap.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheActionProfilesFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, //remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
acp, e := dm.GetActionProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(acp.FilterIDs))
|
|
copy(fltrIDs, acp.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheRateProfilesFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, //remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, grp string) (*[]string, error) {
|
|
rp, e := dm.GetRateProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(rp.FilterIDs))
|
|
copy(fltrIDs, rp.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
|
|
case utils.CacheRateFilterIndexes:
|
|
itemIDs := make(map[string]utils.StringSet)
|
|
for itemID := range indx {
|
|
idSplit := strings.SplitN(itemID, utils.ConcatenatedKeySep, 2)
|
|
if len(idSplit) < 2 {
|
|
return errors.New("Expected to be 2 values")
|
|
}
|
|
if itemIDs[idSplit[1]] == nil {
|
|
itemIDs[idSplit[1]] = make(utils.StringSet)
|
|
}
|
|
itemIDs[idSplit[1]].Add(idSplit[0])
|
|
}
|
|
for rpID, ids := range itemIDs {
|
|
tntGrp := utils.ConcatenatedKey(newFlt.Tenant, rpID)
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, tntGrp,
|
|
removeIndexKeys, ids); err != nil {
|
|
return
|
|
}
|
|
var rp *utils.RateProfile
|
|
if rp, err = dm.GetRateProfile(ctx, newFlt.Tenant, rpID, true, false, utils.NonTransactional); err != nil {
|
|
return
|
|
}
|
|
for itemID := range ids {
|
|
rate, has := rp.Rates[itemID]
|
|
if !has {
|
|
return utils.ErrNotFound
|
|
}
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tntGrp)
|
|
var updIdx map[string]utils.StringSet
|
|
if updIdx, err = newFilterIndex(ctx, dm, idxItmType,
|
|
newFlt.Tenant, rpID, itemID, utils.NonTransactional, rate.FilterIDs, newFlt); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
for _, idx := range updIdx {
|
|
idx.Add(itemID)
|
|
}
|
|
if err = dm.SetIndexes(ctx, idxItmType, tntGrp,
|
|
updIdx, false, utils.NonTransactional); err != nil {
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
return
|
|
}
|
|
guardian.Guardian.UnguardIDs(refID)
|
|
}
|
|
}
|
|
case utils.CacheAttributeFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, _ string) (*[]string, error) {
|
|
ap, e := dm.GetAttributeProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(ap.FilterIDs))
|
|
copy(fltrIDs, ap.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
case utils.CacheDispatcherFilterIndexes:
|
|
if err = removeFilterIndexesForFilter(ctx, dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter
|
|
removeIndexKeys, indx); err != nil {
|
|
return
|
|
}
|
|
idxSlice := indx.AsSlice()
|
|
if _, err = ComputeIndexes(ctx, dm, newFlt.Tenant, utils.EmptyString, idxItmType, // compute all the indexes for afected items
|
|
&idxSlice, utils.NonTransactional, func(tnt, id, _ string) (*[]string, error) {
|
|
dp, e := dm.GetDispatcherProfile(ctx, tnt, id, true, false, utils.NonTransactional)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
fltrIDs := make([]string, len(dp.FilterIDs))
|
|
copy(fltrIDs, dp.FilterIDs)
|
|
return &fltrIDs, nil
|
|
}, newFlt); err != nil && err != utils.ErrDSPProfileNotFound {
|
|
return utils.APIErrorHandler(err)
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// removeFilterIndexesForFilter removes itemIDs from the specified filter index keys.
|
|
// Used to update the index map when a filter is modified.
|
|
func removeFilterIndexesForFilter(ctx *context.Context, dm *DataManager, idxItmType, tnt string,
|
|
removeIndexKeys []string, itemIDs utils.StringSet) error {
|
|
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
|
|
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tnt)
|
|
defer guardian.Guardian.UnguardIDs(refID)
|
|
|
|
// Retrieve current filter indexes.
|
|
fltrIdx, err := dm.GetIndexes(ctx, idxItmType, tnt, utils.EmptyString,
|
|
utils.NonTransactional, true, false)
|
|
if err != nil {
|
|
if err != utils.ErrNotFound {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Remove itemIDs from the specified index keys.
|
|
for _, idxKey := range removeIndexKeys {
|
|
for itemID := range itemIDs {
|
|
fltrIdx[idxKey].Remove(itemID)
|
|
}
|
|
}
|
|
|
|
// Store the updated indexes.
|
|
return dm.SetIndexes(ctx, idxItmType, tnt, fltrIdx, true, utils.NonTransactional)
|
|
}
|
|
|
|
// IsDynamicDPPath check dynamic path with ~*stats, ~*resources, ~*accounts, ~*libphonenumber, ~*asm to not be indexed
|
|
// Used to determine if the rule will be indexed
|
|
func IsDynamicDPPath(path string) bool {
|
|
for _, notIndex := range ToNotBeIndexed {
|
|
if strings.HasPrefix(path, notIndex) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|