uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Added reverse filter indexes for attributes

Browse Source
This commit is contained in:
adi
2022-12-14 18:02:36 +02:00
committed by Dan Christian Bogos
parent ae62ff3b1c
commit 7ea41a6bc7
4 changed files with 132 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
apier/v1/filter_indexes.go
View File

@@ -241,7 +241,7 @@ func (api *APIerSv1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
//AttributeProfile Indexes
var attrIndexes *engine.FilterIndexer
if args.AttributeS {
attrIndexes, err = api.computeAttributeIndexes(args.Tenant, args.Context, nil, transactionID)
attrIndexes, err = engine.ComputeAttributeIndexes(api.DataManager, args.Tenant, args.Context, nil, transactionID)
if err != nil && err != utils.ErrNotFound {
return utils.APIErrorHandler(err)
}
@@ -335,7 +335,7 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs,
return utils.APIErrorHandler(err)
}
//AttributeProfile Indexes
attrIndexes, err := api.computeAttributeIndexes(args.Tenant, args.Context, &args.AttributeIDs, transactionID)
attrIndexes, err := engine.ComputeAttributeIndexes(api.DataManager, args.Tenant, args.Context, &args.AttributeIDs, transactionID)
if err != nil && err != utils.ErrNotFound {
return utils.APIErrorHandler(err)
}
@@ -460,86 +460,6 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs,
return nil
}
func (api *APIerSv1) computeAttributeIndexes(tenant, context string, attrIDs *[]string,
transactionID string) (filterIndexer *engine.FilterIndexer, err error) {
var attributeIDs []string
var attrIndexers *engine.FilterIndexer
if attrIDs == nil {
ids, err := api.DataManager.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix)
if err != nil {
return nil, err
}
for _, id := range ids {
attributeIDs = append(attributeIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1])
}
// this will be on ComputeIndexes that contains empty indexes
attrIndexers = engine.NewFilterIndexer(api.DataManager, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(tenant, context))
} else {
// this will be on ComputeIndexesIDs that contains the old indexes from the next getter
var oldIDx map[string]utils.StringMap
if oldIDx, err = api.DataManager.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant, utils.EmptyString, nil); err != nil || oldIDx == nil {
attrIndexers = engine.NewFilterIndexer(api.DataManager, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context))
} else {
attrIndexers = engine.NewFilterIndexerWithIndexes(api.DataManager, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context), oldIDx)
}
attributeIDs = *attrIDs
transactionID = utils.NonTransactional
}
for _, id := range attributeIDs {
ap, err := api.DataManager.GetAttributeProfile(tenant, id, true, false, utils.NonTransactional)
if err != nil {
return nil, err
}
if !utils.IsSliceMember(ap.Contexts, context) && context != utils.META_ANY {
continue
}
fltrIDs := make([]string, len(ap.FilterIDs))
for i, fltrID := range ap.FilterIDs {
fltrIDs[i] = fltrID
}
if len(fltrIDs) == 0 {
fltrIDs = []string{utils.META_NONE}
}
for _, fltrID := range fltrIDs {
var fltr *engine.Filter
if fltrID == utils.META_NONE {
fltr = &engine.Filter{
Tenant: ap.Tenant,
ID: ap.ID,
Rules: []*engine.FilterRule{
{
Type: utils.META_NONE,
Element: utils.META_ANY,
Values: []string{utils.META_ANY},
},
},
}
} else if fltr, err = engine.GetFilter(api.DataManager, ap.Tenant, fltrID,
true, false, utils.NonTransactional); err != nil {
if err == utils.ErrNotFound {
err = fmt.Errorf("broken reference to filter: %+v for attribute: %+v",
fltrID, ap)
}
return nil, err
}
attrIndexers.IndexTPFilter(engine.FilterToTPFilter(fltr), ap.ID)
}
}
if transactionID == utils.NonTransactional {
if err := attrIndexers.StoreIndexes(true, transactionID); err != nil {
return nil, err
}
return nil, nil
} else {
if err := attrIndexers.StoreIndexes(false, transactionID); err != nil {
return nil, err
}
}
return attrIndexers, nil
}
func (api *APIerSv1) computeDispatcherIndexes(tenant, context string, dspIDs *[]string,
transactionID string) (filterIndexer *engine.FilterIndexer, err error) {
var dispatcherIDs []string

42
engine/datamanager.go
View File

@@ -788,7 +788,7 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
th.Tenant).RemoveItemFromIndex(th.Tenant, th.ID, oldTh.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes,
th.Tenant, th.ID, th.FilterIDs); err != nil {
return
}
@@ -798,7 +798,7 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
utils.EmptyString, th.ID, th.FilterIDs, dm); err != nil {
return err
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString,
if err = addReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes,
th.Tenant, th.ID, th.FilterIDs); err != nil {
return
}
@@ -832,7 +832,7 @@ func (dm *DataManager) RemoveThresholdProfile(tenant, id,
tenant).RemoveItemFromIndex(tenant, id, oldTh.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes,
oldTh.Tenant, oldTh.ID, oldTh.FilterIDs); err != nil {
return
}
@@ -906,7 +906,7 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
sqp.Tenant).RemoveItemFromIndex(sqp.Tenant, sqp.ID, oldSts.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes,
sqp.Tenant, sqp.ID, sqp.FilterIDs); err != nil {
return
}
@@ -916,7 +916,7 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
utils.EmptyString, sqp.ID, sqp.FilterIDs, dm); err != nil {
return
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString,
if err = addReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes,
sqp.Tenant, sqp.ID, sqp.FilterIDs); err != nil {
return
}
@@ -950,7 +950,7 @@ func (dm *DataManager) RemoveStatQueueProfile(tenant, id,
tenant).RemoveItemFromIndex(tenant, id, oldSts.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes,
oldSts.Tenant, oldSts.ID, oldSts.FilterIDs); err != nil {
return
}
@@ -1155,7 +1155,7 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
rp.Tenant).RemoveItemFromIndex(rp.Tenant, rp.ID, oldRes.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes,
rp.Tenant, rp.ID, rp.FilterIDs); err != nil {
return
}
@@ -1164,7 +1164,7 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
if err = createAndIndex(utils.ResourceProfilesPrefix, rp.Tenant, utils.EmptyString, rp.ID, rp.FilterIDs, dm); err != nil {
return
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString,
if err = addReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes,
rp.Tenant, rp.ID, rp.FilterIDs); err != nil {
return
}
@@ -1198,7 +1198,7 @@ func (dm *DataManager) RemoveResourceProfile(tenant, id, transactionID string, w
tenant).RemoveItemFromIndex(tenant, id, oldRes.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes,
oldRes.Tenant, oldRes.ID, oldRes.FilterIDs); err != nil {
return
}
@@ -1945,7 +1945,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool)
supp.Tenant).RemoveItemFromIndex(supp.Tenant, supp.ID, oldSup.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes,
supp.Tenant, supp.ID, supp.FilterIDs); err != nil {
return
}
@@ -1955,7 +1955,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool)
utils.EmptyString, supp.ID, supp.FilterIDs, dm); err != nil {
return
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString,
if err = addReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes,
supp.Tenant, supp.ID, supp.FilterIDs); err != nil {
return
}
@@ -1987,7 +1987,7 @@ func (dm *DataManager) RemoveSupplierProfile(tenant, id, transactionID string, w
tenant).RemoveItemFromIndex(tenant, id, oldSupp.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes,
oldSupp.Tenant, oldSupp.ID, oldSupp.FilterIDs); err != nil {
return
}
@@ -2089,6 +2089,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
}
}
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes,
ap.Tenant, ap.ID, ap.FilterIDs); err != nil {
return
}
}
for _, ctx := range ap.Contexts {
if err = createAndIndex(utils.AttributeProfilePrefix,
@@ -2096,6 +2100,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
return
}
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes,
ap.Tenant, ap.ID, ap.FilterIDs); err != nil {
return
}
}
if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaAttributeProfiles]; itm.Replicate {
err = replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns,
@@ -2125,6 +2133,10 @@ func (dm *DataManager) RemoveAttributeProfile(tenant, id string, transactionID s
return
}
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes,
oldAttr.Tenant, oldAttr.ID, oldAttr.FilterIDs); err != nil {
return
}
}
if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaAttributeProfiles]; itm.Replicate {
replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns,
@@ -2199,7 +2211,7 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e
cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes,
cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil {
return
}
@@ -2209,7 +2221,7 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e
utils.EmptyString, cpp.ID, cpp.FilterIDs, dm); err != nil {
return
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString,
if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes,
cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil {
return
}
@@ -2242,7 +2254,7 @@ func (dm *DataManager) RemoveChargerProfile(tenant, id string,
tenant).RemoveItemFromIndex(tenant, id, oldCpp.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString,
if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes,
oldCpp.Tenant, oldCpp.ID, oldCpp.FilterIDs); err != nil {
return
}

80
engine/filter_indexes.go
View File

@@ -405,3 +405,83 @@ func ComputeStatIndexes(dm *DataManager, tenant string, stIDs *[]string,
}
return sqpIndexers, nil
}
func ComputeAttributeIndexes(dm *DataManager, tenant, context string, attrIDs *[]string,
transactionID string) (filterIndexer *FilterIndexer, err error) {
var attributeIDs []string
var attrIndexers *FilterIndexer
if attrIDs == nil {
ids, err := dm.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix)
if err != nil {
return nil, err
}
for _, id := range ids {
attributeIDs = append(attributeIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1])
}
// this will be on ComputeIndexes that contains empty indexes
attrIndexers = NewFilterIndexer(dm, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(tenant, context))
} else {
// this will be on ComputeIndexesIDs that contains the old indexes from the next getter
var oldIDx map[string]utils.StringMap
if oldIDx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant, utils.EmptyString, nil); err != nil || oldIDx == nil {
attrIndexers = NewFilterIndexer(dm, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context))
} else {
attrIndexers = NewFilterIndexerWithIndexes(dm, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context), oldIDx)
}
attributeIDs = *attrIDs
transactionID = utils.NonTransactional
}
for _, id := range attributeIDs {
ap, err := dm.GetAttributeProfile(tenant, id, true, false, utils.NonTransactional)
if err != nil {
return nil, err
}
if !utils.IsSliceMember(ap.Contexts, context) && context != utils.META_ANY {
continue
}
fltrIDs := make([]string, len(ap.FilterIDs))
for i, fltrID := range ap.FilterIDs {
fltrIDs[i] = fltrID
}
if len(fltrIDs) == 0 {
fltrIDs = []string{utils.META_NONE}
}
for _, fltrID := range fltrIDs {
var fltr *Filter
if fltrID == utils.META_NONE {
fltr = &Filter{
Tenant: ap.Tenant,
ID: ap.ID,
Rules: []*FilterRule{
{
Type: utils.META_NONE,
Element: utils.META_ANY,
Values: []string{utils.META_ANY},
},
},
}
} else if fltr, err = GetFilter(dm, ap.Tenant, fltrID,
true, false, utils.NonTransactional); err != nil {
if err == utils.ErrNotFound {
err = fmt.Errorf("broken reference to filter: %+v for attribute: %+v",
fltrID, ap)
}
return nil, err
}
attrIndexers.IndexTPFilter(FilterToTPFilter(fltr), ap.ID)
}
}
if transactionID == utils.NonTransactional {
if err := attrIndexers.StoreIndexes(true, transactionID); err != nil {
return nil, err
}
return nil, nil
} else {
if err := attrIndexers.StoreIndexes(false, transactionID); err != nil {
return nil, err
}
}
return attrIndexers, nil
}

31
engine/libindex.go
View File

@@ -181,6 +181,27 @@ func UpdateFilterIndexes(dm *DataManager, tnt string, oldFltr *Filter, newFltr *
utils.NonTransactional); err != nil {
return err
}
case utils.CacheAttributeFilterIndexes:
attributeIDs := index.Slice()
for _, attrID := range attributeIDs {
var ap *AttributeProfile
if ap, err = dm.GetAttributeProfile(newFltr.Tenant, attrID,
true, false, utils.NonTransactional); err != nil {
return
}
for _, ctx := range ap.Contexts {
tntCtx := utils.ConcatenatedKey(newFltr.Tenant, ctx)
if err = removeFilterIndexesForFilter(dm, idxItmType, utils.CacheAttributeProfiles,
tntCtx, // remove the indexes for the filter
removeIndexKeys, index); err != nil {
return
}
if _, err = ComputeAttributeIndexes(dm, newFltr.Tenant, ctx, &attributeIDs,
utils.NonTransactional); err != nil {
return err
}
}
}
}
}
return nil
@@ -217,7 +238,7 @@ func removeFilterIndexesForFilter(dm *DataManager, idxItmType, cacheItmType, tnt
}
// addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes
func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt,
func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, tnt,
itemID string, filterIDs []string) (err error) {
for _, fltrID := range filterIDs {
if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters
@@ -246,9 +267,6 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt,
itemID: true,
})
indexerKey := utils.ConcatenatedKey(tnt, fltrID)
if ctx != utils.EmptyString {
indexerKey = utils.ConcatenatedKey(tnt, ctx)
}
fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey)
fltrIndexer.indexes = indexes
if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil { // it will remove from cache the old ones
@@ -261,7 +279,7 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt,
}
// removeReverseFilterIndexForFilter will remove a reference for the filter in reverse filter indexes
func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) {
func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, tnt, itemID string, filterIDs []string) (err error) {
for _, fltrID := range filterIDs {
if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters
continue
@@ -282,9 +300,6 @@ func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, it
delete(indexes[idxItmType], itemID) // delete index from map
indexerKey := utils.ConcatenatedKey(tnt, fltrID)
if ctx != utils.EmptyString {
indexerKey = utils.ConcatenatedKey(tnt, ctx)
}
fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey)
fltrIndexer.indexes = indexes
if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil {