From 7ea41a6bc7bd95941087f04c8b754017a8bebf09 Mon Sep 17 00:00:00 2001 From: adi Date: Wed, 14 Dec 2022 18:02:36 +0200 Subject: [PATCH] Added reverse filter indexes for attributes --- apier/v1/filter_indexes.go | 84 +------------------------------------- engine/datamanager.go | 42 ++++++++++++------- engine/filter_indexes.go | 80 ++++++++++++++++++++++++++++++++++++ engine/libindex.go | 31 ++++++++++---- 4 files changed, 132 insertions(+), 105 deletions(-) diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index 3d20e3b29..2c6e8c68b 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -241,7 +241,7 @@ func (api *APIerSv1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r //AttributeProfile Indexes var attrIndexes *engine.FilterIndexer if args.AttributeS { - attrIndexes, err = api.computeAttributeIndexes(args.Tenant, args.Context, nil, transactionID) + attrIndexes, err = engine.ComputeAttributeIndexes(api.DataManager, args.Tenant, args.Context, nil, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -335,7 +335,7 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs, return utils.APIErrorHandler(err) } //AttributeProfile Indexes - attrIndexes, err := api.computeAttributeIndexes(args.Tenant, args.Context, &args.AttributeIDs, transactionID) + attrIndexes, err := engine.ComputeAttributeIndexes(api.DataManager, args.Tenant, args.Context, &args.AttributeIDs, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -460,86 +460,6 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs, return nil } -func (api *APIerSv1) computeAttributeIndexes(tenant, context string, attrIDs *[]string, - transactionID string) (filterIndexer *engine.FilterIndexer, err error) { - var attributeIDs []string - var attrIndexers *engine.FilterIndexer - if attrIDs == nil { - ids, err := api.DataManager.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix) - if err != nil { - return nil, err - } - for _, id := range ids { - attributeIDs = append(attributeIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) - } - // this will be on ComputeIndexes that contains empty indexes - attrIndexers = engine.NewFilterIndexer(api.DataManager, utils.AttributeProfilePrefix, - utils.ConcatenatedKey(tenant, context)) - } else { - // this will be on ComputeIndexesIDs that contains the old indexes from the next getter - var oldIDx map[string]utils.StringMap - if oldIDx, err = api.DataManager.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], - tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { - attrIndexers = engine.NewFilterIndexer(api.DataManager, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context)) - } else { - attrIndexers = engine.NewFilterIndexerWithIndexes(api.DataManager, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context), oldIDx) - } - attributeIDs = *attrIDs - transactionID = utils.NonTransactional - } - for _, id := range attributeIDs { - ap, err := api.DataManager.GetAttributeProfile(tenant, id, true, false, utils.NonTransactional) - if err != nil { - return nil, err - } - if !utils.IsSliceMember(ap.Contexts, context) && context != utils.META_ANY { - continue - } - fltrIDs := make([]string, len(ap.FilterIDs)) - for i, fltrID := range ap.FilterIDs { - fltrIDs[i] = fltrID - } - if len(fltrIDs) == 0 { - fltrIDs = []string{utils.META_NONE} - } - for _, fltrID := range fltrIDs { - var fltr *engine.Filter - if fltrID == utils.META_NONE { - fltr = &engine.Filter{ - Tenant: ap.Tenant, - ID: ap.ID, - Rules: []*engine.FilterRule{ - { - Type: utils.META_NONE, - Element: utils.META_ANY, - Values: []string{utils.META_ANY}, - }, - }, - } - } else if fltr, err = engine.GetFilter(api.DataManager, ap.Tenant, fltrID, - true, false, utils.NonTransactional); err != nil { - if err == utils.ErrNotFound { - err = fmt.Errorf("broken reference to filter: %+v for attribute: %+v", - fltrID, ap) - } - return nil, err - } - attrIndexers.IndexTPFilter(engine.FilterToTPFilter(fltr), ap.ID) - } - } - if transactionID == utils.NonTransactional { - if err := attrIndexers.StoreIndexes(true, transactionID); err != nil { - return nil, err - } - return nil, nil - } else { - if err := attrIndexers.StoreIndexes(false, transactionID); err != nil { - return nil, err - } - } - return attrIndexers, nil -} - func (api *APIerSv1) computeDispatcherIndexes(tenant, context string, dspIDs *[]string, transactionID string) (filterIndexer *engine.FilterIndexer, err error) { var dispatcherIDs []string diff --git a/engine/datamanager.go b/engine/datamanager.go index bf4004c7e..10183c186 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -788,7 +788,7 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) th.Tenant).RemoveItemFromIndex(th.Tenant, th.ID, oldTh.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, th.Tenant, th.ID, th.FilterIDs); err != nil { return } @@ -798,7 +798,7 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) utils.EmptyString, th.ID, th.FilterIDs, dm); err != nil { return err } - if err = addReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString, + if err = addReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, th.Tenant, th.ID, th.FilterIDs); err != nil { return } @@ -832,7 +832,7 @@ func (dm *DataManager) RemoveThresholdProfile(tenant, id, tenant).RemoveItemFromIndex(tenant, id, oldTh.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheThresholdFilterIndexes, oldTh.Tenant, oldTh.ID, oldTh.FilterIDs); err != nil { return } @@ -906,7 +906,7 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool sqp.Tenant).RemoveItemFromIndex(sqp.Tenant, sqp.ID, oldSts.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, sqp.Tenant, sqp.ID, sqp.FilterIDs); err != nil { return } @@ -916,7 +916,7 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool utils.EmptyString, sqp.ID, sqp.FilterIDs, dm); err != nil { return } - if err = addReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString, + if err = addReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, sqp.Tenant, sqp.ID, sqp.FilterIDs); err != nil { return } @@ -950,7 +950,7 @@ func (dm *DataManager) RemoveStatQueueProfile(tenant, id, tenant).RemoveItemFromIndex(tenant, id, oldSts.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheStatFilterIndexes, oldSts.Tenant, oldSts.ID, oldSts.FilterIDs); err != nil { return } @@ -1155,7 +1155,7 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) ( rp.Tenant).RemoveItemFromIndex(rp.Tenant, rp.ID, oldRes.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, rp.Tenant, rp.ID, rp.FilterIDs); err != nil { return } @@ -1164,7 +1164,7 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) ( if err = createAndIndex(utils.ResourceProfilesPrefix, rp.Tenant, utils.EmptyString, rp.ID, rp.FilterIDs, dm); err != nil { return } - if err = addReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + if err = addReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, rp.Tenant, rp.ID, rp.FilterIDs); err != nil { return } @@ -1198,7 +1198,7 @@ func (dm *DataManager) RemoveResourceProfile(tenant, id, transactionID string, w tenant).RemoveItemFromIndex(tenant, id, oldRes.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, oldRes.Tenant, oldRes.ID, oldRes.FilterIDs); err != nil { return } @@ -1945,7 +1945,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) supp.Tenant).RemoveItemFromIndex(supp.Tenant, supp.ID, oldSup.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, supp.Tenant, supp.ID, supp.FilterIDs); err != nil { return } @@ -1955,7 +1955,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) utils.EmptyString, supp.ID, supp.FilterIDs, dm); err != nil { return } - if err = addReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + if err = addReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, supp.Tenant, supp.ID, supp.FilterIDs); err != nil { return } @@ -1987,7 +1987,7 @@ func (dm *DataManager) RemoveSupplierProfile(tenant, id, transactionID string, w tenant).RemoveItemFromIndex(tenant, id, oldSupp.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, oldSupp.Tenant, oldSupp.ID, oldSupp.FilterIDs); err != nil { return } @@ -2089,6 +2089,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) } } } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes, + ap.Tenant, ap.ID, ap.FilterIDs); err != nil { + return + } } for _, ctx := range ap.Contexts { if err = createAndIndex(utils.AttributeProfilePrefix, @@ -2096,6 +2100,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) return } } + if err = addReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes, + ap.Tenant, ap.ID, ap.FilterIDs); err != nil { + return + } } if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaAttributeProfiles]; itm.Replicate { err = replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, @@ -2125,6 +2133,10 @@ func (dm *DataManager) RemoveAttributeProfile(tenant, id string, transactionID s return } } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheAttributeFilterIndexes, + oldAttr.Tenant, oldAttr.ID, oldAttr.FilterIDs); err != nil { + return + } } if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaAttributeProfiles]; itm.Replicate { replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, @@ -2199,7 +2211,7 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil { return } @@ -2209,7 +2221,7 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e utils.EmptyString, cpp.ID, cpp.FilterIDs, dm); err != nil { return } - if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString, + if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil { return } @@ -2242,7 +2254,7 @@ func (dm *DataManager) RemoveChargerProfile(tenant, id string, tenant).RemoveItemFromIndex(tenant, id, oldCpp.FilterIDs); err != nil { return } - if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString, + if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, oldCpp.Tenant, oldCpp.ID, oldCpp.FilterIDs); err != nil { return } diff --git a/engine/filter_indexes.go b/engine/filter_indexes.go index 860edfee9..a320cb958 100644 --- a/engine/filter_indexes.go +++ b/engine/filter_indexes.go @@ -405,3 +405,83 @@ func ComputeStatIndexes(dm *DataManager, tenant string, stIDs *[]string, } return sqpIndexers, nil } + +func ComputeAttributeIndexes(dm *DataManager, tenant, context string, attrIDs *[]string, + transactionID string) (filterIndexer *FilterIndexer, err error) { + var attributeIDs []string + var attrIndexers *FilterIndexer + if attrIDs == nil { + ids, err := dm.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix) + if err != nil { + return nil, err + } + for _, id := range ids { + attributeIDs = append(attributeIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) + } + // this will be on ComputeIndexes that contains empty indexes + attrIndexers = NewFilterIndexer(dm, utils.AttributeProfilePrefix, + utils.ConcatenatedKey(tenant, context)) + } else { + // this will be on ComputeIndexesIDs that contains the old indexes from the next getter + var oldIDx map[string]utils.StringMap + if oldIDx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], + tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { + attrIndexers = NewFilterIndexer(dm, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context)) + } else { + attrIndexers = NewFilterIndexerWithIndexes(dm, utils.AttributeProfilePrefix, utils.ConcatenatedKey(tenant, context), oldIDx) + } + attributeIDs = *attrIDs + transactionID = utils.NonTransactional + } + for _, id := range attributeIDs { + ap, err := dm.GetAttributeProfile(tenant, id, true, false, utils.NonTransactional) + if err != nil { + return nil, err + } + if !utils.IsSliceMember(ap.Contexts, context) && context != utils.META_ANY { + continue + } + fltrIDs := make([]string, len(ap.FilterIDs)) + for i, fltrID := range ap.FilterIDs { + fltrIDs[i] = fltrID + } + if len(fltrIDs) == 0 { + fltrIDs = []string{utils.META_NONE} + } + for _, fltrID := range fltrIDs { + var fltr *Filter + if fltrID == utils.META_NONE { + fltr = &Filter{ + Tenant: ap.Tenant, + ID: ap.ID, + Rules: []*FilterRule{ + { + Type: utils.META_NONE, + Element: utils.META_ANY, + Values: []string{utils.META_ANY}, + }, + }, + } + } else if fltr, err = GetFilter(dm, ap.Tenant, fltrID, + true, false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for attribute: %+v", + fltrID, ap) + } + return nil, err + } + attrIndexers.IndexTPFilter(FilterToTPFilter(fltr), ap.ID) + } + } + if transactionID == utils.NonTransactional { + if err := attrIndexers.StoreIndexes(true, transactionID); err != nil { + return nil, err + } + return nil, nil + } else { + if err := attrIndexers.StoreIndexes(false, transactionID); err != nil { + return nil, err + } + } + return attrIndexers, nil +} diff --git a/engine/libindex.go b/engine/libindex.go index a41cabdf9..e31d2c3d0 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -181,6 +181,27 @@ func UpdateFilterIndexes(dm *DataManager, tnt string, oldFltr *Filter, newFltr * utils.NonTransactional); err != nil { return err } + case utils.CacheAttributeFilterIndexes: + attributeIDs := index.Slice() + for _, attrID := range attributeIDs { + var ap *AttributeProfile + if ap, err = dm.GetAttributeProfile(newFltr.Tenant, attrID, + true, false, utils.NonTransactional); err != nil { + return + } + for _, ctx := range ap.Contexts { + tntCtx := utils.ConcatenatedKey(newFltr.Tenant, ctx) + if err = removeFilterIndexesForFilter(dm, idxItmType, utils.CacheAttributeProfiles, + tntCtx, // remove the indexes for the filter + removeIndexKeys, index); err != nil { + return + } + if _, err = ComputeAttributeIndexes(dm, newFltr.Tenant, ctx, &attributeIDs, + utils.NonTransactional); err != nil { + return err + } + } + } } } return nil @@ -217,7 +238,7 @@ func removeFilterIndexesForFilter(dm *DataManager, idxItmType, cacheItmType, tnt } // addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes -func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, +func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, tnt, itemID string, filterIDs []string) (err error) { for _, fltrID := range filterIDs { if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters @@ -246,9 +267,6 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID: true, }) indexerKey := utils.ConcatenatedKey(tnt, fltrID) - if ctx != utils.EmptyString { - indexerKey = utils.ConcatenatedKey(tnt, ctx) - } fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey) fltrIndexer.indexes = indexes if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil { // it will remove from cache the old ones @@ -261,7 +279,7 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, } // removeReverseFilterIndexForFilter will remove a reference for the filter in reverse filter indexes -func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) { +func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, tnt, itemID string, filterIDs []string) (err error) { for _, fltrID := range filterIDs { if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters continue @@ -282,9 +300,6 @@ func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, it delete(indexes[idxItmType], itemID) // delete index from map indexerKey := utils.ConcatenatedKey(tnt, fltrID) - if ctx != utils.EmptyString { - indexerKey = utils.ConcatenatedKey(tnt, ctx) - } fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey) fltrIndexer.indexes = indexes if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil {