From 0139f213fca95e56121cb0b552dd8321776b1f7e Mon Sep 17 00:00:00 2001 From: adi Date: Tue, 13 Dec 2022 15:56:15 +0200 Subject: [PATCH] Added resource reverse indexes code --- apier/v1/filter_indexes.go | 80 +------------------------------------- engine/datamanager.go | 12 ++++++ engine/filter_indexes.go | 76 ++++++++++++++++++++++++++++++++++++ engine/libindex.go | 12 ++++++ 4 files changed, 102 insertions(+), 78 deletions(-) diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index 03700106b..5bf904156 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -225,7 +225,7 @@ func (api *APIerSv1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r //ResourceProfile Indexes var rsIndexes *engine.FilterIndexer if args.ResourceS { - rsIndexes, err = api.computeResourceIndexes(args.Tenant, nil, transactionID) + rsIndexes, err = engine.ComputeResourceIndexes(api.DataManager, args.Tenant, nil, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -325,7 +325,7 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs, return utils.APIErrorHandler(err) } //ResourceProfile Indexes - rsIndexes, err := api.computeResourceIndexes(args.Tenant, &args.ResourceIDs, transactionID) + rsIndexes, err := engine.ComputeResourceIndexes(api.DataManager, args.Tenant, &args.ResourceIDs, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -540,82 +540,6 @@ func (api *APIerSv1) computeAttributeIndexes(tenant, context string, attrIDs *[] return attrIndexers, nil } -func (api *APIerSv1) computeResourceIndexes(tenant string, rsIDs *[]string, - transactionID string) (filterIndexer *engine.FilterIndexer, err error) { - var resourceIDs []string - var rpIndexers *engine.FilterIndexer - if rsIDs == nil { - ids, err := api.DataManager.DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix) - if err != nil { - return nil, err - } - for _, id := range ids { - resourceIDs = append(resourceIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) - } - // this will be on ComputeIndexes that contains empty indexes - rpIndexers = engine.NewFilterIndexer(api.DataManager, utils.ResourceProfilesPrefix, tenant) - } else { - // this will be on ComputeIndexesIDs that contains the old indexes from the next getter - var oldIDx map[string]utils.StringMap - if oldIDx, err = api.DataManager.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceFilterIndexes], - tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { - rpIndexers = engine.NewFilterIndexer(api.DataManager, utils.ResourceProfilesPrefix, tenant) - } else { - rpIndexers = engine.NewFilterIndexerWithIndexes(api.DataManager, utils.ResourceProfilesPrefix, tenant, oldIDx) - } - resourceIDs = *rsIDs - transactionID = utils.NonTransactional - } - for _, id := range resourceIDs { - rp, err := api.DataManager.GetResourceProfile(tenant, id, true, false, utils.NonTransactional) - if err != nil { - return nil, err - } - fltrIDs := make([]string, len(rp.FilterIDs)) - for i, fltrID := range rp.FilterIDs { - fltrIDs[i] = fltrID - } - if len(fltrIDs) == 0 { - fltrIDs = []string{utils.META_NONE} - } - for _, fltrID := range fltrIDs { - var fltr *engine.Filter - if fltrID == utils.META_NONE { - fltr = &engine.Filter{ - Tenant: rp.Tenant, - ID: rp.ID, - Rules: []*engine.FilterRule{ - { - Type: utils.META_NONE, - Element: utils.META_ANY, - Values: []string{utils.META_ANY}, - }, - }, - } - } else if fltr, err = engine.GetFilter(api.DataManager, rp.Tenant, fltrID, - true, false, utils.NonTransactional); err != nil { - if err == utils.ErrNotFound { - err = fmt.Errorf("broken reference to filter: %+v for resource: %+v", - fltrID, rp) - } - return nil, err - } - rpIndexers.IndexTPFilter(engine.FilterToTPFilter(fltr), rp.ID) - } - } - if transactionID == utils.NonTransactional { - if err := rpIndexers.StoreIndexes(true, transactionID); err != nil { - return nil, err - } - return nil, nil - } else { - if err := rpIndexers.StoreIndexes(false, transactionID); err != nil { - return nil, err - } - } - return rpIndexers, nil -} - func (api *APIerSv1) computeStatIndexes(tenant string, stIDs *[]string, transactionID string) (filterIndexer *engine.FilterIndexer, err error) { var statIDs []string diff --git a/engine/datamanager.go b/engine/datamanager.go index edd8f5d03..abdae30e6 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -1143,11 +1143,19 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) ( rp.Tenant).RemoveItemFromIndex(rp.Tenant, rp.ID, oldRes.FilterIDs); err != nil { return } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + rp.Tenant, rp.ID, rp.FilterIDs); err != nil { + return + } } } if err = createAndIndex(utils.ResourceProfilesPrefix, rp.Tenant, utils.EmptyString, rp.ID, rp.FilterIDs, dm); err != nil { return } + if err = addReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + rp.Tenant, rp.ID, rp.FilterIDs); err != nil { + return + } Cache.Clear([]string{utils.CacheEventResources}) } if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaResourceProfile]; itm.Replicate { @@ -1178,6 +1186,10 @@ func (dm *DataManager) RemoveResourceProfile(tenant, id, transactionID string, w tenant).RemoveItemFromIndex(tenant, id, oldRes.FilterIDs); err != nil { return } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheResourceFilterIndexes, utils.EmptyString, + oldRes.Tenant, oldRes.ID, oldRes.FilterIDs); err != nil { + return + } } if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaResourceProfile]; itm.Replicate { replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, diff --git a/engine/filter_indexes.go b/engine/filter_indexes.go index 673242613..611563e66 100644 --- a/engine/filter_indexes.go +++ b/engine/filter_indexes.go @@ -177,3 +177,79 @@ func ComputeChargerIndexes(dm *DataManager, tenant string, cppIDs *[]string, } return cppIndexes, nil } + +func ComputeResourceIndexes(dm *DataManager, tenant string, rsIDs *[]string, + transactionID string) (filterIndexer *FilterIndexer, err error) { + var resourceIDs []string + var rpIndexers *FilterIndexer + if rsIDs == nil { + ids, err := dm.DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix) + if err != nil { + return nil, err + } + for _, id := range ids { + resourceIDs = append(resourceIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) + } + // this will be on ComputeIndexes that contains empty indexes + rpIndexers = NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant) + } else { + // this will be on ComputeIndexesIDs that contains the old indexes from the next getter + var oldIDx map[string]utils.StringMap + if oldIDx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceFilterIndexes], + tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { + rpIndexers = NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant) + } else { + rpIndexers = NewFilterIndexerWithIndexes(dm, utils.ResourceProfilesPrefix, tenant, oldIDx) + } + resourceIDs = *rsIDs + transactionID = utils.NonTransactional + } + for _, id := range resourceIDs { + rp, err := dm.GetResourceProfile(tenant, id, true, false, utils.NonTransactional) + if err != nil { + return nil, err + } + fltrIDs := make([]string, len(rp.FilterIDs)) + for i, fltrID := range rp.FilterIDs { + fltrIDs[i] = fltrID + } + if len(fltrIDs) == 0 { + fltrIDs = []string{utils.META_NONE} + } + for _, fltrID := range fltrIDs { + var fltr *Filter + if fltrID == utils.META_NONE { + fltr = &Filter{ + Tenant: rp.Tenant, + ID: rp.ID, + Rules: []*FilterRule{ + { + Type: utils.META_NONE, + Element: utils.META_ANY, + Values: []string{utils.META_ANY}, + }, + }, + } + } else if fltr, err = GetFilter(dm, rp.Tenant, fltrID, + true, false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for resource: %+v", + fltrID, rp) + } + return nil, err + } + rpIndexers.IndexTPFilter(FilterToTPFilter(fltr), rp.ID) + } + } + if transactionID == utils.NonTransactional { + if err := rpIndexers.StoreIndexes(true, transactionID); err != nil { + return nil, err + } + return nil, nil + } else { + if err := rpIndexers.StoreIndexes(false, transactionID); err != nil { + return nil, err + } + } + return rpIndexers, nil +} diff --git a/engine/libindex.go b/engine/libindex.go index e0656b3cf..cbf701263 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -145,6 +145,18 @@ func UpdateFilterIndexes(dm *DataManager, tnt string, oldFltr *Filter, newFltr * utils.NonTransactional); err != nil { return err } + case utils.CacheResourceFilterIndexes: + // remove the indexes from this filter for this partition + if err = removeFilterIndexesForFilter(dm, idxItmType, utils.CacheResourceProfiles, + tnt, removeIndexKeys, index); err != nil { + return + } + // we removed the old reverse indexes, now we have to compute the new ones + resourceIDs := index.Slice() + if _, err = ComputeResourceIndexes(dm, newFltr.Tenant, &resourceIDs, + utils.NonTransactional); err != nil { + return err + } } } return nil