From db4912e33060bc080e44ce9569950ff3141b52e7 Mon Sep 17 00:00:00 2001 From: adi Date: Wed, 14 Dec 2022 13:03:28 +0200 Subject: [PATCH] Added supplier filter indexes --- apier/v1/filter_indexes.go | 80 +------------------ engine/datamanager.go | 12 +++ engine/filter_indexes.go | 76 ++++++++++++++++++ engine/libindex.go | 12 +++ general_tests/filter_indexes_cases_it_test.go | 18 +++-- 5 files changed, 114 insertions(+), 84 deletions(-) diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index 5bf904156..1275dbca0 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -233,7 +233,7 @@ func (api *APIerSv1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r //SupplierProfile Indexes var sppIndexes *engine.FilterIndexer if args.SupplierS { - sppIndexes, err = api.computeSupplierIndexes(args.Tenant, nil, transactionID) + sppIndexes, err = engine.ComputeSupplierIndexes(api.DataManager, args.Tenant, nil, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -330,7 +330,7 @@ func (api *APIerSv1) ComputeFilterIndexIDs(args utils.ArgsComputeFilterIndexIDs, return utils.APIErrorHandler(err) } //SupplierProfile Indexes - sppIndexes, err := api.computeSupplierIndexes(args.Tenant, &args.SupplierIDs, transactionID) + sppIndexes, err := engine.ComputeSupplierIndexes(api.DataManager, args.Tenant, &args.SupplierIDs, transactionID) if err != nil && err != utils.ErrNotFound { return utils.APIErrorHandler(err) } @@ -616,82 +616,6 @@ func (api *APIerSv1) computeStatIndexes(tenant string, stIDs *[]string, return sqpIndexers, nil } -func (api *APIerSv1) computeSupplierIndexes(tenant string, sppIDs *[]string, - transactionID string) (filterIndexer *engine.FilterIndexer, err error) { - var supplierIDs []string - var sppIndexers *engine.FilterIndexer - if sppIDs == nil { - ids, err := api.DataManager.DataDB().GetKeysForPrefix(utils.SupplierProfilePrefix) - if err != nil { - return nil, err - } - for _, id := range ids { - supplierIDs = append(supplierIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) - } - // this will be on ComputeIndexes that contains empty indexes - sppIndexers = engine.NewFilterIndexer(api.DataManager, utils.SupplierProfilePrefix, tenant) - } else { - // this will be on ComputeIndexesIDs that contains the old indexes from the next getter - var oldIDx map[string]utils.StringMap - if oldIDx, err = api.DataManager.GetFilterIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix], - tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { - sppIndexers = engine.NewFilterIndexer(api.DataManager, utils.SupplierProfilePrefix, tenant) - } else { - sppIndexers = engine.NewFilterIndexerWithIndexes(api.DataManager, utils.SupplierProfilePrefix, tenant, oldIDx) - } - supplierIDs = *sppIDs - transactionID = utils.NonTransactional - } - for _, id := range supplierIDs { - spp, err := api.DataManager.GetSupplierProfile(tenant, id, true, false, utils.NonTransactional) - if err != nil { - return nil, err - } - fltrIDs := make([]string, len(spp.FilterIDs)) - for i, fltrID := range spp.FilterIDs { - fltrIDs[i] = fltrID - } - if len(fltrIDs) == 0 { - fltrIDs = []string{utils.META_NONE} - } - for _, fltrID := range fltrIDs { - var fltr *engine.Filter - if fltrID == utils.META_NONE { - fltr = &engine.Filter{ - Tenant: spp.Tenant, - ID: spp.ID, - Rules: []*engine.FilterRule{ - { - Type: utils.META_NONE, - Element: utils.META_ANY, - Values: []string{utils.META_ANY}, - }, - }, - } - } else if fltr, err = engine.GetFilter(api.DataManager, spp.Tenant, fltrID, - true, false, utils.NonTransactional); err != nil { - if err == utils.ErrNotFound { - err = fmt.Errorf("broken reference to filter: %+v for suppliers: %+v", - fltrID, spp) - } - return nil, err - } - sppIndexers.IndexTPFilter(engine.FilterToTPFilter(fltr), spp.ID) - } - } - if transactionID == utils.NonTransactional { - if err := sppIndexers.StoreIndexes(true, transactionID); err != nil { - return nil, err - } - return nil, nil - } else { - if err := sppIndexers.StoreIndexes(false, transactionID); err != nil { - return nil, err - } - } - return sppIndexers, nil -} - func (api *APIerSv1) computeDispatcherIndexes(tenant, context string, dspIDs *[]string, transactionID string) (filterIndexer *engine.FilterIndexer, err error) { var dispatcherIDs []string diff --git a/engine/datamanager.go b/engine/datamanager.go index abdae30e6..c39a9518d 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -1933,12 +1933,20 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) supp.Tenant).RemoveItemFromIndex(supp.Tenant, supp.ID, oldSup.FilterIDs); err != nil { return } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + supp.Tenant, supp.ID, supp.FilterIDs); err != nil { + return + } } } if err = createAndIndex(utils.SupplierProfilePrefix, supp.Tenant, utils.EmptyString, supp.ID, supp.FilterIDs, dm); err != nil { return } + if err = addReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + supp.Tenant, supp.ID, supp.FilterIDs); err != nil { + return + } } if config.CgrConfig().DataDbCfg().Items[utils.MetaSupplierProfiles].Replicate { var reply string @@ -1967,6 +1975,10 @@ func (dm *DataManager) RemoveSupplierProfile(tenant, id, transactionID string, w tenant).RemoveItemFromIndex(tenant, id, oldSupp.FilterIDs); err != nil { return } + if err = removeReverseFilterIndexForFilter(dm, utils.CacheSupplierFilterIndexes, utils.EmptyString, + oldSupp.Tenant, oldSupp.ID, oldSupp.FilterIDs); err != nil { + return + } } if config.CgrConfig().DataDbCfg().Items[utils.MetaSupplierProfiles].Replicate { var reply string diff --git a/engine/filter_indexes.go b/engine/filter_indexes.go index 611563e66..7f702505d 100644 --- a/engine/filter_indexes.go +++ b/engine/filter_indexes.go @@ -253,3 +253,79 @@ func ComputeResourceIndexes(dm *DataManager, tenant string, rsIDs *[]string, } return rpIndexers, nil } + +func ComputeSupplierIndexes(dm *DataManager, tenant string, sppIDs *[]string, + transactionID string) (filterIndexer *FilterIndexer, err error) { + var supplierIDs []string + var sppIndexers *FilterIndexer + if sppIDs == nil { + ids, err := dm.DataDB().GetKeysForPrefix(utils.SupplierProfilePrefix) + if err != nil { + return nil, err + } + for _, id := range ids { + supplierIDs = append(supplierIDs, strings.Split(id, utils.CONCATENATED_KEY_SEP)[1]) + } + // this will be on ComputeIndexes that contains empty indexes + sppIndexers = NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant) + } else { + // this will be on ComputeIndexesIDs that contains the old indexes from the next getter + var oldIDx map[string]utils.StringMap + if oldIDx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix], + tenant, utils.EmptyString, nil); err != nil || oldIDx == nil { + sppIndexers = NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant) + } else { + sppIndexers = NewFilterIndexerWithIndexes(dm, utils.SupplierProfilePrefix, tenant, oldIDx) + } + supplierIDs = *sppIDs + transactionID = utils.NonTransactional + } + for _, id := range supplierIDs { + spp, err := dm.GetSupplierProfile(tenant, id, true, false, utils.NonTransactional) + if err != nil { + return nil, err + } + fltrIDs := make([]string, len(spp.FilterIDs)) + for i, fltrID := range spp.FilterIDs { + fltrIDs[i] = fltrID + } + if len(fltrIDs) == 0 { + fltrIDs = []string{utils.META_NONE} + } + for _, fltrID := range fltrIDs { + var fltr *Filter + if fltrID == utils.META_NONE { + fltr = &Filter{ + Tenant: spp.Tenant, + ID: spp.ID, + Rules: []*FilterRule{ + { + Type: utils.META_NONE, + Element: utils.META_ANY, + Values: []string{utils.META_ANY}, + }, + }, + } + } else if fltr, err = GetFilter(dm, spp.Tenant, fltrID, + true, false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for suppliers: %+v", + fltrID, spp) + } + return nil, err + } + sppIndexers.IndexTPFilter(FilterToTPFilter(fltr), spp.ID) + } + } + if transactionID == utils.NonTransactional { + if err := sppIndexers.StoreIndexes(true, transactionID); err != nil { + return nil, err + } + return nil, nil + } else { + if err := sppIndexers.StoreIndexes(false, transactionID); err != nil { + return nil, err + } + } + return sppIndexers, nil +} diff --git a/engine/libindex.go b/engine/libindex.go index cbf701263..040598388 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -157,6 +157,18 @@ func UpdateFilterIndexes(dm *DataManager, tnt string, oldFltr *Filter, newFltr * utils.NonTransactional); err != nil { return err } + case utils.CacheSupplierFilterIndexes: + // remove the indexes from this filter for this partition + if err = removeFilterIndexesForFilter(dm, idxItmType, utils.CacheSupplierProfiles, + tnt, removeIndexKeys, index); err != nil { + return + } + // we removed the old reverse indexes, now we have to compute the new ones + supplierIDs := index.Slice() + if _, err = ComputeSupplierIndexes(dm, newFltr.Tenant, &supplierIDs, + utils.NonTransactional); err != nil { + return err + } } } return nil diff --git a/general_tests/filter_indexes_cases_it_test.go b/general_tests/filter_indexes_cases_it_test.go index 942a81bad..deadc8876 100644 --- a/general_tests/filter_indexes_cases_it_test.go +++ b/general_tests/filter_indexes_cases_it_test.go @@ -131,13 +131,19 @@ var ( // RESOURCES testFilterIndexesCasesSetResourceWithFltr, testFilterIndexesCasesGetResourcesIndexes, - /* testFilterIndexesCasesOverwriteFilterForResources, - testFilterIndexesCasesGetResourcesIndexesChanged, + /* + testFilterIndexesCasesOverwriteFilterForResources, + testFilterIndexesCasesGetResourcesIndexesChanged, - testFilterIndexesCasesGetReverseFilterIndexes3, - testFilterIndexesCasesRemoveResourcesProfile, - testFilterIndexesCasesGetIndexesAfterRemove3, - testFilterIndexesCasesGetReverseIndexesAfterRemove3, */ + + testFilterIndexesCasesGetReverseFilterIndexes3, + testFilterIndexesCasesRemoveResourcesProfile, + testFilterIndexesCasesGetIndexesAfterRemove3, + testFilterIndexesCasesGetReverseIndexesAfterRemove3, */ + + // SUPPLIER + + // DISPATCHER testFilterIndexesCasesStopEngine, }