From 132d20626616efbef45772718b6849ce2018b47d Mon Sep 17 00:00:00 2001 From: adi Date: Thu, 5 Jan 2023 15:21:51 +0200 Subject: [PATCH] Finished updates for mongo driver and tests --- apier/v1/filter_indexes.go | 2 +- engine/datamanager.go | 4 +- engine/filterindexer_it_test.go | 6 +- engine/storage_mongo_datadb.go | 51 ++++-- general_tests/filter_indexes_cases_it_test.go | 162 ++++++++++-------- 5 files changed, 128 insertions(+), 97 deletions(-) diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index cd066b8ad..374de69f4 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -113,7 +113,7 @@ func (api *APIerSv1) GetFilterIndexes(arg AttrGetFilterIndexes, reply *[]string) arg.ItemType = utils.ReverseFilterIndexes } if indexes, err = api.DataManager.GetFilterIndexes( - utils.PrefixToIndexCache[arg.ItemType], key, "", nil); err != nil { + utils.PrefixToIndexCache[arg.ItemType], key, utils.EmptyString, nil); err != nil { return err } if arg.FilterType != "" { diff --git a/engine/datamanager.go b/engine/datamanager.go index b8492ac9a..dd1dd5c05 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -657,8 +657,8 @@ func (dm *DataManager) RemoveFilter(tenant, id, transactionID string) (err error // we cannot remove a filter if it is referenced to a profile tntFltrID := utils.ConcatenatedKey(tenant, id) var rcvIdx map[string]utils.StringMap - if rcvIdx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntFltrID, - utils.EmptyString, nil); err != nil { + if rcvIdx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], + tntFltrID, utils.EmptyString, nil); err != nil { if err != utils.ErrNotFound { return } diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index f0a22f0ce..053ed3c56 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -197,8 +197,7 @@ func testITGetFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcv) { t.Errorf("Expecting: %+v, received: %+v", eIdxes, rcv) } - if _, err := dataManager.GetFilterIndexes("unknown_key", "unkonwn_tenant", - utils.EmptyString, nil); err == nil || err != utils.ErrNotFound { + if _, err := dataManager.GetFilterIndexes("unknown_key", "unkonwn_tenant", utils.EmptyString, nil); err == nil || err != utils.ErrNotFound { t.Error(err) } } @@ -283,8 +282,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant) if rcvIdx, err := dataManager.GetFilterIndexes( - utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, - utils.EmptyString, nil); err != nil { + utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, utils.EmptyString, nil); err != nil { t.Error(err) } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 056db948c..87da7fde8 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1572,7 +1572,8 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st } var results []result dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix - if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) { // case for reverse filter indexes, key is different + // case for reverse filter indexes, key is different, must not use regex finding for key + if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) { if err = ms.query(func(sctx mongo.SessionContext) (err error) { cur, err := ms.getCol(ColRFI).Find(sctx, bson.M{"key": utils.ConcatenatedKey(dbKey)}) @@ -1593,7 +1594,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st if len(results) == 0 { return nil, utils.ErrNotFound } - } else if len(fldNameVal) != 0 { + } else if len(fldNameVal) != 0 { // case for searching of a field:value for fldName, fldValue := range fldNameVal { if err = ms.query(func(sctx mongo.SessionContext) (err error) { cur, err := ms.getCol(ColRFI).Find(sctx, @@ -1642,6 +1643,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st } } indexes = make(map[string]utils.StringMap) + for _, res := range results { if len(res.Value) == 0 { continue @@ -1661,7 +1663,6 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st indexes[idxTypeItmID[0]].Copy(map[string]bool{ idxTypeItmID[1]: true, }) - } continue } @@ -1672,6 +1673,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st } indexes[indexKey] = utils.StringMapFromSlice(res.Value) } + if len(indexes) == 0 { return nil, utils.ErrNotFound } @@ -1724,34 +1726,49 @@ func (ms *MongoStorage) SetFilterIndexesDrv(cacheID, itemIDPrefix string, }) } else { var lastErr error - for key, itmMp := range indexes { + // forming reverse filter indexes (the format of those is different from normal indexes) + if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) { + replaceItmMP := make(utils.StringMap) if err = ms.query(func(sctx mongo.SessionContext) (err error) { - var idxDbkey string - // reverse filter indexes case - if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) { - idxDbkey = dbKey - replaceItmMP := make(utils.StringMap) + for key, itmMp := range indexes { for itemID := range itmMp { replaceItmMP.Copy(utils.StringMap{ utils.ConcatenatedKey(key, itemID): true, }) } - itmMp = replaceItmMP - } else { // normal filter indexes case - idxDbkey = utils.ConcatenatedKey(dbKey, key) } - if len(itmMp) == 0 { // remove from DB if we set it with empty indexes + if len(replaceItmMP) == 0 { // remove from DB if we set it with empty indexes _, err = ms.getCol(ColRFI).DeleteOne(sctx, - bson.M{"key": idxDbkey}) + bson.M{"key": dbKey}) } else { - _, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": idxDbkey}, - bson.M{"$set": bson.M{"key": idxDbkey, "value": itmMp.Slice()}}, + _, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": dbKey}, + bson.M{"$set": bson.M{"key": dbKey, "value": replaceItmMP.Slice()}}, options.Update().SetUpsert(true), ) } return err }); err != nil { - lastErr = err + return err + } + } else { + // forming normal indexes + for key, itmMp := range indexes { + if err = ms.query(func(sctx mongo.SessionContext) (err error) { + var idxDbkey string + idxDbkey = utils.ConcatenatedKey(dbKey, key) + if len(itmMp) == 0 { // remove from DB if we set it with empty indexes + _, err = ms.getCol(ColRFI).DeleteOne(sctx, + bson.M{"key": idxDbkey}) + } else { + _, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": idxDbkey}, + bson.M{"$set": bson.M{"key": idxDbkey, "value": itmMp.Slice()}}, + options.Update().SetUpsert(true), + ) + } + return err + }); err != nil { + lastErr = err + } } } return lastErr diff --git a/general_tests/filter_indexes_cases_it_test.go b/general_tests/filter_indexes_cases_it_test.go index bebf4fdfa..93153c396 100644 --- a/general_tests/filter_indexes_cases_it_test.go +++ b/general_tests/filter_indexes_cases_it_test.go @@ -94,93 +94,91 @@ var ( testFilterIndexesCasesStartEngine, testFilterIndexesCasesRpcConn, - /* - // ATTRIBUTES - testFilterIndexesCasesSetFilters, - testFilterIndexesCasesSetAttributesWithFilters, - testFilterIndexesCasesGetIndexesAnyContext, - testFilterIndexesCasesGetIndexesSessionsContext, + // ATTRIBUTES + testFilterIndexesCasesSetFilters, + testFilterIndexesCasesSetAttributesWithFilters, + testFilterIndexesCasesGetIndexesAnyContext, + testFilterIndexesCasesGetIndexesSessionsContext, - testFilterIndexesCasesSetDifferentFilters, - testFilterIndexesCasesOverwriteAttributes, - testFilterIndexesCasesComputeAttributesIndexes, - testFilterIndexesCasesGetIndexesAnyContextChanged, - testFilterIndexesCasesGetIndexesSessionsContextChanged, + testFilterIndexesCasesSetDifferentFilters, + testFilterIndexesCasesOverwriteAttributes, + testFilterIndexesCasesComputeAttributesIndexes, + testFilterIndexesCasesGetIndexesAnyContextChanged, + testFilterIndexesCasesGetIndexesSessionsContextChanged, - // CHARGERS - testFilterIndexesCasesSetIndexedFilter, - testFilterIndexesCasesSetChargerWithFltr, - testFilterIndexesCasesGetChargerIndexes, - testFilterIndexesCasesOverwriteFilterForCharger, - testFilterIndexesCasesGetChargerIndexesChanged, + // CHARGERS + testFilterIndexesCasesSetIndexedFilter, + testFilterIndexesCasesSetChargerWithFltr, + testFilterIndexesCasesGetChargerIndexes, + testFilterIndexesCasesOverwriteFilterForCharger, + testFilterIndexesCasesGetChargerIndexesChanged, - testFilterIndexesCasesGetReverseFilterIndexes, // for chargers - testFilterIndexesCasesRemoveChargerProfile, - testFilterIndexesCasesGetIndexesAfterRemove, - testFilterIndexesCasesGetReverseIndexesAfterRemove, + testFilterIndexesCasesGetReverseFilterIndexes, // for chargers + testFilterIndexesCasesRemoveChargerProfile, + testFilterIndexesCasesGetIndexesAfterRemove, + testFilterIndexesCasesGetReverseIndexesAfterRemove, - // THRESHOLDS - testFilterIndexesCasesSetThresholdWithFltr, - testFilterIndexesCasesGetThresholdsIndexes, - testFilterIndexesCasesOverwriteFilterForThresholds, - testFilterIndexesCasesGetThresholdsIndexesChanged, + // THRESHOLDS + testFilterIndexesCasesSetThresholdWithFltr, + testFilterIndexesCasesGetThresholdsIndexes, + testFilterIndexesCasesOverwriteFilterForThresholds, + testFilterIndexesCasesGetThresholdsIndexesChanged, - testFilterIndexesCasesGetReverseFilterIndexes2, - testFilterIndexesCasesRemoveThresholdsProfile, - testFilterIndexesCasesGetIndexesAfterRemove2, - testFilterIndexesCasesGetReverseIndexesAfterRemove2, + testFilterIndexesCasesGetReverseFilterIndexes2, + testFilterIndexesCasesRemoveThresholdsProfile, + testFilterIndexesCasesGetIndexesAfterRemove2, + testFilterIndexesCasesGetReverseIndexesAfterRemove2, + // DISPATCHER + testFilterIndexesCasesSetDispatcherWithFltr, + testFilterIndexesCasesGetDispatchersIndexesAnyContext, + testFilterIndexesCasesGetDispatchersIndexesDifferentContext, + testFilterIndexesCasesOverwriteFilterForDispatchers, + testFilterIndexesCasesGetDispatchersIndexesChangedAnyContext, + testFilterIndexesCasesGetDispatchersIndexesChangedDifferentContext, - // DISPATCHER - testFilterIndexesCasesSetDispatcherWithFltr, - testFilterIndexesCasesGetDispatchersIndexesAnyContext, - testFilterIndexesCasesGetDispatchersIndexesDifferentContext, - testFilterIndexesCasesOverwriteFilterForDispatchers, - testFilterIndexesCasesGetDispatchersIndexesChangedAnyContext, - testFilterIndexesCasesGetDispatchersIndexesChangedDifferentContext, + testFilterIndexesCasesGetReverseFilterIndexes6, + testFilterIndexesCasesRemoveDispatchersProfile, + testFilterIndexesCasesGetIndexesAfterRemoveAnyContext, + testFilterIndexesCasesGetIndexesAfterRemoveDifferentContext, + testFilterIndexesCasesGetReverseIndexesAfterRemove6, + testFilterIndexesCasesOverwriteDispatchersProfile, + testFilterIndexesCasesOverwriteDispatchersGetIndexesEveryContext, + testFilterIndexesCasesOverwriteDispatchersGetReverseIndexes, - testFilterIndexesCasesGetReverseFilterIndexes6, - testFilterIndexesCasesRemoveDispatchersProfile, - testFilterIndexesCasesGetIndexesAfterRemoveAnyContext, - testFilterIndexesCasesGetIndexesAfterRemoveDifferentContext, - testFilterIndexesCasesGetReverseIndexesAfterRemove6, - testFilterIndexesCasesOverwriteDispatchersProfile, - testFilterIndexesCasesOverwriteDispatchersGetIndexesEveryContext, - testFilterIndexesCasesOverwriteDispatchersGetReverseIndexes, + // RESOURCES + testFilterIndexesCasesSetResourceWithFltr, + testFilterIndexesCasesGetResourcesIndexes, + testFilterIndexesCasesOverwriteFilterForResources, + testFilterIndexesCasesGetResourcesIndexesChanged, - // RESOURCES - testFilterIndexesCasesSetResourceWithFltr, - testFilterIndexesCasesGetResourcesIndexes, - testFilterIndexesCasesOverwriteFilterForResources, - testFilterIndexesCasesGetResourcesIndexesChanged, + testFilterIndexesCasesGetReverseFilterIndexes3, + testFilterIndexesCasesRemoveResourcesProfile, + testFilterIndexesCasesGetIndexesAfterRemove3, + testFilterIndexesCasesGetReverseIndexesAfterRemove3, + testFilterIndexesCasesOverwriteResourceProfiles, + testFilterIndexesCasesResourcesGetIndexesAfterOverwrite, + testFilterIndexesCasesResourcesGetReverseIndexesAfterOverwrite, - testFilterIndexesCasesGetReverseFilterIndexes3, - testFilterIndexesCasesRemoveResourcesProfile, - testFilterIndexesCasesGetIndexesAfterRemove3, - testFilterIndexesCasesGetReverseIndexesAfterRemove3, - testFilterIndexesCasesOverwriteResourceProfiles, - testFilterIndexesCasesResourcesGetIndexesAfterOverwrite, - testFilterIndexesCasesResourcesGetReverseIndexesAfterOverwrite, + // SUPPLIER + testFilterIndexesCasesSetSupplierWithFltr, + testFilterIndexesCasesGetSuppliersIndexes, + testFilterIndexesCasesOverwriteFilterForSuppliers, + testFilterIndexesCasesGetSuppliersIndexesChanged, - // SUPPLIER - testFilterIndexesCasesSetSupplierWithFltr, - testFilterIndexesCasesGetSuppliersIndexes, - testFilterIndexesCasesOverwriteFilterForSuppliers, - testFilterIndexesCasesGetSuppliersIndexesChanged, - - testFilterIndexesCasesGetReverseFilterIndexes4, - testFilterIndexesCasesRemoveSuppliersProfile, - testFilterIndexesCasesGetIndexesAfterRemove4, - testFilterIndexesCasesGetReverseIndexesAfterRemove4, - testFilterIndexesCasesOverwriteSupplierProfiles, - testFilterIndexesCasesSuppliersGetIndexesAfterOverwrite, - testFilterIndexesCasesSuppliersGetReverseIndexesAfterOverwrite, */ + testFilterIndexesCasesGetReverseFilterIndexes4, + testFilterIndexesCasesRemoveSuppliersProfile, + testFilterIndexesCasesGetIndexesAfterRemove4, + testFilterIndexesCasesGetReverseIndexesAfterRemove4, + testFilterIndexesCasesOverwriteSupplierProfiles, + testFilterIndexesCasesSuppliersGetIndexesAfterOverwrite, + testFilterIndexesCasesSuppliersGetReverseIndexesAfterOverwrite, // STATS //testFilterIndexesCasesOverwriteFilterForSuppliers1, - /* testFilterIndexesCasesSetStatQueueWithFltr, + testFilterIndexesCasesSetStatQueueWithFltr, testFilterIndexesCasesGetStatQueuesIndexes, testFilterIndexesCasesOverwriteFilterForStatQueues, testFilterIndexesCasesGetStatQueuesIndexesChanged, @@ -191,9 +189,9 @@ var ( testFilterIndexesCasesGetReverseIndexesAfterRemove5, testFilterIndexesCasesOverwriteStatQueueProfiles, testFilterIndexesCasesStatQueuesGetIndexesAfterOverwrite, - testFilterIndexesCasesStatQueuesGetReverseIndexesAfterOverwrite, */ + testFilterIndexesCasesStatQueuesGetReverseIndexesAfterOverwrite, - testMongoFIdx, + //testMongoFIdx, testFilterIndexesCasesStopEngine, } @@ -3720,6 +3718,24 @@ func testMongoFIdx(t *testing.T) { t.Error("Unexpected reply returned", reply) } + tPrfl1 := &engine.ThresholdWithCache{ + ThresholdProfile: &engine.ThresholdProfile{ + Tenant: "cgrates.org", + ID: "TEST_PROFILE1", + FilterIDs: []string{"FLTR_Charger"}, + MaxHits: 1, + MinSleep: time.Duration(5 * time.Minute), + Blocker: false, + Weight: 10.0, + Async: true, + }, + } + if err := fIdxCasesRPC.Call(utils.APIerSv1SetThresholdProfile, tPrfl1, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Error("Unexpected reply returned", reply) + } + arg := &v1.AttrGetFilterIndexes{ Tenant: "cgrates.org", ItemType: utils.MetaStats,