Finished updates for mongo driver and tests

apier/v1/filter_indexes.go

@@ -113,7 +113,7 @@ func (api *APIerSv1) GetFilterIndexes(arg AttrGetFilterIndexes, reply *[]string)
 		arg.ItemType = utils.ReverseFilterIndexes
 	}
 	if indexes, err = api.DataManager.GetFilterIndexes(
-		utils.PrefixToIndexCache[arg.ItemType], key, "", nil); err != nil {
+		utils.PrefixToIndexCache[arg.ItemType], key, utils.EmptyString, nil); err != nil {
 		return err
 	}
 	if arg.FilterType != "" {

engine/datamanager.go

@@ -657,8 +657,8 @@ func (dm *DataManager) RemoveFilter(tenant, id, transactionID string) (err error
 	// we cannot remove a filter if it is referenced to a profile
 	tntFltrID := utils.ConcatenatedKey(tenant, id)
 	var rcvIdx map[string]utils.StringMap
-	if rcvIdx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntFltrID,
-		utils.EmptyString, nil); err != nil {
+	if rcvIdx, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes],
+		tntFltrID, utils.EmptyString, nil); err != nil {
 		if err != utils.ErrNotFound {
 			return
 		}

engine/filterindexer_it_test.go

@@ -197,8 +197,7 @@ func testITGetFilterIndexes(t *testing.T) {
 	} else if !reflect.DeepEqual(eIdxes, rcv) {
 		t.Errorf("Expecting: %+v, received: %+v", eIdxes, rcv)
 	}
-	if _, err := dataManager.GetFilterIndexes("unknown_key", "unkonwn_tenant",
-		utils.EmptyString, nil); err == nil || err != utils.ErrNotFound {
+	if _, err := dataManager.GetFilterIndexes("unknown_key", "unkonwn_tenant", utils.EmptyString, nil); err == nil || err != utils.ErrNotFound {
 		t.Error(err)
 	}
 }
@@ -283,8 +282,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
 	}
 	rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
 	if rcvIdx, err := dataManager.GetFilterIndexes(
-		utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
-		utils.EmptyString, nil); err != nil {
+		utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, utils.EmptyString, nil); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
 		t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)

engine/storage_mongo_datadb.go

@@ -1572,7 +1572,8 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st
 	}
 	var results []result
 	dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix
-	if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) { // case for reverse filter indexes, key is different
+	// case for reverse filter indexes, key is different, must not use regex finding for key
+	if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) {
 		if err = ms.query(func(sctx mongo.SessionContext) (err error) {
 			cur, err := ms.getCol(ColRFI).Find(sctx,
 				bson.M{"key": utils.ConcatenatedKey(dbKey)})
@@ -1593,7 +1594,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st
 		if len(results) == 0 {
 			return nil, utils.ErrNotFound
 		}
-	} else if len(fldNameVal) != 0 {
+	} else if len(fldNameVal) != 0 { // case for searching of a field:value
 		for fldName, fldValue := range fldNameVal {
 			if err = ms.query(func(sctx mongo.SessionContext) (err error) {
 				cur, err := ms.getCol(ColRFI).Find(sctx,
@@ -1642,6 +1643,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st
 		}
 	}
 	indexes = make(map[string]utils.StringMap)
+
 	for _, res := range results {
 		if len(res.Value) == 0 {
 			continue
@@ -1661,7 +1663,6 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st
 				indexes[idxTypeItmID[0]].Copy(map[string]bool{
 					idxTypeItmID[1]: true,
 				})
-
 			}
 			continue
 		}
@@ -1672,6 +1673,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(cacheID, itemIDPrefix, filterType st
 		}
 		indexes[indexKey] = utils.StringMapFromSlice(res.Value)
 	}
+
 	if len(indexes) == 0 {
 		return nil, utils.ErrNotFound
 	}
@@ -1724,34 +1726,49 @@ func (ms *MongoStorage) SetFilterIndexesDrv(cacheID, itemIDPrefix string,
 		})
 	} else {
 		var lastErr error
-		for key, itmMp := range indexes {
+		// forming reverse filter indexes (the format of those is different from normal indexes)
+		if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) {
+			replaceItmMP := make(utils.StringMap)
 			if err = ms.query(func(sctx mongo.SessionContext) (err error) {
-				var idxDbkey string
-				// reverse filter indexes case
-				if strings.HasPrefix(dbKey, utils.ReverseFilterIndexes) {
-					idxDbkey = dbKey
-					replaceItmMP := make(utils.StringMap)
+				for key, itmMp := range indexes {
 					for itemID := range itmMp {
 						replaceItmMP.Copy(utils.StringMap{
 							utils.ConcatenatedKey(key, itemID): true,
 						})
 					}
-					itmMp = replaceItmMP
-				} else { // normal filter indexes case
-					idxDbkey = utils.ConcatenatedKey(dbKey, key)
 				}
-				if len(itmMp) == 0 { // remove from DB if we set it with empty indexes
+				if len(replaceItmMP) == 0 { // remove from DB if we set it with empty indexes
 					_, err = ms.getCol(ColRFI).DeleteOne(sctx,
-						bson.M{"key": idxDbkey})
+						bson.M{"key": dbKey})
 				} else {
-					_, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": idxDbkey},
-						bson.M{"$set": bson.M{"key": idxDbkey, "value": itmMp.Slice()}},
+					_, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": dbKey},
+						bson.M{"$set": bson.M{"key": dbKey, "value": replaceItmMP.Slice()}},
 						options.Update().SetUpsert(true),
 					)
 				}
 				return err
 			}); err != nil {
-				lastErr = err
+				return err
+			}
+		} else {
+			// forming normal indexes
+			for key, itmMp := range indexes {
+				if err = ms.query(func(sctx mongo.SessionContext) (err error) {
+					var idxDbkey string
+					idxDbkey = utils.ConcatenatedKey(dbKey, key)
+					if len(itmMp) == 0 { // remove from DB if we set it with empty indexes
+						_, err = ms.getCol(ColRFI).DeleteOne(sctx,
+							bson.M{"key": idxDbkey})
+					} else {
+						_, err = ms.getCol(ColRFI).UpdateOne(sctx, bson.M{"key": idxDbkey},
+							bson.M{"$set": bson.M{"key": idxDbkey, "value": itmMp.Slice()}},
+							options.Update().SetUpsert(true),
+						)
+					}
+					return err
+				}); err != nil {
+					lastErr = err
+				}
 			}
 		}
 		return lastErr

general_tests/filter_indexes_cases_it_test.go

@@ -94,93 +94,91 @@ var (
 		testFilterIndexesCasesStartEngine,
 		testFilterIndexesCasesRpcConn,
 
-		/*
-			// ATTRIBUTES
-			testFilterIndexesCasesSetFilters,
-			testFilterIndexesCasesSetAttributesWithFilters,
-			testFilterIndexesCasesGetIndexesAnyContext,
-			testFilterIndexesCasesGetIndexesSessionsContext,
+		// ATTRIBUTES
+		testFilterIndexesCasesSetFilters,
+		testFilterIndexesCasesSetAttributesWithFilters,
+		testFilterIndexesCasesGetIndexesAnyContext,
+		testFilterIndexesCasesGetIndexesSessionsContext,
 
-			testFilterIndexesCasesSetDifferentFilters,
-			testFilterIndexesCasesOverwriteAttributes,
-			testFilterIndexesCasesComputeAttributesIndexes,
-			testFilterIndexesCasesGetIndexesAnyContextChanged,
-			testFilterIndexesCasesGetIndexesSessionsContextChanged,
+		testFilterIndexesCasesSetDifferentFilters,
+		testFilterIndexesCasesOverwriteAttributes,
+		testFilterIndexesCasesComputeAttributesIndexes,
+		testFilterIndexesCasesGetIndexesAnyContextChanged,
+		testFilterIndexesCasesGetIndexesSessionsContextChanged,
 
-			// CHARGERS
-			testFilterIndexesCasesSetIndexedFilter,
-			testFilterIndexesCasesSetChargerWithFltr,
-			testFilterIndexesCasesGetChargerIndexes,
-			testFilterIndexesCasesOverwriteFilterForCharger,
-			testFilterIndexesCasesGetChargerIndexesChanged,
+		// CHARGERS
+		testFilterIndexesCasesSetIndexedFilter,
+		testFilterIndexesCasesSetChargerWithFltr,
+		testFilterIndexesCasesGetChargerIndexes,
+		testFilterIndexesCasesOverwriteFilterForCharger,
+		testFilterIndexesCasesGetChargerIndexesChanged,
 
-			testFilterIndexesCasesGetReverseFilterIndexes, // for chargers
-			testFilterIndexesCasesRemoveChargerProfile,
-			testFilterIndexesCasesGetIndexesAfterRemove,
-			testFilterIndexesCasesGetReverseIndexesAfterRemove,
+		testFilterIndexesCasesGetReverseFilterIndexes, // for chargers
+		testFilterIndexesCasesRemoveChargerProfile,
+		testFilterIndexesCasesGetIndexesAfterRemove,
+		testFilterIndexesCasesGetReverseIndexesAfterRemove,
 
-			// THRESHOLDS
-			testFilterIndexesCasesSetThresholdWithFltr,
-			testFilterIndexesCasesGetThresholdsIndexes,
-			testFilterIndexesCasesOverwriteFilterForThresholds,
-			testFilterIndexesCasesGetThresholdsIndexesChanged,
+		// THRESHOLDS
+		testFilterIndexesCasesSetThresholdWithFltr,
+		testFilterIndexesCasesGetThresholdsIndexes,
+		testFilterIndexesCasesOverwriteFilterForThresholds,
+		testFilterIndexesCasesGetThresholdsIndexesChanged,
 
-			testFilterIndexesCasesGetReverseFilterIndexes2,
-			testFilterIndexesCasesRemoveThresholdsProfile,
-			testFilterIndexesCasesGetIndexesAfterRemove2,
-			testFilterIndexesCasesGetReverseIndexesAfterRemove2,
+		testFilterIndexesCasesGetReverseFilterIndexes2,
+		testFilterIndexesCasesRemoveThresholdsProfile,
+		testFilterIndexesCasesGetIndexesAfterRemove2,
+		testFilterIndexesCasesGetReverseIndexesAfterRemove2,
 
+		// DISPATCHER
+		testFilterIndexesCasesSetDispatcherWithFltr,
+		testFilterIndexesCasesGetDispatchersIndexesAnyContext,
+		testFilterIndexesCasesGetDispatchersIndexesDifferentContext,
+		testFilterIndexesCasesOverwriteFilterForDispatchers,
+		testFilterIndexesCasesGetDispatchersIndexesChangedAnyContext,
+		testFilterIndexesCasesGetDispatchersIndexesChangedDifferentContext,
 
-				// DISPATCHER
-				testFilterIndexesCasesSetDispatcherWithFltr,
-				testFilterIndexesCasesGetDispatchersIndexesAnyContext,
-				testFilterIndexesCasesGetDispatchersIndexesDifferentContext,
-				testFilterIndexesCasesOverwriteFilterForDispatchers,
-				testFilterIndexesCasesGetDispatchersIndexesChangedAnyContext,
-				testFilterIndexesCasesGetDispatchersIndexesChangedDifferentContext,
+		testFilterIndexesCasesGetReverseFilterIndexes6,
+		testFilterIndexesCasesRemoveDispatchersProfile,
+		testFilterIndexesCasesGetIndexesAfterRemoveAnyContext,
+		testFilterIndexesCasesGetIndexesAfterRemoveDifferentContext,
+		testFilterIndexesCasesGetReverseIndexesAfterRemove6,
+		testFilterIndexesCasesOverwriteDispatchersProfile,
+		testFilterIndexesCasesOverwriteDispatchersGetIndexesEveryContext,
+		testFilterIndexesCasesOverwriteDispatchersGetReverseIndexes,
 
-				testFilterIndexesCasesGetReverseFilterIndexes6,
-				testFilterIndexesCasesRemoveDispatchersProfile,
-				testFilterIndexesCasesGetIndexesAfterRemoveAnyContext,
-				testFilterIndexesCasesGetIndexesAfterRemoveDifferentContext,
-				testFilterIndexesCasesGetReverseIndexesAfterRemove6,
-				testFilterIndexesCasesOverwriteDispatchersProfile,
-				testFilterIndexesCasesOverwriteDispatchersGetIndexesEveryContext,
-				testFilterIndexesCasesOverwriteDispatchersGetReverseIndexes,
+		// RESOURCES
+		testFilterIndexesCasesSetResourceWithFltr,
+		testFilterIndexesCasesGetResourcesIndexes,
+		testFilterIndexesCasesOverwriteFilterForResources,
+		testFilterIndexesCasesGetResourcesIndexesChanged,
 
-				// RESOURCES
-				testFilterIndexesCasesSetResourceWithFltr,
-				testFilterIndexesCasesGetResourcesIndexes,
-				testFilterIndexesCasesOverwriteFilterForResources,
-				testFilterIndexesCasesGetResourcesIndexesChanged,
+		testFilterIndexesCasesGetReverseFilterIndexes3,
+		testFilterIndexesCasesRemoveResourcesProfile,
+		testFilterIndexesCasesGetIndexesAfterRemove3,
+		testFilterIndexesCasesGetReverseIndexesAfterRemove3,
+		testFilterIndexesCasesOverwriteResourceProfiles,
+		testFilterIndexesCasesResourcesGetIndexesAfterOverwrite,
+		testFilterIndexesCasesResourcesGetReverseIndexesAfterOverwrite,
 
-				testFilterIndexesCasesGetReverseFilterIndexes3,
-				testFilterIndexesCasesRemoveResourcesProfile,
-				testFilterIndexesCasesGetIndexesAfterRemove3,
-				testFilterIndexesCasesGetReverseIndexesAfterRemove3,
-				testFilterIndexesCasesOverwriteResourceProfiles,
-				testFilterIndexesCasesResourcesGetIndexesAfterOverwrite,
-				testFilterIndexesCasesResourcesGetReverseIndexesAfterOverwrite,
+		// SUPPLIER
+		testFilterIndexesCasesSetSupplierWithFltr,
+		testFilterIndexesCasesGetSuppliersIndexes,
+		testFilterIndexesCasesOverwriteFilterForSuppliers,
+		testFilterIndexesCasesGetSuppliersIndexesChanged,
 
-				// SUPPLIER
-				testFilterIndexesCasesSetSupplierWithFltr,
-				testFilterIndexesCasesGetSuppliersIndexes,
-				testFilterIndexesCasesOverwriteFilterForSuppliers,
-				testFilterIndexesCasesGetSuppliersIndexesChanged,
-
-				testFilterIndexesCasesGetReverseFilterIndexes4,
-				testFilterIndexesCasesRemoveSuppliersProfile,
-				testFilterIndexesCasesGetIndexesAfterRemove4,
-				testFilterIndexesCasesGetReverseIndexesAfterRemove4,
-				testFilterIndexesCasesOverwriteSupplierProfiles,
-				testFilterIndexesCasesSuppliersGetIndexesAfterOverwrite,
-				testFilterIndexesCasesSuppliersGetReverseIndexesAfterOverwrite, */
+		testFilterIndexesCasesGetReverseFilterIndexes4,
+		testFilterIndexesCasesRemoveSuppliersProfile,
+		testFilterIndexesCasesGetIndexesAfterRemove4,
+		testFilterIndexesCasesGetReverseIndexesAfterRemove4,
+		testFilterIndexesCasesOverwriteSupplierProfiles,
+		testFilterIndexesCasesSuppliersGetIndexesAfterOverwrite,
+		testFilterIndexesCasesSuppliersGetReverseIndexesAfterOverwrite,
 
 		// STATS
 
 		//testFilterIndexesCasesOverwriteFilterForSuppliers1,
 
-		/* testFilterIndexesCasesSetStatQueueWithFltr,
+		testFilterIndexesCasesSetStatQueueWithFltr,
 		testFilterIndexesCasesGetStatQueuesIndexes,
 		testFilterIndexesCasesOverwriteFilterForStatQueues,
 		testFilterIndexesCasesGetStatQueuesIndexesChanged,
@@ -191,9 +189,9 @@ var (
 		testFilterIndexesCasesGetReverseIndexesAfterRemove5,
 		testFilterIndexesCasesOverwriteStatQueueProfiles,
 		testFilterIndexesCasesStatQueuesGetIndexesAfterOverwrite,
-		testFilterIndexesCasesStatQueuesGetReverseIndexesAfterOverwrite, */
+		testFilterIndexesCasesStatQueuesGetReverseIndexesAfterOverwrite,
 
-		testMongoFIdx,
+		//testMongoFIdx,
 
 		testFilterIndexesCasesStopEngine,
 	}
@@ -3720,6 +3718,24 @@ func testMongoFIdx(t *testing.T) {
 		t.Error("Unexpected reply returned", reply)
 	}
 
+	tPrfl1 := &engine.ThresholdWithCache{
+		ThresholdProfile: &engine.ThresholdProfile{
+			Tenant:    "cgrates.org",
+			ID:        "TEST_PROFILE1",
+			FilterIDs: []string{"FLTR_Charger"},
+			MaxHits:   1,
+			MinSleep:  time.Duration(5 * time.Minute),
+			Blocker:   false,
+			Weight:    10.0,
+			Async:     true,
+		},
+	}
+	if err := fIdxCasesRPC.Call(utils.APIerSv1SetThresholdProfile, tPrfl1, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned", reply)
+	}
+
 	arg := &v1.AttrGetFilterIndexes{
 		Tenant:   "cgrates.org",
 		ItemType: utils.MetaStats,