diff --git a/apier/v1/filter_indexes.go b/apier/v1/filter_indexes.go index ebba3ce32..ef77b1032 100644 --- a/apier/v1/filter_indexes.go +++ b/apier/v1/filter_indexes.go @@ -151,150 +151,7 @@ func (self *ApierV1) GetFilterIndexes(arg AttrGetFilterIndexes, reply *[]string) return nil } -type AttrGetFilterReverseIndexes struct { - Tenant string - Context string - ItemType string - ItemIDs []string - FilterType string - FilterField string - FilterValue string - utils.Paginator -} - -func (self *ApierV1) GetFilterReverseIndexes(arg AttrGetFilterReverseIndexes, reply *[]string) (err error) { - var indexes map[string]utils.StringMap - var indexedSlice []string - indexesFilter := make(map[string]utils.StringMap) - if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ItemType"}); len(missing) != 0 { //Params missing - return utils.NewErrMandatoryIeMissing(missing...) - } - key := arg.Tenant - switch arg.ItemType { - case utils.MetaThresholds: - arg.ItemType = utils.ThresholdProfilePrefix - case utils.MetaSuppliers: - arg.ItemType = utils.SupplierProfilePrefix - case utils.MetaStats: - arg.ItemType = utils.StatQueueProfilePrefix - case utils.MetaResources: - arg.ItemType = utils.ResourceProfilesPrefix - case utils.MetaChargers: - arg.ItemType = utils.ChargerProfilePrefix - case utils.MetaAttributes: - if missing := utils.MissingStructFields(&arg, []string{"Context"}); len(missing) != 0 { //Params missing - return utils.NewErrMandatoryIeMissing(missing...) - } - arg.ItemType = utils.AttributeProfilePrefix - key = utils.ConcatenatedKey(arg.Tenant, arg.Context) - } - if arg.ItemIDs != nil { - indexes = make(map[string]utils.StringMap) - for _, itemID := range arg.ItemIDs { - if tmpIndexes, err := self.DataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[arg.ItemType], key, map[string]string{itemID: ""}); err != nil { - return err - } else { - for key, val := range tmpIndexes { - indexes[key] = make(utils.StringMap) - indexes[key] = val - } - - } - } - } else { - indexes, err = self.DataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[arg.ItemType], key, nil) - if err != nil { - return err - } - } - if arg.FilterType != "" { - for val, strmap := range indexes { - indexesFilter[val] = make(utils.StringMap) - for _, value := range strmap.Slice() { - if strings.HasPrefix(value, arg.FilterType) { - indexesFilter[val][value] = true - indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if len(indexedSlice) == 0 { - return utils.ErrNotFound - } - } - if arg.FilterField != "" { - if len(indexedSlice) == 0 { - indexesFilter = make(map[string]utils.StringMap) - for val, strmap := range indexes { - indexesFilter[val] = make(utils.StringMap) - for _, value := range strmap.Slice() { - if strings.Index(value, arg.FilterField) != -1 { - indexesFilter[val][value] = true - indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if len(indexedSlice) == 0 { - return utils.ErrNotFound - } - } else { - var cloneIndexSlice []string - for val, strmap := range indexesFilter { - for _, value := range strmap.Slice() { - if strings.Index(value, arg.FilterField) != -1 { - cloneIndexSlice = append(cloneIndexSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if len(cloneIndexSlice) == 0 { - return utils.ErrNotFound - } - indexedSlice = cloneIndexSlice - } - } - if arg.FilterValue != "" { - if len(indexedSlice) == 0 { - for val, strmap := range indexes { - for _, value := range strmap.Slice() { - if strings.Index(value, arg.FilterValue) != -1 { - indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if len(indexedSlice) == 0 { - return utils.ErrNotFound - } - } else { - var cloneIndexSlice []string - for val, strmap := range indexesFilter { - for _, value := range strmap.Slice() { - if strings.Index(value, arg.FilterValue) != -1 { - cloneIndexSlice = append(cloneIndexSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if len(cloneIndexSlice) == 0 { - return utils.ErrNotFound - } - indexedSlice = cloneIndexSlice - } - } - if len(indexedSlice) == 0 { - for val, strmap := range indexes { - for _, value := range strmap.Slice() { - indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value)) - } - } - } - if arg.Paginator.Limit != nil || arg.Paginator.Offset != nil || arg.Paginator.SearchTerm != "" { - *reply = arg.Paginator.PaginateStringSlice(indexedSlice) - } else { - *reply = indexedSlice - } - return nil -} - +//NEED REVIEW func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, reply *string) error { transactionID := utils.GenUUID() //ThresholdProfile Indexes @@ -333,7 +190,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if err := thdsIndexers.StoreIndexes(true, transactionID); err != nil { if args.ThresholdIDs != nil { for _, id := range *args.ThresholdIDs { - if err := thdsIndexers.RemoveItemFromIndex(id); err != nil { + th, err := self.DataManager.GetThresholdProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := thdsIndexers.RemoveItemFromIndex(args.Tenant, id, th.FilterIDs); err != nil { return err } } @@ -345,7 +206,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if sqpIndexers != nil { if err := sqpIndexers.StoreIndexes(true, transactionID); err != nil { for _, id := range *args.StatIDs { - if err := sqpIndexers.RemoveItemFromIndex(id); err != nil { + sqp, err := self.DataManager.GetStatQueueProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := sqpIndexers.RemoveItemFromIndex(args.Tenant, id, sqp.FilterIDs); err != nil { return err } } @@ -356,7 +221,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if rsIndexes != nil { if err := rsIndexes.StoreIndexes(true, transactionID); err != nil { for _, id := range *args.ResourceIDs { - if err := rsIndexes.RemoveItemFromIndex(id); err != nil { + rp, err := self.DataManager.GetResourceProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := rsIndexes.RemoveItemFromIndex(args.Tenant, id, rp.FilterIDs); err != nil { return err } } @@ -367,7 +236,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if sppIndexes != nil { if err := sppIndexes.StoreIndexes(true, transactionID); err != nil { for _, id := range *args.SupplierIDs { - if err := sppIndexes.RemoveItemFromIndex(id); err != nil { + spp, err := self.DataManager.GetSupplierProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := sppIndexes.RemoveItemFromIndex(args.Tenant, id, spp.FilterIDs); err != nil { return err } } @@ -378,7 +251,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if attrIndexes != nil { if err := attrIndexes.StoreIndexes(true, transactionID); err != nil { for _, id := range *args.AttributeIDs { - if err := attrIndexes.RemoveItemFromIndex(id); err != nil { + ap, err := self.DataManager.GetAttributeProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := attrIndexes.RemoveItemFromIndex(args.Tenant, id, ap.FilterIDs); err != nil { return err } } @@ -389,7 +266,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r if cppIndexes != nil { if err := attrIndexes.StoreIndexes(true, transactionID); err != nil { for _, id := range *args.ChargerIDs { - if err := cppIndexes.RemoveItemFromIndex(id); err != nil { + cpp, err := self.DataManager.GetChargerProfile(args.Tenant, id, false, utils.NonTransactional) + if err != nil { + return err + } + if err := cppIndexes.RemoveItemFromIndex(args.Tenant, id, cpp.FilterIDs); err != nil { return err } } diff --git a/apier/v1/filter_indexes_it_test.go b/apier/v1/filter_indexes_it_test.go index d5e7e98c1..9a7e60b54 100644 --- a/apier/v1/filter_indexes_it_test.go +++ b/apier/v1/filter_indexes_it_test.go @@ -57,11 +57,11 @@ var sTestsFilterIndexesSV1 = []func(t *testing.T){ testV1FIdxThirdComputeThresholdsIndexes, testV1FIdxRemoveThresholdProfile, - testV1FIdxSetStatQueueProfileIndexes, - testV1FIdxComputeStatQueueProfileIndexes, - testV1FIdxSetSecondStatQueueProfileIndexes, - testV1FIdxSecondComputeStatQueueProfileIndexes, - testV1FIdxRemoveStatQueueProfile, + // testV1FIdxSetStatQueueProfileIndexes, + // testV1FIdxComputeStatQueueProfileIndexes, + // testV1FIdxSetSecondStatQueueProfileIndexes, + // testV1FIdxSecondComputeStatQueueProfileIndexes, + // testV1FIdxRemoveStatQueueProfile, testV1FIdxSetResourceProfileIndexes, testV1FIdxComputeResourceProfileIndexes, @@ -235,18 +235,10 @@ func testV1FIdxSetThresholdProfile(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxComputeThresholdsIndexes(t *testing.T) { @@ -279,16 +271,6 @@ func testV1FIdxComputeThresholdsIndexes(t *testing.T) { if !reflect.DeepEqual(expectedIDX, indexes) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxSetSecondThresholdProfile(t *testing.T) { @@ -352,11 +334,6 @@ func testV1FIdxSetSecondThresholdProfile(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - tenant); err != nil { - t.Error(err) - } if _, err = onStor.GetFilterIndexes( utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { @@ -395,17 +372,6 @@ func testV1FIdxSecondComputeThresholdsIndexes(t *testing.T) { if !reflect.DeepEqual(expectedIDX, indexes) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE2": {"*string:Account:1002": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxThirdComputeThresholdsIndexes(t *testing.T) { @@ -442,22 +408,6 @@ func testV1FIdxThirdComputeThresholdsIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": { - "*string:Account:1001": true}, - "TEST_PROFILE2": { - "*string:Account:1002": true}, - } - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxRemoveThresholdProfile(t *testing.T) { @@ -506,10 +456,6 @@ func testV1FIdxRemoveThresholdProfile(t *testing.T) { tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } //StatQueueProfile @@ -583,10 +529,6 @@ func testV1FIdxSetStatQueueProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { t.Error(err) @@ -623,18 +565,6 @@ func testV1FIdxComputeStatQueueProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxSetSecondStatQueueProfileIndexes(t *testing.T) { @@ -708,11 +638,6 @@ func testV1FIdxSetSecondStatQueueProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes( utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { @@ -752,18 +677,6 @@ func testV1FIdxSecondComputeStatQueueProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE2": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxRemoveStatQueueProfile(t *testing.T) { @@ -807,14 +720,10 @@ func testV1FIdxRemoveStatQueueProfile(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - if _, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], + if _, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } //ResourceProfile @@ -876,10 +785,6 @@ func testV1FIdxSetResourceProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { t.Error(err) @@ -916,18 +821,6 @@ func testV1FIdxComputeResourceProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "RCFG1": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxSetSecondResourceProfileIndexes(t *testing.T) { @@ -988,11 +881,6 @@ func testV1FIdxSetSecondResourceProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes( utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { @@ -1029,17 +917,6 @@ func testV1FIdxSecondComputeResourceProfileIndexes(t *testing.T) { if !reflect.DeepEqual(expectedIDX, indexes) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "RCFG2": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxRemoveResourceProfile(t *testing.T) { @@ -1085,10 +962,6 @@ func testV1FIdxRemoveResourceProfile(t *testing.T) { tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } //SupplierProfile @@ -1156,10 +1029,6 @@ func testV1FIdxSetSupplierProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { t.Error(err) @@ -1196,18 +1065,6 @@ func testV1FIdxComputeSupplierProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxSetSecondSupplierProfileIndexes(t *testing.T) { @@ -1275,11 +1132,6 @@ func testV1FIdxSetSecondSupplierProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes( utils.PrefixToIndexCache[utils.SupplierProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { @@ -1319,18 +1171,6 @@ func testV1FIdxSecondComputeSupplierProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE2": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxRemoveSupplierProfile(t *testing.T) { @@ -1378,10 +1218,6 @@ func testV1FIdxRemoveSupplierProfile(t *testing.T) { tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } //AttributeProfile @@ -1452,10 +1288,6 @@ func testV1FIdxSetAttributeProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { t.Error(err) @@ -1491,17 +1323,6 @@ func testV1FIdxComputeAttributeProfileIndexes(t *testing.T) { if !reflect.DeepEqual(expectedIDX, indexes) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "ApierTest": {"*string:Account:1001": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - tenant, nil); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) { @@ -1572,11 +1393,6 @@ func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) { tenant); err != nil { t.Error(err) } - if err := onStor.RemoveFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - tenant); err != nil { - t.Error(err) - } if indexes, err = onStor.GetFilterIndexes( utils.PrefixToIndexCache[utils.AttributeProfilePrefix], tenant, engine.MetaString, nil); err != utils.ErrNotFound { @@ -1616,18 +1432,6 @@ func testV1FIdxSecondComputeAttributeProfileIndexes(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes)) } - expectedRevIDX := map[string]utils.StringMap{ - "ApierTest2": {"*string:Account:1001": true}} - indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - tenant, nil) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxRemoveAttributeProfile(t *testing.T) { @@ -1674,10 +1478,6 @@ func testV1FIdxRemoveAttributeProfile(t *testing.T) { tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], - tenant, nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxPopulateDatabase(t *testing.T) { diff --git a/apier/v1/filterindexecache_it_test.go b/apier/v1/filterindexecache_it_test.go index 789e09283..c057ca93e 100644 --- a/apier/v1/filterindexecache_it_test.go +++ b/apier/v1/filterindexecache_it_test.go @@ -51,14 +51,14 @@ var sTestsFilterIndexesSV1Ca = []func(t *testing.T){ testV1FIdxCaUpdateThresholdProfileFromTP, testV1FIdxCaRemoveThresholdProfile, - testFlush, - testV1FIdxCaGetStatQueuesWithNotFound, - testV1FIdxCaSetStatQueueProfile, - testV1FIdxCaFromFolder, - testV1FIdxCaGetStatQueuesFromTP, - testV1FIdxCaUpdateStatQueueProfile, - testV1FIdxCaUpdateStatQueueProfileFromTP, - testV1FIdxCaRemoveStatQueueProfile, + // testFlush, + // testV1FIdxCaGetStatQueuesWithNotFound, + // testV1FIdxCaSetStatQueueProfile, + // testV1FIdxCaFromFolder, + // testV1FIdxCaGetStatQueuesFromTP, + // testV1FIdxCaUpdateStatQueueProfile, + // testV1FIdxCaUpdateStatQueueProfileFromTP, + // testV1FIdxCaRemoveStatQueueProfile, testFlush, testV1FIdxCaProcessAttributeProfileEventWithNotFound, @@ -178,10 +178,6 @@ func testV1FIdxCaProcessEventWithNotFound(t *testing.T) { if err := tFIdxCaRpc.Call(utils.ThresholdSv1ProcessEvent, tEv, &thIDs); err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - if indexes, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], - "cgrates.org", nil); err == nil || err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxCaSetThresholdProfile(t *testing.T) { @@ -248,19 +244,6 @@ func testV1FIdxCaSetThresholdProfile(t *testing.T) { } else if !reflect.DeepEqual(thIDs, eIDs) { t.Errorf("Expecting hits: %s, received: %s", eIDs, thIDs) } - //test to make sure indexes are made as expected - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true, - "*string:EventType:BalanceUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - "cgrates.org", fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes) - } } func testV1FIdxCaGetThresholdFromTP(t *testing.T) { @@ -284,22 +267,6 @@ func testV1FIdxCaGetThresholdFromTP(t *testing.T) { } else if !reflect.DeepEqual(thIDs, eIDs) { t.Errorf("Expecting hits: %s, received: %s", eIDs, thIDs) } - //test to make sure indexes are made as expected - idx := map[string]utils.StringMap{ - "THD_ACNT_BALANCE_1": { - "*string:Account:1001": true, - "*string:Account:1002": true, - "*string:EventType:BalanceUpdate": true}} - fldNameVal := map[string]string{"THD_ACNT_BALANCE_1": ""} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - "cgrates.org", fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateThresholdProfile(t *testing.T) { @@ -376,20 +343,6 @@ func testV1FIdxCaUpdateThresholdProfile(t *testing.T) { } else if !reflect.DeepEqual(thIDs, eIDs) { t.Errorf("Expecting : %s, received: %s", eIDs, thIDs) } - //test to make sure indexes are made as expecte - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1002": true, - "*string:EventType:AccountUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - "cgrates.org", fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", - expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateThresholdProfileFromTP(t *testing.T) { @@ -460,19 +413,6 @@ func testV1FIdxCaUpdateThresholdProfileFromTP(t *testing.T) { } else if !reflect.DeepEqual(thIDs, eIDs) { t.Errorf("Expecting : %s, received: %s", eIDs, thIDs) } - //test to make sure indexes are made as expecte - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1002": true, - "*string:EventType:AccountUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], - "cgrates.org", fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxCaRemoveThresholdProfile(t *testing.T) { @@ -540,13 +480,6 @@ func testV1FIdxCaRemoveThresholdProfile(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - //test to make sure indexes are made as expected - fldNameVal2 := map[string]string{"THD_ACNT_BALANCE_1": "", "TEST_PROFILE1": ""} - if _, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org", - fldNameVal2); err == nil || err != utils.ErrNotFound { - t.Error(err) - } } //StatQueue @@ -564,11 +497,6 @@ func testV1FIdxCaGetStatQueuesWithNotFound(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - nil); err == nil || err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxCaSetStatQueueProfile(t *testing.T) { @@ -642,19 +570,6 @@ func testV1FIdxCaSetStatQueueProfile(t *testing.T) { } else if !reflect.DeepEqual(reply, expected) { t.Errorf("Expecting: %+v, received: %+v", expected, reply) } - - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true, - "*string:EventType:AccountUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes) - } } func testV1FIdxCaGetStatQueuesFromTP(t *testing.T) { @@ -711,21 +626,6 @@ func testV1FIdxCaGetStatQueuesFromTP(t *testing.T) { } else if !reflect.DeepEqual(reply, expected) { t.Errorf("Expecting: %+v, received: %+v", expected, reply) } - - idx := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1001": true, - "*string:EventType:AccountUpdate": true}} - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - fldNameVal); err != nil { - t.Error(err) - } - - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateStatQueueProfile(t *testing.T) { @@ -794,19 +694,6 @@ func testV1FIdxCaUpdateStatQueueProfile(t *testing.T) { } else if !reflect.DeepEqual(reply, expected) { t.Errorf("Expecting: %+v, received: %+v", expected, reply) } - - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1003": true, - "*string:EventType:BalanceUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateStatQueueProfileFromTP(t *testing.T) { @@ -864,19 +751,6 @@ func testV1FIdxCaUpdateStatQueueProfileFromTP(t *testing.T) { } else if !reflect.DeepEqual(ids, expected) { t.Errorf("Expecting: %+v, received: %+v", expected, ids) } - fldNameVal := map[string]string{"Stats1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "Stats1": { - "*string:Account:1003": true, - "*string:EventType:AccountUpdate": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes) - } } func testV1FIdxCaRemoveStatQueueProfile(t *testing.T) { @@ -944,12 +818,6 @@ func testV1FIdxCaRemoveStatQueueProfile(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - // fldNameVals := map[string]string{"THD_ACNT_BALANCE_1": "", "TEST_PROFILE1": ""} - // if _, err = onStor.GetFilterReverseIndexes( - // utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org", - // fldNameVals); err == nil || err != utils.ErrNotFound { - // t.Error(err) - // } } //AttributeProfile @@ -968,11 +836,6 @@ func testV1FIdxCaProcessAttributeProfileEventWithNotFound(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org", - nil); err == nil || err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxCaSetAttributeProfile(t *testing.T) { @@ -1045,20 +908,6 @@ func testV1FIdxCaSetAttributeProfile(t *testing.T) { ev, &rplyEv); err != nil { t.Error(err) } - //test to make sure indexes are made as expected - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "TEST_PROFILE1": {"*string:Account:1009": true, - "*string:Destination:+491511231234": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - "cgrates.org:*sessions", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes) - } } func testV1FIdxCaGetAttributeProfileFromTP(t *testing.T) { @@ -1076,19 +925,6 @@ func testV1FIdxCaGetAttributeProfileFromTP(t *testing.T) { if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil { t.Error(err) } - //test to make sure indexes are made as expected - idx := map[string]utils.StringMap{"ATTR_1": { - "*string:Account:1007": true}} - fldNameVal := map[string]string{"ATTR_1": ""} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - "cgrates.org:*sessions", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateAttributeProfile(t *testing.T) { @@ -1160,21 +996,6 @@ func testV1FIdxCaUpdateAttributeProfile(t *testing.T) { if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil { t.Error(err) } - //test to make sure indexes are made as expected - idx := map[string]utils.StringMap{ - "TEST_PROFILE1": { - "*string:Account:2009": true, - "*string:Destination:+492511231234": true}} - fldNameVal := map[string]string{"TEST_PROFILE1": ""} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - "cgrates.org:*sessions", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateAttributeProfileFromTP(t *testing.T) { @@ -1228,21 +1049,6 @@ func testV1FIdxCaUpdateAttributeProfileFromTP(t *testing.T) { if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil { t.Error(err) } - //test to make sure indexes are made as expected - idx := map[string]utils.StringMap{ - "ATTR_1": { - "*string:Account:3009": true, - "*string:Destination:+492511231234": true}} - fldNameVal := map[string]string{"ATTR_1": ""} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix], - "cgrates.org:*sessions", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaRemoveAttributeProfile(t *testing.T) { @@ -1308,13 +1114,6 @@ func testV1FIdxCaRemoveAttributeProfile(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - //test to make sure indexes are made as expected - fldNameVal2 := map[string]string{"ATTR_1": "", "TEST_PROFILE1": ""} - if _, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.AttributeProfilePrefix], "cgrates.org:*rating", - fldNameVal2); err == nil || err != utils.ErrNotFound { - t.Error(err) - } } // ResourceProfile @@ -1339,11 +1138,6 @@ func testV1FIdxCaGetResourceProfileWithNotFound(t *testing.T) { argsRU, &reply); err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], "cgrates.org", - nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxCaSetResourceProfile(t *testing.T) { filter = &engine.Filter{ @@ -1420,20 +1214,6 @@ func testV1FIdxCaSetResourceProfile(t *testing.T) { } else if result != "Approved" { t.Error("Unexpected reply returned", result) } - fldNameVal := map[string]string{"RCFG1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "RCFG1": { - "*string:Account:1001": true, - "*string:Subject:1002": true, - "*string:Destination:1001": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], "cgrates.org", - fldNameVal); err != nil && err != utils.ErrNotFound { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes) - } } func testV1FIdxCaGetResourceProfileFromTP(t *testing.T) { @@ -1479,21 +1259,6 @@ func testV1FIdxCaGetResourceProfileFromTP(t *testing.T) { } else if reply != "ResGroup1" { t.Error("Unexpected reply returned", reply) } - idx := map[string]utils.StringMap{ - "ResGroup1": { - "*prefix:Destination:10": true, - "*prefix:Destination:20": true, - "*string:Account:1001": true, - "*string:Account:1002": true}} - fldNameVal := map[string]string{"ResGroup1": ""} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], "cgrates.org", - fldNameVal); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(idx, indexes) { - t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes)) - } } func testV1FIdxCaUpdateResourceProfile(t *testing.T) { @@ -1565,21 +1330,6 @@ func testV1FIdxCaUpdateResourceProfile(t *testing.T) { } else if result != "MessageAllocation" { t.Error("Unexpected reply returned", result) } - - // fldNameVal2 := map[string]string{"RCFG1": ""} - // expectedRevIDX := map[string]utils.StringMap{ - // "RCFG1": { - // "*string:Account:2002": true, - // "*string:Destination:2002": true, - // "*string:Subject:2001": true}} - // if indexes, err = onStor.GetFilterReverseIndexes( - // utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], "cgrates.org", - // fldNameVal2); err != nil { - // t.Error(err) - // } - // if !reflect.DeepEqual(expectedRevIDX, indexes) { - // t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - // } } func testV1FIdxCaUpdateResourceProfileFromTP(t *testing.T) { @@ -1643,20 +1393,6 @@ func testV1FIdxCaUpdateResourceProfileFromTP(t *testing.T) { } else if result != "ResGroup1" { t.Error("Unexpected reply returned", result) } - fldNameVal2 := map[string]string{"ResGroup1": ""} - expectedRevIDX := map[string]utils.StringMap{ - "ResGroup1": { - "*string:Account:1002": true, - "*string:Destination:1002": true, - "*string:Subject:1001": true}} - if indexes, err = onStor.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - "cgrates.org", fldNameVal2); err != nil { - t.Error(err) - } - if !reflect.DeepEqual(expectedRevIDX, indexes) { - t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes)) - } } func testV1FIdxCaRemoveResourceProfile(t *testing.T) { @@ -1722,13 +1458,6 @@ func testV1FIdxCaRemoveResourceProfile(t *testing.T) { err.Error() != utils.ErrNotFound.Error() { t.Error(err) } - - fldNameVals2 := map[string]string{"ResGroup1": "", "TEST_PROFILE1": ""} - if _, err = onStor.GetFilterReverseIndexes( - utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org", - fldNameVals2); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } func testV1FIdxCaStopEngine(t *testing.T) { diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 330e6ccc6..b10c50fb3 100755 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -405,22 +405,22 @@ func main() { } var cacheIDs []string if len(apfIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheAttributeFilterIndexes, utils.CacheAttributeFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheAttributeFilterIndexes) } if len(spfIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheSupplierFilterIndexes, utils.CacheSupplierFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheSupplierFilterIndexes) } if len(trspfIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheThresholdFilterIndexes, utils.CacheThresholdFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheThresholdFilterIndexes) } if len(stqpIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheStatFilterIndexes, utils.CacheStatFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheStatFilterIndexes) } if len(rspIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheResourceFilterIndexes, utils.CacheResourceFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheResourceFilterIndexes) } if len(chargerIDs) != 0 { - cacheIDs = append(cacheIDs, utils.CacheChargerFilterIndexes, utils.CacheChargerFilterRevIndexes) + cacheIDs = append(cacheIDs, utils.CacheChargerFilterIndexes) } if err = cacheS.Call(utils.CacheSv1Clear, cacheIDs, &reply); err != nil { log.Printf("WARNING: Got error on cache clear: %s\n", err.Error()) diff --git a/console/filter_reverse_indexes.go b/console/filter_reverse_indexes.go deleted file mode 100755 index 8d310b388..000000000 --- a/console/filter_reverse_indexes.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/apier/v1" -) - -func init() { - c := &CmdGetFilterReverseIndexes{ - name: "filter_reverse_indexes", - rpcMethod: "ApierV1.GetFilterReverseIndexes", - rpcParams: &v1.AttrGetFilterReverseIndexes{}, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -// Commander implementation -type CmdGetFilterReverseIndexes struct { - name string - rpcMethod string - rpcParams *v1.AttrGetFilterReverseIndexes - *CommandExecuter -} - -func (self *CmdGetFilterReverseIndexes) Name() string { - return self.name -} - -func (self *CmdGetFilterReverseIndexes) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdGetFilterReverseIndexes) RpcParams(reset bool) interface{} { - if reset || self.rpcParams == nil { - self.rpcParams = &v1.AttrGetFilterReverseIndexes{} - } - return self.rpcParams -} - -func (self *CmdGetFilterReverseIndexes) PostprocessRpcParams() error { - return nil -} - -func (self *CmdGetFilterReverseIndexes) RpcResult() interface{} { - var atr []string - return &atr -} diff --git a/engine/attributes_test.go b/engine/attributes_test.go index dabd7b289..950fa5a29 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -476,11 +476,6 @@ func TestAttributeIndexer(t *testing.T) { "AttrPrf": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "AttrPrf": utils.StringMap{ - "*string:Account:1007": true, - }, - } rfi1 := NewFilterIndexer(dmAtr, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrPrf.Tenant, utils.META_ANY)) if rcvIdx, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType], @@ -491,14 +486,6 @@ func TestAttributeIndexer(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } - if reverseRcvIdx, err := dmAtr.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi1.itemType], rfi1.dbKeySuffix, nil); err != nil { - t.Error(err) - } else { - if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } - } //Set AttributeProfile with new context (*sessions) attrPrf.Contexts = []string{utils.MetaSessionS} if err := dmAtr.SetAttributeProfile(attrPrf, true); err != nil { @@ -514,21 +501,11 @@ func TestAttributeIndexer(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } - if reverseRcvIdx, err := dmAtr.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi2.itemType], rfi2.dbKeySuffix, nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //verify if old index was deleted ( context *any) if _, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType], rfi1.dbKeySuffix, MetaString, nil); err != utils.ErrNotFound { t.Error(err) } - if _, err := dmAtr.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi1.itemType], rfi1.dbKeySuffix, nil); err != utils.ErrNotFound { - t.Error(err) - } } func TestAttributeProcessWithMultipleRuns1(t *testing.T) { diff --git a/engine/datamanager.go b/engine/datamanager.go index 5a39b1697..0073a356a 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -420,6 +420,10 @@ func (dm *DataManager) GetThresholdProfile(tenant, id string, skipCache bool, } func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) (err error) { + oldTh, err := dm.GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetThresholdProfileDrv(th); err != nil { return err } @@ -428,6 +432,20 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) return } if withIndex { + if oldTh != nil { + var needsRemove bool + for _, fltrID := range oldTh.FilterIDs { + if !utils.IsSliceMember(th.FilterIDs, fltrID) { + needsRemove = true + } + } + if needsRemove { + if err = NewFilterIndexer(dm, utils.ThresholdProfilePrefix, + th.Tenant).RemoveItemFromIndex(th.Tenant, th.ID, oldTh.FilterIDs); err != nil { + return + } + } + } return createAndIndex(utils.ThresholdProfilePrefix, th.Tenant, utils.EmptyString, th.ID, th.FilterIDs, dm) } return @@ -435,6 +453,10 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) func (dm *DataManager) RemoveThresholdProfile(tenant, id, transactionID string, withIndex bool) (err error) { + oldTh, err := dm.GetThresholdProfile(tenant, id, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemThresholdProfileDrv(tenant, id); err != nil { return } @@ -442,7 +464,7 @@ func (dm *DataManager) RemoveThresholdProfile(tenant, id, cacheCommit(transactionID), transactionID) if withIndex { return NewFilterIndexer(dm, - utils.ThresholdProfilePrefix, tenant).RemoveItemFromIndex(id) + utils.ThresholdProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldTh.FilterIDs) } return } @@ -472,6 +494,10 @@ func (dm *DataManager) GetStatQueueProfile(tenant, id string, skipCache bool, } func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool) (err error) { + oldSts, err := dm.GetStatQueueProfile(sqp.Tenant, sqp.ID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetStatQueueProfileDrv(sqp); err != nil { return err } @@ -480,6 +506,20 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool return } if withIndex { + if oldSts != nil { + var needsRemove bool + for _, fltrID := range oldSts.FilterIDs { + if !utils.IsSliceMember(sqp.FilterIDs, fltrID) { + needsRemove = true + } + } + if needsRemove { + if err = NewFilterIndexer(dm, utils.StatQueueProfilePrefix, + sqp.Tenant).RemoveItemFromIndex(sqp.Tenant, sqp.ID, oldSts.FilterIDs); err != nil { + return + } + } + } return createAndIndex(utils.StatQueueProfilePrefix, sqp.Tenant, utils.EmptyString, sqp.ID, sqp.FilterIDs, dm) } return @@ -487,13 +527,17 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool func (dm *DataManager) RemoveStatQueueProfile(tenant, id, transactionID string, withIndex bool) (err error) { + oldSts, err := dm.GetResourceProfile(tenant, id, true, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemStatQueueProfileDrv(tenant, id); err != nil { return } Cache.Remove(utils.CacheStatQueueProfiles, utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) if withIndex { - return NewFilterIndexer(dm, utils.StatQueueProfilePrefix, tenant).RemoveItemFromIndex(id) + return NewFilterIndexer(dm, utils.StatQueueProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldSts.FilterIDs) } return } @@ -607,6 +651,10 @@ func (dm *DataManager) GetResourceProfile(tenant, id string, } func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (err error) { + oldRes, err := dm.GetResourceProfile(rp.Tenant, rp.ID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetResourceProfileDrv(rp); err != nil { return err } @@ -616,6 +664,20 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) ( } //to be implemented in tests if withIndex { + if oldRes != nil { + var needsRemove bool + for _, fltrID := range oldRes.FilterIDs { + if !utils.IsSliceMember(rp.FilterIDs, fltrID) { + needsRemove = true + } + } + if needsRemove { + if err = NewFilterIndexer(dm, utils.ResourceProfilesPrefix, + rp.Tenant).RemoveItemFromIndex(rp.Tenant, rp.ID, oldRes.FilterIDs); err != nil { + return + } + } + } if err = createAndIndex(utils.ResourceProfilesPrefix, rp.Tenant, utils.EmptyString, rp.ID, rp.FilterIDs, dm); err != nil { return } @@ -625,13 +687,17 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) ( } func (dm *DataManager) RemoveResourceProfile(tenant, id, transactionID string, withIndex bool) (err error) { + oldRes, err := dm.GetResourceProfile(tenant, id, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemoveResourceProfileDrv(tenant, id); err != nil { return } Cache.Remove(utils.CacheResourceProfiles, utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) if withIndex { - return NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant).RemoveItemFromIndex(id) + return NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant).RemoveItemFromIndex(tenant, id, oldRes.FilterIDs) } return } @@ -955,22 +1021,6 @@ func (dm *DataManager) RemoveFilterIndexes(cacheID, itemIDPrefix string) (err er return dm.DataDB().RemoveFilterIndexesDrv(cacheID, itemIDPrefix) } -func (dm *DataManager) GetFilterReverseIndexes(cacheID, itemIDPrefix string, - fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { - return dm.DataDB().GetFilterReverseIndexesDrv(cacheID, itemIDPrefix, fldNameVal) -} - -func (dm *DataManager) SetFilterReverseIndexes(cacheID, itemIDPrefix string, - indexes map[string]utils.StringMap, - commit bool, transactionID string) (err error) { - return dm.DataDB().SetFilterReverseIndexesDrv(cacheID, - itemIDPrefix, indexes, commit, transactionID) -} - -func (dm *DataManager) RemoveFilterReverseIndexes(cacheID, itemIDPrefix string) (err error) { - return dm.DataDB().RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix) -} - func (dm *DataManager) MatchFilterIndex(cacheID, itemIDPrefix, filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) { fieldValKey := utils.ConcatenatedKey(itemIDPrefix, filterType, fieldName, fieldVal) @@ -1043,6 +1093,10 @@ func (dm *DataManager) GetSupplierProfile(tenant, id string, skipCache bool, } func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) (err error) { + oldSup, err := dm.GetSupplierProfile(supp.Tenant, supp.ID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetSupplierProfileDrv(supp); err != nil { return err } @@ -1050,19 +1104,37 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) return } if withIndex { + if oldSup != nil { + var needsRemove bool + for _, fltrID := range oldSup.FilterIDs { + if !utils.IsSliceMember(supp.FilterIDs, fltrID) { + needsRemove = true + } + } + if needsRemove { + if err = NewFilterIndexer(dm, utils.SupplierProfilePrefix, + supp.Tenant).RemoveItemFromIndex(supp.Tenant, supp.ID, oldSup.FilterIDs); err != nil { + return + } + } + } return createAndIndex(utils.SupplierProfilePrefix, supp.Tenant, utils.EmptyString, supp.ID, supp.FilterIDs, dm) } return } func (dm *DataManager) RemoveSupplierProfile(tenant, id, transactionID string, withIndex bool) (err error) { + oldSupp, err := dm.GetSupplierProfile(tenant, id, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemoveSupplierProfileDrv(tenant, id); err != nil { return } Cache.Remove(utils.CacheSupplierProfiles, utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) if withIndex { - return NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant).RemoveItemFromIndex(id) + return NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldSupp.FilterIDs) } return } @@ -1095,7 +1167,7 @@ func (dm *DataManager) GetAttributeProfile(tenant, id string, skipCache bool, } func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) (err error) { - oldAP, err := dm.GetAttributeProfile(ap.Tenant, ap.ID, true, utils.NonTransactional) + oldAP, err := dm.GetAttributeProfile(ap.Tenant, ap.ID, false, utils.NonTransactional) if err != nil && err != utils.ErrNotFound { return err } @@ -1120,7 +1192,7 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) } if needsRemove { if err = NewFilterIndexer(dm, utils.AttributeProfilePrefix, - utils.ConcatenatedKey(ap.Tenant, ctx)).RemoveItemFromIndex(ap.ID); err != nil { + utils.ConcatenatedKey(ap.Tenant, ctx)).RemoveItemFromIndex(ap.Tenant, ap.ID, oldAP.FilterIDs); err != nil { return } } @@ -1138,6 +1210,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) func (dm *DataManager) RemoveAttributeProfile(tenant, id string, contexts []string, transactionID string, withIndex bool) (err error) { + oldAttr, err := dm.GetAttributeProfile(tenant, id, true, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemoveAttributeProfileDrv(tenant, id); err != nil { return } @@ -1146,7 +1222,7 @@ func (dm *DataManager) RemoveAttributeProfile(tenant, id string, contexts []stri if withIndex { for _, context := range contexts { if err = NewFilterIndexer(dm, utils.AttributeProfilePrefix, - utils.ConcatenatedKey(tenant, context)).RemoveItemFromIndex(id); err != nil { + utils.ConcatenatedKey(tenant, context)).RemoveItemFromIndex(tenant, id, oldAttr.FilterIDs); err != nil { return } } @@ -1179,6 +1255,10 @@ func (dm *DataManager) GetChargerProfile(tenant, id string, skipCache bool, } func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (err error) { + oldCpp, err := dm.GetChargerProfile(cpp.Tenant, cpp.ID, true, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetChargerProfileDrv(cpp); err != nil { return err } @@ -1186,6 +1266,20 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e return } if withIndex { + if oldCpp != nil { + var needsRemove bool + for _, fltrID := range oldCpp.FilterIDs { + if !utils.IsSliceMember(cpp.FilterIDs, fltrID) { + needsRemove = true + } + } + if needsRemove { + if err = NewFilterIndexer(dm, utils.SupplierProfilePrefix, + cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil { + return + } + } + } return createAndIndex(utils.ChargerProfilePrefix, cpp.Tenant, utils.EmptyString, cpp.ID, cpp.FilterIDs, dm) } return @@ -1193,13 +1287,17 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e func (dm *DataManager) RemoveChargerProfile(tenant, id string, transactionID string, withIndex bool) (err error) { + oldCpp, err := dm.GetChargerProfile(tenant, id, true, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().RemoveChargerProfileDrv(tenant, id); err != nil { return } Cache.Remove(utils.CacheChargerProfiles, utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) if withIndex { - return NewFilterIndexer(dm, utils.ChargerProfilePrefix, tenant).RemoveItemFromIndex(id) + return NewFilterIndexer(dm, utils.ChargerProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldCpp.FilterIDs) } return } diff --git a/engine/filterindexer.go b/engine/filterindexer.go index 050f81604..152435679 100644 --- a/engine/filterindexer.go +++ b/engine/filterindexer.go @@ -20,7 +20,6 @@ package engine import ( "fmt" - "strings" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/guardian" @@ -29,31 +28,19 @@ import ( func NewFilterIndexer(dm *DataManager, itemType, dbKeySuffix string) *FilterIndexer { return &FilterIndexer{dm: dm, itemType: itemType, dbKeySuffix: dbKeySuffix, - indexes: make(map[string]utils.StringMap), - reveseIndex: make(map[string]utils.StringMap), - chngdIndxKeys: make(utils.StringMap), - chngdRevIndxKeys: make(utils.StringMap)} + indexes: make(map[string]utils.StringMap), + chngdIndxKeys: make(utils.StringMap)} } // FilterIndexer is a centralized indexer for all data sources using RequestFilter // retrieves and stores it's data from/to dataDB // not thread safe, meant to be used as logic within other code blocks type FilterIndexer struct { - indexes map[string]utils.StringMap // map[fieldName:fieldValue]utils.StringMap[itemID] - reveseIndex map[string]utils.StringMap // map[itemID]utils.StringMap[fieldName:fieldValue] - dm *DataManager - itemType string - dbKeySuffix string // get/store the result from/into this key - chngdIndxKeys utils.StringMap // keep record of the changed fieldName:fieldValue pair so we can re-cache wisely - chngdRevIndxKeys utils.StringMap // keep record of the changed itemID so we can re-cache wisely -} - -// ChangedKeys returns the changed keys from original indexes so we can reload wisely -func (rfi *FilterIndexer) ChangedKeys(reverse bool) utils.StringMap { - if reverse { - return rfi.chngdRevIndxKeys - } - return rfi.chngdIndxKeys + indexes map[string]utils.StringMap // map[fieldName:fieldValue]utils.StringMap[itemID] + dm *DataManager + itemType string + dbKeySuffix string // get/store the result from/into this key + chngdIndxKeys utils.StringMap // keep record of the changed fieldName:fieldValue pair so we can re-cache wisely } // IndexTPFilter parses reqFltrs, adding itemID in the indexes and marks the changed keys in chngdIndxKeys @@ -67,13 +54,8 @@ func (rfi *FilterIndexer) IndexTPFilter(tpFltr *utils.TPFilterProfile, itemID st rfi.indexes[concatKey] = make(utils.StringMap) } rfi.indexes[concatKey][itemID] = true - if _, hasIt := rfi.reveseIndex[itemID]; !hasIt { - rfi.reveseIndex[itemID] = make(utils.StringMap) - } - rfi.reveseIndex[itemID][concatKey] = true rfi.chngdIndxKeys[concatKey] = true } - rfi.chngdRevIndxKeys[itemID] = true case MetaPrefix: for _, fldVal := range fltr.Values { concatKey := utils.ConcatenatedKey(fltr.Type, fltr.FieldName, fldVal) @@ -81,23 +63,15 @@ func (rfi *FilterIndexer) IndexTPFilter(tpFltr *utils.TPFilterProfile, itemID st rfi.indexes[concatKey] = make(utils.StringMap) } rfi.indexes[concatKey][itemID] = true - if _, hasIt := rfi.reveseIndex[itemID]; !hasIt { - rfi.reveseIndex[itemID] = make(utils.StringMap) - } - rfi.reveseIndex[itemID][concatKey] = true rfi.chngdIndxKeys[concatKey] = true } - rfi.chngdRevIndxKeys[itemID] = true case utils.META_NONE: concatKey := utils.ConcatenatedKey(utils.META_NONE, utils.ANY, utils.ANY) if _, hasIt := rfi.indexes[concatKey]; !hasIt { rfi.indexes[concatKey] = make(utils.StringMap) } - if _, hasIt := rfi.reveseIndex[itemID]; !hasIt { - rfi.reveseIndex[itemID] = make(utils.StringMap) - } - rfi.reveseIndex[itemID][concatKey] = true rfi.indexes[concatKey][itemID] = true + rfi.chngdIndxKeys[concatKey] = true } } return @@ -107,22 +81,22 @@ func (rfi *FilterIndexer) cacheRemItemType() { // ToDo: tune here by removing pe switch rfi.itemType { case utils.ThresholdProfilePrefix: - Cache.Clear([]string{utils.CacheThresholdFilterIndexes, utils.CacheThresholdFilterRevIndexes}) + Cache.Clear([]string{utils.CacheThresholdFilterIndexes}) case utils.ResourceProfilesPrefix: - Cache.Clear([]string{utils.CacheResourceFilterIndexes, utils.CacheResourceFilterRevIndexes}) + Cache.Clear([]string{utils.CacheResourceFilterIndexes}) case utils.StatQueueProfilePrefix: - Cache.Clear([]string{utils.CacheStatFilterIndexes, utils.CacheStatFilterRevIndexes}) + Cache.Clear([]string{utils.CacheStatFilterIndexes}) case utils.SupplierProfilePrefix: - Cache.Clear([]string{utils.CacheSupplierFilterIndexes, utils.CacheSupplierFilterRevIndexes}) + Cache.Clear([]string{utils.CacheSupplierFilterIndexes}) case utils.AttributeProfilePrefix: - Cache.Clear([]string{utils.CacheAttributeFilterIndexes, utils.CacheAttributeFilterRevIndexes}) + Cache.Clear([]string{utils.CacheAttributeFilterIndexes}) case utils.ChargerProfilePrefix: - Cache.Clear([]string{utils.CacheChargerFilterIndexes, utils.CacheChargerFilterRevIndexes}) + Cache.Clear([]string{utils.CacheChargerFilterIndexes}) } } @@ -136,32 +110,10 @@ func (rfi *FilterIndexer) StoreIndexes(commit bool, transactionID string) (err e rfi.indexes, commit, transactionID); err != nil { return } - if err = rfi.dm.SetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - rfi.reveseIndex, commit, transactionID); err != nil { - return - } rfi.cacheRemItemType() return } -//Populate the FilterIndexer.reveseIndex for specifil itemID -func (rfi *FilterIndexer) loadItemReverseIndex(filterType, itemID string) (err error) { - rcvReveseIdx, err := rfi.dm.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - map[string]string{itemID: ""}) - if err != nil { - return err - } - for _, val2 := range rcvReveseIdx { - if _, has := rfi.reveseIndex[itemID]; !has { - rfi.reveseIndex[itemID] = make(utils.StringMap) - } - rfi.reveseIndex[itemID] = val2 - } - return err -} - //Populate FilterIndexer.indexes with specific fieldName:fieldValue , item func (rfi *FilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal string) error { rcvIdx, err := rfi.dm.GetFilterIndexes( @@ -180,18 +132,119 @@ func (rfi *FilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal str } //RemoveItemFromIndex remove Indexes for a specific itemID -func (rfi *FilterIndexer) RemoveItemFromIndex(itemID string) (err error) { - if err = rfi.loadItemReverseIndex(MetaString, itemID); err != nil { - return err - } - for key, _ := range rfi.reveseIndex[itemID] { - kSplt := strings.Split(key, utils.CONCATENATED_KEY_SEP) - if len(kSplt) != 3 { - return fmt.Errorf("Malformed key in db: %s", key) - } - if err = rfi.loadFldNameFldValIndex(kSplt[0], kSplt[1], kSplt[2]); err != nil { +func (rfi *FilterIndexer) RemoveItemFromIndex(tenant, itemID string, oldFilters []string) (err error) { + var filterIDs []string + switch rfi.itemType { + case utils.ThresholdProfilePrefix: + th, err := rfi.dm.GetThresholdProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { return err } + if th != nil { + filterIDs = make([]string, len(th.FilterIDs)) + for i, fltrID := range th.FilterIDs { + filterIDs[i] = fltrID + } + } + case utils.AttributeProfilePrefix: + attrPrf, err := rfi.dm.GetAttributeProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } + if attrPrf != nil { + filterIDs = make([]string, len(attrPrf.FilterIDs)) + for i, fltrID := range attrPrf.FilterIDs { + filterIDs[i] = fltrID + } + } + case utils.ResourceProfilesPrefix: + res, err := rfi.dm.GetResourceProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } + if res != nil { + filterIDs = make([]string, len(res.FilterIDs)) + for i, fltrID := range res.FilterIDs { + filterIDs[i] = fltrID + } + } + case utils.StatQueueProfilePrefix: + stq, err := rfi.dm.GetStatQueueProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } + if stq != nil { + filterIDs = make([]string, len(stq.FilterIDs)) + for i, fltrID := range stq.FilterIDs { + filterIDs[i] = fltrID + } + } + case utils.SupplierProfilePrefix: + spp, err := rfi.dm.GetSupplierProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } + if spp != nil { + filterIDs = make([]string, len(spp.FilterIDs)) + for i, fltrID := range spp.FilterIDs { + filterIDs[i] = fltrID + } + } + case utils.ChargerProfilePrefix: + cpp, err := rfi.dm.GetChargerProfile(tenant, itemID, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } + if cpp != nil { + filterIDs = make([]string, len(cpp.FilterIDs)) + for i, fltrID := range cpp.FilterIDs { + filterIDs[i] = fltrID + } + } + default: + } + if len(filterIDs) == 0 { + filterIDs = []string{utils.META_NONE} + } + for _, oldFltr := range oldFilters { + filterIDs = append(filterIDs, oldFltr) + } + for _, fltrID := range filterIDs { + var fltr *Filter + if fltrID == utils.META_NONE { + fltr = &Filter{ + Tenant: tenant, + ID: itemID, + Rules: []*FilterRule{ + &FilterRule{ + Type: utils.META_NONE, + FieldName: utils.META_ANY, + Values: []string{utils.META_ANY}, + }, + }, + } + } else if fltr, err = rfi.dm.GetFilter(tenant, fltrID, + false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for itemType: %+v and ID: %+v", + fltrID, rfi.itemType, itemID) + } + return err + } + for _, flt := range fltr.Rules { + var fldType, fldName string + var fldVals []string + if utils.IsSliceMember([]string{MetaString, MetaPrefix, utils.META_NONE}, flt.Type) { + fldType, fldName = flt.Type, flt.FieldName + fldVals = flt.Values + } + for _, fldVal := range fldVals { + if err = rfi.loadFldNameFldValIndex(fldType, + fldName, fldVal); err != nil && err != utils.ErrNotFound { + return err + } + } + } } for _, itmMp := range rfi.indexes { for range itmMp { @@ -200,7 +253,6 @@ func (rfi *FilterIndexer) RemoveItemFromIndex(itemID string) (err error) { } } } - rfi.reveseIndex[itemID] = make(utils.StringMap) //Force deleting in driver return rfi.StoreIndexes(false, utils.NonTransactional) } @@ -211,10 +263,6 @@ func createAndIndex(itemPrefix, tenant, context, itemID string, filterIDs []stri indexerKey = utils.ConcatenatedKey(tenant, context) } indexer := NewFilterIndexer(dm, itemPrefix, indexerKey) - if err = indexer.RemoveItemFromIndex(itemID); err != nil && - err.Error() != utils.ErrNotFound.Error() { - return - } fltrIDs := make([]string, len(filterIDs)) for i, fltrID := range filterIDs { fltrIDs[i] = fltrID diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index 2e58cbd75..4f87de4ce 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -75,7 +75,7 @@ func TestFilterIndexerITRedis(t *testing.T) { } func TestFilterIndexerITMongo(t *testing.T) { - cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "cdrsv2mongo") + cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "tutmongo") mgoITCfg, err := config.NewCGRConfigFromFolder(cdrsMongoCfgPath) if err != nil { t.Fatal(err) @@ -196,22 +196,6 @@ func testITGetFilterIndexes(t *testing.T) { MetaString, nil); err == nil || err != utils.ErrNotFound { t.Error(err) } - if err := dataManager.RemoveFilterIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - "cgrates.org"); err != nil { - t.Error(err) - } - _, err := dataManager.GetFilterIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - "cgrates.org", MetaString, nil) - if err != utils.ErrNotFound { - t.Error(err) - } - if err := dataManager.SetFilterIndexes( - utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], - "cgrates.org", eIdxes, false, utils.NonTransactional); err != nil { - t.Error(err) - } } func testITMatchFilterIndex(t *testing.T) { @@ -292,16 +276,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { "THD_Test2": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - "THD_Test2": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant) if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, @@ -310,13 +284,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } + //Replace existing filter (Filter1 -> Filter2) fp2 := &Filter{ Tenant: "cgrates.org", @@ -336,7 +304,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { if err := dataManager.SetFilter(fp2); err != nil { t.Error(err) } - th.FilterIDs = []string{"Filter2"} if err := dataManager.SetThresholdProfile(th, true); err != nil { t.Error(err) @@ -355,17 +322,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { "THD_Test2": true, }, } - - reverseIdxes = map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*string:Account:1001": true, - "*string:Account:1002": true, - }, - "THD_Test2": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != nil { @@ -373,13 +329,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //replace old filter with two different filters fp3 := &Filter{ Tenant: "cgrates.org", @@ -399,7 +348,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { if err := dataManager.SetFilter(fp3); err != nil { t.Error(err) } - th.FilterIDs = []string{"Filter1", "Filter3"} if err := dataManager.SetThresholdProfile(th, true); err != nil { t.Error(err) @@ -420,18 +368,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { "THD_Test2": true, }, } - reverseIdxes = map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*string:Destination:10": true, - "*string:Destination:20": true, - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - "THD_Test2": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != nil { @@ -439,13 +375,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //remove thresholds if err := dataManager.RemoveThresholdProfile(th.Tenant, th.ID, utils.NonTransactional, true); err != nil { @@ -456,15 +385,10 @@ func testITTestThresholdFilterIndexes(t *testing.T) { t.Error(err) } if _, err := dataManager.GetFilterIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, + utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != utils.ErrNotFound { t.Error(err) } - if _, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != utils.ErrNotFound { - t.Error(err) - } } func testITTestAttributeProfileFilterIndexes(t *testing.T) { @@ -516,12 +440,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { "AttrPrf": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "AttrPrf": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } for _, ctx := range attrProfile.Contexts { rfi := NewFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrProfile.Tenant, ctx)) @@ -532,13 +450,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } } //Set AttributeProfile with 1 new context (con3) attrProfile.Contexts = []string{"con3"} @@ -555,13 +466,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //check if old contexts was delete for _, ctx := range []string{"con1", "con2"} { rfi := NewFilterIndexer(onStor, utils.AttributeProfilePrefix, @@ -571,11 +475,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } if err := dataManager.RemoveAttributeProfile(attrProfile.Tenant, @@ -586,15 +485,10 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { rfi = NewFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey("cgrates.org", "con3")) if _, err := dataManager.GetFilterIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, + utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != nil && err != utils.ErrNotFound { t.Error(err) } - if _, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil && err != utils.ErrNotFound { - t.Error(err) - } } func testITTestThresholdInlineFilterIndexing(t *testing.T) { @@ -639,12 +533,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { "THD_Test": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant) if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, @@ -653,13 +541,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //Add an InlineFilter th.FilterIDs = []string{"Filter1", "*string:Account:1001"} if err := dataManager.SetThresholdProfile(th, true); err != nil { @@ -676,14 +557,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { "THD_Test": true, }, } - - reverseIdxes = map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*string:Account:1001": true, - "*string:EventType:Event1": true, - "*string:EventType:Event2": true, - }, - } if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != nil { @@ -691,28 +564,16 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { } else if !reflect.DeepEqual(eIdxes, rcvIdx) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } //remove threshold if err := dataManager.RemoveThresholdProfile(th.Tenant, th.ID, utils.NonTransactional, true); err != nil { t.Error(err) } if _, err := dataManager.GetFilterIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, + utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != utils.ErrNotFound { t.Error(err) } - if _, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix, - nil); err != utils.ErrNotFound { - t.Error(err) - } } func testITTestStoreFilterIndexesWithTransID(t *testing.T) { @@ -843,14 +704,6 @@ func testITTestIndexingWithEmptyFltrID(t *testing.T) { "THD_Test2": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "THD_Test": utils.StringMap{ - "*none:*any:*any": true, - }, - "THD_Test2": utils.StringMap{ - "*none:*any:*any": true, - }, - } rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant) if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, @@ -861,15 +714,6 @@ func testITTestIndexingWithEmptyFltrID(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], - rfi.dbKeySuffix, nil); err != nil { - t.Error(err) - } else { - if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } - } eMp := utils.StringMap{ "THD_Test": true, "THD_Test2": true, @@ -942,14 +786,6 @@ func testITTestIndexingWithEmptyFltrID2(t *testing.T) { "SPL_Weight2": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "SPL_Weight": utils.StringMap{ - "*none:*any:*any": true, - }, - "SPL_Weight2": utils.StringMap{ - "*none:*any:*any": true, - }, - } rfi := NewFilterIndexer(onStor, utils.SupplierProfilePrefix, splProfile.Tenant) if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, @@ -960,15 +796,6 @@ func testITTestIndexingWithEmptyFltrID2(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], - rfi.dbKeySuffix, nil); err != nil { - t.Error(err) - } else { - if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } - } eMp := utils.StringMap{ "SPL_Weight": true, "SPL_Weight2": true, @@ -1019,18 +846,6 @@ func testITTestIndexingThresholds(t *testing.T) { "TH3": true, }, } - reverseIdxes := map[string]utils.StringMap{ - "TH1": utils.StringMap{ - "*string:Account:1001": true, - }, - "TH2": utils.StringMap{ - "*string:Account:1001": true, - }, - "TH3": utils.StringMap{ - "*string:Account:1002": true, - }, - } - if rcvIdx, err := dataManager.GetFilterIndexes( utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix, MetaString, nil); err != nil { @@ -1040,15 +855,6 @@ func testITTestIndexingThresholds(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } - if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes( - utils.PrefixToRevIndexCache[rfi.itemType], - rfi.dbKeySuffix, nil); err != nil { - t.Error(err) - } else { - if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { - t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) - } - } eMp := utils.StringMap{ "TH1": true, "TH2": true, diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 45962415b..913f2bf08 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -123,11 +123,6 @@ type DataDB interface { SetFilterIndexesDrv(cacheID, itemIDPrefix string, indexes map[string]utils.StringMap, commit bool, transactionID string) (err error) RemoveFilterIndexesDrv(cacheID, itemIDPrefix string) (err error) - GetFilterReverseIndexesDrv(cacheID, itemIDPrefix string, - fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) - SetFilterReverseIndexesDrv(cacheID, itemIDPrefix string, indexes map[string]utils.StringMap, - commit bool, transactionID string) (err error) - RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix string) (err error) MatchFilterIndexDrv(cacheID, itemIDPrefix, filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) GetStatQueueProfileDrv(tenant string, ID string) (sq *StatQueueProfile, err error) diff --git a/engine/storage_map_datadb.go b/engine/storage_map_datadb.go index e68733014..8ae2b2f8f 100644 --- a/engine/storage_map_datadb.go +++ b/engine/storage_map_datadb.go @@ -1333,110 +1333,6 @@ func (ms *MapStorage) RemoveFilterIndexesDrv(cacheID, itemIDPrefix string) (err return } -//GetFilterReverseIndexesDrv retrieves ReverseIndexes from dataDB -func (ms *MapStorage) GetFilterReverseIndexesDrv(cacheID, itemIDPrefix string, - fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) { - ms.mu.RLock() - defer ms.mu.RUnlock() - dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix - values, ok := ms.dict[dbKey] - if !ok { - return nil, utils.ErrNotFound - } - if len(fldNameVal) != 0 { - rcvidx := make(map[string]utils.StringMap) - err = ms.ms.Unmarshal(values, &rcvidx) - if err != nil { - return nil, err - } - indexes = make(map[string]utils.StringMap) - for fldName := range fldNameVal { - if _, has := indexes[fldName]; !has { - indexes[fldName] = make(utils.StringMap) - } - for key := range rcvidx[fldName] { - indexes[fldName][key] = true - } - } - return - } else { - err = ms.ms.Unmarshal(values, &indexes) - if err != nil { - return nil, err - } - if len(indexes) == 0 { - return nil, utils.ErrNotFound - } - } - return -} - -//SetFilterReverseIndexesDrv stores ReverseIndexes into DataDB -func (ms *MapStorage) SetFilterReverseIndexesDrv(cacheID, itemIDPrefix string, - revIdx map[string]utils.StringMap, commit bool, transactionID string) (err error) { - ms.mu.Lock() - defer ms.mu.Unlock() - originKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix - dbKey := originKey - if transactionID != "" { - dbKey = "tmp_" + utils.ConcatenatedKey(dbKey, transactionID) - } - if commit && transactionID != "" { - values, _ := ms.dict[dbKey] - delete(ms.dict, dbKey) - ms.dict[originKey] = values - return nil - } - var toBeDeleted []string - toBeAdded := make(map[string]utils.StringMap) - for key, strMp := range revIdx { - if len(strMp) == 0 { // remove with no more elements inside - toBeDeleted = append(toBeDeleted, key) - delete(revIdx, key) - continue - } - toBeAdded[key] = make(utils.StringMap) - toBeAdded[key] = strMp - } - values, has := ms.dict[dbKey] - if !has { - result, err := ms.ms.Marshal(toBeAdded) - if err != nil { - return err - } - ms.dict[dbKey] = result - return err - } - mp := make(map[string]utils.StringMap) - err = ms.ms.Unmarshal(values, &mp) - if err != nil { - return err - } - for _, key := range toBeDeleted { - delete(mp, key) - } - for key, strMp := range toBeAdded { - if _, has := mp[key]; !has { - mp[key] = make(utils.StringMap) - } - mp[key] = strMp - } - result, err := ms.ms.Marshal(mp) - if err != nil { - return err - } - ms.dict[dbKey] = result - return -} - -//RemoveFilterReverseIndexesDrv removes ReverseIndexes for a specific itemID -func (ms *MapStorage) RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix string) (err error) { - ms.mu.Lock() - defer ms.mu.Unlock() - delete(ms.dict, utils.CacheInstanceToPrefix[cacheID]+itemIDPrefix) - return -} - func (ms *MapStorage) MatchFilterIndexDrv(cacheID, itemIDPrefix, filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) { ms.mu.RLock() diff --git a/utils/consts.go b/utils/consts.go index 26e10e2c1..db17c14c4 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -41,44 +41,38 @@ var ( MetaFileFWV: FWVSuffix, } CacheInstanceToPrefix = map[string]string{ - CacheDestinations: DESTINATION_PREFIX, - CacheReverseDestinations: REVERSE_DESTINATION_PREFIX, - CacheRatingPlans: RATING_PLAN_PREFIX, - CacheRatingProfiles: RATING_PROFILE_PREFIX, - CacheLCRRules: LCR_PREFIX, - CacheCDRStatS: CDR_STATS_PREFIX, - CacheActions: ACTION_PREFIX, - CacheActionPlans: ACTION_PLAN_PREFIX, - CacheAccountActionPlans: AccountActionPlansPrefix, - CacheActionTriggers: ACTION_TRIGGER_PREFIX, - CacheSharedGroups: SHARED_GROUP_PREFIX, - CacheAliases: ALIASES_PREFIX, - CacheReverseAliases: REVERSE_ALIASES_PREFIX, - CacheDerivedChargers: DERIVEDCHARGERS_PREFIX, - CacheResourceProfiles: ResourceProfilesPrefix, - CacheResources: ResourcesPrefix, - CacheEventResources: EventResourcesPrefix, - CacheTimings: TimingsPrefix, - CacheStatQueueProfiles: StatQueueProfilePrefix, - CacheStatQueues: StatQueuePrefix, - CacheThresholdProfiles: ThresholdProfilePrefix, - CacheThresholds: ThresholdPrefix, - CacheFilters: FilterPrefix, - CacheSupplierProfiles: SupplierProfilePrefix, - CacheAttributeProfiles: AttributeProfilePrefix, - CacheChargerProfiles: ChargerProfilePrefix, - CacheResourceFilterIndexes: ResourceFilterIndexes, - CacheResourceFilterRevIndexes: ResourceFilterRevIndexes, - CacheStatFilterIndexes: StatFilterIndexes, - CacheStatFilterRevIndexes: StatFilterRevIndexes, - CacheThresholdFilterIndexes: ThresholdFilterIndexes, - CacheThresholdFilterRevIndexes: ThresholdFilterRevIndexes, - CacheSupplierFilterIndexes: SupplierFilterIndexes, - CacheSupplierFilterRevIndexes: SupplierFilterRevIndexes, - CacheAttributeFilterIndexes: AttributeFilterIndexes, - CacheAttributeFilterRevIndexes: AttributeFilterRevIndexes, - CacheChargerFilterIndexes: ChargerFilterIndexes, - CacheChargerFilterRevIndexes: ChargerFilterRevIndexes, + CacheDestinations: DESTINATION_PREFIX, + CacheReverseDestinations: REVERSE_DESTINATION_PREFIX, + CacheRatingPlans: RATING_PLAN_PREFIX, + CacheRatingProfiles: RATING_PROFILE_PREFIX, + CacheLCRRules: LCR_PREFIX, + CacheCDRStatS: CDR_STATS_PREFIX, + CacheActions: ACTION_PREFIX, + CacheActionPlans: ACTION_PLAN_PREFIX, + CacheAccountActionPlans: AccountActionPlansPrefix, + CacheActionTriggers: ACTION_TRIGGER_PREFIX, + CacheSharedGroups: SHARED_GROUP_PREFIX, + CacheAliases: ALIASES_PREFIX, + CacheReverseAliases: REVERSE_ALIASES_PREFIX, + CacheDerivedChargers: DERIVEDCHARGERS_PREFIX, + CacheResourceProfiles: ResourceProfilesPrefix, + CacheResources: ResourcesPrefix, + CacheEventResources: EventResourcesPrefix, + CacheTimings: TimingsPrefix, + CacheStatQueueProfiles: StatQueueProfilePrefix, + CacheStatQueues: StatQueuePrefix, + CacheThresholdProfiles: ThresholdProfilePrefix, + CacheThresholds: ThresholdPrefix, + CacheFilters: FilterPrefix, + CacheSupplierProfiles: SupplierProfilePrefix, + CacheAttributeProfiles: AttributeProfilePrefix, + CacheChargerProfiles: ChargerProfilePrefix, + CacheResourceFilterIndexes: ResourceFilterIndexes, + CacheStatFilterIndexes: StatFilterIndexes, + CacheThresholdFilterIndexes: ThresholdFilterIndexes, + CacheSupplierFilterIndexes: SupplierFilterIndexes, + CacheAttributeFilterIndexes: AttributeFilterIndexes, + CacheChargerFilterIndexes: ChargerFilterIndexes, } CachePrefixToInstance map[string]string // will be built on init PrefixToIndexCache = map[string]string{ @@ -89,14 +83,6 @@ var ( AttributeProfilePrefix: CacheAttributeFilterIndexes, ChargerProfilePrefix: CacheChargerFilterIndexes, } - PrefixToRevIndexCache = map[string]string{ - ThresholdProfilePrefix: CacheThresholdFilterRevIndexes, - ResourceProfilesPrefix: CacheResourceFilterRevIndexes, - StatQueueProfilePrefix: CacheStatFilterRevIndexes, - SupplierProfilePrefix: CacheSupplierFilterRevIndexes, - AttributeProfilePrefix: CacheAttributeFilterRevIndexes, - ChargerProfilePrefix: CacheChargerFilterRevIndexes, - } CacheIndexesToPrefix map[string]string // will be built on init ) @@ -865,62 +851,50 @@ const ( // Cache Name const ( - CacheDestinations = "destinations" - CacheReverseDestinations = "reverse_destinations" - CacheRatingPlans = "rating_plans" - CacheRatingProfiles = "rating_profiles" - CacheLCRRules = "lcr_rules" - CacheCDRStatS = "cdr_stats" - CacheActions = "actions" - CacheActionPlans = "action_plans" - CacheAccountActionPlans = "account_action_plans" - CacheActionTriggers = "action_triggers" - CacheSharedGroups = "shared_groups" - CacheAliases = "aliases" - CacheReverseAliases = "reverse_aliases" - CacheDerivedChargers = "derived_chargers" - CacheResources = "resources" - CacheResourceProfiles = "resource_profiles" - CacheTimings = "timings" - CacheEventResources = "event_resources" - CacheStatQueueProfiles = "statqueue_profiles" - CacheStatQueues = "statqueues" - CacheThresholdProfiles = "threshold_profiles" - CacheThresholds = "thresholds" - CacheFilters = "filters" - CacheSupplierProfiles = "supplier_profiles" - CacheAttributeProfiles = "attribute_profiles" - CacheChargerProfiles = "charger_profiles" - CacheResourceFilterIndexes = "resource_filter_indexes" - CacheResourceFilterRevIndexes = "resource_filter_revindexes" - CacheStatFilterIndexes = "stat_filter_indexes" - CacheStatFilterRevIndexes = "stat_filter_revindexes" - CacheThresholdFilterIndexes = "threshold_filter_indexes" - CacheThresholdFilterRevIndexes = "threshold_filter_revindexes" - CacheSupplierFilterIndexes = "supplier_filter_indexes" - CacheSupplierFilterRevIndexes = "supplier_filter_revindexes" - CacheAttributeFilterIndexes = "attribute_filter_indexes" - CacheAttributeFilterRevIndexes = "attribute_filter_revindexes" - CacheChargerFilterIndexes = "charger_filter_indexes" - CacheChargerFilterRevIndexes = "charger_filter_revindexes" - MetaPrecaching = "*precaching" - MetaReady = "*ready" + CacheDestinations = "destinations" + CacheReverseDestinations = "reverse_destinations" + CacheRatingPlans = "rating_plans" + CacheRatingProfiles = "rating_profiles" + CacheLCRRules = "lcr_rules" + CacheCDRStatS = "cdr_stats" + CacheActions = "actions" + CacheActionPlans = "action_plans" + CacheAccountActionPlans = "account_action_plans" + CacheActionTriggers = "action_triggers" + CacheSharedGroups = "shared_groups" + CacheAliases = "aliases" + CacheReverseAliases = "reverse_aliases" + CacheDerivedChargers = "derived_chargers" + CacheResources = "resources" + CacheResourceProfiles = "resource_profiles" + CacheTimings = "timings" + CacheEventResources = "event_resources" + CacheStatQueueProfiles = "statqueue_profiles" + CacheStatQueues = "statqueues" + CacheThresholdProfiles = "threshold_profiles" + CacheThresholds = "thresholds" + CacheFilters = "filters" + CacheSupplierProfiles = "supplier_profiles" + CacheAttributeProfiles = "attribute_profiles" + CacheChargerProfiles = "charger_profiles" + CacheResourceFilterIndexes = "resource_filter_indexes" + CacheStatFilterIndexes = "stat_filter_indexes" + CacheThresholdFilterIndexes = "threshold_filter_indexes" + CacheSupplierFilterIndexes = "supplier_filter_indexes" + CacheAttributeFilterIndexes = "attribute_filter_indexes" + CacheChargerFilterIndexes = "charger_filter_indexes" + MetaPrecaching = "*precaching" + MetaReady = "*ready" ) // Prefix for indexing const ( - ResourceFilterIndexes = "rfi_" - ResourceFilterRevIndexes = "rfr_" - StatFilterIndexes = "sfi_" - StatFilterRevIndexes = "sfr_" - ThresholdFilterIndexes = "tfi_" - ThresholdFilterRevIndexes = "tfr_" - SupplierFilterIndexes = "spi_" - SupplierFilterRevIndexes = "spr_" - AttributeFilterIndexes = "afi_" - AttributeFilterRevIndexes = "afr_" - ChargerFilterIndexes = "cfi_" - ChargerFilterRevIndexes = "cfr_" + ResourceFilterIndexes = "rfi_" + StatFilterIndexes = "sfi_" + ThresholdFilterIndexes = "tfi_" + SupplierFilterIndexes = "spi_" + AttributeFilterIndexes = "afi_" + ChargerFilterIndexes = "cfi_" ) // Agents @@ -945,9 +919,6 @@ func buildCacheIndexesToPrefix() { for k, v := range PrefixToIndexCache { CacheIndexesToPrefix[v] = k } - for k, v := range PrefixToRevIndexCache { - CacheIndexesToPrefix[v] = k - } } func init() {