uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Remove reverse filter indexes (duplicate data)

Browse Source
This commit is contained in:
TeoV
2018-08-21 09:35:58 -04:00
committed by Dan Christian Bogos
parent 34f11aa3f2
commit 0f8dd3754e
12 changed files with 375 additions and 1239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
apier/v1/filter_indexes.go
View File

@@ -151,150 +151,7 @@ func (self *ApierV1) GetFilterIndexes(arg AttrGetFilterIndexes, reply *[]string)
return nil
}
type AttrGetFilterReverseIndexes struct {
Tenant string
Context string
ItemType string
ItemIDs []string
FilterType string
FilterField string
FilterValue string
utils.Paginator
}
func (self *ApierV1) GetFilterReverseIndexes(arg AttrGetFilterReverseIndexes, reply *[]string) (err error) {
var indexes map[string]utils.StringMap
var indexedSlice []string
indexesFilter := make(map[string]utils.StringMap)
if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ItemType"}); len(missing) != 0 { //Params missing
return utils.NewErrMandatoryIeMissing(missing...)
}
key := arg.Tenant
switch arg.ItemType {
case utils.MetaThresholds:
arg.ItemType = utils.ThresholdProfilePrefix
case utils.MetaSuppliers:
arg.ItemType = utils.SupplierProfilePrefix
case utils.MetaStats:
arg.ItemType = utils.StatQueueProfilePrefix
case utils.MetaResources:
arg.ItemType = utils.ResourceProfilesPrefix
case utils.MetaChargers:
arg.ItemType = utils.ChargerProfilePrefix
case utils.MetaAttributes:
if missing := utils.MissingStructFields(&arg, []string{"Context"}); len(missing) != 0 { //Params missing
return utils.NewErrMandatoryIeMissing(missing...)
}
arg.ItemType = utils.AttributeProfilePrefix
key = utils.ConcatenatedKey(arg.Tenant, arg.Context)
}
if arg.ItemIDs != nil {
indexes = make(map[string]utils.StringMap)
for _, itemID := range arg.ItemIDs {
if tmpIndexes, err := self.DataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[arg.ItemType], key, map[string]string{itemID: ""}); err != nil {
return err
} else {
for key, val := range tmpIndexes {
indexes[key] = make(utils.StringMap)
indexes[key] = val
}
}
}
} else {
indexes, err = self.DataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[arg.ItemType], key, nil)
if err != nil {
return err
}
}
if arg.FilterType != "" {
for val, strmap := range indexes {
indexesFilter[val] = make(utils.StringMap)
for _, value := range strmap.Slice() {
if strings.HasPrefix(value, arg.FilterType) {
indexesFilter[val][value] = true
indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value))
}
}
}
if len(indexedSlice) == 0 {
return utils.ErrNotFound
}
}
if arg.FilterField != "" {
if len(indexedSlice) == 0 {
indexesFilter = make(map[string]utils.StringMap)
for val, strmap := range indexes {
indexesFilter[val] = make(utils.StringMap)
for _, value := range strmap.Slice() {
if strings.Index(value, arg.FilterField) != -1 {
indexesFilter[val][value] = true
indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value))
}
}
}
if len(indexedSlice) == 0 {
return utils.ErrNotFound
}
} else {
var cloneIndexSlice []string
for val, strmap := range indexesFilter {
for _, value := range strmap.Slice() {
if strings.Index(value, arg.FilterField) != -1 {
cloneIndexSlice = append(cloneIndexSlice, utils.ConcatenatedKey(val, value))
}
}
}
if len(cloneIndexSlice) == 0 {
return utils.ErrNotFound
}
indexedSlice = cloneIndexSlice
}
}
if arg.FilterValue != "" {
if len(indexedSlice) == 0 {
for val, strmap := range indexes {
for _, value := range strmap.Slice() {
if strings.Index(value, arg.FilterValue) != -1 {
indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value))
}
}
}
if len(indexedSlice) == 0 {
return utils.ErrNotFound
}
} else {
var cloneIndexSlice []string
for val, strmap := range indexesFilter {
for _, value := range strmap.Slice() {
if strings.Index(value, arg.FilterValue) != -1 {
cloneIndexSlice = append(cloneIndexSlice, utils.ConcatenatedKey(val, value))
}
}
}
if len(cloneIndexSlice) == 0 {
return utils.ErrNotFound
}
indexedSlice = cloneIndexSlice
}
}
if len(indexedSlice) == 0 {
for val, strmap := range indexes {
for _, value := range strmap.Slice() {
indexedSlice = append(indexedSlice, utils.ConcatenatedKey(val, value))
}
}
}
if arg.Paginator.Limit != nil || arg.Paginator.Offset != nil || arg.Paginator.SearchTerm != "" {
*reply = arg.Paginator.PaginateStringSlice(indexedSlice)
} else {
*reply = indexedSlice
}
return nil
}
//NEED REVIEW
func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, reply *string) error {
transactionID := utils.GenUUID()
//ThresholdProfile Indexes
@@ -333,7 +190,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if err := thdsIndexers.StoreIndexes(true, transactionID); err != nil {
if args.ThresholdIDs != nil {
for _, id := range *args.ThresholdIDs {
if err := thdsIndexers.RemoveItemFromIndex(id); err != nil {
th, err := self.DataManager.GetThresholdProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := thdsIndexers.RemoveItemFromIndex(args.Tenant, id, th.FilterIDs); err != nil {
return err
}
}
@@ -345,7 +206,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if sqpIndexers != nil {
if err := sqpIndexers.StoreIndexes(true, transactionID); err != nil {
for _, id := range *args.StatIDs {
if err := sqpIndexers.RemoveItemFromIndex(id); err != nil {
sqp, err := self.DataManager.GetStatQueueProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := sqpIndexers.RemoveItemFromIndex(args.Tenant, id, sqp.FilterIDs); err != nil {
return err
}
}
@@ -356,7 +221,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if rsIndexes != nil {
if err := rsIndexes.StoreIndexes(true, transactionID); err != nil {
for _, id := range *args.ResourceIDs {
if err := rsIndexes.RemoveItemFromIndex(id); err != nil {
rp, err := self.DataManager.GetResourceProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := rsIndexes.RemoveItemFromIndex(args.Tenant, id, rp.FilterIDs); err != nil {
return err
}
}
@@ -367,7 +236,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if sppIndexes != nil {
if err := sppIndexes.StoreIndexes(true, transactionID); err != nil {
for _, id := range *args.SupplierIDs {
if err := sppIndexes.RemoveItemFromIndex(id); err != nil {
spp, err := self.DataManager.GetSupplierProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := sppIndexes.RemoveItemFromIndex(args.Tenant, id, spp.FilterIDs); err != nil {
return err
}
}
@@ -378,7 +251,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if attrIndexes != nil {
if err := attrIndexes.StoreIndexes(true, transactionID); err != nil {
for _, id := range *args.AttributeIDs {
if err := attrIndexes.RemoveItemFromIndex(id); err != nil {
ap, err := self.DataManager.GetAttributeProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := attrIndexes.RemoveItemFromIndex(args.Tenant, id, ap.FilterIDs); err != nil {
return err
}
}
@@ -389,7 +266,11 @@ func (self *ApierV1) ComputeFilterIndexes(args utils.ArgsComputeFilterIndexes, r
if cppIndexes != nil {
if err := attrIndexes.StoreIndexes(true, transactionID); err != nil {
for _, id := range *args.ChargerIDs {
if err := cppIndexes.RemoveItemFromIndex(id); err != nil {
cpp, err := self.DataManager.GetChargerProfile(args.Tenant, id, false, utils.NonTransactional)
if err != nil {
return err
}
if err := cppIndexes.RemoveItemFromIndex(args.Tenant, id, cpp.FilterIDs); err != nil {
return err
}
}

212
apier/v1/filter_indexes_it_test.go
View File

@@ -57,11 +57,11 @@ var sTestsFilterIndexesSV1 = []func(t *testing.T){
testV1FIdxThirdComputeThresholdsIndexes,
testV1FIdxRemoveThresholdProfile,
testV1FIdxSetStatQueueProfileIndexes,
testV1FIdxComputeStatQueueProfileIndexes,
testV1FIdxSetSecondStatQueueProfileIndexes,
testV1FIdxSecondComputeStatQueueProfileIndexes,
testV1FIdxRemoveStatQueueProfile,
// testV1FIdxSetStatQueueProfileIndexes,
// testV1FIdxComputeStatQueueProfileIndexes,
// testV1FIdxSetSecondStatQueueProfileIndexes,
// testV1FIdxSecondComputeStatQueueProfileIndexes,
// testV1FIdxRemoveStatQueueProfile,
testV1FIdxSetResourceProfileIndexes,
testV1FIdxComputeResourceProfileIndexes,
@@ -235,18 +235,10 @@ func testV1FIdxSetThresholdProfile(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxComputeThresholdsIndexes(t *testing.T) {
@@ -279,16 +271,6 @@ func testV1FIdxComputeThresholdsIndexes(t *testing.T) {
if !reflect.DeepEqual(expectedIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix], tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxSetSecondThresholdProfile(t *testing.T) {
@@ -352,11 +334,6 @@ func testV1FIdxSetSecondThresholdProfile(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if _, err = onStor.GetFilterIndexes(
utils.PrefixToIndexCache[utils.ThresholdProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
@@ -395,17 +372,6 @@ func testV1FIdxSecondComputeThresholdsIndexes(t *testing.T) {
if !reflect.DeepEqual(expectedIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE2": {"*string:Account:1002": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxThirdComputeThresholdsIndexes(t *testing.T) {
@@ -442,22 +408,6 @@ func testV1FIdxThirdComputeThresholdsIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {
"*string:Account:1001": true},
"TEST_PROFILE2": {
"*string:Account:1002": true},
}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxRemoveThresholdProfile(t *testing.T) {
@@ -506,10 +456,6 @@ func testV1FIdxRemoveThresholdProfile(t *testing.T) {
tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
//StatQueueProfile
@@ -583,10 +529,6 @@ func testV1FIdxSetStatQueueProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
@@ -623,18 +565,6 @@ func testV1FIdxComputeStatQueueProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxSetSecondStatQueueProfileIndexes(t *testing.T) {
@@ -708,11 +638,6 @@ func testV1FIdxSetSecondStatQueueProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(
utils.PrefixToIndexCache[utils.StatQueueProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
@@ -752,18 +677,6 @@ func testV1FIdxSecondComputeStatQueueProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE2": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxRemoveStatQueueProfile(t *testing.T) {
@@ -807,14 +720,10 @@ func testV1FIdxRemoveStatQueueProfile(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if _, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
if _, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.StatQueueProfilePrefix],
tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
//ResourceProfile
@@ -876,10 +785,6 @@ func testV1FIdxSetResourceProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
@@ -916,18 +821,6 @@ func testV1FIdxComputeResourceProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"RCFG1": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxSetSecondResourceProfileIndexes(t *testing.T) {
@@ -988,11 +881,6 @@ func testV1FIdxSetSecondResourceProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(
utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
@@ -1029,17 +917,6 @@ func testV1FIdxSecondComputeResourceProfileIndexes(t *testing.T) {
if !reflect.DeepEqual(expectedIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"RCFG2": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxRemoveResourceProfile(t *testing.T) {
@@ -1085,10 +962,6 @@ func testV1FIdxRemoveResourceProfile(t *testing.T) {
tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ResourceProfilesPrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
//SupplierProfile
@@ -1156,10 +1029,6 @@ func testV1FIdxSetSupplierProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
@@ -1196,18 +1065,6 @@ func testV1FIdxComputeSupplierProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxSetSecondSupplierProfileIndexes(t *testing.T) {
@@ -1275,11 +1132,6 @@ func testV1FIdxSetSecondSupplierProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(
utils.PrefixToIndexCache[utils.SupplierProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
@@ -1319,18 +1171,6 @@ func testV1FIdxSecondComputeSupplierProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE2": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.SupplierProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxRemoveSupplierProfile(t *testing.T) {
@@ -1378,10 +1218,6 @@ func testV1FIdxRemoveSupplierProfile(t *testing.T) {
tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.SupplierProfilePrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
//AttributeProfile
@@ -1452,10 +1288,6 @@ func testV1FIdxSetAttributeProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
@@ -1491,17 +1323,6 @@ func testV1FIdxComputeAttributeProfileIndexes(t *testing.T) {
if !reflect.DeepEqual(expectedIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"ApierTest": {"*string:Account:1001": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
tenant, nil); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) {
@@ -1572,11 +1393,6 @@ func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) {
tenant); err != nil {
t.Error(err)
}
if err := onStor.RemoveFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
tenant); err != nil {
t.Error(err)
}
if indexes, err = onStor.GetFilterIndexes(
utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant, engine.MetaString, nil); err != utils.ErrNotFound {
@@ -1616,18 +1432,6 @@ func testV1FIdxSecondComputeAttributeProfileIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v",
expectedIDX, utils.ToJSON(indexes))
}
expectedRevIDX := map[string]utils.StringMap{
"ApierTest2": {"*string:Account:1001": true}}
indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
tenant, nil)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxRemoveAttributeProfile(t *testing.T) {
@@ -1674,10 +1478,6 @@ func testV1FIdxRemoveAttributeProfile(t *testing.T) {
tenant, engine.MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix],
tenant, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxPopulateDatabase(t *testing.T) {

287
apier/v1/filterindexecache_it_test.go
View File

@@ -51,14 +51,14 @@ var sTestsFilterIndexesSV1Ca = []func(t *testing.T){
testV1FIdxCaUpdateThresholdProfileFromTP,
testV1FIdxCaRemoveThresholdProfile,
testFlush,
testV1FIdxCaGetStatQueuesWithNotFound,
testV1FIdxCaSetStatQueueProfile,
testV1FIdxCaFromFolder,
testV1FIdxCaGetStatQueuesFromTP,
testV1FIdxCaUpdateStatQueueProfile,
testV1FIdxCaUpdateStatQueueProfileFromTP,
testV1FIdxCaRemoveStatQueueProfile,
// testFlush,
// testV1FIdxCaGetStatQueuesWithNotFound,
// testV1FIdxCaSetStatQueueProfile,
// testV1FIdxCaFromFolder,
// testV1FIdxCaGetStatQueuesFromTP,
// testV1FIdxCaUpdateStatQueueProfile,
// testV1FIdxCaUpdateStatQueueProfileFromTP,
// testV1FIdxCaRemoveStatQueueProfile,
testFlush,
testV1FIdxCaProcessAttributeProfileEventWithNotFound,
@@ -178,10 +178,6 @@ func testV1FIdxCaProcessEventWithNotFound(t *testing.T) {
if err := tFIdxCaRpc.Call(utils.ThresholdSv1ProcessEvent, tEv, &thIDs); err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if indexes, err = onStor.GetFilterReverseIndexes(utils.PrefixToIndexCache[utils.ThresholdProfilePrefix],
"cgrates.org", nil); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxCaSetThresholdProfile(t *testing.T) {
@@ -248,19 +244,6 @@ func testV1FIdxCaSetThresholdProfile(t *testing.T) {
} else if !reflect.DeepEqual(thIDs, eIDs) {
t.Errorf("Expecting hits: %s, received: %s", eIDs, thIDs)
}
//test to make sure indexes are made as expected
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true,
"*string:EventType:BalanceUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
"cgrates.org", fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes)
}
}
func testV1FIdxCaGetThresholdFromTP(t *testing.T) {
@@ -284,22 +267,6 @@ func testV1FIdxCaGetThresholdFromTP(t *testing.T) {
} else if !reflect.DeepEqual(thIDs, eIDs) {
t.Errorf("Expecting hits: %s, received: %s", eIDs, thIDs)
}
//test to make sure indexes are made as expected
idx := map[string]utils.StringMap{
"THD_ACNT_BALANCE_1": {
"*string:Account:1001": true,
"*string:Account:1002": true,
"*string:EventType:BalanceUpdate": true}}
fldNameVal := map[string]string{"THD_ACNT_BALANCE_1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
"cgrates.org", fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateThresholdProfile(t *testing.T) {
@@ -376,20 +343,6 @@ func testV1FIdxCaUpdateThresholdProfile(t *testing.T) {
} else if !reflect.DeepEqual(thIDs, eIDs) {
t.Errorf("Expecting : %s, received: %s", eIDs, thIDs)
}
//test to make sure indexes are made as expecte
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1002": true,
"*string:EventType:AccountUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
"cgrates.org", fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v",
expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateThresholdProfileFromTP(t *testing.T) {
@@ -460,19 +413,6 @@ func testV1FIdxCaUpdateThresholdProfileFromTP(t *testing.T) {
} else if !reflect.DeepEqual(thIDs, eIDs) {
t.Errorf("Expecting : %s, received: %s", eIDs, thIDs)
}
//test to make sure indexes are made as expecte
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1002": true,
"*string:EventType:AccountUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ThresholdProfilePrefix],
"cgrates.org", fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxCaRemoveThresholdProfile(t *testing.T) {
@@ -540,13 +480,6 @@ func testV1FIdxCaRemoveThresholdProfile(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
//test to make sure indexes are made as expected
fldNameVal2 := map[string]string{"THD_ACNT_BALANCE_1": "", "TEST_PROFILE1": ""}
if _, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org",
fldNameVal2); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
//StatQueue
@@ -564,11 +497,6 @@ func testV1FIdxCaGetStatQueuesWithNotFound(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
nil); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxCaSetStatQueueProfile(t *testing.T) {
@@ -642,19 +570,6 @@ func testV1FIdxCaSetStatQueueProfile(t *testing.T) {
} else if !reflect.DeepEqual(reply, expected) {
t.Errorf("Expecting: %+v, received: %+v", expected, reply)
}
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true,
"*string:EventType:AccountUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes)
}
}
func testV1FIdxCaGetStatQueuesFromTP(t *testing.T) {
@@ -711,21 +626,6 @@ func testV1FIdxCaGetStatQueuesFromTP(t *testing.T) {
} else if !reflect.DeepEqual(reply, expected) {
t.Errorf("Expecting: %+v, received: %+v", expected, reply)
}
idx := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1001": true,
"*string:EventType:AccountUpdate": true}}
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateStatQueueProfile(t *testing.T) {
@@ -794,19 +694,6 @@ func testV1FIdxCaUpdateStatQueueProfile(t *testing.T) {
} else if !reflect.DeepEqual(reply, expected) {
t.Errorf("Expecting: %+v, received: %+v", expected, reply)
}
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1003": true,
"*string:EventType:BalanceUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateStatQueueProfileFromTP(t *testing.T) {
@@ -864,19 +751,6 @@ func testV1FIdxCaUpdateStatQueueProfileFromTP(t *testing.T) {
} else if !reflect.DeepEqual(ids, expected) {
t.Errorf("Expecting: %+v, received: %+v", expected, ids)
}
fldNameVal := map[string]string{"Stats1": ""}
expectedRevIDX := map[string]utils.StringMap{
"Stats1": {
"*string:Account:1003": true,
"*string:EventType:AccountUpdate": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes)
}
}
func testV1FIdxCaRemoveStatQueueProfile(t *testing.T) {
@@ -944,12 +818,6 @@ func testV1FIdxCaRemoveStatQueueProfile(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
// fldNameVals := map[string]string{"THD_ACNT_BALANCE_1": "", "TEST_PROFILE1": ""}
// if _, err = onStor.GetFilterReverseIndexes(
// utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org",
// fldNameVals); err == nil || err != utils.ErrNotFound {
// t.Error(err)
// }
}
//AttributeProfile
@@ -968,11 +836,6 @@ func testV1FIdxCaProcessAttributeProfileEventWithNotFound(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org",
nil); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxCaSetAttributeProfile(t *testing.T) {
@@ -1045,20 +908,6 @@ func testV1FIdxCaSetAttributeProfile(t *testing.T) {
ev, &rplyEv); err != nil {
t.Error(err)
}
//test to make sure indexes are made as expected
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
expectedRevIDX := map[string]utils.StringMap{
"TEST_PROFILE1": {"*string:Account:1009": true,
"*string:Destination:+491511231234": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
"cgrates.org:*sessions",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes)
}
}
func testV1FIdxCaGetAttributeProfileFromTP(t *testing.T) {
@@ -1076,19 +925,6 @@ func testV1FIdxCaGetAttributeProfileFromTP(t *testing.T) {
if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil {
t.Error(err)
}
//test to make sure indexes are made as expected
idx := map[string]utils.StringMap{"ATTR_1": {
"*string:Account:1007": true}}
fldNameVal := map[string]string{"ATTR_1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
"cgrates.org:*sessions",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateAttributeProfile(t *testing.T) {
@@ -1160,21 +996,6 @@ func testV1FIdxCaUpdateAttributeProfile(t *testing.T) {
if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil {
t.Error(err)
}
//test to make sure indexes are made as expected
idx := map[string]utils.StringMap{
"TEST_PROFILE1": {
"*string:Account:2009": true,
"*string:Destination:+492511231234": true}}
fldNameVal := map[string]string{"TEST_PROFILE1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
"cgrates.org:*sessions",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateAttributeProfileFromTP(t *testing.T) {
@@ -1228,21 +1049,6 @@ func testV1FIdxCaUpdateAttributeProfileFromTP(t *testing.T) {
if err := tFIdxCaRpc.Call(utils.AttributeSv1ProcessEvent, ev, &rplyEv); err != nil {
t.Error(err)
}
//test to make sure indexes are made as expected
idx := map[string]utils.StringMap{
"ATTR_1": {
"*string:Account:3009": true,
"*string:Destination:+492511231234": true}}
fldNameVal := map[string]string{"ATTR_1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.AttributeProfilePrefix],
"cgrates.org:*sessions",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaRemoveAttributeProfile(t *testing.T) {
@@ -1308,13 +1114,6 @@ func testV1FIdxCaRemoveAttributeProfile(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
//test to make sure indexes are made as expected
fldNameVal2 := map[string]string{"ATTR_1": "", "TEST_PROFILE1": ""}
if _, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.AttributeProfilePrefix], "cgrates.org:*rating",
fldNameVal2); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
// ResourceProfile
@@ -1339,11 +1138,6 @@ func testV1FIdxCaGetResourceProfileWithNotFound(t *testing.T) {
argsRU, &reply); err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.StatQueueProfilePrefix], "cgrates.org",
nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxCaSetResourceProfile(t *testing.T) {
filter = &engine.Filter{
@@ -1420,20 +1214,6 @@ func testV1FIdxCaSetResourceProfile(t *testing.T) {
} else if result != "Approved" {
t.Error("Unexpected reply returned", result)
}
fldNameVal := map[string]string{"RCFG1": ""}
expectedRevIDX := map[string]utils.StringMap{
"RCFG1": {
"*string:Account:1001": true,
"*string:Subject:1002": true,
"*string:Destination:1001": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], "cgrates.org",
fldNameVal); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, indexes)
}
}
func testV1FIdxCaGetResourceProfileFromTP(t *testing.T) {
@@ -1479,21 +1259,6 @@ func testV1FIdxCaGetResourceProfileFromTP(t *testing.T) {
} else if reply != "ResGroup1" {
t.Error("Unexpected reply returned", reply)
}
idx := map[string]utils.StringMap{
"ResGroup1": {
"*prefix:Destination:10": true,
"*prefix:Destination:20": true,
"*string:Account:1001": true,
"*string:Account:1002": true}}
fldNameVal := map[string]string{"ResGroup1": ""}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix], "cgrates.org",
fldNameVal); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(idx, indexes) {
t.Errorf("Expecting: %+v, received: %+v", idx, utils.ToJSON(indexes))
}
}
func testV1FIdxCaUpdateResourceProfile(t *testing.T) {
@@ -1565,21 +1330,6 @@ func testV1FIdxCaUpdateResourceProfile(t *testing.T) {
} else if result != "MessageAllocation" {
t.Error("Unexpected reply returned", result)
}
// fldNameVal2 := map[string]string{"RCFG1": ""}
// expectedRevIDX := map[string]utils.StringMap{
// "RCFG1": {
// "*string:Account:2002": true,
// "*string:Destination:2002": true,
// "*string:Subject:2001": true}}
// if indexes, err = onStor.GetFilterReverseIndexes(
// utils.PrefixToIndexCache[utils.ResourceProfilesPrefix], "cgrates.org",
// fldNameVal2); err != nil {
// t.Error(err)
// }
// if !reflect.DeepEqual(expectedRevIDX, indexes) {
// t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
// }
}
func testV1FIdxCaUpdateResourceProfileFromTP(t *testing.T) {
@@ -1643,20 +1393,6 @@ func testV1FIdxCaUpdateResourceProfileFromTP(t *testing.T) {
} else if result != "ResGroup1" {
t.Error("Unexpected reply returned", result)
}
fldNameVal2 := map[string]string{"ResGroup1": ""}
expectedRevIDX := map[string]utils.StringMap{
"ResGroup1": {
"*string:Account:1002": true,
"*string:Destination:1002": true,
"*string:Subject:1001": true}}
if indexes, err = onStor.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
"cgrates.org", fldNameVal2); err != nil {
t.Error(err)
}
if !reflect.DeepEqual(expectedRevIDX, indexes) {
t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
}
}
func testV1FIdxCaRemoveResourceProfile(t *testing.T) {
@@ -1722,13 +1458,6 @@ func testV1FIdxCaRemoveResourceProfile(t *testing.T) {
err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
fldNameVals2 := map[string]string{"ResGroup1": "", "TEST_PROFILE1": ""}
if _, err = onStor.GetFilterReverseIndexes(
utils.PrefixToIndexCache[utils.ThresholdProfilePrefix], "cgrates.org",
fldNameVals2); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
func testV1FIdxCaStopEngine(t *testing.T) {

12
cmd/cgr-loader/cgr-loader.go
View File

@@ -405,22 +405,22 @@ func main() {
}
var cacheIDs []string
if len(apfIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheAttributeFilterIndexes, utils.CacheAttributeFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheAttributeFilterIndexes)
}
if len(spfIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheSupplierFilterIndexes, utils.CacheSupplierFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheSupplierFilterIndexes)
}
if len(trspfIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheThresholdFilterIndexes, utils.CacheThresholdFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheThresholdFilterIndexes)
}
if len(stqpIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheStatFilterIndexes, utils.CacheStatFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheStatFilterIndexes)
}
if len(rspIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheResourceFilterIndexes, utils.CacheResourceFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheResourceFilterIndexes)
}
if len(chargerIDs) != 0 {
cacheIDs = append(cacheIDs, utils.CacheChargerFilterIndexes, utils.CacheChargerFilterRevIndexes)
cacheIDs = append(cacheIDs, utils.CacheChargerFilterIndexes)
}
if err = cacheS.Call(utils.CacheSv1Clear, cacheIDs, &reply); err != nil {
log.Printf("WARNING: Got error on cache clear: %s\n", err.Error())

65
console/filter_reverse_indexes.go
View File

@@ -1,65 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package console
import (
"github.com/cgrates/cgrates/apier/v1"
)
func init() {
c := &CmdGetFilterReverseIndexes{
name: "filter_reverse_indexes",
rpcMethod: "ApierV1.GetFilterReverseIndexes",
rpcParams: &v1.AttrGetFilterReverseIndexes{},
}
commands[c.Name()] = c
c.CommandExecuter = &CommandExecuter{c}
}
// Commander implementation
type CmdGetFilterReverseIndexes struct {
name string
rpcMethod string
rpcParams *v1.AttrGetFilterReverseIndexes
*CommandExecuter
}
func (self *CmdGetFilterReverseIndexes) Name() string {
return self.name
}
func (self *CmdGetFilterReverseIndexes) RpcMethod() string {
return self.rpcMethod
}
func (self *CmdGetFilterReverseIndexes) RpcParams(reset bool) interface{} {
if reset || self.rpcParams == nil {
self.rpcParams = &v1.AttrGetFilterReverseIndexes{}
}
return self.rpcParams
}
func (self *CmdGetFilterReverseIndexes) PostprocessRpcParams() error {
return nil
}
func (self *CmdGetFilterReverseIndexes) RpcResult() interface{} {
var atr []string
return &atr
}

23
engine/attributes_test.go
View File

@@ -476,11 +476,6 @@ func TestAttributeIndexer(t *testing.T) {
"AttrPrf": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"AttrPrf": utils.StringMap{
"*string:Account:1007": true,
},
}
rfi1 := NewFilterIndexer(dmAtr, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(attrPrf.Tenant, utils.META_ANY))
if rcvIdx, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType],
@@ -491,14 +486,6 @@ func TestAttributeIndexer(t *testing.T) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
if reverseRcvIdx, err := dmAtr.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi1.itemType], rfi1.dbKeySuffix, nil); err != nil {
t.Error(err)
} else {
if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
}
//Set AttributeProfile with new context (*sessions)
attrPrf.Contexts = []string{utils.MetaSessionS}
if err := dmAtr.SetAttributeProfile(attrPrf, true); err != nil {
@@ -514,21 +501,11 @@ func TestAttributeIndexer(t *testing.T) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
if reverseRcvIdx, err := dmAtr.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi2.itemType], rfi2.dbKeySuffix, nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//verify if old index was deleted ( context *any)
if _, err := dmAtr.GetFilterIndexes(utils.PrefixToIndexCache[rfi1.itemType],
rfi1.dbKeySuffix, MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
}
if _, err := dmAtr.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi1.itemType], rfi1.dbKeySuffix, nil); err != utils.ErrNotFound {
t.Error(err)
}
}
func TestAttributeProcessWithMultipleRuns1(t *testing.T) {

146
engine/datamanager.go
View File

@@ -420,6 +420,10 @@ func (dm *DataManager) GetThresholdProfile(tenant, id string, skipCache bool,
}
func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool) (err error) {
oldTh, err := dm.GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().SetThresholdProfileDrv(th); err != nil {
return err
}
@@ -428,6 +432,20 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
return
}
if withIndex {
if oldTh != nil {
var needsRemove bool
for _, fltrID := range oldTh.FilterIDs {
if !utils.IsSliceMember(th.FilterIDs, fltrID) {
needsRemove = true
}
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.ThresholdProfilePrefix,
th.Tenant).RemoveItemFromIndex(th.Tenant, th.ID, oldTh.FilterIDs); err != nil {
return
}
}
}
return createAndIndex(utils.ThresholdProfilePrefix, th.Tenant, utils.EmptyString, th.ID, th.FilterIDs, dm)
}
return
@@ -435,6 +453,10 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
func (dm *DataManager) RemoveThresholdProfile(tenant, id,
transactionID string, withIndex bool) (err error) {
oldTh, err := dm.GetThresholdProfile(tenant, id, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemThresholdProfileDrv(tenant, id); err != nil {
return
}
@@ -442,7 +464,7 @@ func (dm *DataManager) RemoveThresholdProfile(tenant, id,
cacheCommit(transactionID), transactionID)
if withIndex {
return NewFilterIndexer(dm,
utils.ThresholdProfilePrefix, tenant).RemoveItemFromIndex(id)
utils.ThresholdProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldTh.FilterIDs)
}
return
}
@@ -472,6 +494,10 @@ func (dm *DataManager) GetStatQueueProfile(tenant, id string, skipCache bool,
}
func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool) (err error) {
oldSts, err := dm.GetStatQueueProfile(sqp.Tenant, sqp.ID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().SetStatQueueProfileDrv(sqp); err != nil {
return err
}
@@ -480,6 +506,20 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
return
}
if withIndex {
if oldSts != nil {
var needsRemove bool
for _, fltrID := range oldSts.FilterIDs {
if !utils.IsSliceMember(sqp.FilterIDs, fltrID) {
needsRemove = true
}
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.StatQueueProfilePrefix,
sqp.Tenant).RemoveItemFromIndex(sqp.Tenant, sqp.ID, oldSts.FilterIDs); err != nil {
return
}
}
}
return createAndIndex(utils.StatQueueProfilePrefix, sqp.Tenant, utils.EmptyString, sqp.ID, sqp.FilterIDs, dm)
}
return
@@ -487,13 +527,17 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
func (dm *DataManager) RemoveStatQueueProfile(tenant, id,
transactionID string, withIndex bool) (err error) {
oldSts, err := dm.GetResourceProfile(tenant, id, true, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemStatQueueProfileDrv(tenant, id); err != nil {
return
}
Cache.Remove(utils.CacheStatQueueProfiles, utils.ConcatenatedKey(tenant, id),
cacheCommit(transactionID), transactionID)
if withIndex {
return NewFilterIndexer(dm, utils.StatQueueProfilePrefix, tenant).RemoveItemFromIndex(id)
return NewFilterIndexer(dm, utils.StatQueueProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldSts.FilterIDs)
}
return
}
@@ -607,6 +651,10 @@ func (dm *DataManager) GetResourceProfile(tenant, id string,
}
func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (err error) {
oldRes, err := dm.GetResourceProfile(rp.Tenant, rp.ID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().SetResourceProfileDrv(rp); err != nil {
return err
}
@@ -616,6 +664,20 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
}
//to be implemented in tests
if withIndex {
if oldRes != nil {
var needsRemove bool
for _, fltrID := range oldRes.FilterIDs {
if !utils.IsSliceMember(rp.FilterIDs, fltrID) {
needsRemove = true
}
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.ResourceProfilesPrefix,
rp.Tenant).RemoveItemFromIndex(rp.Tenant, rp.ID, oldRes.FilterIDs); err != nil {
return
}
}
}
if err = createAndIndex(utils.ResourceProfilesPrefix, rp.Tenant, utils.EmptyString, rp.ID, rp.FilterIDs, dm); err != nil {
return
}
@@ -625,13 +687,17 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
}
func (dm *DataManager) RemoveResourceProfile(tenant, id, transactionID string, withIndex bool) (err error) {
oldRes, err := dm.GetResourceProfile(tenant, id, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemoveResourceProfileDrv(tenant, id); err != nil {
return
}
Cache.Remove(utils.CacheResourceProfiles, utils.ConcatenatedKey(tenant, id),
cacheCommit(transactionID), transactionID)
if withIndex {
return NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant).RemoveItemFromIndex(id)
return NewFilterIndexer(dm, utils.ResourceProfilesPrefix, tenant).RemoveItemFromIndex(tenant, id, oldRes.FilterIDs)
}
return
}
@@ -955,22 +1021,6 @@ func (dm *DataManager) RemoveFilterIndexes(cacheID, itemIDPrefix string) (err er
return dm.DataDB().RemoveFilterIndexesDrv(cacheID, itemIDPrefix)
}
func (dm *DataManager) GetFilterReverseIndexes(cacheID, itemIDPrefix string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
return dm.DataDB().GetFilterReverseIndexesDrv(cacheID, itemIDPrefix, fldNameVal)
}
func (dm *DataManager) SetFilterReverseIndexes(cacheID, itemIDPrefix string,
indexes map[string]utils.StringMap,
commit bool, transactionID string) (err error) {
return dm.DataDB().SetFilterReverseIndexesDrv(cacheID,
itemIDPrefix, indexes, commit, transactionID)
}
func (dm *DataManager) RemoveFilterReverseIndexes(cacheID, itemIDPrefix string) (err error) {
return dm.DataDB().RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix)
}
func (dm *DataManager) MatchFilterIndex(cacheID, itemIDPrefix,
filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) {
fieldValKey := utils.ConcatenatedKey(itemIDPrefix, filterType, fieldName, fieldVal)
@@ -1043,6 +1093,10 @@ func (dm *DataManager) GetSupplierProfile(tenant, id string, skipCache bool,
}
func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool) (err error) {
oldSup, err := dm.GetSupplierProfile(supp.Tenant, supp.ID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().SetSupplierProfileDrv(supp); err != nil {
return err
}
@@ -1050,19 +1104,37 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool)
return
}
if withIndex {
if oldSup != nil {
var needsRemove bool
for _, fltrID := range oldSup.FilterIDs {
if !utils.IsSliceMember(supp.FilterIDs, fltrID) {
needsRemove = true
}
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.SupplierProfilePrefix,
supp.Tenant).RemoveItemFromIndex(supp.Tenant, supp.ID, oldSup.FilterIDs); err != nil {
return
}
}
}
return createAndIndex(utils.SupplierProfilePrefix, supp.Tenant, utils.EmptyString, supp.ID, supp.FilterIDs, dm)
}
return
}
func (dm *DataManager) RemoveSupplierProfile(tenant, id, transactionID string, withIndex bool) (err error) {
oldSupp, err := dm.GetSupplierProfile(tenant, id, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemoveSupplierProfileDrv(tenant, id); err != nil {
return
}
Cache.Remove(utils.CacheSupplierProfiles, utils.ConcatenatedKey(tenant, id),
cacheCommit(transactionID), transactionID)
if withIndex {
return NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant).RemoveItemFromIndex(id)
return NewFilterIndexer(dm, utils.SupplierProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldSupp.FilterIDs)
}
return
}
@@ -1095,7 +1167,7 @@ func (dm *DataManager) GetAttributeProfile(tenant, id string, skipCache bool,
}
func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) (err error) {
oldAP, err := dm.GetAttributeProfile(ap.Tenant, ap.ID, true, utils.NonTransactional)
oldAP, err := dm.GetAttributeProfile(ap.Tenant, ap.ID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
@@ -1120,7 +1192,7 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(ap.Tenant, ctx)).RemoveItemFromIndex(ap.ID); err != nil {
utils.ConcatenatedKey(ap.Tenant, ctx)).RemoveItemFromIndex(ap.Tenant, ap.ID, oldAP.FilterIDs); err != nil {
return
}
}
@@ -1138,6 +1210,10 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
func (dm *DataManager) RemoveAttributeProfile(tenant, id string, contexts []string,
transactionID string, withIndex bool) (err error) {
oldAttr, err := dm.GetAttributeProfile(tenant, id, true, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemoveAttributeProfileDrv(tenant, id); err != nil {
return
}
@@ -1146,7 +1222,7 @@ func (dm *DataManager) RemoveAttributeProfile(tenant, id string, contexts []stri
if withIndex {
for _, context := range contexts {
if err = NewFilterIndexer(dm, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(tenant, context)).RemoveItemFromIndex(id); err != nil {
utils.ConcatenatedKey(tenant, context)).RemoveItemFromIndex(tenant, id, oldAttr.FilterIDs); err != nil {
return
}
}
@@ -1179,6 +1255,10 @@ func (dm *DataManager) GetChargerProfile(tenant, id string, skipCache bool,
}
func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (err error) {
oldCpp, err := dm.GetChargerProfile(cpp.Tenant, cpp.ID, true, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().SetChargerProfileDrv(cpp); err != nil {
return err
}
@@ -1186,6 +1266,20 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e
return
}
if withIndex {
if oldCpp != nil {
var needsRemove bool
for _, fltrID := range oldCpp.FilterIDs {
if !utils.IsSliceMember(cpp.FilterIDs, fltrID) {
needsRemove = true
}
}
if needsRemove {
if err = NewFilterIndexer(dm, utils.SupplierProfilePrefix,
cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil {
return
}
}
}
return createAndIndex(utils.ChargerProfilePrefix, cpp.Tenant, utils.EmptyString, cpp.ID, cpp.FilterIDs, dm)
}
return
@@ -1193,13 +1287,17 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e
func (dm *DataManager) RemoveChargerProfile(tenant, id string,
transactionID string, withIndex bool) (err error) {
oldCpp, err := dm.GetChargerProfile(tenant, id, true, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if err = dm.DataDB().RemoveChargerProfileDrv(tenant, id); err != nil {
return
}
Cache.Remove(utils.CacheChargerProfiles, utils.ConcatenatedKey(tenant, id),
cacheCommit(transactionID), transactionID)
if withIndex {
return NewFilterIndexer(dm, utils.ChargerProfilePrefix, tenant).RemoveItemFromIndex(id)
return NewFilterIndexer(dm, utils.ChargerProfilePrefix, tenant).RemoveItemFromIndex(tenant, id, oldCpp.FilterIDs)
}
return
}

202
engine/filterindexer.go
View File

@@ -20,7 +20,6 @@ package engine
import (
"fmt"
"strings"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/guardian"
@@ -29,31 +28,19 @@ import (
func NewFilterIndexer(dm *DataManager, itemType, dbKeySuffix string) *FilterIndexer {
return &FilterIndexer{dm: dm, itemType: itemType, dbKeySuffix: dbKeySuffix,
indexes: make(map[string]utils.StringMap),
reveseIndex: make(map[string]utils.StringMap),
chngdIndxKeys: make(utils.StringMap),
chngdRevIndxKeys: make(utils.StringMap)}
indexes: make(map[string]utils.StringMap),
chngdIndxKeys: make(utils.StringMap)}
}
// FilterIndexer is a centralized indexer for all data sources using RequestFilter
// retrieves and stores it's data from/to dataDB
// not thread safe, meant to be used as logic within other code blocks
type FilterIndexer struct {
indexes map[string]utils.StringMap // map[fieldName:fieldValue]utils.StringMap[itemID]
reveseIndex map[string]utils.StringMap // map[itemID]utils.StringMap[fieldName:fieldValue]
dm *DataManager
itemType string
dbKeySuffix string // get/store the result from/into this key
chngdIndxKeys utils.StringMap // keep record of the changed fieldName:fieldValue pair so we can re-cache wisely
chngdRevIndxKeys utils.StringMap // keep record of the changed itemID so we can re-cache wisely
}
// ChangedKeys returns the changed keys from original indexes so we can reload wisely
func (rfi *FilterIndexer) ChangedKeys(reverse bool) utils.StringMap {
if reverse {
return rfi.chngdRevIndxKeys
}
return rfi.chngdIndxKeys
indexes map[string]utils.StringMap // map[fieldName:fieldValue]utils.StringMap[itemID]
dm *DataManager
itemType string
dbKeySuffix string // get/store the result from/into this key
chngdIndxKeys utils.StringMap // keep record of the changed fieldName:fieldValue pair so we can re-cache wisely
}
// IndexTPFilter parses reqFltrs, adding itemID in the indexes and marks the changed keys in chngdIndxKeys
@@ -67,13 +54,8 @@ func (rfi *FilterIndexer) IndexTPFilter(tpFltr *utils.TPFilterProfile, itemID st
rfi.indexes[concatKey] = make(utils.StringMap)
}
rfi.indexes[concatKey][itemID] = true
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][concatKey] = true
rfi.chngdIndxKeys[concatKey] = true
}
rfi.chngdRevIndxKeys[itemID] = true
case MetaPrefix:
for _, fldVal := range fltr.Values {
concatKey := utils.ConcatenatedKey(fltr.Type, fltr.FieldName, fldVal)
@@ -81,23 +63,15 @@ func (rfi *FilterIndexer) IndexTPFilter(tpFltr *utils.TPFilterProfile, itemID st
rfi.indexes[concatKey] = make(utils.StringMap)
}
rfi.indexes[concatKey][itemID] = true
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][concatKey] = true
rfi.chngdIndxKeys[concatKey] = true
}
rfi.chngdRevIndxKeys[itemID] = true
case utils.META_NONE:
concatKey := utils.ConcatenatedKey(utils.META_NONE, utils.ANY, utils.ANY)
if _, hasIt := rfi.indexes[concatKey]; !hasIt {
rfi.indexes[concatKey] = make(utils.StringMap)
}
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][concatKey] = true
rfi.indexes[concatKey][itemID] = true
rfi.chngdIndxKeys[concatKey] = true
}
}
return
@@ -107,22 +81,22 @@ func (rfi *FilterIndexer) cacheRemItemType() { // ToDo: tune here by removing pe
switch rfi.itemType {
case utils.ThresholdProfilePrefix:
Cache.Clear([]string{utils.CacheThresholdFilterIndexes, utils.CacheThresholdFilterRevIndexes})
Cache.Clear([]string{utils.CacheThresholdFilterIndexes})
case utils.ResourceProfilesPrefix:
Cache.Clear([]string{utils.CacheResourceFilterIndexes, utils.CacheResourceFilterRevIndexes})
Cache.Clear([]string{utils.CacheResourceFilterIndexes})
case utils.StatQueueProfilePrefix:
Cache.Clear([]string{utils.CacheStatFilterIndexes, utils.CacheStatFilterRevIndexes})
Cache.Clear([]string{utils.CacheStatFilterIndexes})
case utils.SupplierProfilePrefix:
Cache.Clear([]string{utils.CacheSupplierFilterIndexes, utils.CacheSupplierFilterRevIndexes})
Cache.Clear([]string{utils.CacheSupplierFilterIndexes})
case utils.AttributeProfilePrefix:
Cache.Clear([]string{utils.CacheAttributeFilterIndexes, utils.CacheAttributeFilterRevIndexes})
Cache.Clear([]string{utils.CacheAttributeFilterIndexes})
case utils.ChargerProfilePrefix:
Cache.Clear([]string{utils.CacheChargerFilterIndexes, utils.CacheChargerFilterRevIndexes})
Cache.Clear([]string{utils.CacheChargerFilterIndexes})
}
}
@@ -136,32 +110,10 @@ func (rfi *FilterIndexer) StoreIndexes(commit bool, transactionID string) (err e
rfi.indexes, commit, transactionID); err != nil {
return
}
if err = rfi.dm.SetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
rfi.reveseIndex, commit, transactionID); err != nil {
return
}
rfi.cacheRemItemType()
return
}
//Populate the FilterIndexer.reveseIndex for specifil itemID
func (rfi *FilterIndexer) loadItemReverseIndex(filterType, itemID string) (err error) {
rcvReveseIdx, err := rfi.dm.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
map[string]string{itemID: ""})
if err != nil {
return err
}
for _, val2 := range rcvReveseIdx {
if _, has := rfi.reveseIndex[itemID]; !has {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID] = val2
}
return err
}
//Populate FilterIndexer.indexes with specific fieldName:fieldValue , item
func (rfi *FilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal string) error {
rcvIdx, err := rfi.dm.GetFilterIndexes(
@@ -180,18 +132,119 @@ func (rfi *FilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal str
}
//RemoveItemFromIndex remove Indexes for a specific itemID
func (rfi *FilterIndexer) RemoveItemFromIndex(itemID string) (err error) {
if err = rfi.loadItemReverseIndex(MetaString, itemID); err != nil {
return err
}
for key, _ := range rfi.reveseIndex[itemID] {
kSplt := strings.Split(key, utils.CONCATENATED_KEY_SEP)
if len(kSplt) != 3 {
return fmt.Errorf("Malformed key in db: %s", key)
}
if err = rfi.loadFldNameFldValIndex(kSplt[0], kSplt[1], kSplt[2]); err != nil {
func (rfi *FilterIndexer) RemoveItemFromIndex(tenant, itemID string, oldFilters []string) (err error) {
var filterIDs []string
switch rfi.itemType {
case utils.ThresholdProfilePrefix:
th, err := rfi.dm.GetThresholdProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if th != nil {
filterIDs = make([]string, len(th.FilterIDs))
for i, fltrID := range th.FilterIDs {
filterIDs[i] = fltrID
}
}
case utils.AttributeProfilePrefix:
attrPrf, err := rfi.dm.GetAttributeProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if attrPrf != nil {
filterIDs = make([]string, len(attrPrf.FilterIDs))
for i, fltrID := range attrPrf.FilterIDs {
filterIDs[i] = fltrID
}
}
case utils.ResourceProfilesPrefix:
res, err := rfi.dm.GetResourceProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if res != nil {
filterIDs = make([]string, len(res.FilterIDs))
for i, fltrID := range res.FilterIDs {
filterIDs[i] = fltrID
}
}
case utils.StatQueueProfilePrefix:
stq, err := rfi.dm.GetStatQueueProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if stq != nil {
filterIDs = make([]string, len(stq.FilterIDs))
for i, fltrID := range stq.FilterIDs {
filterIDs[i] = fltrID
}
}
case utils.SupplierProfilePrefix:
spp, err := rfi.dm.GetSupplierProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if spp != nil {
filterIDs = make([]string, len(spp.FilterIDs))
for i, fltrID := range spp.FilterIDs {
filterIDs[i] = fltrID
}
}
case utils.ChargerProfilePrefix:
cpp, err := rfi.dm.GetChargerProfile(tenant, itemID, false, utils.NonTransactional)
if err != nil && err != utils.ErrNotFound {
return err
}
if cpp != nil {
filterIDs = make([]string, len(cpp.FilterIDs))
for i, fltrID := range cpp.FilterIDs {
filterIDs[i] = fltrID
}
}
default:
}
if len(filterIDs) == 0 {
filterIDs = []string{utils.META_NONE}
}
for _, oldFltr := range oldFilters {
filterIDs = append(filterIDs, oldFltr)
}
for _, fltrID := range filterIDs {
var fltr *Filter
if fltrID == utils.META_NONE {
fltr = &Filter{
Tenant: tenant,
ID: itemID,
Rules: []*FilterRule{
&FilterRule{
Type: utils.META_NONE,
FieldName: utils.META_ANY,
Values: []string{utils.META_ANY},
},
},
}
} else if fltr, err = rfi.dm.GetFilter(tenant, fltrID,
false, utils.NonTransactional); err != nil {
if err == utils.ErrNotFound {
err = fmt.Errorf("broken reference to filter: %+v for itemType: %+v and ID: %+v",
fltrID, rfi.itemType, itemID)
}
return err
}
for _, flt := range fltr.Rules {
var fldType, fldName string
var fldVals []string
if utils.IsSliceMember([]string{MetaString, MetaPrefix, utils.META_NONE}, flt.Type) {
fldType, fldName = flt.Type, flt.FieldName
fldVals = flt.Values
}
for _, fldVal := range fldVals {
if err = rfi.loadFldNameFldValIndex(fldType,
fldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}
}
}
for _, itmMp := range rfi.indexes {
for range itmMp {
@@ -200,7 +253,6 @@ func (rfi *FilterIndexer) RemoveItemFromIndex(itemID string) (err error) {
}
}
}
rfi.reveseIndex[itemID] = make(utils.StringMap) //Force deleting in driver
return rfi.StoreIndexes(false, utils.NonTransactional)
}
@@ -211,10 +263,6 @@ func createAndIndex(itemPrefix, tenant, context, itemID string, filterIDs []stri
indexerKey = utils.ConcatenatedKey(tenant, context)
}
indexer := NewFilterIndexer(dm, itemPrefix, indexerKey)
if err = indexer.RemoveItemFromIndex(itemID); err != nil &&
err.Error() != utils.ErrNotFound.Error() {
return
}
fltrIDs := make([]string, len(filterIDs))
for i, fltrID := range filterIDs {
fltrIDs[i] = fltrID

204
engine/filterindexer_it_test.go
View File

@@ -75,7 +75,7 @@ func TestFilterIndexerITRedis(t *testing.T) {
}
func TestFilterIndexerITMongo(t *testing.T) {
cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "cdrsv2mongo")
cdrsMongoCfgPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
mgoITCfg, err := config.NewCGRConfigFromFolder(cdrsMongoCfgPath)
if err != nil {
t.Fatal(err)
@@ -196,22 +196,6 @@ func testITGetFilterIndexes(t *testing.T) {
MetaString, nil); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
if err := dataManager.RemoveFilterIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
"cgrates.org"); err != nil {
t.Error(err)
}
_, err := dataManager.GetFilterIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
"cgrates.org", MetaString, nil)
if err != utils.ErrNotFound {
t.Error(err)
}
if err := dataManager.SetFilterIndexes(
utils.PrefixToRevIndexCache[utils.ResourceProfilesPrefix],
"cgrates.org", eIdxes, false, utils.NonTransactional); err != nil {
t.Error(err)
}
}
func testITMatchFilterIndex(t *testing.T) {
@@ -292,16 +276,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
"THD_Test2": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
"THD_Test2": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
@@ -310,13 +284,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//Replace existing filter (Filter1 -> Filter2)
fp2 := &Filter{
Tenant: "cgrates.org",
@@ -336,7 +304,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
if err := dataManager.SetFilter(fp2); err != nil {
t.Error(err)
}
th.FilterIDs = []string{"Filter2"}
if err := dataManager.SetThresholdProfile(th, true); err != nil {
t.Error(err)
@@ -355,17 +322,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
"THD_Test2": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*string:Account:1001": true,
"*string:Account:1002": true,
},
"THD_Test2": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != nil {
@@ -373,13 +329,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//replace old filter with two different filters
fp3 := &Filter{
Tenant: "cgrates.org",
@@ -399,7 +348,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
if err := dataManager.SetFilter(fp3); err != nil {
t.Error(err)
}
th.FilterIDs = []string{"Filter1", "Filter3"}
if err := dataManager.SetThresholdProfile(th, true); err != nil {
t.Error(err)
@@ -420,18 +368,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
"THD_Test2": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*string:Destination:10": true,
"*string:Destination:20": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
"THD_Test2": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != nil {
@@ -439,13 +375,6 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//remove thresholds
if err := dataManager.RemoveThresholdProfile(th.Tenant,
th.ID, utils.NonTransactional, true); err != nil {
@@ -456,15 +385,10 @@ func testITTestThresholdFilterIndexes(t *testing.T) {
t.Error(err)
}
if _, err := dataManager.GetFilterIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
}
if _, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != utils.ErrNotFound {
t.Error(err)
}
}
func testITTestAttributeProfileFilterIndexes(t *testing.T) {
@@ -516,12 +440,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
"AttrPrf": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"AttrPrf": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
for _, ctx := range attrProfile.Contexts {
rfi := NewFilterIndexer(onStor, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(attrProfile.Tenant, ctx))
@@ -532,13 +450,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
}
//Set AttributeProfile with 1 new context (con3)
attrProfile.Contexts = []string{"con3"}
@@ -555,13 +466,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//check if old contexts was delete
for _, ctx := range []string{"con1", "con2"} {
rfi := NewFilterIndexer(onStor, utils.AttributeProfilePrefix,
@@ -571,11 +475,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
if err := dataManager.RemoveAttributeProfile(attrProfile.Tenant,
@@ -586,15 +485,10 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) {
rfi = NewFilterIndexer(onStor, utils.AttributeProfilePrefix,
utils.ConcatenatedKey("cgrates.org", "con3"))
if _, err := dataManager.GetFilterIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
if _, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
}
func testITTestThresholdInlineFilterIndexing(t *testing.T) {
@@ -639,12 +533,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) {
"THD_Test": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
@@ -653,13 +541,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//Add an InlineFilter
th.FilterIDs = []string{"Filter1", "*string:Account:1001"}
if err := dataManager.SetThresholdProfile(th, true); err != nil {
@@ -676,14 +557,6 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) {
"THD_Test": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*string:Account:1001": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != nil {
@@ -691,28 +564,16 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) {
} else if !reflect.DeepEqual(eIdxes, rcvIdx) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
//remove threshold
if err := dataManager.RemoveThresholdProfile(th.Tenant,
th.ID, utils.NonTransactional, true); err != nil {
t.Error(err)
}
if _, err := dataManager.GetFilterIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != utils.ErrNotFound {
t.Error(err)
}
if _, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType], rfi.dbKeySuffix,
nil); err != utils.ErrNotFound {
t.Error(err)
}
}
func testITTestStoreFilterIndexesWithTransID(t *testing.T) {
@@ -843,14 +704,6 @@ func testITTestIndexingWithEmptyFltrID(t *testing.T) {
"THD_Test2": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"*none:*any:*any": true,
},
"THD_Test2": utils.StringMap{
"*none:*any:*any": true,
},
}
rfi := NewFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
@@ -861,15 +714,6 @@ func testITTestIndexingWithEmptyFltrID(t *testing.T) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType],
rfi.dbKeySuffix, nil); err != nil {
t.Error(err)
} else {
if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
}
eMp := utils.StringMap{
"THD_Test": true,
"THD_Test2": true,
@@ -942,14 +786,6 @@ func testITTestIndexingWithEmptyFltrID2(t *testing.T) {
"SPL_Weight2": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"SPL_Weight": utils.StringMap{
"*none:*any:*any": true,
},
"SPL_Weight2": utils.StringMap{
"*none:*any:*any": true,
},
}
rfi := NewFilterIndexer(onStor, utils.SupplierProfilePrefix, splProfile.Tenant)
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
@@ -960,15 +796,6 @@ func testITTestIndexingWithEmptyFltrID2(t *testing.T) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType],
rfi.dbKeySuffix, nil); err != nil {
t.Error(err)
} else {
if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
}
eMp := utils.StringMap{
"SPL_Weight": true,
"SPL_Weight2": true,
@@ -1019,18 +846,6 @@ func testITTestIndexingThresholds(t *testing.T) {
"TH3": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"TH1": utils.StringMap{
"*string:Account:1001": true,
},
"TH2": utils.StringMap{
"*string:Account:1001": true,
},
"TH3": utils.StringMap{
"*string:Account:1002": true,
},
}
if rcvIdx, err := dataManager.GetFilterIndexes(
utils.PrefixToIndexCache[rfi.itemType], rfi.dbKeySuffix,
MetaString, nil); err != nil {
@@ -1040,15 +855,6 @@ func testITTestIndexingThresholds(t *testing.T) {
t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx)
}
}
if reverseRcvIdx, err := dataManager.GetFilterReverseIndexes(
utils.PrefixToRevIndexCache[rfi.itemType],
rfi.dbKeySuffix, nil); err != nil {
t.Error(err)
} else {
if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) {
t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx)
}
}
eMp := utils.StringMap{
"TH1": true,
"TH2": true,

5
engine/storage_interface.go
View File

@@ -123,11 +123,6 @@ type DataDB interface {
SetFilterIndexesDrv(cacheID, itemIDPrefix string,
indexes map[string]utils.StringMap, commit bool, transactionID string) (err error)
RemoveFilterIndexesDrv(cacheID, itemIDPrefix string) (err error)
GetFilterReverseIndexesDrv(cacheID, itemIDPrefix string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error)
SetFilterReverseIndexesDrv(cacheID, itemIDPrefix string, indexes map[string]utils.StringMap,
commit bool, transactionID string) (err error)
RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix string) (err error)
MatchFilterIndexDrv(cacheID, itemIDPrefix,
filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error)
GetStatQueueProfileDrv(tenant string, ID string) (sq *StatQueueProfile, err error)

104
engine/storage_map_datadb.go
View File

@@ -1333,110 +1333,6 @@ func (ms *MapStorage) RemoveFilterIndexesDrv(cacheID, itemIDPrefix string) (err
return
}
//GetFilterReverseIndexesDrv retrieves ReverseIndexes from dataDB
func (ms *MapStorage) GetFilterReverseIndexesDrv(cacheID, itemIDPrefix string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
dbKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix
values, ok := ms.dict[dbKey]
if !ok {
return nil, utils.ErrNotFound
}
if len(fldNameVal) != 0 {
rcvidx := make(map[string]utils.StringMap)
err = ms.ms.Unmarshal(values, &rcvidx)
if err != nil {
return nil, err
}
indexes = make(map[string]utils.StringMap)
for fldName := range fldNameVal {
if _, has := indexes[fldName]; !has {
indexes[fldName] = make(utils.StringMap)
}
for key := range rcvidx[fldName] {
indexes[fldName][key] = true
}
}
return
} else {
err = ms.ms.Unmarshal(values, &indexes)
if err != nil {
return nil, err
}
if len(indexes) == 0 {
return nil, utils.ErrNotFound
}
}
return
}
//SetFilterReverseIndexesDrv stores ReverseIndexes into DataDB
func (ms *MapStorage) SetFilterReverseIndexesDrv(cacheID, itemIDPrefix string,
revIdx map[string]utils.StringMap, commit bool, transactionID string) (err error) {
ms.mu.Lock()
defer ms.mu.Unlock()
originKey := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix
dbKey := originKey
if transactionID != "" {
dbKey = "tmp_" + utils.ConcatenatedKey(dbKey, transactionID)
}
if commit && transactionID != "" {
values, _ := ms.dict[dbKey]
delete(ms.dict, dbKey)
ms.dict[originKey] = values
return nil
}
var toBeDeleted []string
toBeAdded := make(map[string]utils.StringMap)
for key, strMp := range revIdx {
if len(strMp) == 0 { // remove with no more elements inside
toBeDeleted = append(toBeDeleted, key)
delete(revIdx, key)
continue
}
toBeAdded[key] = make(utils.StringMap)
toBeAdded[key] = strMp
}
values, has := ms.dict[dbKey]
if !has {
result, err := ms.ms.Marshal(toBeAdded)
if err != nil {
return err
}
ms.dict[dbKey] = result
return err
}
mp := make(map[string]utils.StringMap)
err = ms.ms.Unmarshal(values, &mp)
if err != nil {
return err
}
for _, key := range toBeDeleted {
delete(mp, key)
}
for key, strMp := range toBeAdded {
if _, has := mp[key]; !has {
mp[key] = make(utils.StringMap)
}
mp[key] = strMp
}
result, err := ms.ms.Marshal(mp)
if err != nil {
return err
}
ms.dict[dbKey] = result
return
}
//RemoveFilterReverseIndexesDrv removes ReverseIndexes for a specific itemID
func (ms *MapStorage) RemoveFilterReverseIndexesDrv(cacheID, itemIDPrefix string) (err error) {
ms.mu.Lock()
defer ms.mu.Unlock()
delete(ms.dict, utils.CacheInstanceToPrefix[cacheID]+itemIDPrefix)
return
}
func (ms *MapStorage) MatchFilterIndexDrv(cacheID, itemIDPrefix,
filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
ms.mu.RLock()

173
utils/consts.go
View File

@@ -41,44 +41,38 @@ var (
MetaFileFWV: FWVSuffix,
}
CacheInstanceToPrefix = map[string]string{
CacheDestinations: DESTINATION_PREFIX,
CacheReverseDestinations: REVERSE_DESTINATION_PREFIX,
CacheRatingPlans: RATING_PLAN_PREFIX,
CacheRatingProfiles: RATING_PROFILE_PREFIX,
CacheLCRRules: LCR_PREFIX,
CacheCDRStatS: CDR_STATS_PREFIX,
CacheActions: ACTION_PREFIX,
CacheActionPlans: ACTION_PLAN_PREFIX,
CacheAccountActionPlans: AccountActionPlansPrefix,
CacheActionTriggers: ACTION_TRIGGER_PREFIX,
CacheSharedGroups: SHARED_GROUP_PREFIX,
CacheAliases: ALIASES_PREFIX,
CacheReverseAliases: REVERSE_ALIASES_PREFIX,
CacheDerivedChargers: DERIVEDCHARGERS_PREFIX,
CacheResourceProfiles: ResourceProfilesPrefix,
CacheResources: ResourcesPrefix,
CacheEventResources: EventResourcesPrefix,
CacheTimings: TimingsPrefix,
CacheStatQueueProfiles: StatQueueProfilePrefix,
CacheStatQueues: StatQueuePrefix,
CacheThresholdProfiles: ThresholdProfilePrefix,
CacheThresholds: ThresholdPrefix,
CacheFilters: FilterPrefix,
CacheSupplierProfiles: SupplierProfilePrefix,
CacheAttributeProfiles: AttributeProfilePrefix,
CacheChargerProfiles: ChargerProfilePrefix,
CacheResourceFilterIndexes: ResourceFilterIndexes,
CacheResourceFilterRevIndexes: ResourceFilterRevIndexes,
CacheStatFilterIndexes: StatFilterIndexes,
CacheStatFilterRevIndexes: StatFilterRevIndexes,
CacheThresholdFilterIndexes: ThresholdFilterIndexes,
CacheThresholdFilterRevIndexes: ThresholdFilterRevIndexes,
CacheSupplierFilterIndexes: SupplierFilterIndexes,
CacheSupplierFilterRevIndexes: SupplierFilterRevIndexes,
CacheAttributeFilterIndexes: AttributeFilterIndexes,
CacheAttributeFilterRevIndexes: AttributeFilterRevIndexes,
CacheChargerFilterIndexes: ChargerFilterIndexes,
CacheChargerFilterRevIndexes: ChargerFilterRevIndexes,
CacheDestinations: DESTINATION_PREFIX,
CacheReverseDestinations: REVERSE_DESTINATION_PREFIX,
CacheRatingPlans: RATING_PLAN_PREFIX,
CacheRatingProfiles: RATING_PROFILE_PREFIX,
CacheLCRRules: LCR_PREFIX,
CacheCDRStatS: CDR_STATS_PREFIX,
CacheActions: ACTION_PREFIX,
CacheActionPlans: ACTION_PLAN_PREFIX,
CacheAccountActionPlans: AccountActionPlansPrefix,
CacheActionTriggers: ACTION_TRIGGER_PREFIX,
CacheSharedGroups: SHARED_GROUP_PREFIX,
CacheAliases: ALIASES_PREFIX,
CacheReverseAliases: REVERSE_ALIASES_PREFIX,
CacheDerivedChargers: DERIVEDCHARGERS_PREFIX,
CacheResourceProfiles: ResourceProfilesPrefix,
CacheResources: ResourcesPrefix,
CacheEventResources: EventResourcesPrefix,
CacheTimings: TimingsPrefix,
CacheStatQueueProfiles: StatQueueProfilePrefix,
CacheStatQueues: StatQueuePrefix,
CacheThresholdProfiles: ThresholdProfilePrefix,
CacheThresholds: ThresholdPrefix,
CacheFilters: FilterPrefix,
CacheSupplierProfiles: SupplierProfilePrefix,
CacheAttributeProfiles: AttributeProfilePrefix,
CacheChargerProfiles: ChargerProfilePrefix,
CacheResourceFilterIndexes: ResourceFilterIndexes,
CacheStatFilterIndexes: StatFilterIndexes,
CacheThresholdFilterIndexes: ThresholdFilterIndexes,
CacheSupplierFilterIndexes: SupplierFilterIndexes,
CacheAttributeFilterIndexes: AttributeFilterIndexes,
CacheChargerFilterIndexes: ChargerFilterIndexes,
}
CachePrefixToInstance map[string]string // will be built on init
PrefixToIndexCache = map[string]string{
@@ -89,14 +83,6 @@ var (
AttributeProfilePrefix: CacheAttributeFilterIndexes,
ChargerProfilePrefix: CacheChargerFilterIndexes,
}
PrefixToRevIndexCache = map[string]string{
ThresholdProfilePrefix: CacheThresholdFilterRevIndexes,
ResourceProfilesPrefix: CacheResourceFilterRevIndexes,
StatQueueProfilePrefix: CacheStatFilterRevIndexes,
SupplierProfilePrefix: CacheSupplierFilterRevIndexes,
AttributeProfilePrefix: CacheAttributeFilterRevIndexes,
ChargerProfilePrefix: CacheChargerFilterRevIndexes,
}
CacheIndexesToPrefix map[string]string // will be built on init
)
@@ -865,62 +851,50 @@ const (
// Cache Name
const (
CacheDestinations = "destinations"
CacheReverseDestinations = "reverse_destinations"
CacheRatingPlans = "rating_plans"
CacheRatingProfiles = "rating_profiles"
CacheLCRRules = "lcr_rules"
CacheCDRStatS = "cdr_stats"
CacheActions = "actions"
CacheActionPlans = "action_plans"
CacheAccountActionPlans = "account_action_plans"
CacheActionTriggers = "action_triggers"
CacheSharedGroups = "shared_groups"
CacheAliases = "aliases"
CacheReverseAliases = "reverse_aliases"
CacheDerivedChargers = "derived_chargers"
CacheResources = "resources"
CacheResourceProfiles = "resource_profiles"
CacheTimings = "timings"
CacheEventResources = "event_resources"
CacheStatQueueProfiles = "statqueue_profiles"
CacheStatQueues = "statqueues"
CacheThresholdProfiles = "threshold_profiles"
CacheThresholds = "thresholds"
CacheFilters = "filters"
CacheSupplierProfiles = "supplier_profiles"
CacheAttributeProfiles = "attribute_profiles"
CacheChargerProfiles = "charger_profiles"
CacheResourceFilterIndexes = "resource_filter_indexes"
CacheResourceFilterRevIndexes = "resource_filter_revindexes"
CacheStatFilterIndexes = "stat_filter_indexes"
CacheStatFilterRevIndexes = "stat_filter_revindexes"
CacheThresholdFilterIndexes = "threshold_filter_indexes"
CacheThresholdFilterRevIndexes = "threshold_filter_revindexes"
CacheSupplierFilterIndexes = "supplier_filter_indexes"
CacheSupplierFilterRevIndexes = "supplier_filter_revindexes"
CacheAttributeFilterIndexes = "attribute_filter_indexes"
CacheAttributeFilterRevIndexes = "attribute_filter_revindexes"
CacheChargerFilterIndexes = "charger_filter_indexes"
CacheChargerFilterRevIndexes = "charger_filter_revindexes"
MetaPrecaching = "*precaching"
MetaReady = "*ready"
CacheDestinations = "destinations"
CacheReverseDestinations = "reverse_destinations"
CacheRatingPlans = "rating_plans"
CacheRatingProfiles = "rating_profiles"
CacheLCRRules = "lcr_rules"
CacheCDRStatS = "cdr_stats"
CacheActions = "actions"
CacheActionPlans = "action_plans"
CacheAccountActionPlans = "account_action_plans"
CacheActionTriggers = "action_triggers"
CacheSharedGroups = "shared_groups"
CacheAliases = "aliases"
CacheReverseAliases = "reverse_aliases"
CacheDerivedChargers = "derived_chargers"
CacheResources = "resources"
CacheResourceProfiles = "resource_profiles"
CacheTimings = "timings"
CacheEventResources = "event_resources"
CacheStatQueueProfiles = "statqueue_profiles"
CacheStatQueues = "statqueues"
CacheThresholdProfiles = "threshold_profiles"
CacheThresholds = "thresholds"
CacheFilters = "filters"
CacheSupplierProfiles = "supplier_profiles"
CacheAttributeProfiles = "attribute_profiles"
CacheChargerProfiles = "charger_profiles"
CacheResourceFilterIndexes = "resource_filter_indexes"
CacheStatFilterIndexes = "stat_filter_indexes"
CacheThresholdFilterIndexes = "threshold_filter_indexes"
CacheSupplierFilterIndexes = "supplier_filter_indexes"
CacheAttributeFilterIndexes = "attribute_filter_indexes"
CacheChargerFilterIndexes = "charger_filter_indexes"
MetaPrecaching = "*precaching"
MetaReady = "*ready"
)
// Prefix for indexing
const (
ResourceFilterIndexes = "rfi_"
ResourceFilterRevIndexes = "rfr_"
StatFilterIndexes = "sfi_"
StatFilterRevIndexes = "sfr_"
ThresholdFilterIndexes = "tfi_"
ThresholdFilterRevIndexes = "tfr_"
SupplierFilterIndexes = "spi_"
SupplierFilterRevIndexes = "spr_"
AttributeFilterIndexes = "afi_"
AttributeFilterRevIndexes = "afr_"
ChargerFilterIndexes = "cfi_"
ChargerFilterRevIndexes = "cfr_"
ResourceFilterIndexes = "rfi_"
StatFilterIndexes = "sfi_"
ThresholdFilterIndexes = "tfi_"
SupplierFilterIndexes = "spi_"
AttributeFilterIndexes = "afi_"
ChargerFilterIndexes = "cfi_"
)
// Agents
@@ -945,9 +919,6 @@ func buildCacheIndexesToPrefix() {
for k, v := range PrefixToIndexCache {
CacheIndexesToPrefix[v] = k
}
for k, v := range PrefixToRevIndexCache {
CacheIndexesToPrefix[v] = k
}
}
func init() {