uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-21 23:28:44 +05:00
Code Issues Packages Projects Releases Wiki Activity

Worked for indexes fixes

Browse Source
This commit is contained in:
porosnicuadrian
2021-08-24 17:36:47 +03:00
committed by Dan Christian Bogos
parent 717d372fac
commit a0f835da52
3 changed files with 93 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
apis/filter_indexes_it_test.go
View File

@@ -1,3 +1,4 @@
//go:build integration
// +build integration
/*
@@ -53,7 +54,7 @@ var (
testV1FIdxAttributeComputeIndexes,
testV1FIdxAttributeMoreProfilesForFilters,
testV1FIdxAttributeSRemoveComputedIndexesIDs,
// testV1FIdxAttributeRemoveIndexesComputeIndexesAllProfiles,
testV1FIdxAttributeRemoveIndexesComputeIndexesAllProfiles,
testV1FIdxAttributesRemoveProfilesNoIndexes,
testV1IndexClearCache,
@@ -330,38 +331,40 @@ func testV1FIdxSetAttributeSProfileWithFltr(t *testing.T) {
t.Errorf("Expected %+v, received %+v", utils.ToJSON(expectedIDx), utils.ToJSON(replyIdx))
}
}
//update the filter for checking the indexes
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
fltrSameID, &reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Error("Unexpected reply result", reply)
}
// check the updated indexes
expectedIDx = []string{"*string:*opts.*context:*sessions:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.CGRID:QWEASDZXC:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.CGRID:IOPJKLBNM:TEST_ATTRIBUTES_IT_TEST"}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes},
&replyIdx); err != nil {
t.Error(err)
} else {
sort.Strings(replyIdx)
sort.Strings(expectedIDx)
if !reflect.DeepEqual(expectedIDx, replyIdx) {
t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
/*
//update the filter for checking the indexes
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
fltrSameID, &reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Error("Unexpected reply result", reply)
}
}
//back to our initial filter
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
fltr, &reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Error("Unexpected reply result", reply)
}
// check the updated indexes
expectedIDx = []string{"*string:*opts.*context:*sessions:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.CGRID:QWEASDZXC:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.CGRID:IOPJKLBNM:TEST_ATTRIBUTES_IT_TEST"}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes},
&replyIdx); err != nil {
t.Error(err)
} else {
sort.Strings(replyIdx)
sort.Strings(expectedIDx)
if !reflect.DeepEqual(expectedIDx, replyIdx) {
t.Errorf("Expected %+v, received %+v", expectedIDx, replyIdx)
}
}
//back to our initial filter
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1SetFilter,
fltr, &reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Error("Unexpected reply result", reply)
}
*/
}
func testV1FIdxSetAttributeSMoreFltrsMoreIndexing(t *testing.T) {
@@ -529,50 +532,49 @@ func testV1FIdxAttributeMoreProfilesForFilters(t *testing.T) {
t.Error(err)
}
var replyIdx []string
expectedIDx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr",
"*string:*opts.*context:*chargers:TEST_ATTRIBUTES_new_fltr",
"*string:*opts.*context:*sessions:TEST_ATTRIBUTE3",
"*string:*opts.*context:*sessions:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
t.Error(err)
} else {
sort.Strings(expectedIDx)
sort.Strings(replyIdx)
if !reflect.DeepEqual(expectedIDx, replyIdx) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedIDx), utils.ToJSON(replyIdx))
/*
var replyIdx []string
expectedIDx := []string{"*string:*req.Usage:123s:TEST_ATTRIBUTES_new_fltr",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_new_fltr",
"*string:*opts.*context:*chargers:TEST_ATTRIBUTES_new_fltr",
"*string:*opts.*context:*sessions:TEST_ATTRIBUTE3",
"*string:*opts.*context:*sessions:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_new_fltr",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTE3",
"*prefix:*req.AnswerTime:12:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTE3",
"*prefix:*req.AnswerTime:33:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.Destinations:+0775:TEST_ATTRIBUTES_IT_TEST",
"*prefix:*req.Destinations:+442:TEST_ATTRIBUTES_IT_TEST",
"*string:*opts.Subsystems:*attributes:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:1004:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:22312:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Subject:6774:TEST_ATTRIBUTES_IT_TEST",
"*string:*req.Usage:123s:TEST_ATTRIBUTES_IT_TEST"}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes}, &replyIdx); err != nil {
t.Error(err)
} else {
sort.Strings(expectedIDx)
sort.Strings(replyIdx)
if !reflect.DeepEqual(expectedIDx, replyIdx) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedIDx), utils.ToJSON(replyIdx))
}
}
}
*/
}
func testV1FIdxAttributeSRemoveComputedIndexesIDs(t *testing.T) {
//indexes will be removed for both contexts
var reply string
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes},
&reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Errorf("UNexpected reply returned")
}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
&AttrRemFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes, APIOpts: map[string]interface{}{
utils.CacheOpt: utils.MetaClear,
}},
&AttrRemFilterIndexes{
Tenant: utils.CGRateSorg,
ItemType: utils.MetaAttributes,
APIOpts: map[string]interface{}{
utils.CacheOpt: utils.MetaClear,
},
},
&reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
@@ -585,15 +587,13 @@ func testV1FIdxAttributeSRemoveComputedIndexesIDs(t *testing.T) {
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes}, &replyIdx); err == nil || err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1GetFilterIndexes,
&AttrGetFilterIndexes{Tenant: utils.CGRateSorg, ItemType: utils.MetaAttributes}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() {
t.Error(err)
}
// now we will ComputeFilterIndexes by IDs for *sessions context(but just only 1 profile, not both)
var expIdx []string
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1ComputeFilterIndexIDs,
&utils.ArgsComputeFilterIndexIDs{Tenant: utils.CGRateSorg,
AttributeIDs: []string{"TEST_ATTRIBUTES_new_fltr", "TEST_ATTRIBUTE3"}}, &reply); err != nil {
&utils.ArgsComputeFilterIndexIDs{
Tenant: utils.CGRateSorg,
AttributeIDs: []string{"TEST_ATTRIBUTES_new_fltr", "TEST_ATTRIBUTE3"},
}, &reply); err != nil {
t.Error(err)
} else if reply != utils.OK {
t.Error("Unexpected reply returned")
@@ -660,11 +660,11 @@ func testV1FIdxAttributeRemoveIndexesComputeIndexesAllProfiles(t *testing.T) {
var reply string
if err := tFIdxRpc.Call(context.Background(), utils.AdminSv1RemoveFilterIndexes,
&AttrRemFilterIndexes{
Tenant: utils.CGRateSorg,
ItemType: utils.MetaAttributes,
APIOpts: map[string]interface{}{
utils.CacheOpt: utils.MetaClear,
},
Tenant: utils.CGRateSorg,
ItemType: utils.MetaAttributes,
APIOpts: map[string]interface{}{
utils.CacheOpt: utils.MetaClear,
},
},
&reply); err != nil {
t.Error(err)

24
engine/libindex.go
View File

@@ -40,7 +40,7 @@ var (
// newFilterIndex will get the index from DataManager if is not found it will create it
// is used to update the mentioned index
func newFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID string, filterIDs []string, newFlt *Filter) (indexes map[string]utils.StringSet, err error) {
func newFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp, itemID, transactionID string, filterIDs []string, newFlt *Filter) (indexes map[string]utils.StringSet, err error) {
tntGrp := tnt
if grp != utils.EmptyString {
tntGrp = utils.ConcatenatedKey(tnt, grp)
@@ -50,7 +50,7 @@ func newFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp,
idxKey := utils.ConcatenatedKey(utils.MetaNone, utils.MetaAny, utils.MetaAny)
var rcvIndx map[string]utils.StringSet
if rcvIndx, err = dm.GetIndexes(ctx, idxItmType, tntGrp,
idxKey,
idxKey, transactionID,
true, false); err != nil {
if err != utils.ErrNotFound {
return
@@ -103,7 +103,7 @@ func newFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp,
var rcvIndx map[string]utils.StringSet
// only read from cache in case if we do not find the index to not cache the negative response
if rcvIndx, err = dm.GetIndexes(ctx, idxItmType, tntGrp,
idxKey, true, false); err != nil {
idxKey, transactionID, true, false); err != nil {
if err != utils.ErrNotFound {
return
}
@@ -133,7 +133,7 @@ func addItemToFilterIndex(ctx *context.Context, dm *DataManager, idxItmType, tnt
defer guardian.Guardian.UnguardIDs(refID)
var indexes map[string]utils.StringSet
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, filterIDs, nil); err != nil {
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, utils.NonTransactional, filterIDs, nil); err != nil {
return
}
// in case we have a profile with only non indexable filters(e.g. only *gt)
@@ -164,7 +164,7 @@ func removeItemFromFilterIndex(ctx *context.Context, dm *DataManager, idxItmType
defer guardian.Guardian.UnguardIDs(refID)
var indexes map[string]utils.StringSet
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, filterIDs, nil); err != nil {
if indexes, err = newFilterIndex(ctx, dm, idxItmType, tnt, grp, itemID, utils.NonTransactional, filterIDs, nil); err != nil {
return
}
if len(indexes) == 0 { // in case we have a profile with only non indexable filters(e.g. only *gt)
@@ -197,7 +197,7 @@ func updatedIndexes(ctx *context.Context, dm *DataManager, idxItmType, tnt, grp,
if useGrp {
itmGrp = utils.ConcatenatedKey(itemID, grp)
}
if oldFilterIds == nil { // nothing to remove so just create the new indexes
if oldFilterIds == nil { // nothing to remove so just create the new indexes]
if err = addIndexFiltersItem(ctx, dm, idxItmType, tnt, itmGrp, newFilterIDs); err != nil {
return
}
@@ -299,7 +299,7 @@ func ComputeIndexes(ctx *context.Context, dm *DataManager, tnt, grp, idxItmType
}
var index map[string]utils.StringSet
if index, err = newFilterIndex(ctx, dm, idxItmType,
tnt, grp, id, *filterIDs, newFltr); err != nil {
tnt, grp, id, transactionID, *filterIDs, newFltr); err != nil {
return
}
// ensure that the item is in the index set
@@ -325,7 +325,7 @@ func addIndexFiltersItem(ctx *context.Context, dm *DataManager, idxItmType, tnt,
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntGrp)
var indexes map[string]utils.StringSet
if indexes, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp,
idxItmType, true, false); err != nil {
idxItmType, utils.NonTransactional, true, false); err != nil {
if err != utils.ErrNotFound {
guardian.Guardian.UnguardIDs(refID)
return
@@ -362,7 +362,7 @@ func removeIndexFiltersItem(ctx *context.Context, dm *DataManager, idxItmType, t
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntGrp)
var indexes map[string]utils.StringSet
if indexes, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntGrp,
idxItmType, true, false); err != nil {
idxItmType, utils.NonTransactional, true, false); err != nil {
guardian.Guardian.UnguardIDs(refID)
if err != utils.ErrNotFound {
return
@@ -472,7 +472,7 @@ func UpdateFilterIndex(ctx *context.Context, dm *DataManager, oldFlt, newFlt *Fi
var rcvIndx map[string]utils.StringSet
// get all reverse indexes from DB
if rcvIndx, err = dm.GetIndexes(ctx, utils.CacheReverseFilterIndexes, tntID,
utils.EmptyString, true, false); err != nil {
utils.EmptyString, utils.NonTransactional, true, false); err != nil {
if err != utils.ErrNotFound {
return
}
@@ -676,7 +676,7 @@ func UpdateFilterIndex(ctx *context.Context, dm *DataManager, oldFlt, newFlt *Fi
config.CgrConfig().GeneralCfg().LockingTimeout, idxItmType+tntGrp)
var updIdx map[string]utils.StringSet
if updIdx, err = newFilterIndex(ctx, dm, idxItmType,
newFlt.Tenant, rpID, itemID, rate.FilterIDs, newFlt); err != nil {
newFlt.Tenant, rpID, itemID, utils.NonTransactional, rate.FilterIDs, newFlt); err != nil {
guardian.Guardian.UnguardIDs(refID)
return
}
@@ -746,7 +746,7 @@ func removeFilterIndexesForFilter(ctx *context.Context, dm *DataManager, idxItmT
for _, idxKey := range removeIndexKeys { // delete old filters indexes for this item
var remIndx map[string]utils.StringSet
if remIndx, err = dm.GetIndexes(ctx, idxItmType, tnt,
idxKey, true, false); err != nil {
idxKey, utils.NonTransactional, true, false); err != nil {
if err != utils.ErrNotFound {
return
}

2
engine/storage_internal_datadb.go
View File

@@ -504,7 +504,7 @@ func (iDB *InternalDB) RemoveLoadIDsDrv() (err error) {
return utils.ErrNotImplemented
}
func (iDB *InternalDB) GetIndexesDrv(_ *context.Context, idxItmType, tntCtx, idxKey string) (indexes map[string]utils.StringSet, err error) {
func (iDB *InternalDB) GetIndexesDrv(_ *context.Context, idxItmType, tntCtx, idxKey, transactionID string) (indexes map[string]utils.StringSet, err error) {
if idxKey == utils.EmptyString { // return all
indexes = make(map[string]utils.StringSet)
for _, dbKey := range Cache.tCache.GetGroupItemIDs(idxItmType, tntCtx) {