uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-24 08:38:45 +05:00
Code Issues Packages Projects Releases Wiki Activity

Reverse filter indexes updates + tests

Browse Source
This commit is contained in:
adi
2022-12-05 17:12:35 +02:00
committed by Dan Christian Bogos
parent 4d3544aabc
commit aeb1cf349a
6 changed files with 142 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config/config_defaults.go
View File

@@ -216,6 +216,7 @@ const CGRATES_CFG_JSON = `
"*attribute_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control attribute filter indexes caching
"*charger_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control charger filter indexes caching
"*dispatcher_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control dispatcher filter indexes caching
"*reverse_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false}, // control reverse filter indexes caching
"*dispatcher_routes": {"limit": -1, "ttl": "", "static_ttl": false}, // control dispatcher routes caching
"*diameter_messages": {"limit": -1, "ttl": "3h", "static_ttl": false}, // diameter messages caching
"*rpc_responses": {"limit": 0, "ttl": "2s", "static_ttl": false}, // RPC responses caching

2
config/config_json_test.go
View File

@@ -74,6 +74,8 @@ func TestDfGeneralJsonCfg(t *testing.T) {
func TestCacheJsonCfg(t *testing.T) {
eCfg := &CacheJsonCfg{
utils.CacheReverseFilterIndexes: &CacheParamJsonCfg{Limit: utils.IntPointer(-1),
Ttl: utils.StringPointer(""), Static_ttl: utils.BoolPointer(false)},
utils.CacheDestinations: &CacheParamJsonCfg{Limit: utils.IntPointer(-1),
Ttl: utils.StringPointer(""), Static_ttl: utils.BoolPointer(false),
Precache: utils.BoolPointer(false)},

2
config/config_test.go
View File

@@ -624,6 +624,8 @@ func TestCgrCfgJSONDefaultsSMGenericCfg(t *testing.T) {
func TestCgrCfgJSONDefaultsCacheCFG(t *testing.T) {
eCacheCfg := CacheCfg{
utils.CacheReverseFilterIndexes: &CacheParamCfg{Limit: -1,
TTL: time.Duration(0), StaticTTL: false, Precache: false},
utils.CacheDestinations: &CacheParamCfg{Limit: -1,
TTL: time.Duration(0), StaticTTL: false, Precache: false},
utils.CacheReverseDestinations: &CacheParamCfg{Limit: -1,

8
engine/datamanager.go
View File

@@ -2152,12 +2152,20 @@ func (dm *DataManager) SetChargerProfile(cpp *ChargerProfile, withIndex bool) (e
cpp.Tenant).RemoveItemFromIndex(cpp.Tenant, cpp.ID, oldCpp.FilterIDs); err != nil {
return
}
if err = removeReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString,
cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil {
return
}
}
}
if err = createAndIndex(utils.ChargerProfilePrefix, cpp.Tenant,
utils.EmptyString, cpp.ID, cpp.FilterIDs, dm); err != nil {
return
}
if err = addReverseFilterIndexForFilter(dm, utils.CacheChargerFilterIndexes, utils.EmptyString,
cpp.Tenant, cpp.ID, cpp.FilterIDs); err != nil {
return
}
}
if config.CgrConfig().DataDbCfg().Items[utils.MetaChargerProfiles].Replicate {
var reply string

66
engine/libindex.go
View File

@@ -34,7 +34,55 @@ func UpdateFilterIndexes(dm *DataManager, oldFltr *Filter, newFltr *Filter) (err
}
// addReverseFilterIndexForFilter will add a reference for the filter in reverse filter indexes
func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) {
func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt,
itemID string, filterIDs []string) (err error) {
for _, fltrID := range filterIDs {
if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters
continue
}
tntFltrID := utils.ConcatenatedKey(tnt, fltrID)
refID := guardian.Guardian.GuardIDs(utils.EmptyString,
config.CgrConfig().GeneralCfg().LockingTimeout, utils.CacheReverseFilterIndexes+tntFltrID)
var indexes map[string]utils.StringMap
if indexes, err = dm.GetFilterIndexes(utils.PrefixToIndexCache[utils.ReverseFilterIndexes], tntFltrID,
utils.EmptyString, nil); err != nil {
if err != utils.ErrNotFound {
guardian.Guardian.UnguardIDs(refID)
return
}
err = nil
indexes = map[string]utils.StringMap{
idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps
}
}
indexes[idxItmType] = map[string]bool{
itemID: true,
}
// it is removed in StoreIndexes
/* // remove the old reference from cache in case
for idxKeyItmType := range indexes {
Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType),
true, utils.NonTransactional)
} */
indexerKey := utils.ConcatenatedKey(tnt, fltrID)
if ctx != utils.EmptyString {
indexerKey = utils.ConcatenatedKey(tnt, ctx)
}
fltrIndexer := NewFilterIndexer(dm, utils.ReverseFilterIndexes, indexerKey)
fltrIndexer.indexes = indexes
if err = fltrIndexer.StoreIndexes(true, utils.NonTransactional); err != nil {
guardian.Guardian.UnguardIDs(refID)
return
}
guardian.Guardian.UnguardIDs(refID)
}
return
}
// removeReverseFilterIndexForFilter will remove a reference for the filter in reverse filter indexes
func removeReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemID string, filterIDs []string) (err error) {
for _, fltrID := range filterIDs {
if strings.HasPrefix(fltrID, utils.Meta) { // we do not reverse for inline filters
continue
@@ -50,19 +98,11 @@ func addReverseFilterIndexForFilter(dm *DataManager, idxItmType, ctx, tnt, itemI
return
}
err = nil
indexes = map[string]utils.StringMap{
idxItmType: make(map[string]bool), // not found in database any reverse, we declare them to add in the next steps
}
continue // already removed
}
indexes[idxItmType] = map[string]bool{
itemID: true,
}
// IT IS REMOVED IN StoreIndexes
/* // remove the old reference from cache in case
for idxKeyItmType := range indexes {
Cache.Remove(utils.CacheReverseFilterIndexes, utils.ConcatenatedKey(tntCtx, idxKeyItmType),
true, utils.NonTransactional)
} */
delete(indexes[idxItmType], itemID) // delete index from map
indexerKey := tnt
if ctx != utils.EmptyString {
indexerKey = utils.ConcatenatedKey(tnt, ctx)

77
general_tests/filter_indexes_cases_it_test.go
View File

@@ -480,6 +480,65 @@ func testFilterIndexesCasesSetIndexedFilter(t *testing.T) {
} else if result != utils.OK {
t.Error("Unexpected reply returned", result)
}
/*
filter1 = &v1.FilterWithCache{
Filter: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_Charger12312",
Rules: []*engine.FilterRule{
{
Type: utils.MetaString,
Element: "~*req.CGRID",
Values: []string{"tester_id"},
},
{
Type: utils.MetaPrefix,
Element: "~*req.AnswerTime",
Values: []string{"2022"},
},
{
Type: utils.MetaSuffix,
Element: "~*req.AnswerTime",
Values: []string{"202"},
},
},
},
}
filter2 := &v1.FilterWithCache{
Filter: &engine.Filter{
Tenant: "cgrates.org",
ID: "FLTR_Charger4564",
Rules: []*engine.FilterRule{
{
Type: utils.MetaString,
Element: "~*req.CGRID",
Values: []string{"tester_id"},
},
{
Type: utils.MetaPrefix,
Element: "~*req.AnswerTime",
Values: []string{"2022"},
},
{
Type: utils.MetaSuffix,
Element: "~*req.AnswerTime",
Values: []string{"202"},
},
},
},
}
if err := fIdxCasesRPC.Call(utils.APIerSv1SetFilter, filter1, &result); err != nil {
t.Error(err)
} else if result != utils.OK {
t.Error("Unexpected reply returned", result)
}
if err := fIdxCasesRPC.Call(utils.APIerSv1SetFilter, filter2, &result); err != nil {
t.Error(err)
} else if result != utils.OK {
t.Error("Unexpected reply returned", result)
}
*/
}
func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) {
@@ -487,7 +546,7 @@ func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) {
ChargerProfile: &engine.ChargerProfile{
Tenant: "cgrates.org",
ID: "ChrgerIndexable",
FilterIDs: []string{"FLTR_Charger"},
FilterIDs: []string{"FLTR_Charger" /*"FLTR_Charger12312", "FLTR_Charger4564"*/},
RunID: utils.MetaRaw,
AttributeIDs: []string{"ATTR_FLTR1"},
Weight: 20,
@@ -499,6 +558,22 @@ func testFilterIndexesCasesSetChargerWithFltr(t *testing.T) {
} else if result != utils.OK {
t.Error("Unexpected reply returned", result)
}
/* chargerProfile =
&v1.ChargerWithCache{
ChargerProfile: &engine.ChargerProfile{
Tenant: "cgrates.org",
ID: "ChrgerIndexable222",
FilterIDs: []string{"FLTR_Charger", "FLTR_Charger4564"},
RunID: utils.MetaRaw,
AttributeIDs: []string{"ATTR_FLTR1"},
Weight: 20,
},
}
if err := fIdxCasesRPC.Call(utils.APIerSv1SetChargerProfile, chargerProfile, &result); err != nil {
t.Error(err)
} else if result != utils.OK {
t.Error("Unexpected reply returned", result)
} */
}
func testFilterIndexesCasesGetChargerIndexes(t *testing.T) {