uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-22 23:58:44 +05:00
Code Issues Packages Projects Releases Wiki Activity

Updated indexes to use *string tag

Browse Source
This commit is contained in:
edwardro22
2018-01-21 19:08:38 +02:00
committed by Dan Christian Bogos
parent 8ba3474fe2
commit ed5851f3f2
17 changed files with 211 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
engine/attributes.go
View File

@@ -62,7 +62,7 @@ func (alS *AttributeService) matchingAttributeProfilesForEvent(ev *utils.CGREven
attrIdxKey = utils.ConcatenatedKey(ev.Tenant, contextVal)
matchingAPs := make(map[string]*AttributeProfile)
aPrflIDs, err := matchingItemIDsForEvent(ev.Event, alS.indexedFields,
alS.dm, utils.AttributeProfilesStringIndex+attrIdxKey)
alS.dm, utils.AttributeProfilesStringIndex+attrIdxKey, MetaString)
if err != nil {
return nil, err
}

20
engine/datamanager.go
View File

@@ -435,7 +435,7 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
continue
}
for _, fldVal := range flt.Values {
if err = indexer.loadFldNameFldValIndex(flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
if err = indexer.loadFldNameFldValIndex(flt.Type, flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}
@@ -521,7 +521,7 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
continue
}
for _, fldVal := range flt.Values {
if err = indexer.loadFldNameFldValIndex(flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
if err = indexer.loadFldNameFldValIndex(flt.Type, flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}
@@ -685,7 +685,7 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
continue
}
for _, fldVal := range flt.Values {
if err = indexer.loadFldNameFldValIndex(flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
if err = indexer.loadFldNameFldValIndex(flt.Type, flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}
@@ -992,8 +992,8 @@ func (dm *DataManager) HasData(category, subject string) (has bool, err error) {
return dm.DataDB().HasDataDrv(category, subject)
}
func (dm *DataManager) GetFilterIndexes(dbKey string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
return dm.DataDB().GetFilterIndexesDrv(dbKey, fldNameVal)
func (dm *DataManager) GetFilterIndexes(dbKey, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
return dm.DataDB().GetFilterIndexesDrv(dbKey, filterType, fldNameVal)
}
func (dm *DataManager) SetFilterIndexes(dbKey string, indexes map[string]utils.StringMap) (err error) {
@@ -1016,8 +1016,8 @@ func (dm *DataManager) RemoveFilterReverseIndexes(dbKey string) (err error) {
return dm.DataDB().RemoveFilterReverseIndexesDrv(dbKey)
}
func (dm *DataManager) MatchFilterIndex(dbKey, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) {
fieldValKey := utils.ConcatenatedKey(fieldName, fieldVal)
func (dm *DataManager) MatchFilterIndex(dbKey, filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) {
fieldValKey := utils.ConcatenatedKey(filterType, fieldName, fieldVal)
cacheKey := dbKey + fieldValKey
if x, ok := cache.Get(cacheKey); ok { // Attempt to find in cache first
if x == nil {
@@ -1026,7 +1026,7 @@ func (dm *DataManager) MatchFilterIndex(dbKey, fieldName, fieldVal string) (item
return x.(utils.StringMap), nil
}
// Not found in cache, check in DB
itemIDs, err = dm.DataDB().MatchFilterIndexDrv(dbKey, fieldName, fieldVal)
itemIDs, err = dm.DataDB().MatchFilterIndexDrv(dbKey, filterType, fieldName, fieldVal)
if err != nil {
if err == utils.ErrNotFound {
cache.Set(cacheKey, nil, true, utils.NonTransactional)
@@ -1122,7 +1122,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool)
continue
}
for _, fldVal := range flt.Values {
if err = indexer.loadFldNameFldValIndex(flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
if err = indexer.loadFldNameFldValIndex(flt.Type, flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}
@@ -1229,7 +1229,7 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
continue
}
for _, fldVal := range flt.Values {
if err = indexer.loadFldNameFldValIndex(flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
if err = indexer.loadFldNameFldValIndex(flt.Type, flt.FieldName, fldVal); err != nil && err != utils.ErrNotFound {
return err
}
}

73
engine/filterindexer.go
View File

@@ -57,34 +57,47 @@ func (rfi *ReqFilterIndexer) ChangedKeys(reverse bool) utils.StringMap {
// IndexTPFilter parses reqFltrs, adding itemID in the indexes and marks the changed keys in chngdIndxKeys
func (rfi *ReqFilterIndexer) IndexTPFilter(tpFltr *utils.TPFilterProfile, itemID string) {
var hasMetaString bool
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
for _, fltr := range tpFltr.Filters {
if fltr.Type != MetaString {
continue
}
hasMetaString = true // Mark that we found at least one metatring so we don't index globally
for _, fldVal := range fltr.Values {
concatKey := utils.ConcatenatedKey(fltr.FieldName, fldVal)
if _, hasIt := rfi.indexes[concatKey]; !hasIt {
rfi.indexes[concatKey] = make(utils.StringMap)
switch fltr.Type {
case MetaString:
for _, fldVal := range fltr.Values {
concatKey := utils.ConcatenatedKey(fltr.Type, fltr.FieldName, fldVal)
if _, hasIt := rfi.indexes[concatKey]; !hasIt {
rfi.indexes[concatKey] = make(utils.StringMap)
}
rfi.indexes[concatKey][itemID] = true
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][concatKey] = true
rfi.chngdIndxKeys[concatKey] = true
}
rfi.indexes[concatKey][itemID] = true
rfi.reveseIndex[itemID][concatKey] = true
rfi.chngdIndxKeys[concatKey] = true
rfi.chngdRevIndxKeys[itemID] = true
// case MetaStringPrefix:
// for _, fldVal := range fltr.Values {
// concatKey := utils.ConcatenatedKey(fltr.FieldName, fldVal)
// if _, hasIt := rfi.indexes[concatKey]; !hasIt {
// rfi.indexes[concatKey] = make(utils.StringMap)
// }
// rfi.indexes[concatKey][itemID] = true
// rfi.reveseIndex[itemID][concatKey] = true
// rfi.chngdIndxKeys[concatKey] = true
// }
// rfi.chngdRevIndxKeys[itemID] = true
default:
naConcatKey := utils.ConcatenatedKey(utils.MetaDefault, utils.NOT_AVAILABLE, utils.NOT_AVAILABLE)
if _, hasIt := rfi.indexes[naConcatKey]; !hasIt {
rfi.indexes[naConcatKey] = make(utils.StringMap)
}
if _, hasIt := rfi.reveseIndex[itemID]; !hasIt {
rfi.reveseIndex[itemID] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][naConcatKey] = true
rfi.indexes[naConcatKey][itemID] = true // Fields without real field index will be located in map[NOT_AVAILABLE:NOT_AVAILABLE][rl.ID]
}
rfi.chngdRevIndxKeys[itemID] = true
}
if !hasMetaString {
naConcatKey := utils.ConcatenatedKey(utils.NOT_AVAILABLE, utils.NOT_AVAILABLE)
if _, hasIt := rfi.indexes[naConcatKey]; !hasIt {
rfi.indexes[naConcatKey] = make(utils.StringMap)
}
rfi.reveseIndex[itemID][naConcatKey] = true
rfi.indexes[naConcatKey][itemID] = true // Fields without real field index will be located in map[NOT_AVAILABLE:NOT_AVAILABLE][rl.ID]
}
return
}
@@ -129,7 +142,7 @@ func (rfi *ReqFilterIndexer) StoreIndexes() (err error) {
}
//Populate the ReqFilterIndexer.reveseIndex for specifil itemID
func (rfi *ReqFilterIndexer) loadItemReverseIndex(itemID string) (err error) {
func (rfi *ReqFilterIndexer) loadItemReverseIndex(filterType, itemID string) (err error) {
rcvReveseIdx, err := rfi.dm.GetFilterReverseIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, true),
map[string]string{itemID: ""})
@@ -146,9 +159,9 @@ func (rfi *ReqFilterIndexer) loadItemReverseIndex(itemID string) (err error) {
}
//Populate ReqFilterIndexer.indexes with specific fieldName:fieldValue , item
func (rfi *ReqFilterIndexer) loadFldNameFldValIndex(fldName, fldVal string) error {
func (rfi *ReqFilterIndexer) loadFldNameFldValIndex(filterType, fldName, fldVal string) error {
rcvIdx, err := rfi.dm.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), filterType,
map[string]string{fldName: fldVal})
if err != nil {
return err
@@ -164,15 +177,15 @@ func (rfi *ReqFilterIndexer) loadFldNameFldValIndex(fldName, fldVal string) erro
//RemoveItemFromIndex remove Indexes for a specific itemID
func (rfi *ReqFilterIndexer) RemoveItemFromIndex(itemID string) (err error) {
if err = rfi.loadItemReverseIndex(itemID); err != nil {
if err = rfi.loadItemReverseIndex(MetaString, itemID); err != nil {
return err
}
for key, _ := range rfi.reveseIndex[itemID] {
kSplt := strings.Split(key, utils.CONCATENATED_KEY_SEP)
if len(kSplt) != 2 {
if len(kSplt) != 3 {
return fmt.Errorf("Malformed key in db: %s", key)
}
if err = rfi.loadFldNameFldValIndex(kSplt[0], kSplt[1]); err != nil {
if err = rfi.loadFldNameFldValIndex(kSplt[0], kSplt[1], kSplt[2]); err != nil {
return err
}
}

129
engine/onstor_it_test.go
View File

@@ -192,17 +192,17 @@ func testOnStorITSetGetDerivedCharges(t *testing.T) {
func testOnStorITSetFilterIndexes(t *testing.T) {
idxes := map[string]utils.StringMap{
"Account:1001": utils.StringMap{
"*string:Account:1001": utils.StringMap{
"RL1": true,
},
"Account:1002": utils.StringMap{
"*string:Account:1002": utils.StringMap{
"RL1": true,
"RL2": true,
},
"Account:dan": utils.StringMap{
"*string:Account:dan": utils.StringMap{
"RL2": true,
},
"Subject:dan": utils.StringMap{
"*string:Subject:dan": utils.StringMap{
"RL2": true,
"RL3": true,
},
@@ -220,17 +220,17 @@ func testOnStorITSetFilterIndexes(t *testing.T) {
func testOnStorITGetFilterIndexes(t *testing.T) {
eIdxes := map[string]utils.StringMap{
"Account:1001": utils.StringMap{
"*string:Account:1001": utils.StringMap{
"RL1": true,
},
"Account:1002": utils.StringMap{
"*string:Account:1002": utils.StringMap{
"RL1": true,
"RL2": true,
},
"Account:dan": utils.StringMap{
"*string:Account:dan": utils.StringMap{
"RL2": true,
},
"Subject:dan": utils.StringMap{
"*string:Subject:dan": utils.StringMap{
"RL2": true,
"RL3": true,
},
@@ -243,20 +243,20 @@ func testOnStorITGetFilterIndexes(t *testing.T) {
"Subject": "dan",
}
expectedsbjDan := map[string]utils.StringMap{
"Subject:dan": utils.StringMap{
"*string:Subject:dan": utils.StringMap{
"RL2": true,
"RL3": true,
},
}
if exsbjDan, err := onStor.GetFilterIndexes(
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false),
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), MetaString,
sbjDan); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expectedsbjDan, exsbjDan) {
t.Errorf("Expecting: %+v, received: %+v", expectedsbjDan, exsbjDan)
}
if rcv, err := onStor.GetFilterIndexes(
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false),
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -264,7 +264,7 @@ func testOnStorITGetFilterIndexes(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v", eIdxes, rcv)
}
}
if _, err := onStor.GetFilterIndexes("unknown_key", nil); err == nil || err != utils.ErrNotFound {
if _, err := onStor.GetFilterIndexes("unknown_key", MetaString, nil); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
if err := onStor.RemoveFilterIndexes(
@@ -272,7 +272,7 @@ func testOnStorITGetFilterIndexes(t *testing.T) {
t.Error(err)
}
_, err := onStor.GetFilterIndexes(
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), nil)
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false), MetaString, nil)
if err != utils.ErrNotFound {
t.Error(err)
}
@@ -290,14 +290,14 @@ func testOnStorITMatchFilterIndex(t *testing.T) {
}
if rcvMp, err := onStor.MatchFilterIndex(
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false),
"Account", "1002"); err != nil {
MetaString, "Account", "1002"); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eMp, rcvMp) {
t.Errorf("Expecting: %+v, received: %+v", eMp, rcvMp)
}
if _, err := onStor.MatchFilterIndex(
GetDBIndexKey(utils.ResourceProfilesPrefix, "cgrates.org", false),
"NonexistentField", "1002"); err == nil || err != utils.ErrNotFound {
MetaString, "NonexistentField", "1002"); err == nil || err != utils.ErrNotFound {
t.Error(err)
}
}
@@ -2676,28 +2676,28 @@ func testOnStorITTestThresholdFilterIndexes(t *testing.T) {
t.Error(err)
}
eIdxes := map[string]utils.StringMap{
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"THD_Test": true,
"THD_Test2": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"THD_Test": true,
"THD_Test2": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
"THD_Test2": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
rfi := NewReqFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -2739,32 +2739,32 @@ func testOnStorITTestThresholdFilterIndexes(t *testing.T) {
t.Error(err)
}
eIdxes = map[string]utils.StringMap{
"Account:1001": utils.StringMap{
"*string:Account:1001": utils.StringMap{
"THD_Test": true,
},
"Account:1002": utils.StringMap{
"*string:Account:1002": utils.StringMap{
"THD_Test": true,
},
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"THD_Test2": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"THD_Test2": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"Account:1001": true,
"Account:1002": true,
"*string:Account:1001": true,
"*string:Account:1002": true,
},
"THD_Test2": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -2806,35 +2806,35 @@ func testOnStorITTestThresholdFilterIndexes(t *testing.T) {
t.Error(err)
}
eIdxes = map[string]utils.StringMap{
"Destination:10": utils.StringMap{
"*string:Destination:10": utils.StringMap{
"THD_Test": true,
},
"Destination:20": utils.StringMap{
"*string:Destination:20": utils.StringMap{
"THD_Test": true,
},
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"THD_Test": true,
"THD_Test2": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"THD_Test": true,
"THD_Test2": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"Destination:10": true,
"Destination:20": true,
"EventType:Event1": true,
"EventType:Event2": true,
"*string:Destination:10": true,
"*string:Destination:20": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
"THD_Test2": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -2861,7 +2861,7 @@ func testOnStorITTestThresholdFilterIndexes(t *testing.T) {
t.Error(err)
}
if _, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != utils.ErrNotFound {
t.Error(err)
}
@@ -2870,6 +2870,7 @@ func testOnStorITTestThresholdFilterIndexes(t *testing.T) {
nil); err != utils.ErrNotFound {
t.Error(err)
}
}
func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) {
@@ -2915,24 +2916,24 @@ func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) {
t.Error(err)
}
eIdxes := map[string]utils.StringMap{
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"AttrPrf": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"AttrPrf": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"AttrPrf": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
for _, ctx := range attrProfile.Contexts {
rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(attrProfile.Tenant, ctx))
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -2959,7 +2960,7 @@ func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) {
rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(attrProfile.Tenant, "con3"))
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -2982,7 +2983,7 @@ func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) {
rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix,
utils.ConcatenatedKey(attrProfile.Tenant, ctx))
if _, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
@@ -3000,7 +3001,7 @@ func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) {
//check if index is removed
rfi = NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey("cgrates.org", "con3"))
if _, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil && err != utils.ErrNotFound {
t.Error(err)
}
@@ -3046,22 +3047,22 @@ func testOnStorITTestThresholdInlineFilterIndexing(t *testing.T) {
t.Error(err)
}
eIdxes := map[string]utils.StringMap{
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"THD_Test": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"THD_Test": true,
},
}
reverseIdxes := map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"EventType:Event1": true,
"EventType:Event2": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
rfi := NewReqFilterIndexer(onStor, utils.ThresholdProfilePrefix, th.Tenant)
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -3084,26 +3085,26 @@ func testOnStorITTestThresholdInlineFilterIndexing(t *testing.T) {
t.Error(err)
}
eIdxes = map[string]utils.StringMap{
"Account:1001": utils.StringMap{
"*string:Account:1001": utils.StringMap{
"THD_Test": true,
},
"EventType:Event1": utils.StringMap{
"*string:EventType:Event1": utils.StringMap{
"THD_Test": true,
},
"EventType:Event2": utils.StringMap{
"*string:EventType:Event2": utils.StringMap{
"THD_Test": true,
},
}
reverseIdxes = map[string]utils.StringMap{
"THD_Test": utils.StringMap{
"Account:1001": true,
"EventType:Event1": true,
"EventType:Event2": true,
"*string:Account:1001": true,
"*string:EventType:Event1": true,
"*string:EventType:Event2": true,
},
}
if rcvIdx, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != nil {
t.Error(err)
} else {
@@ -3126,7 +3127,7 @@ func testOnStorITTestThresholdInlineFilterIndexing(t *testing.T) {
t.Error(err)
}
if _, err := onStor.GetFilterIndexes(
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false),
GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), MetaString,
nil); err != utils.ErrNotFound {
t.Error(err)
}

6
engine/reqfilterhelpers.go
View File

@@ -29,7 +29,7 @@ import (
// helper on top of dataDB.MatchReqFilterIndex, adding utils.NOT_AVAILABLE to list of fields queried
// executes a number of $(len(fields) + 1) queries to dataDB so the size of event influences the speed of return
func matchingItemIDsForEvent(ev map[string]interface{}, fieldIDs []string,
dm *DataManager, dbIdxKey string) (itemIDs utils.StringMap, err error) {
dm *DataManager, dbIdxKey, filterType string) (itemIDs utils.StringMap, err error) {
if len(fieldIDs) == 0 {
fieldIDs = make([]string, len(ev))
i := 0
@@ -50,7 +50,7 @@ func matchingItemIDsForEvent(ev map[string]interface{}, fieldIDs []string,
fmt.Sprintf("<%s> cannot cast field: %s into string", utils.FilterS, fldName))
continue
}
dbItemIDs, err := dm.MatchFilterIndex(dbIdxKey, fldName, fldVal)
dbItemIDs, err := dm.MatchFilterIndex(dbIdxKey, filterType, fldName, fldVal)
if err != nil {
if err == utils.ErrNotFound {
continue
@@ -63,7 +63,7 @@ func matchingItemIDsForEvent(ev map[string]interface{}, fieldIDs []string,
}
}
}
dbItemIDs, err := dm.MatchFilterIndex(dbIdxKey, utils.NOT_AVAILABLE, utils.NOT_AVAILABLE) // add unindexed itemIDs to be checked
dbItemIDs, err := dm.MatchFilterIndex(dbIdxKey, filterType, utils.NOT_AVAILABLE, utils.NOT_AVAILABLE) // add unindexed itemIDs to be checked
if err != nil {
if err != utils.ErrNotFound {
return nil, err

2
engine/resources.go
View File

@@ -434,7 +434,7 @@ func (rS *ResourceService) cachedResourcesForEvent(evUUID string) (rs Resources)
// matchingResourcesForEvent returns ordered list of matching resources which are active by the time of the call
func (rS *ResourceService) matchingResourcesForEvent(tenant string, ev map[string]interface{}) (rs Resources, err error) {
matchingResources := make(map[string]*Resource)
rIDs, err := matchingItemIDsForEvent(ev, rS.indexedFields, rS.dm, utils.ResourceProfilesStringIndex+tenant)
rIDs, err := matchingItemIDsForEvent(ev, rS.indexedFields, rS.dm, utils.ResourceProfilesStringIndex+tenant, MetaString)
if err != nil {
return nil, err
}

2
engine/stats.go
View File

@@ -141,7 +141,7 @@ func (sS *StatService) StoreStatQueue(sq *StatQueue) (err error) {
// matchingStatQueuesForEvent returns ordered list of matching resources which are active by the time of the call
func (sS *StatService) matchingStatQueuesForEvent(ev *utils.CGREvent) (sqs StatQueues, err error) {
matchingSQs := make(map[string]*StatQueue)
sqIDs, err := matchingItemIDsForEvent(ev.Event, sS.indexedFields, sS.dm, utils.StatQueuesStringIndex+ev.Tenant)
sqIDs, err := matchingItemIDsForEvent(ev.Event, sS.indexedFields, sS.dm, utils.StatQueuesStringIndex+ev.Tenant, MetaString)
if err != nil {
return nil, err
}

4
engine/storage_interface.go
View File

@@ -117,13 +117,13 @@ type DataDB interface {
RemoveTimingDrv(string) error
GetLoadHistory(int, bool, string) ([]*utils.LoadInstance, error)
AddLoadHistory(*utils.LoadInstance, int, string) error
GetFilterIndexesDrv(dbKey string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error)
GetFilterIndexesDrv(dbKey, filterType string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error)
SetFilterIndexesDrv(dbKey string, indexes map[string]utils.StringMap) (err error)
RemoveFilterIndexesDrv(id string) (err error)
GetFilterReverseIndexesDrv(dbKey string, fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error)
SetFilterReverseIndexesDrv(dbKey string, indexes map[string]utils.StringMap) (err error)
RemoveFilterReverseIndexesDrv(dbKey string) (err error)
MatchFilterIndexDrv(dbKey, fieldName, fieldVal string) (itemIDs utils.StringMap, err error)
MatchFilterIndexDrv(dbKey, filterType, fieldName, fieldVal string) (itemIDs utils.StringMap, err error)
GetStatQueueProfileDrv(tenant string, ID string) (sq *StatQueueProfile, err error)
SetStatQueueProfileDrv(sq *StatQueueProfile) (err error)
RemStatQueueProfileDrv(tenant, id string) (err error)

16
engine/storage_map.go
View File

@@ -1233,7 +1233,7 @@ func (ms *MapStorage) RemoveTimingDrv(id string) error {
}
//GetFilterIndexesDrv retrieves Indexes from dataDB
func (ms *MapStorage) GetFilterIndexesDrv(dbKey string,
func (ms *MapStorage) GetFilterIndexesDrv(dbKey, filterType string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
@@ -1249,11 +1249,11 @@ func (ms *MapStorage) GetFilterIndexesDrv(dbKey string,
}
indexes = make(map[string]utils.StringMap)
for fldName, fldVal := range fldNameVal {
if _, has := indexes[utils.ConcatenatedKey(fldName, fldVal)]; !has {
indexes[utils.ConcatenatedKey(fldName, fldVal)] = make(utils.StringMap)
if _, has := indexes[utils.ConcatenatedKey(filterType, fldName, fldVal)]; !has {
indexes[utils.ConcatenatedKey(filterType, fldName, fldVal)] = make(utils.StringMap)
}
if len(rcvidx[utils.ConcatenatedKey(fldName, fldVal)]) != 0 {
indexes[utils.ConcatenatedKey(fldName, fldVal)] = rcvidx[utils.ConcatenatedKey(fldName, fldVal)]
if len(rcvidx[utils.ConcatenatedKey(filterType, fldName, fldVal)]) != 0 {
indexes[utils.ConcatenatedKey(filterType, fldName, fldVal)] = rcvidx[utils.ConcatenatedKey(fldName, fldVal)]
}
}
@@ -1338,7 +1338,7 @@ func (ms *MapStorage) RemoveFilterReverseIndexesDrv(dbKey string) (err error) {
return
}
func (ms *MapStorage) MatchFilterIndexDrv(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
func (ms *MapStorage) MatchFilterIndexDrv(dbKey, filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
values, ok := ms.dict[dbKey]
@@ -1349,8 +1349,8 @@ func (ms *MapStorage) MatchFilterIndexDrv(dbKey, fldName, fldVal string) (itemID
if err = ms.ms.Unmarshal(values, &indexes); err != nil {
return nil, err
}
if _, hasIt := indexes[utils.ConcatenatedKey(fldName, fldVal)]; hasIt {
itemIDs = indexes[utils.ConcatenatedKey(fldName, fldVal)]
if _, hasIt := indexes[utils.ConcatenatedKey(filterType, fldName, fldVal)]; hasIt {
itemIDs = indexes[utils.ConcatenatedKey(filterType, fldName, fldVal)]
}
if len(itemIDs) == 0 {
return nil, utils.ErrNotFound

10
engine/storage_mongo_datadb.go
View File

@@ -1907,7 +1907,7 @@ func (ms *MongoStorage) RemoveTimingDrv(id string) (err error) {
}
// GetFilterIndexesDrv retrieves Indexes from dataDB
func (ms *MongoStorage) GetFilterIndexesDrv(dbKey string,
func (ms *MongoStorage) GetFilterIndexesDrv(dbKey, filterType string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
session, col := ms.conn(colRFI)
defer session.Close()
@@ -1918,7 +1918,7 @@ func (ms *MongoStorage) GetFilterIndexesDrv(dbKey string,
findParam := bson.M{"key": dbKey}
if len(fldNameVal) != 0 {
for fldName, fldValue := range fldNameVal {
qryFltr := fmt.Sprintf("value.%s", utils.ConcatenatedKey(fldName, fldValue))
qryFltr := fmt.Sprintf("value.%s", utils.ConcatenatedKey(filterType, fldName, fldValue))
if err = col.Find(bson.M{"key": dbKey, qryFltr: bson.M{"$exists": true}}).Select(
bson.M{qryFltr: true}).One(&result); err != nil {
if err == mgo.ErrNotFound {
@@ -2055,14 +2055,14 @@ func (ms *MongoStorage) RemoveFilterReverseIndexesDrv(dbKey string) (err error)
return
}
func (ms *MongoStorage) MatchFilterIndexDrv(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
func (ms *MongoStorage) MatchFilterIndexDrv(dbKey, filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
session, col := ms.conn(colRFI)
defer session.Close()
var result struct {
Key string
Value map[string][]string
}
fldKey := fmt.Sprintf("value.%s", utils.ConcatenatedKey(fldName, fldVal))
fldKey := fmt.Sprintf("value.%s", utils.ConcatenatedKey(filterType, fldName, fldVal))
if err = col.Find(
bson.M{"key": dbKey, fldKey: bson.M{"$exists": true}}).Select(
bson.M{fldKey: true}).One(&result); err != nil {
@@ -2071,7 +2071,7 @@ func (ms *MongoStorage) MatchFilterIndexDrv(dbKey, fldName, fldVal string) (item
}
return nil, err
}
itemIDs = utils.StringMapFromSlice(result.Value[utils.ConcatenatedKey(fldName, fldVal)])
itemIDs = utils.StringMapFromSlice(result.Value[utils.ConcatenatedKey(filterType, fldName, fldVal)])
return
}

12
engine/storage_redis.go
View File

@@ -1347,7 +1347,7 @@ func (rs *RedisStorage) RemoveTimingDrv(id string) (err error) {
}
//GetFilterIndexesDrv retrieves Indexes from dataDB
func (rs *RedisStorage) GetFilterIndexesDrv(dbKey string,
func (rs *RedisStorage) GetFilterIndexesDrv(dbKey, filterType string,
fldNameVal map[string]string) (indexes map[string]utils.StringMap, err error) {
mp := make(map[string]string)
if len(fldNameVal) == 0 {
@@ -1360,14 +1360,14 @@ func (rs *RedisStorage) GetFilterIndexesDrv(dbKey string,
} else {
var itmMpStrLst []string
for fldName, fldVal := range fldNameVal {
concatNameVal := utils.ConcatenatedKey(fldName, fldVal)
itmMpStrLst, err = rs.Cmd("HMGET", dbKey, concatNameVal).List()
concatTypeNameVal := utils.ConcatenatedKey(filterType, fldName, fldVal)
itmMpStrLst, err = rs.Cmd("HMGET", dbKey, concatTypeNameVal).List()
if err != nil {
return
} else if itmMpStrLst[0] == "" {
return nil, utils.ErrNotFound
}
mp[concatNameVal] = itmMpStrLst[0]
mp[concatTypeNameVal] = itmMpStrLst[0]
}
}
indexes = make(map[string]utils.StringMap)
@@ -1483,8 +1483,8 @@ func (rs *RedisStorage) RemoveFilterReverseIndexesDrv(dbKey string) (err error)
return rs.Cmd("DEL", dbKey).Err
}
func (rs *RedisStorage) MatchFilterIndexDrv(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
fieldValKey := utils.ConcatenatedKey(fldName, fldVal)
func (rs *RedisStorage) MatchFilterIndexDrv(dbKey, filterType, fldName, fldVal string) (itemIDs utils.StringMap, err error) {
fieldValKey := utils.ConcatenatedKey(filterType, fldName, fldVal)
fldValBytes, err := rs.Cmd("HGET", dbKey, fieldValKey).Bytes()
if err != nil {
if err == redis.ErrRespNil { // did not find the destination

2
engine/suppliers.go
View File

@@ -115,7 +115,7 @@ func (spS *SupplierService) Shutdown() error {
func (spS *SupplierService) matchingSupplierProfilesForEvent(ev *utils.CGREvent) (sPrfls SupplierProfiles, err error) {
matchingLPs := make(map[string]*SupplierProfile)
sPrflIDs, err := matchingItemIDsForEvent(ev.Event, spS.indexedFields,
spS.dm, utils.SupplierProfilesStringIndex+ev.Tenant)
spS.dm, utils.SupplierProfilesStringIndex+ev.Tenant, MetaString)
if err != nil {
return nil, err
}

3
engine/thresholds.go
View File

@@ -219,8 +219,9 @@ func (tS *ThresholdService) matchingThresholdsForEvent(args *ArgsProcessEvent) (
if len(args.ThresholdIDs) != 0 {
tIDs = args.ThresholdIDs
} else {
tIDsMap, err := matchingItemIDsForEvent(args.Event, tS.indexedFields, tS.dm, utils.ThresholdStringIndex+args.Tenant)
tIDsMap, err := matchingItemIDsForEvent(args.Event, tS.indexedFields, tS.dm, utils.ThresholdStringIndex+args.Tenant, MetaString)
if err != nil {
utils.Logger.Debug(fmt.Sprintf("here gets NOTFOUND"))
return nil, err
}
tIDs = tIDsMap.Slice()