diff --git a/apier/v1/smgenericv1_it_test.go b/apier/v1/smgenericv1_it_test.go index 0e473f35b..c9e14a20f 100644 --- a/apier/v1/smgenericv1_it_test.go +++ b/apier/v1/smgenericv1_it_test.go @@ -102,7 +102,7 @@ func TestSMGV1CacheStats(t *testing.T) { var rcvStats *utils.CacheStats expectedStats := &utils.CacheStats{Destinations: 5, ReverseDestinations: 7, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, AccountActionPlans: 5, SharedGroups: 1, DerivedChargers: 1, - LcrProfiles: 5, CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 2} + LcrProfiles: 5, CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 3} var args utils.AttrCacheStats if err := smgV1Rpc.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) diff --git a/data/tariffplans/tutorial/ResourceLimits.csv b/data/tariffplans/tutorial/ResourceLimits.csv index 2bcef8a58..c95d6b485 100644 --- a/data/tariffplans/tutorial/ResourceLimits.csv +++ b/data/tariffplans/tutorial/ResourceLimits.csv @@ -3,4 +3,4 @@ ResGroup1,*string,Account,1001;1002,2014-07-29T15:00:00Z,1s,7,,20, ResGroup1,*string_prefix,Destination,10;20,,,,,, ResGroup1,*rsr_fields,,Subject(~^1.*1$);Destination(1002),,,,,, ResGroup2,*destinations,Destination,DST_FS,2014-07-29T15:00:00Z,3600s,8,SPECIAL_1002,10, -ResGroup2,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,, +ResGroup3,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,, diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 8b7726d24..4b832488e 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1929,8 +1929,10 @@ func APItoResourceLimit(tpRL *utils.TPResourceLimit, timezone string) (rl *Resou return nil, err } } - if rl.Limit, err = strconv.ParseFloat(tpRL.Limit, 64); err != nil { - return nil, err + if tpRL.Limit != "" { + if rl.Limit, err = strconv.ParseFloat(tpRL.Limit, 64); err != nil { + return nil, err + } } return rl, nil } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index abf4e1e4c..245266f6d 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -237,14 +237,12 @@ func testOnStorITMatchReqFilterIndex(t *testing.T) { "RL1": true, "RL2": true, } - if rcvMp, err := onStor.MatchReqFilterIndex(utils.ResourceLimitsIndex, - utils.ConcatenatedKey("Account", "1002")); err != nil { + if rcvMp, err := onStor.MatchReqFilterIndex(utils.ResourceLimitsIndex, "Account", "1002"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { t.Errorf("Expecting: %+v, received: %+v", eMp, rcvMp) } - if _, err := onStor.MatchReqFilterIndex(utils.ResourceLimitsIndex, - utils.ConcatenatedKey("NonexistentField", "1002")); err == nil || err != utils.ErrNotFound { + if _, err := onStor.MatchReqFilterIndex(utils.ResourceLimitsIndex, "NonexistentField", "1002"); err == nil || err != utils.ErrNotFound { t.Error(err) } } diff --git a/engine/reqfilterhelpers.go b/engine/reqfilterhelpers.go new file mode 100644 index 000000000..4c092f476 --- /dev/null +++ b/engine/reqfilterhelpers.go @@ -0,0 +1,62 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package engine + +import ( + "fmt" + + "github.com/cgrates/cgrates/utils" +) + +// matchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event +// helper on top of dataDB.MatchReqFilterIndex, adding utils.NOT_AVAILABLE to list of fields queried +func matchingItemIDsForEvent(ev map[string]interface{}, dataDB DataDB, dbIdxKey string) (itemIDs utils.StringMap, err error) { + fmt.Printf("Event: %+v, dbIdxKey: %s\n", ev, dbIdxKey) + itemIDs = make(utils.StringMap) + for fldName, fieldValIf := range ev { + fldVal, canCast := utils.CastFieldIfToString(fieldValIf) + if !canCast { + return nil, fmt.Errorf("Cannot cast field: %s into string", fldName) + } + dbItemIDs, err := dataDB.MatchReqFilterIndex(dbIdxKey, fldName, fldVal) + if err != nil { + if err == utils.ErrNotFound { + continue + } + return nil, err + } + for itemID := range dbItemIDs { + if _, hasIt := itemIDs[itemID]; !hasIt { // Add it to list if not already there + itemIDs[itemID] = dbItemIDs[itemID] + } + } + } + dbItemIDs, err := dataDB.MatchReqFilterIndex(dbIdxKey, utils.NOT_AVAILABLE, utils.NOT_AVAILABLE) // add unindexed itemIDs to be checked + if err != nil { + if err != utils.ErrNotFound { + return nil, err + } + err = nil // not found is ignored + } + for itemID := range dbItemIDs { + if _, hasIt := itemIDs[itemID]; !hasIt { + itemIDs[itemID] = dbItemIDs[itemID] + } + } + return +} diff --git a/engine/reslimiter.go b/engine/reslimiter.go index 01d008581..0c1e115b9 100644 --- a/engine/reslimiter.go +++ b/engine/reslimiter.go @@ -192,55 +192,12 @@ func (rls *ResourceLimiterService) ServiceShutdown() error { // matchingResourceLimitsForEvent returns ordered list of matching resources which are active by the time of the call func (rls *ResourceLimiterService) matchingResourceLimitsForEvent(ev map[string]interface{}) (resLimits ResourceLimits, err error) { matchingResources := make(map[string]*ResourceLimit) - for fldName, fieldValIf := range ev { - fldVal, canCast := utils.CastFieldIfToString(fieldValIf) - if !canCast { - return nil, fmt.Errorf("Cannot cast field: %s into string", fldName) - } - rlIDs, err := rls.dataDB.MatchReqFilterIndex(utils.ResourceLimitsIndex, utils.ConcatenatedKey(fldName, fldVal)) - if err != nil { - if err == utils.ErrNotFound { - continue - } - return nil, err - } - for resName := range rlIDs { - if _, hasIt := matchingResources[resName]; hasIt { // Already checked this RL - continue - } - rl, err := rls.dataDB.GetResourceLimit(resName, false, utils.NonTransactional) - if err != nil { - if err == utils.ErrNotFound { - continue - } - return nil, err - } - if rl.ActivationInterval != nil && !rl.ActivationInterval.IsActiveAtTime(time.Now()) { // not active - continue - } - passAllFilters := true - for _, fltr := range rl.Filters { - if pass, err := fltr.Pass(ev, "", rls.cdrStatS); err != nil { - return nil, utils.NewErrServerError(err) - } else if !pass { - passAllFilters = false - continue - } - } - if passAllFilters { - matchingResources[rl.ID] = rl // Cannot save it here since we could have errors after and resource will remain unused - } - } - } - // Check un-indexed resources - uIdxRLIDs, err := rls.dataDB.MatchReqFilterIndex(utils.ResourceLimitsIndex, utils.ConcatenatedKey(utils.NOT_AVAILABLE, utils.NOT_AVAILABLE)) - if err != nil && err != utils.ErrNotFound { + rlIDs, err := matchingItemIDsForEvent(ev, rls.dataDB, utils.ResourceLimitsIndex) + if err != nil { return nil, err } - for resName := range uIdxRLIDs { - if _, hasIt := matchingResources[resName]; hasIt { // Already checked this RL - continue - } + utils.Logger.Debug(fmt.Sprintf("### #0 rlIDs: %+v\n", rlIDs)) + for resName := range rlIDs { rl, err := rls.dataDB.GetResourceLimit(resName, false, utils.NonTransactional) if err != nil { if err == utils.ErrNotFound { @@ -248,18 +205,32 @@ func (rls *ResourceLimiterService) matchingResourceLimitsForEvent(ev map[string] } return nil, err } - if rl.ActivationInterval != nil && !rl.ActivationInterval.IsActiveAtTime(time.Now()) { // not active + utils.Logger.Debug(fmt.Sprintf("### #1 RL, rl: %+v\n", rl)) + if rl.ActivationInterval != nil && + !rl.ActivationInterval.IsActiveAtTime(time.Now()) { // not active + utils.Logger.Debug(fmt.Sprintf("### #2 RL, rl: %+v, not active: %+v\n", rl, rl.ActivationInterval)) continue } + passAllFilters := true for _, fltr := range rl.Filters { + utils.Logger.Debug(fmt.Sprintf("### #3 RL, rl: %+v, fltr: %+v\n", rl, fltr)) if pass, err := fltr.Pass(ev, "", rls.cdrStatS); err != nil { + utils.Logger.Debug(fmt.Sprintf("### RL, rl: %+v, fltr: %+v, not passing, err: %v\n", rl, fltr, err)) return nil, utils.NewErrServerError(err) } else if !pass { + passAllFilters = false continue } - matchingResources[rl.ID] = rl // Cannot save it here since we could have errors after and resource will remain unused + utils.Logger.Debug(fmt.Sprintf("### #4 RL, rl: %+v, fltr: %+v, passing\n", rl, fltr)) } + if !passAllFilters { + continue + } + utils.Logger.Debug(fmt.Sprintf("### #4.1 RL, passing all filters, adding rlID: %s, rl: %+v, matchingResources: %+v\n", rl.ID, rl, matchingResources)) + matchingResources[rl.ID] = rl // Cannot save it here since we could have errors after and resource will remain unused } + utils.Logger.Debug(fmt.Sprintf("### #5 RL, matchingResources: %+v\n", matchingResources)) + // All good, convert from Map to Slice so we can sort resLimits = make(ResourceLimits, len(matchingResources)) i := 0 for _, rl := range matchingResources { @@ -271,7 +242,8 @@ func (rls *ResourceLimiterService) matchingResourceLimitsForEvent(ev map[string] } // V1ResourceLimitsForEvent returns active resource limits matching the event -func (rls *ResourceLimiterService) V1ResourceLimitsForEvent(ev map[string]interface{}, reply *[]*ResourceLimit) error { +func (rls *ResourceLimiterService) V1ResourceLimitsForEvent(ev map[string]interface{}, + reply *[]*ResourceLimit) error { matchingRLForEv, err := rls.matchingResourceLimitsForEvent(ev) if err != nil { return utils.NewErrServerError(err) @@ -280,7 +252,8 @@ func (rls *ResourceLimiterService) V1ResourceLimitsForEvent(ev map[string]interf return nil } -func (rls *ResourceLimiterService) V1AllowUsage(args utils.AttrRLsResourceUsage, allow *bool) (err error) { +func (rls *ResourceLimiterService) V1AllowUsage(args utils.AttrRLsResourceUsage, + allow *bool) (err error) { mtcRLs, err := rls.matchingResourceLimitsForEvent(args.Event) if err != nil { return utils.NewErrServerError(err) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 767b40597..6df1ab266 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -109,7 +109,7 @@ type DataDB interface { SetStructVersion(*StructVersion) error GetReqFilterIndexes(dbKey string) (indexes map[string]map[string]utils.StringMap, err error) SetReqFilterIndexes(dbKey string, indexes map[string]map[string]utils.StringMap) (err error) - MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs utils.StringMap, err error) + MatchReqFilterIndex(dbKey, fieldName, fieldVal string) (itemIDs utils.StringMap, err error) GetStatsQueue(sqID string, skipCache bool, transactionID string) (sq *StatsQueue, err error) SetStatsQueue(sq *StatsQueue) (err error) RemStatsQueue(sqID string, transactionID string) (err error) diff --git a/engine/storage_map.go b/engine/storage_map.go index be2999aac..b40154064 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1504,8 +1504,8 @@ func (ms *MapStorage) SetReqFilterIndexes(dbKey string, indexes map[string]map[s ms.dict[dbKey] = result return } -func (ms *MapStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs utils.StringMap, err error) { - cacheKey := dbKey + fieldValKey +func (ms *MapStorage) MatchReqFilterIndex(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) { + cacheKey := dbKey + utils.ConcatenatedKey(fldName, fldVal) ms.mu.RLock() defer ms.mu.RUnlock() if x, ok := cache.Get(cacheKey); ok { // Attempt to find in cache first @@ -1524,9 +1524,8 @@ func (ms *MapStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs ut if err = ms.ms.Unmarshal(values, &indexes); err != nil { return nil, err } - keySplt := strings.Split(fieldValKey, ":") - if _, hasIt := indexes[keySplt[0]]; hasIt { - itemIDs = indexes[keySplt[0]][keySplt[1]] + if _, hasIt := indexes[fldName]; hasIt { + itemIDs = indexes[fldName][fldVal] } //Verify items if len(itemIDs) == 0 { diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 882637ef5..10a458fd1 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1976,11 +1976,8 @@ func (ms *MongoStorage) SetReqFilterIndexes(dbKey string, indexes map[string]map return } -func (ms *MongoStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs utils.StringMap, err error) { - fldValSplt := strings.Split(fieldValKey, utils.CONCATENATED_KEY_SEP) - if len(fldValSplt) != 2 { - return nil, fmt.Errorf("malformed key in query: %s", fldValSplt) - } +func (ms *MongoStorage) MatchReqFilterIndex(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) { + fieldValKey := utils.ConcatenatedKey(fldName, fldVal) cacheKey := dbKey + fieldValKey if x, ok := cache.Get(cacheKey); ok { // Attempt to find in cache first if x == nil { @@ -1994,7 +1991,7 @@ func (ms *MongoStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs Key string Value map[string]map[string]utils.StringMap } - fldKey := fmt.Sprintf("value.%s.%s", fldValSplt[0], fldValSplt[1]) + fldKey := fmt.Sprintf("value.%s.%s", fldName, fldVal) if err = col.Find( bson.M{"key": dbKey, fldKey: bson.M{"$exists": true}}).Select( bson.M{fldKey: true}).One(&result); err != nil { @@ -2004,7 +2001,7 @@ func (ms *MongoStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs } return nil, err } - itemIDs = result.Value[fldValSplt[0]][fldValSplt[1]] + itemIDs = result.Value[fldName][fldVal] cache.Set(cacheKey, itemIDs, true, utils.NonTransactional) return } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 6893a0140..d07f1c8b1 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1532,7 +1532,8 @@ func (rs *RedisStorage) SetReqFilterIndexes(dbKey string, indexes map[string]map return rs.Cmd("HMSET", dbKey, mp).Err } -func (rs *RedisStorage) MatchReqFilterIndex(dbKey, fieldValKey string) (itemIDs utils.StringMap, err error) { +func (rs *RedisStorage) MatchReqFilterIndex(dbKey, fldName, fldVal string) (itemIDs utils.StringMap, err error) { + fieldValKey := utils.ConcatenatedKey(fldName, fldVal) cacheKey := dbKey + fieldValKey if x, ok := cache.Get(cacheKey); ok { // Attempt to find in cache first if x == nil { diff --git a/general_tests/tut_smgeneric_it_test.go b/general_tests/tut_smgeneric_it_test.go index a496c751c..9aa69ae6b 100644 --- a/general_tests/tut_smgeneric_it_test.go +++ b/general_tests/tut_smgeneric_it_test.go @@ -100,7 +100,7 @@ func TestTutSMGCacheStats(t *testing.T) { var rcvStats *utils.CacheStats expectedStats := &utils.CacheStats{Destinations: 5, ReverseDestinations: 7, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, AccountActionPlans: 5, SharedGroups: 1, DerivedChargers: 1, LcrProfiles: 5, - CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 2} + CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 3} var args utils.AttrCacheStats if err := tutSMGRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV2.GetCacheStats: ", err.Error()) diff --git a/general_tests/tutorial_it_test.go b/general_tests/tutorial_it_test.go index f29a6bdfc..b9e866877 100644 --- a/general_tests/tutorial_it_test.go +++ b/general_tests/tutorial_it_test.go @@ -104,7 +104,7 @@ func TestTutITCacheStats(t *testing.T) { var rcvStats *utils.CacheStats expectedStats := &utils.CacheStats{Destinations: 5, ReverseDestinations: 7, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, AccountActionPlans: 5, SharedGroups: 1, DerivedChargers: 1, LcrProfiles: 5, - CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 2} + CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourceLimits: 3} var args utils.AttrCacheStats if err := tutLocalRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV2.GetCacheStats: ", err.Error()) diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 621917a5c..e9e737311 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1313,7 +1313,7 @@ type ActivationInterval struct { } func (ai *ActivationInterval) IsActiveAtTime(atTime time.Time) bool { - return ai.ActivationTime.Before(atTime) && + return (ai.ActivationTime.IsZero() || ai.ActivationTime.Before(atTime)) && (ai.ExpiryTime.IsZero() || ai.ExpiryTime.After(atTime)) }