From b0c354c7c11d44daf73f769d7abe02908034cf5f Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 24 Jul 2020 15:31:55 +0300 Subject: [PATCH] Updated filter indexes to only index dynamic path --- apier/v1/api_interfaces.go | 2 +- apier/v1/chargers.go | 2 +- apier/v1/chargers_it_test.go | 2 +- apier/v1/dispatcher.go | 2 +- apier/v1/filter_indexes_it_test.go | 180 +++++------ apier/v1/remote_it_test.go | 26 +- config/attributescfg_test.go | 8 +- config/chargerscfg_test.go | 4 +- config/config_it_test.go | 14 +- config/dispatchercfg_test.go | 8 +- config/resourcescfg_test.go | 8 +- config/routescfg_test.go | 8 +- config/statscfg_test.go | 8 +- config/thresholdscfg_test.go | 12 +- data/conf/samples/docker/cgrates.json | 2 +- .../cgrates.json | 2 +- data/conf/samples/loader_mysql/cgrates.json | 2 +- .../samples/loaders/tutinternal/cgrates.json | 2 +- .../samples/loaders/tutmysql/cgrates.json | 4 +- data/conf/samples/sessinternal/cgrates.json | 2 +- data/conf/samples/sessintjson/cgrates.json | 2 +- data/conf/samples/tutinternal/cgrates.json | 2 +- .../conf/samples/tutinternal_gob/cgrates.json | 2 +- .../conf/samples/tutinternal_new/cgrates.json | 2 +- data/conf/samples/tutmongo2/cgrates.json | 14 +- data/conf/samples/tutmongo2_gob/cgrates.json | 14 +- data/conf/samples/tutmysql/cgrates.json | 2 +- data/conf/samples/tutmysql2/cgrates.json | 14 +- data/conf/samples/tutmysql2_gob/cgrates.json | 14 +- .../samples/tutmysql_internal/cgrates.json | 2 +- .../cgrates/etc/cgrates/cgrates.json | 10 +- .../cgrates/etc/cgrates/cgrates.json | 10 +- .../kamevapi/cgrates/etc/cgrates/cgrates.json | 10 +- .../osips/cgrates/etc/cgrates/cgrates.json | 14 +- .../cgrates/etc/cgrates/cgrates.json | 10 +- .../cgrates/etc/cgrates/cgrates.json | 10 +- .../kamevapi/cgrates/etc/cgrates/cgrates.json | 10 +- .../osips/cgrates/etc/cgrates/cgrates.json | 14 +- .../cgrates/etc/cgrates/cgrates.json | 10 +- dispatchers/chargers.go | 2 +- dispatchers/dispatchers.go | 6 +- engine/attributes.go | 4 +- engine/chargers.go | 19 +- engine/chargers_test.go | 8 +- engine/filterhelpers.go | 10 +- engine/libindex.go | 66 +++- engine/resources.go | 25 +- engine/routes.go | 4 +- engine/stats.go | 18 +- engine/thresholds.go | 6 +- engine/z_attributes_test.go | 3 +- engine/z_filterhelpers_test.go | 16 +- engine/z_filterindexer_it_test.go | 285 ++++++++++++++---- migrator/alias_it_test.go | 4 +- migrator/derived_chargers_it_test.go | 4 +- migrator/filters_it_test.go | 54 ++-- migrator/user_it_test.go | 2 +- rates/rates.go | 6 +- utils/mapstorage.go | 76 +++-- utils/mapstorage_test.go | 25 +- 60 files changed, 673 insertions(+), 434 deletions(-) diff --git a/apier/v1/api_interfaces.go b/apier/v1/api_interfaces.go index b44ad9333..80de94ae7 100644 --- a/apier/v1/api_interfaces.go +++ b/apier/v1/api_interfaces.go @@ -69,7 +69,7 @@ type AttributeSv1Interface interface { type ChargerSv1Interface interface { Ping(ign *utils.CGREventWithArgDispatcher, reply *string) error - GetChargersForEvent(cgrEv *utils.CGREventWithArgDispatcher, reply *engine.ChargerProfiles) error + GetChargersForEvent(cgrEv *utils.CGREventWithOpts, reply *engine.ChargerProfiles) error ProcessEvent(args *utils.CGREventWithOpts, reply *[]*engine.ChrgSProcessEventReply) error } diff --git a/apier/v1/chargers.go b/apier/v1/chargers.go index b17632aef..c5be872a1 100644 --- a/apier/v1/chargers.go +++ b/apier/v1/chargers.go @@ -134,7 +134,7 @@ func (cSv1 *ChargerSv1) Ping(ign *utils.CGREventWithArgDispatcher, reply *string } // GetChargerForEvent returns matching ChargerProfile for Event -func (cSv1 *ChargerSv1) GetChargersForEvent(cgrEv *utils.CGREventWithArgDispatcher, +func (cSv1 *ChargerSv1) GetChargersForEvent(cgrEv *utils.CGREventWithOpts, reply *engine.ChargerProfiles) error { return cSv1.cS.V1GetChargersForEvent(cgrEv, reply) } diff --git a/apier/v1/chargers_it_test.go b/apier/v1/chargers_it_test.go index 6ccd7f3cf..adc97912f 100755 --- a/apier/v1/chargers_it_test.go +++ b/apier/v1/chargers_it_test.go @@ -232,7 +232,7 @@ func testChargerSGetChargersForEvent(t *testing.T) { } var result *engine.ChargerProfiles if err := chargerRPC.Call(utils.ChargerSv1GetChargersForEvent, - &utils.CGREventWithArgDispatcher{ + &utils.CGREventWithOpts{ CGREvent: chargerEvent[1].CGREvent, ArgDispatcher: chargerEvent[1].ArgDispatcher, }, &result); err == nil || diff --git a/apier/v1/dispatcher.go b/apier/v1/dispatcher.go index ad4f383d3..d4af6ac93 100755 --- a/apier/v1/dispatcher.go +++ b/apier/v1/dispatcher.go @@ -389,7 +389,7 @@ func (dC *DispatcherChargerSv1) Ping(args *utils.CGREventWithArgDispatcher, repl } // GetChargersForEvent implements ChargerSv1GetChargersForEvent -func (dC *DispatcherChargerSv1) GetChargersForEvent(args *utils.CGREventWithArgDispatcher, +func (dC *DispatcherChargerSv1) GetChargersForEvent(args *utils.CGREventWithOpts, reply *engine.ChargerProfiles) (err error) { return dC.dC.ChargerSv1GetChargersForEvent(args, reply) } diff --git a/apier/v1/filter_indexes_it_test.go b/apier/v1/filter_indexes_it_test.go index fb14a5777..050014afd 100644 --- a/apier/v1/filter_indexes_it_test.go +++ b/apier/v1/filter_indexes_it_test.go @@ -234,7 +234,7 @@ func testV1FIdxComputeThresholdsIndexes(t *testing.T) { if reply2 != utils.OK { t.Errorf("Error: %+v", reply2) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE1"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE1"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaThresholds, Tenant: tenant, FilterType: utils.MetaString}, @@ -330,7 +330,7 @@ func testV1FIdxSecondComputeThresholdsIndexes(t *testing.T) { if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1002:TEST_PROFILE2"} + expectedIDX := []string{"*string:*req.Account:1002:TEST_PROFILE2"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaThresholds, Tenant: tenant, FilterType: utils.MetaString}, @@ -353,7 +353,7 @@ func testV1FIdxThirdComputeThresholdsIndexes(t *testing.T) { if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE1", "*string:~*req.Account:1002:TEST_PROFILE2"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE1", "*string:*req.Account:1002:TEST_PROFILE2"} sort.Strings(expectedIDX) var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ @@ -451,10 +451,10 @@ func testV1FIdxSetStatQueueProfileIndexes(t *testing.T) { QueueLength: 10, TTL: time.Duration(10) * time.Second, Metrics: []*engine.MetricWithFilters{ - &engine.MetricWithFilters{ + { MetricID: utils.MetaSum, }, - &engine.MetricWithFilters{ + { MetricID: utils.MetaACD, }, }, @@ -501,7 +501,7 @@ func testV1FIdxComputeStatQueueProfileIndexes(t *testing.T) { if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE1"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE1"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaStats, Tenant: tenant, FilterType: utils.MetaString}, @@ -554,10 +554,10 @@ func testV1FIdxSetSecondStatQueueProfileIndexes(t *testing.T) { QueueLength: 10, TTL: time.Duration(10) * time.Second, Metrics: []*engine.MetricWithFilters{ - &engine.MetricWithFilters{ + { MetricID: "*sum", }, - &engine.MetricWithFilters{ + { MetricID: utils.MetaACD, }, }, @@ -605,7 +605,7 @@ func testV1FIdxSecondComputeStatQueueProfileIndexes(t *testing.T) { if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE2"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE2"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaStats, Tenant: tenant, FilterType: utils.MetaString}, @@ -740,7 +740,7 @@ func testV1FIdxComputeResourceProfileIndexes(t *testing.T) { if reply2 != utils.OK { t.Errorf("Error: %+v", reply2) } - expectedIDX := []string{"*string:~*req.Account:1001:RCFG1"} + expectedIDX := []string{"*string:*req.Account:1001:RCFG1"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaResources, Tenant: tenant, FilterType: utils.MetaString}, @@ -835,7 +835,7 @@ func testV1FIdxSecondComputeResourceProfileIndexes(t *testing.T) { if reply2 != utils.OK { t.Errorf("Error: %+v", reply2) } - expectedIDX := []string{"*string:~*req.Account:1001:RCFG2"} + expectedIDX := []string{"*string:*req.Account:1001:RCFG2"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaResources, Tenant: tenant, FilterType: utils.MetaString}, @@ -975,7 +975,7 @@ func testV1FIdxComputeRouteProfileIndexes(t *testing.T) { if reply2 != utils.OK { t.Errorf("Error: %+v", reply2) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE1"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE1"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaRoutes, Tenant: tenant, FilterType: utils.MetaString}, @@ -1074,7 +1074,7 @@ func testV1FIdxSecondComputeRouteProfileIndexes(t *testing.T) { if reply2 != utils.OK { t.Errorf("Error: %+v", reply2) } - expectedIDX := []string{"*string:~*req.Account:1001:TEST_PROFILE2"} + expectedIDX := []string{"*string:*req.Account:1001:TEST_PROFILE2"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaRoutes, Tenant: tenant, FilterType: utils.MetaString}, @@ -1225,7 +1225,7 @@ func testV1FIdxComputeAttributeProfileIndexes(t *testing.T) { } else if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1001:ApierTest"} + expectedIDX := []string{"*string:*req.Account:1001:ApierTest"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaAttributes, @@ -1338,7 +1338,7 @@ func testV1FIdxSecondComputeAttributeProfileIndexes(t *testing.T) { } else if result != utils.OK { t.Errorf("Error: %+v", result) } - expectedIDX := []string{"*string:~*req.Account:1001:ApierTest2"} + expectedIDX := []string{"*string:*req.Account:1001:ApierTest2"} var indexes []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaAttributes, @@ -1494,27 +1494,27 @@ func testV1FIdxGetFilterIndexes1(t *testing.T) { ItemType: utils.MetaResources, } expectedIndexes := []string{ - "*string:~*req.Account:3001:ResProfile3", - "*string:~*req.Destination:1001:ResProfile1", - "*string:~*req.Destination:1001:ResProfile2", - "*string:~*req.Destination:1001:ResProfile3", - "*string:~*req.Account:1002:ResProfile1", - "*string:~*req.Account:1002:ResProfile2", - "*string:~*req.Account:1002:ResProfile3", - "*string:~*req.Account:1003:ResProfile3", - "*prefix:~*req.Destination:20:ResProfile1", - "*prefix:~*req.Destination:20:ResProfile2", - "*string:~*req.Account:1001:ResProfile1", - "*string:~*req.Account:1001:ResProfile2", - "*string:~*req.Account:2002:ResProfile2", - "*prefix:~*req.Destination:1001:ResProfile3", - "*prefix:~*req.Destination:200:ResProfile3", - "*string:~*req.Destination:2001:ResProfile1", - "*string:~*req.Destination:2001:ResProfile2", - "*string:~*req.Destination:2001:ResProfile3", - "*prefix:~*req.Account:10:ResProfile1", - "*prefix:~*req.Account:10:ResProfile2", - "*prefix:~*req.Account:10:ResProfile3"} + "*string:*req.Account:3001:ResProfile3", + "*string:*req.Destination:1001:ResProfile1", + "*string:*req.Destination:1001:ResProfile2", + "*string:*req.Destination:1001:ResProfile3", + "*string:*req.Account:1002:ResProfile1", + "*string:*req.Account:1002:ResProfile2", + "*string:*req.Account:1002:ResProfile3", + "*string:*req.Account:1003:ResProfile3", + "*prefix:*req.Destination:20:ResProfile1", + "*prefix:*req.Destination:20:ResProfile2", + "*string:*req.Account:1001:ResProfile1", + "*string:*req.Account:1001:ResProfile2", + "*string:*req.Account:2002:ResProfile2", + "*prefix:*req.Destination:1001:ResProfile3", + "*prefix:*req.Destination:200:ResProfile3", + "*string:*req.Destination:2001:ResProfile1", + "*string:*req.Destination:2001:ResProfile2", + "*string:*req.Destination:2001:ResProfile3", + "*prefix:*req.Account:10:ResProfile1", + "*prefix:*req.Account:10:ResProfile2", + "*prefix:*req.Account:10:ResProfile3"} sort.Strings(expectedIndexes) var reply []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil { @@ -1531,20 +1531,20 @@ func testV1FIdxGetFilterIndexes2(t *testing.T) { FilterType: utils.MetaString, } expectedIndexes := []string{ - "*string:~*req.Account:1003:ResProfile3", - "*string:~*req.Account:3001:ResProfile3", - "*string:~*req.Destination:1001:ResProfile1", - "*string:~*req.Destination:1001:ResProfile2", - "*string:~*req.Destination:1001:ResProfile3", - "*string:~*req.Account:1002:ResProfile1", - "*string:~*req.Account:1002:ResProfile2", - "*string:~*req.Account:1002:ResProfile3", - "*string:~*req.Account:1001:ResProfile1", - "*string:~*req.Account:1001:ResProfile2", - "*string:~*req.Destination:2001:ResProfile3", - "*string:~*req.Destination:2001:ResProfile1", - "*string:~*req.Destination:2001:ResProfile2", - "*string:~*req.Account:2002:ResProfile2"} + "*string:*req.Account:1003:ResProfile3", + "*string:*req.Account:3001:ResProfile3", + "*string:*req.Destination:1001:ResProfile1", + "*string:*req.Destination:1001:ResProfile2", + "*string:*req.Destination:1001:ResProfile3", + "*string:*req.Account:1002:ResProfile1", + "*string:*req.Account:1002:ResProfile2", + "*string:*req.Account:1002:ResProfile3", + "*string:*req.Account:1001:ResProfile1", + "*string:*req.Account:1001:ResProfile2", + "*string:*req.Destination:2001:ResProfile3", + "*string:*req.Destination:2001:ResProfile1", + "*string:*req.Destination:2001:ResProfile2", + "*string:*req.Account:2002:ResProfile2"} sort.Strings(expectedIndexes) var reply []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil { @@ -1561,13 +1561,13 @@ func testV1FIdxGetFilterIndexes3(t *testing.T) { FilterType: utils.MetaPrefix, } expectedIndexes := []string{ - "*prefix:~*req.Destination:20:ResProfile1", - "*prefix:~*req.Destination:20:ResProfile2", - "*prefix:~*req.Account:10:ResProfile1", - "*prefix:~*req.Account:10:ResProfile2", - "*prefix:~*req.Account:10:ResProfile3", - "*prefix:~*req.Destination:200:ResProfile3", - "*prefix:~*req.Destination:1001:ResProfile3"} + "*prefix:*req.Destination:20:ResProfile1", + "*prefix:*req.Destination:20:ResProfile2", + "*prefix:*req.Account:10:ResProfile1", + "*prefix:*req.Account:10:ResProfile2", + "*prefix:*req.Account:10:ResProfile3", + "*prefix:*req.Destination:200:ResProfile3", + "*prefix:*req.Destination:1001:ResProfile3"} sort.Strings(expectedIndexes) var reply []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil { @@ -1585,14 +1585,14 @@ func testV1FIdxGetFilterIndexes4(t *testing.T) { FilterField: "Account", } expectedIndexes := []string{ - "*string:~*req.Account:1003:ResProfile3", - "*string:~*req.Account:3001:ResProfile3", - "*string:~*req.Account:1002:ResProfile1", - "*string:~*req.Account:1002:ResProfile2", - "*string:~*req.Account:1002:ResProfile3", - "*string:~*req.Account:1001:ResProfile1", - "*string:~*req.Account:1001:ResProfile2", - "*string:~*req.Account:2002:ResProfile2"} + "*string:*req.Account:1003:ResProfile3", + "*string:*req.Account:3001:ResProfile3", + "*string:*req.Account:1002:ResProfile1", + "*string:*req.Account:1002:ResProfile2", + "*string:*req.Account:1002:ResProfile3", + "*string:*req.Account:1001:ResProfile1", + "*string:*req.Account:1001:ResProfile2", + "*string:*req.Account:2002:ResProfile2"} sort.Strings(expectedIndexes) var reply []string if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &reply); err != nil { @@ -1632,8 +1632,8 @@ func testV1FIdxSetDispatcherProfile(t *testing.T) { FilterType: utils.MetaString, } expectedIndexes := []string{ - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) var idx []string @@ -1651,8 +1651,8 @@ func testV1FIdxSetDispatcherProfile(t *testing.T) { FilterType: utils.MetaString, } expectedIndexes = []string{ - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &idx); err != nil { @@ -1668,9 +1668,9 @@ func testV1FIdxSetDispatcherProfile(t *testing.T) { ItemType: utils.MetaDispatchers, } expectedIndexes = []string{ - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &idx); err != nil { @@ -1702,9 +1702,9 @@ func testV1FIdxSetDispatcherProfile(t *testing.T) { ItemType: utils.MetaDispatchers, } expectedIndexes = []string{ - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &idx); err != nil { @@ -1728,9 +1728,9 @@ func testV1FIdxComputeDispatcherProfileIndexes(t *testing.T) { t.Errorf("Error: %+v", result) } expectedIndexes := []string{ - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) var indexes []string @@ -1792,9 +1792,9 @@ func testV1FIdxSetDispatcherProfile2(t *testing.T) { } expectedIndexes := []string{ "*none:*any:*any:DSP_Test2", - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) var idx []string @@ -1812,9 +1812,9 @@ func testV1FIdxSetDispatcherProfile2(t *testing.T) { } expectedIndexes = []string{ "*none:*any:*any:DSP_Test3", - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, arg, &idx); err != nil { @@ -1876,9 +1876,9 @@ func testV1FIdxComputeDispatcherProfileIndexes2(t *testing.T) { } expectedIndexes := []string{ "*none:*any:*any:DSP_Test3", - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) var indexes []string @@ -1904,9 +1904,9 @@ func testV1FIdxComputeDispatcherProfileIndexes2(t *testing.T) { } expectedIndexes = []string{ "*none:*any:*any:DSP_Test2", - "*prefix:~*req.RandomField:RandomValue:DSP_Test1", - "*string:~*req.Account:1001:DSP_Test1", - "*string:~*req.Subject:2012:DSP_Test1", + "*prefix:*req.RandomField:RandomValue:DSP_Test1", + "*string:*req.Account:1001:DSP_Test1", + "*string:*req.Subject:2012:DSP_Test1", } sort.Strings(expectedIndexes) if err := tFIdxRpc.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ diff --git a/apier/v1/remote_it_test.go b/apier/v1/remote_it_test.go index 13c962615..96c6fedc5 100644 --- a/apier/v1/remote_it_test.go +++ b/apier/v1/remote_it_test.go @@ -802,8 +802,8 @@ func testInternalReplicationSetThreshold(t *testing.T) { } //verify indexes on engine2 before adding new threshold profile var indexes []string - expectedIDX := []string{"*string:~*req.Account:1001:THD_ACNT_1001", - "*string:~*req.Account:1002:THD_ACNT_1002"} + expectedIDX := []string{"*string:*req.Account:1001:THD_ACNT_1001", + "*string:*req.Account:1002:THD_ACNT_1002"} if err := engineTwoRPC.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ ItemType: utils.MetaThresholds, Tenant: "cgrates.org", FilterType: utils.MetaString}, &indexes); err != nil { @@ -855,10 +855,10 @@ func testInternalReplicationSetThreshold(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", tPrfl.ThresholdProfile, reply) } expectedIDX = []string{ - "*string:~*req.Account:1001:THD_ACNT_1001", - "*string:~*req.Account:1001:THD_Replication", - "*string:~*req.Account:1002:THD_ACNT_1002", - "*string:~*req.CustomField:CustomValue:THD_Replication", + "*string:*req.Account:1001:THD_ACNT_1001", + "*string:*req.Account:1001:THD_Replication", + "*string:*req.Account:1002:THD_ACNT_1002", + "*string:*req.CustomField:CustomValue:THD_Replication", } // verify index on internal sort.Strings(expectedIDX) @@ -880,9 +880,9 @@ func testInternalReplicationSetThreshold(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", tPrfl.ThresholdProfile, reply) } expectedIDX2 := []string{ - "*string:~*req.Account:1001:THD_ACNT_1001", - "*string:~*req.Account:1001:THD_Replication", - "*string:~*req.CustomField:CustomValue:THD_Replication", + "*string:*req.Account:1001:THD_ACNT_1001", + "*string:*req.Account:1001:THD_Replication", + "*string:*req.CustomField:CustomValue:THD_Replication", } // verify indexes on engine1 (should be the same as internal) if err := engineOneRPC.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ @@ -902,10 +902,10 @@ func testInternalReplicationSetThreshold(t *testing.T) { } else if !reflect.DeepEqual(tPrfl.ThresholdProfile, reply) { t.Errorf("Expecting: %+v, received: %+v", tPrfl.ThresholdProfile, reply) } - expectedIDX = []string{"*string:~*req.Account:1001:THD_ACNT_1001", - "*string:~*req.Account:1001:THD_Replication", - "*string:~*req.Account:1002:THD_ACNT_1002", - "*string:~*req.CustomField:CustomValue:THD_Replication", + expectedIDX = []string{"*string:*req.Account:1001:THD_ACNT_1001", + "*string:*req.Account:1001:THD_Replication", + "*string:*req.Account:1002:THD_ACNT_1002", + "*string:*req.CustomField:CustomValue:THD_Replication", } // check if indexes was created correctly on engine2 if err := engineTwoRPC.Call(utils.APIerSv1GetFilterIndexes, &AttrGetFilterIndexes{ diff --git a/config/attributescfg_test.go b/config/attributescfg_test.go index 9ac471fd3..7faaabf2e 100644 --- a/config/attributescfg_test.go +++ b/config/attributescfg_test.go @@ -40,13 +40,13 @@ func TestAttributeSCfgloadFromJsonCfg(t *testing.T) { "attributes": { // Attribute service "enabled": true, // starts attribute service: . //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1","index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1","*req.index2"], // query indexes based on these fields for faster processing "process_runs": 1, // number of run loops when processing event }, }` expected = AttributeSCfg{ Enabled: true, - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, ProcessRuns: 1, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { @@ -65,13 +65,13 @@ func TestAttributeSCfgAsMapInterface(t *testing.T) { cfgJSONStr := `{ "attributes": { "enabled": true, - "prefix_indexed_fields": ["index1","index2"], + "prefix_indexed_fields": ["*req.index1","*req.index2"], "process_runs": 3, }, }` eMap := map[string]interface{}{ "enabled": true, - "prefix_indexed_fields": []string{"index1", "index2"}, + "prefix_indexed_fields": []string{"*req.index1", "*req.index2"}, "process_runs": 3, "indexed_selects": false, "nested_fields": false, diff --git a/config/chargerscfg_test.go b/config/chargerscfg_test.go index ff9447e0e..396b7ce09 100644 --- a/config/chargerscfg_test.go +++ b/config/chargerscfg_test.go @@ -41,13 +41,13 @@ func TestChargerSCfgloadFromJsonCfg(t *testing.T) { "enabled": true, // starts charger service: . "attributes_conns": [], // address where to reach the AttributeS <""|127.0.0.1:2013> //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1", "index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1", "*req.index2"], // query indexes based on these fields for faster processing }, }` expected = ChargerSCfg{ Enabled: true, AttributeSConns: []string{}, - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) diff --git a/config/config_it_test.go b/config/config_it_test.go index 1acf6ccfd..36722dcf1 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -128,7 +128,7 @@ func testCGRConfigReloadAttributeS(t *testing.T) { } expAttr := &AttributeSCfg{ Enabled: true, - StringIndexedFields: &[]string{utils.Account}, + StringIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Account}, PrefixIndexedFields: &[]string{}, IndexedSelects: true, ProcessRuns: 1, @@ -154,7 +154,7 @@ func testCGRConfigReloadChargerS(t *testing.T) { } expAttr := &ChargerSCfg{ Enabled: true, - StringIndexedFields: &[]string{utils.Account}, + StringIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Account}, PrefixIndexedFields: &[]string{}, IndexedSelects: true, AttributeSConns: []string{"*localhost"}, @@ -180,7 +180,7 @@ func testCGRConfigReloadThresholdS(t *testing.T) { } expAttr := &ThresholdSCfg{ Enabled: true, - StringIndexedFields: &[]string{utils.Account}, + StringIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Account}, PrefixIndexedFields: &[]string{}, IndexedSelects: true, } @@ -205,7 +205,7 @@ func testCGRConfigReloadStatS(t *testing.T) { } expAttr := &StatSCfg{ Enabled: true, - StringIndexedFields: &[]string{utils.Account}, + StringIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Account}, PrefixIndexedFields: &[]string{}, IndexedSelects: true, ThresholdSConns: []string{utils.MetaLocalHost}, @@ -231,7 +231,7 @@ func testCGRConfigReloadResourceS(t *testing.T) { } expAttr := &ResourceSConfig{ Enabled: true, - StringIndexedFields: &[]string{utils.Account}, + StringIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Account}, PrefixIndexedFields: &[]string{}, IndexedSelects: true, ThresholdSConns: []string{utils.MetaLocalHost}, @@ -257,8 +257,8 @@ func testCGRConfigReloadSupplierS(t *testing.T) { } expAttr := &RouteSCfg{ Enabled: true, - StringIndexedFields: &[]string{"LCRProfile"}, - PrefixIndexedFields: &[]string{utils.Destination}, + StringIndexedFields: &[]string{"*req.LCRProfile"}, + PrefixIndexedFields: &[]string{utils.MetaReq + utils.NestingSep + utils.Destination}, ResourceSConns: []string{}, StatSConns: []string{}, AttributeSConns: []string{}, diff --git a/config/dispatchercfg_test.go b/config/dispatchercfg_test.go index da386593d..6ede83554 100644 --- a/config/dispatchercfg_test.go +++ b/config/dispatchercfg_test.go @@ -111,8 +111,8 @@ func TestDispatcherSCfgAsMapInterface(t *testing.T) { "dispatchers":{ "enabled": false, "indexed_selects":true, - "string_indexed_fields": ["string","indexed","fields"], - "prefix_indexed_fields": ["prefix","indexed","fields"], + "string_indexed_fields": ["*req.string","*req.indexed","*req.fields"], + "prefix_indexed_fields": ["*req.prefix","*req.indexed","*req.fields"], "nested_fields": false, "attributes_conns": ["*internal"], }, @@ -121,10 +121,10 @@ func TestDispatcherSCfgAsMapInterface(t *testing.T) { eMap = map[string]interface{}{ "enabled": false, "indexed_selects": true, - "prefix_indexed_fields": []string{"prefix", "indexed", "fields"}, + "prefix_indexed_fields": []string{"*req.prefix", "*req.indexed", "*req.fields"}, "nested_fields": false, "attributes_conns": []string{"*internal"}, - "string_indexed_fields": []string{"string", "indexed", "fields"}, + "string_indexed_fields": []string{"*req.string", "*req.indexed", "*req.fields"}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) diff --git a/config/resourcescfg_test.go b/config/resourcescfg_test.go index 48f90654b..7b285c973 100644 --- a/config/resourcescfg_test.go +++ b/config/resourcescfg_test.go @@ -43,14 +43,14 @@ func TestResourceSConfigloadFromJsonCfg(t *testing.T) { "store_interval": "1s", // dump cache regularly to dataDB, 0 - dump at start/shutdown: <""|$dur> "thresholds_conns": [], // address where to reach the thresholds service, empty to disable thresholds functionality: <""|*internal|x.y.z.y:1234> //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1", "index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1", "*req.index2"], // query indexes based on these fields for faster processing }, }` expected = ResourceSConfig{ Enabled: true, StoreInterval: time.Duration(time.Second), ThresholdSConns: []string{}, - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -101,7 +101,7 @@ func TestResourceSConfigAsMapInterface(t *testing.T) { "store_interval": "7m", "thresholds_conns": ["*internal"], "indexed_selects":true, - "prefix_indexed_fields": ["prefix_indexed_fields1","prefix_indexed_fields2"], + "prefix_indexed_fields": ["*req.prefix_indexed_fields1","*req.prefix_indexed_fields2"], "nested_fields": false, }, }` @@ -111,7 +111,7 @@ func TestResourceSConfigAsMapInterface(t *testing.T) { "thresholds_conns": []string{"*internal"}, "indexed_selects": true, "string_indexed_fields": []string{}, - "prefix_indexed_fields": []string{"prefix_indexed_fields1", "prefix_indexed_fields2"}, + "prefix_indexed_fields": []string{"*req.prefix_indexed_fields1", "*req.prefix_indexed_fields2"}, "nested_fields": false, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { diff --git a/config/routescfg_test.go b/config/routescfg_test.go index a3539c381..74d474b23 100644 --- a/config/routescfg_test.go +++ b/config/routescfg_test.go @@ -40,7 +40,7 @@ func TestRouteSCfgloadFromJsonCfg(t *testing.T) { "routes": { // Route service "enabled": false, // starts RouteS service: . //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1", "index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1", "*req.index2"], // query indexes based on these fields for faster processing "attributes_conns": [], // address where to reach the AttributeS <""|127.0.0.1:2013> "resources_conns": [], // address where to reach the Resource service, empty to disable functionality: <""|*internal|x.y.z.y:1234> "stats_conns": [], // address where to reach the Stat service, empty to disable stats functionality: <""|*internal|x.y.z.y:1234> @@ -48,7 +48,7 @@ func TestRouteSCfgloadFromJsonCfg(t *testing.T) { }, }` expected = RouteSCfg{ - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, AttributeSConns: []string{}, ResourceSConns: []string{}, StatSConns: []string{}, @@ -106,7 +106,7 @@ func TestRouteSCfgAsMapInterface(t *testing.T) { "routes": { "enabled": false, "indexed_selects":true, - "prefix_indexed_fields": ["prefix","indexed","fields"], + "prefix_indexed_fields": ["*req.prefix","*req.indexed","*req.fields"], "nested_fields": false, "attributes_conns": ["*internal"], "resources_conns": ["*internal"], @@ -118,7 +118,7 @@ func TestRouteSCfgAsMapInterface(t *testing.T) { eMap = map[string]interface{}{ "enabled": false, "indexed_selects": true, - "prefix_indexed_fields": []string{"prefix", "indexed", "fields"}, + "prefix_indexed_fields": []string{"*req.prefix", "*req.indexed", "*req.fields"}, "string_indexed_fields": []string{}, "nested_fields": false, "attributes_conns": []string{"*internal"}, diff --git a/config/statscfg_test.go b/config/statscfg_test.go index c2c56cb47..ba1a8a3cc 100644 --- a/config/statscfg_test.go +++ b/config/statscfg_test.go @@ -43,13 +43,13 @@ func TestStatSCfgloadFromJsonCfg(t *testing.T) { "store_interval": "2s", // dump cache regularly to dataDB, 0 - dump at start/shutdown: <""|$dur> "thresholds_conns": [], // address where to reach the thresholds service, empty to disable thresholds functionality: <""|*internal|x.y.z.y:1234> //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1", "index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1", "*req.index2"], // query indexes based on these fields for faster processing }, }` expected = StatSCfg{ StoreInterval: time.Duration(time.Second * 2), ThresholdSConns: []string{}, - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -112,7 +112,7 @@ func TestStatSCfgAsMapInterface(t *testing.T) { "store_uncompressed_limit": 0, "thresholds_conns": ["*internal"], "indexed_selects":true, - "prefix_indexed_fields": ["prefix_indexed_fields1","prefix_indexed_fields2"], + "prefix_indexed_fields": ["*req.prefix_indexed_fields1","*req.prefix_indexed_fields2"], "nested_fields": false, }, }` @@ -122,7 +122,7 @@ func TestStatSCfgAsMapInterface(t *testing.T) { "store_uncompressed_limit": 0, "thresholds_conns": []string{"*internal"}, "indexed_selects": true, - "prefix_indexed_fields": []string{"prefix_indexed_fields1", "prefix_indexed_fields2"}, + "prefix_indexed_fields": []string{"*req.prefix_indexed_fields1", "*req.prefix_indexed_fields2"}, "nested_fields": false, "string_indexed_fields": []string{}, } diff --git a/config/thresholdscfg_test.go b/config/thresholdscfg_test.go index b42984d4f..dc362b87a 100644 --- a/config/thresholdscfg_test.go +++ b/config/thresholdscfg_test.go @@ -42,12 +42,12 @@ func TestThresholdSCfgloadFromJsonCfg(t *testing.T) { "enabled": false, // starts ThresholdS service: . "store_interval": "2h", // dump cache regularly to dataDB, 0 - dump at start/shutdown: <""|$dur> //"string_indexed_fields": [], // query indexes based on these fields for faster processing - "prefix_indexed_fields": ["index1", "index2"], // query indexes based on these fields for faster processing + "prefix_indexed_fields": ["*req.index1", "*req.index2"], // query indexes based on these fields for faster processing }, }` expected = ThresholdSCfg{ StoreInterval: time.Duration(time.Hour * 2), - PrefixIndexedFields: &[]string{"index1", "index2"}, + PrefixIndexedFields: &[]string{"*req.index1", "*req.index2"}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -95,8 +95,8 @@ func TestThresholdSCfgAsMapInterface(t *testing.T) { "enabled": true, "store_interval": "96h", "indexed_selects":true, - "string_indexed_fields": ["string","indexed","fields"], - "prefix_indexed_fields": ["prefix_indexed_fields1","prefix_indexed_fields2"], + "string_indexed_fields": ["*req.string","*req.indexed","*req.fields"], + "prefix_indexed_fields": ["*req.prefix_indexed_fields1","*req.prefix_indexed_fields2"], "nested_fields": true, }, }` @@ -104,8 +104,8 @@ func TestThresholdSCfgAsMapInterface(t *testing.T) { "enabled": true, "store_interval": "96h0m0s", "indexed_selects": true, - "string_indexed_fields": []string{"string", "indexed", "fields"}, - "prefix_indexed_fields": []string{"prefix_indexed_fields1", "prefix_indexed_fields2"}, + "string_indexed_fields": []string{"*req.string", "*req.indexed", "*req.fields"}, + "prefix_indexed_fields": []string{"*req.prefix_indexed_fields1", "*req.prefix_indexed_fields2"}, "nested_fields": true, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { diff --git a/data/conf/samples/docker/cgrates.json b/data/conf/samples/docker/cgrates.json index cab6a517b..3faebe5ce 100644 --- a/data/conf/samples/docker/cgrates.json +++ b/data/conf/samples/docker/cgrates.json @@ -91,7 +91,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], "rals_conns": ["*internal"], diff --git a/data/conf/samples/internal_broadcast_replication/cgrates.json b/data/conf/samples/internal_broadcast_replication/cgrates.json index d781ae0f7..69ffcde84 100644 --- a/data/conf/samples/internal_broadcast_replication/cgrates.json +++ b/data/conf/samples/internal_broadcast_replication/cgrates.json @@ -87,7 +87,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["broadcast_conn"], "resources_conns": ["broadcast_conn"], }, diff --git a/data/conf/samples/loader_mysql/cgrates.json b/data/conf/samples/loader_mysql/cgrates.json index 98935ff76..4bd77807b 100644 --- a/data/conf/samples/loader_mysql/cgrates.json +++ b/data/conf/samples/loader_mysql/cgrates.json @@ -77,7 +77,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], "rals_conns": ["*internal"], diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json index 08556d511..69ccfd469 100644 --- a/data/conf/samples/loaders/tutinternal/cgrates.json +++ b/data/conf/samples/loaders/tutinternal/cgrates.json @@ -81,7 +81,7 @@ "attributes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], }, diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json index 650c222a6..54e006c47 100644 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -82,7 +82,7 @@ "attributes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], }, @@ -113,7 +113,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], }, diff --git a/data/conf/samples/sessinternal/cgrates.json b/data/conf/samples/sessinternal/cgrates.json index 596c088b3..f9e66d0c5 100644 --- a/data/conf/samples/sessinternal/cgrates.json +++ b/data/conf/samples/sessinternal/cgrates.json @@ -69,7 +69,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], }, diff --git a/data/conf/samples/sessintjson/cgrates.json b/data/conf/samples/sessintjson/cgrates.json index 28815e9a2..86e8ef44c 100644 --- a/data/conf/samples/sessintjson/cgrates.json +++ b/data/conf/samples/sessintjson/cgrates.json @@ -69,7 +69,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*localhost"], "resources_conns": ["*localhost"], }, diff --git a/data/conf/samples/tutinternal/cgrates.json b/data/conf/samples/tutinternal/cgrates.json index 31c23b882..5886ae32f 100644 --- a/data/conf/samples/tutinternal/cgrates.json +++ b/data/conf/samples/tutinternal/cgrates.json @@ -77,7 +77,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], "rals_conns": ["*internal"], diff --git a/data/conf/samples/tutinternal_gob/cgrates.json b/data/conf/samples/tutinternal_gob/cgrates.json index 254acfce6..25015e6e3 100644 --- a/data/conf/samples/tutinternal_gob/cgrates.json +++ b/data/conf/samples/tutinternal_gob/cgrates.json @@ -83,7 +83,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["conn1"], "resources_conns": ["conn1"], }, diff --git a/data/conf/samples/tutinternal_new/cgrates.json b/data/conf/samples/tutinternal_new/cgrates.json index 045c4e7b5..70682c476 100644 --- a/data/conf/samples/tutinternal_new/cgrates.json +++ b/data/conf/samples/tutinternal_new/cgrates.json @@ -78,7 +78,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], }, diff --git a/data/conf/samples/tutmongo2/cgrates.json b/data/conf/samples/tutmongo2/cgrates.json index 940df3f82..66d1bb417 100644 --- a/data/conf/samples/tutmongo2/cgrates.json +++ b/data/conf/samples/tutmongo2/cgrates.json @@ -99,41 +99,41 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "chargers": { "enabled": true, "attributes_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "resources": { "enabled": true, "thresholds_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "stats": { "enabled": true, "thresholds_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "routes": { "enabled": true, - "string_indexed_fields": ["LCRProfile"], - "prefix_indexed_fields":["Destination"], + "string_indexed_fields": ["*req.LCRProfile"], + "prefix_indexed_fields":["*req.Destination"], }, diff --git a/data/conf/samples/tutmongo2_gob/cgrates.json b/data/conf/samples/tutmongo2_gob/cgrates.json index 40f7f1230..13af37029 100644 --- a/data/conf/samples/tutmongo2_gob/cgrates.json +++ b/data/conf/samples/tutmongo2_gob/cgrates.json @@ -106,41 +106,41 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "chargers": { "enabled": true, "attributes_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "resources": { "enabled": true, "thresholds_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "stats": { "enabled": true, "thresholds_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "routes": { "enabled": true, - "string_indexed_fields": ["LCRProfile"], - "prefix_indexed_fields":["Destination"], + "string_indexed_fields": ["*req.LCRProfile"], + "prefix_indexed_fields":["*req.Destination"], }, diff --git a/data/conf/samples/tutmysql/cgrates.json b/data/conf/samples/tutmysql/cgrates.json index b19fbb774..72cf47eed 100644 --- a/data/conf/samples/tutmysql/cgrates.json +++ b/data/conf/samples/tutmysql/cgrates.json @@ -77,7 +77,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], "resources_conns": ["*internal"], "rals_conns": ["*internal"], diff --git a/data/conf/samples/tutmysql2/cgrates.json b/data/conf/samples/tutmysql2/cgrates.json index fd5bb53b0..0e10dd27b 100644 --- a/data/conf/samples/tutmysql2/cgrates.json +++ b/data/conf/samples/tutmysql2/cgrates.json @@ -56,41 +56,41 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "chargers": { "enabled": true, "attributes_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "resources": { "enabled": true, "thresholds_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "stats": { "enabled": true, "thresholds_conns": ["*localhost"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "routes": { "enabled": true, - "string_indexed_fields": ["LCRProfile"], - "prefix_indexed_fields":["Destination"], + "string_indexed_fields": ["*req.LCRProfile"], + "prefix_indexed_fields":["*req.Destination"], }, diff --git a/data/conf/samples/tutmysql2_gob/cgrates.json b/data/conf/samples/tutmysql2_gob/cgrates.json index 2e6a76dd2..da0348e75 100644 --- a/data/conf/samples/tutmysql2_gob/cgrates.json +++ b/data/conf/samples/tutmysql2_gob/cgrates.json @@ -65,41 +65,41 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "chargers": { "enabled": true, "attributes_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "resources": { "enabled": true, "thresholds_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "stats": { "enabled": true, "thresholds_conns": ["conn1"], - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"] + "string_indexed_fields": ["*req.Account"] }, "routes": { "enabled": true, - "string_indexed_fields": ["LCRProfile"], - "prefix_indexed_fields":["Destination"], + "string_indexed_fields": ["*req.LCRProfile"], + "prefix_indexed_fields":["*req.Destination"], }, diff --git a/data/conf/samples/tutmysql_internal/cgrates.json b/data/conf/samples/tutmysql_internal/cgrates.json index 4259a1c97..8d503fa10 100644 --- a/data/conf/samples/tutmysql_internal/cgrates.json +++ b/data/conf/samples/tutmysql_internal/cgrates.json @@ -193,7 +193,7 @@ "routes": { "enabled": true, - "prefix_indexed_fields":["Destination"], + "prefix_indexed_fields":["*req.Destination"], "stats_conns": ["*internal"], }, diff --git a/data/tutorial_tests/asterisk_ari/cgrates/etc/cgrates/cgrates.json b/data/tutorial_tests/asterisk_ari/cgrates/etc/cgrates/cgrates.json index 7975e6f98..754fa89fd 100644 --- a/data/tutorial_tests/asterisk_ari/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorial_tests/asterisk_ari/cgrates/etc/cgrates/cgrates.json @@ -77,25 +77,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -104,7 +104,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorial_tests/fs_evsock/cgrates/etc/cgrates/cgrates.json b/data/tutorial_tests/fs_evsock/cgrates/etc/cgrates/cgrates.json index 8389365da..c8b242a18 100644 --- a/data/tutorial_tests/fs_evsock/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorial_tests/fs_evsock/cgrates/etc/cgrates/cgrates.json @@ -75,25 +75,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -102,7 +102,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorial_tests/kamevapi/cgrates/etc/cgrates/cgrates.json b/data/tutorial_tests/kamevapi/cgrates/etc/cgrates/cgrates.json index 13a0ea78c..63bf5737a 100644 --- a/data/tutorial_tests/kamevapi/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorial_tests/kamevapi/cgrates/etc/cgrates/cgrates.json @@ -76,25 +76,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -103,7 +103,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorial_tests/osips/cgrates/etc/cgrates/cgrates.json b/data/tutorial_tests/osips/cgrates/etc/cgrates/cgrates.json index 9bf1d50ad..ea2a0ddc5 100644 --- a/data/tutorial_tests/osips/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorial_tests/osips/cgrates/etc/cgrates/cgrates.json @@ -64,28 +64,28 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, "thresholds_conns": ["*internal"], - "string_indexed_fields": ["Account"], - "prefix_indexed_fields": ["Destination"], + "string_indexed_fields": ["*req.Account"], + "prefix_indexed_fields": ["*req.Destination"], }, "stats": { "enabled": true, "thresholds_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -93,8 +93,8 @@ "enabled": true, "resources_conns": ["*internal"], "stats_conns": ["*internal"], - "string_indexed_fields": ["Account"], - "prefix_indexed_fields": ["Destination"], + "string_indexed_fields": ["*req.Account"], + "prefix_indexed_fields": ["*req.Destination"], }, diff --git a/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json b/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json index a1385d1c3..4ca7c50c8 100644 --- a/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json @@ -76,25 +76,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -103,7 +103,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json index 8389365da..c8b242a18 100644 --- a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json @@ -75,25 +75,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -102,7 +102,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json index 4e1c4122d..54ee74836 100644 --- a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json @@ -76,25 +76,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -103,7 +103,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json b/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json index 4c3b6e6b9..447e22911 100644 --- a/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json @@ -63,28 +63,28 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, "thresholds_conns": ["*internal"], - "string_indexed_fields": ["Account"], - "prefix_indexed_fields": ["Destination"], + "string_indexed_fields": ["*req.Account"], + "prefix_indexed_fields": ["*req.Destination"], }, "stats": { "enabled": true, "thresholds_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -92,8 +92,8 @@ "enabled": true, "resources_conns": ["*internal"], "stats_conns": ["*internal"], - "string_indexed_fields": ["Account"], - "prefix_indexed_fields": ["Destination"], + "string_indexed_fields": ["*req.Account"], + "prefix_indexed_fields": ["*req.Destination"], }, diff --git a/data/tutorials/sip_redirect/cgrates/etc/cgrates/cgrates.json b/data/tutorials/sip_redirect/cgrates/etc/cgrates/cgrates.json index 0e44d990c..f37a400c5 100644 --- a/data/tutorials/sip_redirect/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/sip_redirect/cgrates/etc/cgrates/cgrates.json @@ -65,25 +65,25 @@ "attributes": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "resources": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, "stats": { "enabled": true, - "string_indexed_fields": ["Account","RunID","Destination"], + "string_indexed_fields": ["*req.Account","*req.RunID","*req.Destination"], }, "thresholds": { "enabled": true, - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, @@ -92,7 +92,7 @@ "resources_conns": ["*internal"], "stats_conns": ["*internal"], "rals_conns": ["*internal"], - "string_indexed_fields": ["Account"], + "string_indexed_fields": ["*req.Account"], }, diff --git a/dispatchers/chargers.go b/dispatchers/chargers.go index b764ce453..1ca374be6 100755 --- a/dispatchers/chargers.go +++ b/dispatchers/chargers.go @@ -45,7 +45,7 @@ func (dS *DispatcherService) ChargerSv1Ping(args *utils.CGREventWithArgDispatche utils.ChargerSv1Ping, args, reply) } -func (dS *DispatcherService) ChargerSv1GetChargersForEvent(args *utils.CGREventWithArgDispatcher, +func (dS *DispatcherService) ChargerSv1GetChargersForEvent(args *utils.CGREventWithOpts, reply *engine.ChargerProfiles) (err error) { tnt := dS.cfg.GeneralCfg().DefaultTenant if args.CGREvent != nil && args.CGREvent.Tenant != utils.EmptyString { diff --git a/dispatchers/dispatchers.go b/dispatchers/dispatchers.go index dc24e7348..a8476d619 100755 --- a/dispatchers/dispatchers.go +++ b/dispatchers/dispatchers.go @@ -116,7 +116,8 @@ func (dS *DispatcherService) dispatcherProfileForEvent(ev *utils.CGREvent, if subsys != "" { idxKeyPrfx = utils.ConcatenatedKey(ev.Tenant, subsys) } - prflIDs, err := engine.MatchingItemIDsForEvent(ev.Event, + evNm := utils.MapStorage{utils.MetaReq: ev.Event} + prflIDs, err := engine.MatchingItemIDsForEvent(evNm, dS.cfg.DispatcherSCfg().StringIndexedFields, dS.cfg.DispatcherSCfg().PrefixIndexedFields, dS.dm, utils.CacheDispatcherFilterIndexes, idxKeyPrfx, @@ -128,7 +129,7 @@ func (dS *DispatcherService) dispatcherProfileForEvent(ev *utils.CGREvent, if err != utils.ErrNotFound { return nil, err } - prflIDs, err = engine.MatchingItemIDsForEvent(ev.Event, + prflIDs, err = engine.MatchingItemIDsForEvent(evNm, dS.cfg.DispatcherSCfg().StringIndexedFields, dS.cfg.DispatcherSCfg().PrefixIndexedFields, dS.dm, utils.CacheDispatcherFilterIndexes, anyIdxPrfx, @@ -139,7 +140,6 @@ func (dS *DispatcherService) dispatcherProfileForEvent(ev *utils.CGREvent, return nil, err } } - evNm := utils.MapStorage{utils.MetaReq: ev.Event} for prflID := range prflIDs { prfl, err := dS.dm.GetDispatcherProfile(ev.Tenant, prflID, true, true, utils.NonTransactional) if err != nil { diff --git a/engine/attributes.go b/engine/attributes.go index d234092a1..6cb692d52 100644 --- a/engine/attributes.go +++ b/engine/attributes.go @@ -72,7 +72,7 @@ func (alS *AttributeService) attributeProfileForEvent(args *AttrArgsProcessEvent if len(args.AttributeIDs) != 0 { attrIDs = args.AttributeIDs } else { - aPrflIDs, err := MatchingItemIDsForEvent(args.Event, + aPrflIDs, err := MatchingItemIDsForEvent(evNm, alS.cgrcfg.AttributeSCfg().StringIndexedFields, alS.cgrcfg.AttributeSCfg().PrefixIndexedFields, alS.dm, utils.CacheAttributeFilterIndexes, attrIdxKey, @@ -83,7 +83,7 @@ func (alS *AttributeService) attributeProfileForEvent(args *AttrArgsProcessEvent if err != utils.ErrNotFound { return nil, err } - if aPrflIDs, err = MatchingItemIDsForEvent(args.Event, + if aPrflIDs, err = MatchingItemIDsForEvent(evNm, alS.cgrcfg.AttributeSCfg().StringIndexedFields, alS.cgrcfg.AttributeSCfg().PrefixIndexedFields, alS.dm, utils.CacheAttributeFilterIndexes, diff --git a/engine/chargers.go b/engine/chargers.go index 6e95395d5..f564a658d 100644 --- a/engine/chargers.go +++ b/engine/chargers.go @@ -56,8 +56,12 @@ func (cS *ChargerService) Shutdown() (err error) { } // matchingChargingProfilesForEvent returns ordered list of matching chargers which are active by the time of the function call -func (cS *ChargerService) matchingChargerProfilesForEvent(cgrEv *utils.CGREvent) (cPs ChargerProfiles, err error) { - cpIDs, err := MatchingItemIDsForEvent(cgrEv.Event, +func (cS *ChargerService) matchingChargerProfilesForEvent(cgrEv *utils.CGREventWithOpts) (cPs ChargerProfiles, err error) { + evNm := utils.MapStorage{ + utils.MetaReq: cgrEv.Event, + utils.MetaOpts: cgrEv.Opts, + } + cpIDs, err := MatchingItemIDsForEvent(evNm, cS.cfg.ChargerSCfg().StringIndexedFields, cS.cfg.ChargerSCfg().PrefixIndexedFields, cS.dm, utils.CacheChargerFilterIndexes, cgrEv.Tenant, @@ -68,7 +72,6 @@ func (cS *ChargerService) matchingChargerProfilesForEvent(cgrEv *utils.CGREvent) return nil, err } matchingCPs := make(map[string]*ChargerProfile) - evNm := utils.MapStorage{utils.MetaReq: cgrEv.Event} for cpID := range cpIDs { cP, err := cS.dm.GetChargerProfile(cgrEv.Tenant, cpID, true, true, utils.NonTransactional) if err != nil { @@ -110,13 +113,13 @@ type ChrgSProcessEventReply struct { func (cS *ChargerService) processEvent(cgrEv *utils.CGREventWithOpts) (rply []*ChrgSProcessEventReply, err error) { var cPs ChargerProfiles - if cPs, err = cS.matchingChargerProfilesForEvent(cgrEv.CGREvent); err != nil { - return nil, err - } if cgrEv.Opts == nil { cgrEv.Opts = make(map[string]interface{}) } cgrEv.Opts[utils.Subsys] = utils.MetaChargers + if cPs, err = cS.matchingChargerProfilesForEvent(cgrEv); err != nil { + return nil, err + } rply = make([]*ChrgSProcessEventReply, len(cPs)) for i, cP := range cPs { clonedEv := cgrEv.Clone() @@ -176,9 +179,9 @@ func (cS *ChargerService) V1ProcessEvent(args *utils.CGREventWithOpts, } // V1GetChargersForEvent exposes the list of ordered matching ChargingProfiles for an event -func (cS *ChargerService) V1GetChargersForEvent(args *utils.CGREventWithArgDispatcher, +func (cS *ChargerService) V1GetChargersForEvent(args *utils.CGREventWithOpts, rply *ChargerProfiles) (err error) { - cPs, err := cS.matchingChargerProfilesForEvent(args.CGREvent) + cPs, err := cS.matchingChargerProfilesForEvent(args) if err != nil { if err != utils.ErrNotFound { err = utils.NewErrServerError(err) diff --git a/engine/chargers_test.go b/engine/chargers_test.go index 8372dacbb..68bc89a28 100755 --- a/engine/chargers_test.go +++ b/engine/chargers_test.go @@ -64,7 +64,7 @@ var ( Weight: 20, }, } - chargerEvents = []*utils.CGREventWithArgDispatcher{ + chargerEvents = []*utils.CGREventWithOpts{ { CGREvent: &utils.CGREvent{ Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, @@ -188,18 +188,18 @@ func TestChargerSetChargerProfiles(t *testing.T) { } func TestChargerMatchingChargerProfilesForEvent(t *testing.T) { - if _, err = chargerSrv.matchingChargerProfilesForEvent(chargerEvents[2].CGREvent); err == nil || + if _, err = chargerSrv.matchingChargerProfilesForEvent(chargerEvents[2]); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Errorf("Error: %+v", err) } - if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[0].CGREvent); err != nil { + if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[0]); err != nil { t.Errorf("Error: %+v", err) } else if !reflect.DeepEqual(cPPs[0], rcv[0]) { t.Errorf("Expecting: %+v, received: %+v ", cPPs[0], rcv[0]) } - if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[1].CGREvent); err != nil { + if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[1]); err != nil { t.Errorf("Error: %+v", err) } else if !reflect.DeepEqual(cPPs[1], rcv[0]) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(cPPs[1]), utils.ToJSON(rcv)) diff --git a/engine/filterhelpers.go b/engine/filterhelpers.go index b9f36d0be..ac99d2607 100644 --- a/engine/filterhelpers.go +++ b/engine/filterhelpers.go @@ -29,13 +29,12 @@ import ( // MatchingItemIDsForEvent returns the list of item IDs matching fieldName/fieldValue for an event // fieldIDs limits the fields which are checked against indexes // helper on top of dataDB.GetIndexes, adding utils.ANY to list of fields queried -func MatchingItemIDsForEvent(ev map[string]interface{}, stringFldIDs, prefixFldIDs *[]string, +func MatchingItemIDsForEvent(ev utils.MapStorage, stringFldIDs, prefixFldIDs *[]string, dm *DataManager, cacheID, itemIDPrefix string, indexedSelects, nestedFields bool) (itemIDs utils.StringSet, err error) { itemIDs = make(utils.StringSet) var allFieldIDs []string - navEv := utils.MapStorage(ev) if indexedSelects && (stringFldIDs == nil || prefixFldIDs == nil) { - allFieldIDs = navEv.GetKeys(nestedFields) + allFieldIDs = ev.GetKeys(nestedFields, 2, utils.EmptyString) } // Guard will protect the function with automatic locking lockID := utils.CacheInstanceToPrefix[cacheID] + itemIDPrefix @@ -59,7 +58,7 @@ func MatchingItemIDsForEvent(ev map[string]interface{}, stringFldIDs, prefixFldI fieldIDs = &allFieldIDs } for _, fldName := range *fieldIDs { - fieldValIf, err := navEv.FieldAsInterface(strings.Split(fldName, utils.NestingSep)) + fieldValIf, err := ev.FieldAsInterface(strings.Split(fldName, utils.NestingSep)) if err != nil && filterIndexTypes[i] != utils.META_NONE { continue } @@ -72,9 +71,6 @@ func MatchingItemIDsForEvent(ev map[string]interface{}, stringFldIDs, prefixFldI if filterIndexTypes[i] == utils.MetaPrefix { fldVals = utils.SplitPrefix(fldVal, 1) // all prefixes till last digit } - if fldName != utils.META_ANY { - fldName = utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + fldName - } var dbItemIDs utils.StringSet // list of items matched in DB for _, val := range fldVals { var dbIndexes map[string]utils.StringSet // list of items matched in DB diff --git a/engine/libindex.go b/engine/libindex.go index ba5a72e52..24d77a2da 100644 --- a/engine/libindex.go +++ b/engine/libindex.go @@ -67,8 +67,20 @@ func newFilterIndex(dm *DataManager, idxItmType, tnt, ctx, itemID string, filter continue } + isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix) for _, fldVal := range flt.Values { - idxKey := utils.ConcatenatedKey(flt.Type, flt.Element, fldVal) + var idxKey string + if isDyn { + if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic + continue + } + idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal) + } else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { + idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element) + } else { + // do not index not dynamic filters + continue + } var rcvIndx map[string]utils.StringSet if rcvIndx, err = dm.GetIndexes(idxItmType, tntCtx, idxKey, true, false); err != nil { @@ -464,19 +476,45 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { if !utils.SliceHasMember([]string{utils.MetaPrefix, utils.MetaString}, flt.Type) { continue } + isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix) for _, fldVal := range flt.Values { - newRules.Add(utils.ConcatenatedKey(flt.Type, flt.Element, fldVal)) + var idxKey string + if isDyn { + if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic + continue + } + idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal) + } else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { + idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element) + } else { + // do not index not dynamic filters + continue + } + newRules.Add(idxKey) } } for _, flt := range oldFlt.Rules { if !utils.SliceHasMember([]string{utils.MetaPrefix, utils.MetaString}, flt.Type) { continue } + isDyn := strings.HasPrefix(flt.Element, utils.DynamicDataPrefix) for _, fldVal := range flt.Values { - if key := utils.ConcatenatedKey(flt.Type, flt.Element, fldVal); !newRules.Has(key) { - removeRules.Add(key) + var idxKey string + if isDyn { + if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { // do not index if both the element and the value is dynamic + continue + } + idxKey = utils.ConcatenatedKey(flt.Type, flt.Element[1:], fldVal) + } else if strings.HasPrefix(fldVal, utils.DynamicDataPrefix) { + idxKey = utils.ConcatenatedKey(flt.Type, fldVal[1:], flt.Element) } else { - oldRules.Add(key) + // do not index not dynamic filters + continue + } + if !newRules.Has(idxKey) { + removeRules.Add(idxKey) + } else { + oldRules.Add(idxKey) } } } @@ -506,7 +544,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { for idxItmType, indx := range rcvIndx { switch idxItmType { case utils.CacheThresholdFilterIndexes: - if err = removeFilterIndexesForFilrer(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter + if err = removeFilterIndexesForFilter(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter removeIndexKeys, indx); err != nil { return } @@ -526,7 +564,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return utils.APIErrorHandler(err) } case utils.CacheStatFilterIndexes: - if err = removeFilterIndexesForFilrer(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter + if err = removeFilterIndexesForFilter(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter removeIndexKeys, indx); err != nil { return } @@ -546,7 +584,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return utils.APIErrorHandler(err) } case utils.CacheResourceFilterIndexes: - if err = removeFilterIndexesForFilrer(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter + if err = removeFilterIndexesForFilter(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter removeIndexKeys, indx); err != nil { return } @@ -566,7 +604,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return utils.APIErrorHandler(err) } case utils.CacheRouteFilterIndexes: - if err = removeFilterIndexesForFilrer(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter + if err = removeFilterIndexesForFilter(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter removeIndexKeys, indx); err != nil { return } @@ -586,7 +624,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return utils.APIErrorHandler(err) } case utils.CacheChargerFilterIndexes: - if err = removeFilterIndexesForFilrer(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter + if err = removeFilterIndexesForFilter(dm, idxItmType, newFlt.Tenant, // remove the indexes for the filter removeIndexKeys, indx); err != nil { return } @@ -613,7 +651,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return } for _, ctx := range ap.Contexts { - if err = removeFilterIndexesForFilrer(dm, idxItmType, + if err = removeFilterIndexesForFilter(dm, idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx), // remove the indexes for the filter removeIndexKeys, indx); err != nil { return @@ -640,7 +678,7 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return } for _, ctx := range dp.Subsystems { - if err = removeFilterIndexesForFilrer(dm, idxItmType, + if err = removeFilterIndexesForFilter(dm, idxItmType, utils.ConcatenatedKey(newFlt.Tenant, ctx), // remove the indexes for the filter removeIndexKeys, indx); err != nil { return @@ -664,8 +702,8 @@ func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) { return } -// removeFilterIndexesForFilrer removes the itemID for the index keys -func removeFilterIndexesForFilrer(dm *DataManager, idxItmType, tnt string, +// removeFilterIndexesForFilter removes the itemID for the index keys +func removeFilterIndexesForFilter(dm *DataManager, idxItmType, tnt string, removeIndexKeys []string, itemIDs utils.StringSet) (err error) { for _, idxKey := range removeIndexKeys { // delete old filters indexes for this item var remIndx map[string]utils.StringSet diff --git a/engine/resources.go b/engine/resources.go index 3fc95389d..d686c2705 100644 --- a/engine/resources.go +++ b/engine/resources.go @@ -69,6 +69,7 @@ type ResourceUsage struct { Units float64 // Number of units used } +// TenantID returns the concatenated key between tenant and ID func (ru *ResourceUsage) TenantID() string { return utils.ConcatenatedKey(ru.Tenant, ru.ID) } @@ -78,7 +79,7 @@ func (ru *ResourceUsage) isActive(atTime time.Time) bool { return ru.ExpiryTime.IsZero() || ru.ExpiryTime.Sub(atTime) > 0 } -// clone duplicates ru +// Clone duplicates ru func (ru *ResourceUsage) Clone() (cln *ResourceUsage) { cln = new(ResourceUsage) *cln = *ru @@ -116,7 +117,7 @@ func (r *Resource) removeExpiredUnits() { if r, has := r.Usages[rID]; has && r.isActive(time.Now()) { break } - firstActive += 1 + firstActive++ } if firstActive == 0 { return @@ -205,7 +206,7 @@ func (r *Resource) clearUsage(ruID string) (err error) { // Resources is an orderable list of Resources based on Weight type Resources []*Resource -// sort based on Weight +// Sort sorts based on Weight func (rs Resources) Sort() { sort.Slice(rs, func(i, j int) bool { return rs[i].rPrf.Weight > rs[j].rPrf.Weight }) } @@ -218,7 +219,7 @@ func (rs Resources) recordUsage(ru *ResourceUsage) (err error) { utils.Logger.Warning(fmt.Sprintf("<%s>cannot record usage, err: %s", utils.ResourceS, err.Error())) break } - nonReservedIdx += 1 + nonReservedIdx++ } if err != nil { for _, r := range rs[:nonReservedIdx] { @@ -337,7 +338,7 @@ type ResourceService struct { connMgr *ConnManager } -// Called to start the service +// ListenAndServe is called to start the service func (rS *ResourceService) ListenAndServe(exitChan chan bool) error { utils.Logger.Info(fmt.Sprintf("<%s> starting <%s> subsystem", utils.CoreS, utils.ResourceS)) go rS.runBackup() // start backup loop @@ -346,7 +347,7 @@ func (rS *ResourceService) ListenAndServe(exitChan chan bool) error { return nil } -// Called to shutdown the service +// Shutdown is called to shutdown the service func (rS *ResourceService) Shutdown() error { utils.Logger.Info(" service shutdown initialized") close(rS.stopBackup) @@ -466,6 +467,7 @@ func (rS *ResourceService) matchingResourcesForEvent(ev *utils.CGREvent, matchingResources := make(map[string]*Resource) var isCached bool var rIDs utils.StringSet + evNm := utils.MapStorage{utils.MetaReq: ev.Event} if x, ok := Cache.Get(utils.CacheEventResources, evUUID); ok { // The ResourceIDs were cached as utils.StringSet{"resID":bool} isCached = true if x == nil { @@ -473,7 +475,7 @@ func (rS *ResourceService) matchingResourcesForEvent(ev *utils.CGREvent, } rIDs = x.(utils.StringSet) } else { // select the resourceIDs out of dataDB - rIDs, err = MatchingItemIDsForEvent(ev.Event, + rIDs, err = MatchingItemIDsForEvent(evNm, rS.cgrcfg.ResourceSCfg().StringIndexedFields, rS.cgrcfg.ResourceSCfg().PrefixIndexedFields, rS.dm, utils.CacheResourceFilterIndexes, ev.Tenant, @@ -489,7 +491,6 @@ func (rS *ResourceService) matchingResourcesForEvent(ev *utils.CGREvent, return } } - evNm := utils.MapStorage{utils.MetaReq: ev.Event} lockIDs := utils.PrefixSliceItems(rs.IDs(), utils.ResourcesPrefix) guardian.Guardian.Guard(func() (gIface interface{}, gErr error) { for resName := range rIDs { @@ -778,16 +779,16 @@ func (rS *ResourceService) V1ReleaseResource(args utils.ArgRSv1ResourceUsage, re return } -// GetResource returns a resource configuration +// V1GetResource returns a resource configuration func (rS *ResourceService) V1GetResource(arg *utils.TenantIDWithArgDispatcher, reply *Resource) error { if missing := utils.MissingStructFields(arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if res, err := rS.dm.GetResource(arg.Tenant, arg.ID, true, true, utils.NonTransactional); err != nil { + res, err := rS.dm.GetResource(arg.Tenant, arg.ID, true, true, utils.NonTransactional) + if err != nil { return err - } else { - *reply = *res } + *reply = *res return nil } diff --git a/engine/routes.go b/engine/routes.go index e9999f1de..ff6078114 100644 --- a/engine/routes.go +++ b/engine/routes.go @@ -154,7 +154,8 @@ func (rpS *RouteService) Shutdown() error { // matchingRouteProfilesForEvent returns ordered list of matching resources which are active by the time of the call func (rpS *RouteService) matchingRouteProfilesForEvent(ev *utils.CGREvent, singleResult bool) (matchingRPrf []*RouteProfile, err error) { - rPrfIDs, err := MatchingItemIDsForEvent(ev.Event, + evNm := utils.MapStorage{utils.MetaReq: ev.Event} + rPrfIDs, err := MatchingItemIDsForEvent(evNm, rpS.cgrcfg.RouteSCfg().StringIndexedFields, rpS.cgrcfg.RouteSCfg().PrefixIndexedFields, rpS.dm, utils.CacheRouteFilterIndexes, ev.Tenant, @@ -169,7 +170,6 @@ func (rpS *RouteService) matchingRouteProfilesForEvent(ev *utils.CGREvent, singl } else { matchingRPrf = make([]*RouteProfile, 0, len(rPrfIDs)) } - evNm := utils.MapStorage{utils.MetaReq: ev.Event} for lpID := range rPrfIDs { rPrf, err := rpS.dm.GetRouteProfile(ev.Tenant, lpID, true, true, utils.NonTransactional) if err != nil { diff --git a/engine/stats.go b/engine/stats.go index 27d234c3d..bd22f68ac 100644 --- a/engine/stats.go +++ b/engine/stats.go @@ -151,9 +151,10 @@ func (sS *StatService) StoreStatQueue(sq *StatQueue) (err error) { // matchingStatQueuesForEvent returns ordered list of matching resources which are active by the time of the call func (sS *StatService) matchingStatQueuesForEvent(args *StatsArgsProcessEvent) (sqs StatQueues, err error) { + evNm := utils.MapStorage{utils.MetaReq: args.Event} sqIDs := utils.NewStringSet(args.StatIDs) if len(sqIDs) == 0 { - sqIDs, err = MatchingItemIDsForEvent(args.Event, + sqIDs, err = MatchingItemIDsForEvent(evNm, sS.cgrcfg.StatSCfg().StringIndexedFields, sS.cgrcfg.StatSCfg().PrefixIndexedFields, sS.dm, utils.CacheStatFilterIndexes, args.Tenant, @@ -164,7 +165,6 @@ func (sS *StatService) matchingStatQueuesForEvent(args *StatsArgsProcessEvent) ( return } } - evNm := utils.MapStorage{utils.MetaReq: args.Event} sqs = make(StatQueues, 0, len(sqIDs)) for sqID := range sqIDs { sqPrfl, err := sS.dm.GetStatQueueProfile(args.Tenant, sqID, true, true, utils.NonTransactional) @@ -218,8 +218,8 @@ func (sS *StatService) matchingStatQueuesForEvent(args *StatsArgsProcessEvent) ( // Call implements rpcclient.ClientConnector interface for internal RPC // here for cases when passing StatsService as rpccclient.RpcClientConnection -func (ss *StatService) Call(serviceMethod string, args interface{}, reply interface{}) error { - return utils.RPCCall(ss, serviceMethod, args, reply) +func (sS *StatService) Call(serviceMethod string, args interface{}, reply interface{}) error { + return utils.RPCCall(sS, serviceMethod, args, reply) } type StatsArgsProcessEvent struct { @@ -326,7 +326,7 @@ func (sS *StatService) V1ProcessEvent(args *StatsArgsProcessEvent, reply *[]stri return } -// V1StatQueuesForEvent implements StatV1 method for processing an Event +// V1GetStatQueuesForEvent implements StatV1 method for processing an Event func (sS *StatService) V1GetStatQueuesForEvent(args *StatsArgsProcessEvent, reply *[]string) (err error) { if args.CGREvent == nil { return utils.NewErrMandatoryIeMissing(utils.CGREventString) @@ -351,11 +351,11 @@ func (sS *StatService) V1GetStatQueuesForEvent(args *StatsArgsProcessEvent, repl // V1GetStatQueue returns a StatQueue object func (sS *StatService) V1GetStatQueue(args *utils.TenantIDWithArgDispatcher, reply *StatQueue) (err error) { - if sq, err := sS.dm.GetStatQueue(args.Tenant, args.ID, true, true, ""); err != nil { + sq, err := sS.dm.GetStatQueue(args.Tenant, args.ID, true, true, "") + if err != nil { return err - } else { - *reply = *sq } + *reply = *sq return } @@ -381,7 +381,7 @@ func (sS *StatService) V1GetQueueStringMetrics(args *utils.TenantID, reply *map[ return } -// V1GetFloatMetrics returns the metrics as float64 values +// V1GetQueueFloatMetrics returns the metrics as float64 values func (sS *StatService) V1GetQueueFloatMetrics(args *utils.TenantID, reply *map[string]float64) (err error) { if missing := utils.MissingStructFields(args, []string{utils.Tenant, utils.ID}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) diff --git a/engine/thresholds.go b/engine/thresholds.go index 1819e6cc9..31a899cbb 100644 --- a/engine/thresholds.go +++ b/engine/thresholds.go @@ -235,9 +235,10 @@ func (tS *ThresholdService) StoreThreshold(t *Threshold) (err error) { // matchingThresholdsForEvent returns ordered list of matching thresholds which are active for an Event func (tS *ThresholdService) matchingThresholdsForEvent(args *ArgsProcessEvent) (ts Thresholds, err error) { + evNm := utils.MapStorage{utils.MetaReq: args.Event} tIDs := utils.NewStringSet(args.ThresholdIDs) if len(tIDs) == 0 { - tIDs, err = MatchingItemIDsForEvent(args.Event, + tIDs, err = MatchingItemIDsForEvent(evNm, tS.cgrcfg.ThresholdSCfg().StringIndexedFields, tS.cgrcfg.ThresholdSCfg().PrefixIndexedFields, tS.dm, utils.CacheThresholdFilterIndexes, args.Tenant, @@ -248,9 +249,6 @@ func (tS *ThresholdService) matchingThresholdsForEvent(args *ArgsProcessEvent) ( return nil, err } } - evNm := utils.MapStorage{ - utils.MetaReq: args.Event, - } ts = make(Thresholds, 0, len(tIDs)) for tID := range tIDs { tPrfl, err := tS.dm.GetThresholdProfile(args.Tenant, tID, true, true, utils.NonTransactional) diff --git a/engine/z_attributes_test.go b/engine/z_attributes_test.go index c1c8746a0..9f51fc1cf 100644 --- a/engine/z_attributes_test.go +++ b/engine/z_attributes_test.go @@ -451,7 +451,7 @@ func TestAttributeIndexer(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:~*req.Account:1007": { + "*string:*req.Account:1007": { "AttrPrf": struct{}{}, }, } @@ -807,6 +807,7 @@ func TestAttributeProcessWithMultipleRuns3(t *testing.T) { if !reflect.DeepEqual(eRply.MatchedProfiles, reply.MatchedProfiles) { t.Errorf("Expecting %+v, received: %+v", eRply.MatchedProfiles, reply.MatchedProfiles) } + sort.Strings(reply.AlteredFields) if !reflect.DeepEqual(eRply.AlteredFields, reply.AlteredFields) { t.Errorf("Expecting %+v, received: %+v", eRply.AlteredFields, reply.AlteredFields) } diff --git a/engine/z_filterhelpers_test.go b/engine/z_filterhelpers_test.go index 6072defaa..0052dbcae 100644 --- a/engine/z_filterhelpers_test.go +++ b/engine/z_filterhelpers_test.go @@ -82,10 +82,10 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) { } tntCtx := utils.ConcatenatedKey(tnt, context) - matchEV = map[string]interface{}{ + matchEV = utils.MapStorage{utils.MetaReq: map[string]interface{}{ utils.AnswerTime: time.Date(2014, 7, 14, 14, 30, 0, 0, time.UTC), "Field": "profile", - } + }} aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, false) if err != nil { @@ -95,9 +95,9 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) { if !has { t.Errorf("Expecting: %+v, received: %+v", stringFilterID, aPrflIDs) } - matchEV = map[string]interface{}{ + matchEV = utils.MapStorage{utils.MetaReq: map[string]interface{}{ "Field": "profilePrefix", - } + }} aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, false) if err != nil { @@ -158,10 +158,10 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) { } tntCtx := utils.ConcatenatedKey(config.CgrConfig().GeneralCfg().DefaultTenant, context) - matchEV = map[string]interface{}{ + matchEV = utils.MapStorage{utils.MetaReq: map[string]interface{}{ utils.AnswerTime: time.Date(2014, 7, 14, 14, 30, 0, 0, time.UTC), "CallCost": map[string]interface{}{"Account": 1001}, - } + }} aPrflIDs, err := MatchingItemIDsForEvent(matchEV, nil, nil, dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, true) if err != nil { @@ -171,9 +171,9 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) { if !has { t.Errorf("Expecting: %+v, received: %+v", stringFilterID, aPrflIDs) } - matchEV = map[string]interface{}{ + matchEV = utils.MapStorage{utils.MetaReq: map[string]interface{}{ "CallCost": map[string]interface{}{"Field": "profilePrefix"}, - } + }} aPrflIDs, err = MatchingItemIDsForEvent(matchEV, nil, nil, dmMatch, utils.CacheAttributeFilterIndexes, tntCtx, true, true) if err != nil { diff --git a/engine/z_filterindexer_it_test.go b/engine/z_filterindexer_it_test.go index a8e0c89cf..0f9198a5a 100644 --- a/engine/z_filterindexer_it_test.go +++ b/engine/z_filterindexer_it_test.go @@ -46,6 +46,7 @@ var sTests = []func(t *testing.T){ testITIsDBEmpty, testITTestThresholdFilterIndexes, testITTestAttributeProfileFilterIndexes, + testITTestAttributeProfileFilterIndexes2, testITTestThresholdInlineFilterIndexing, testITFlush, testITIsDBEmpty, @@ -232,7 +233,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { ID: "Filter1", Rules: []*FilterRule{ { - Element: "EventType", + Element: "~*req.EventType", Type: utils.MetaString, Values: []string{"Event1", "Event2"}, }, @@ -275,11 +276,11 @@ func testITTestThresholdFilterIndexes(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, @@ -298,7 +299,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { ID: "Filter2", Rules: []*FilterRule{ { - Element: "Account", + Element: "~*req.Account", Type: utils.MetaString, Values: []string{"1001", "1002"}, }, @@ -318,16 +319,16 @@ func testITTestThresholdFilterIndexes(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "THD_Test": struct{}{}, }, - "*string:Account:1002": { + "*string:*req.Account:1002": { "THD_Test": struct{}{}, }, - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test2": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test2": struct{}{}, }, } @@ -344,7 +345,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { ID: "Filter3", Rules: []*FilterRule{ { - Element: "Destination", + Element: "~*req.Destination", Type: utils.MetaString, Values: []string{"10", "20"}, }, @@ -365,17 +366,17 @@ func testITTestThresholdFilterIndexes(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:Destination:10": { + "*string:*req.Destination:10": { "THD_Test": struct{}{}, }, - "*string:Destination:20": { + "*string:*req.Destination:20": { "THD_Test": struct{}{}, }, - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, @@ -385,7 +386,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { utils.EmptyString, false, false); err != nil { t.Error(err) } else if !reflect.DeepEqual(eIdxes, rcvIdx) { - t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + t.Errorf("Expecting %+v, received: %+v", utils.ToJSON(eIdxes), utils.ToJSON(rcvIdx)) } //replace old filter with two different filters @@ -394,7 +395,7 @@ func testITTestThresholdFilterIndexes(t *testing.T) { ID: "Filter3", Rules: []*FilterRule{ { - Element: "Destination", + Element: "~*req.Destination", Type: utils.MetaString, Values: []string{"30", "50"}, }, @@ -422,17 +423,17 @@ func testITTestThresholdFilterIndexes(t *testing.T) { } eIdxes = map[string]utils.StringSet{ - "*string:Destination:30": { + "*string:*req.Destination:30": { "THD_Test": struct{}{}, }, - "*string:Destination:50": { + "*string:*req.Destination:50": { "THD_Test": struct{}{}, }, - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test": struct{}{}, "THD_Test2": struct{}{}, }, @@ -472,7 +473,7 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { ID: "AttrFilter", Rules: []*FilterRule{ { - Element: "EventType", + Element: "~*req.EventType", Type: utils.MetaString, Values: []string{"Event1", "Event2"}, }, @@ -506,10 +507,10 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "AttrPrf": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "AttrPrf": struct{}{}, }, } @@ -568,9 +569,9 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { ID: "AttrFilter", Rules: []*FilterRule{ { - Element: "EventType", + Element: "~*req.EventType", Type: utils.MetaString, - Values: []string{"Event3", "Event4"}, + Values: []string{"Event3", "~*req.Event4"}, }, }, ActivationInterval: &utils.ActivationInterval{ @@ -582,10 +583,99 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:EventType:Event3": { + "*string:*req.EventType:Event3": { "AttrPrf": struct{}{}, }, - "*string:EventType:Event4": { + } + for _, ctx := range attrProfile.Contexts { + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProfile.Tenant, ctx), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", utils.ToJSON(eIdxes), rcvIdx) + } + } + + eIdxes = map[string]utils.StringSet{ + "*attribute_filter_indexes": { + "AttrPrf": struct{}{}, + }, + } + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheReverseFilterIndexes, + fp.TenantID(), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + + if err := dataManager.RemoveAttributeProfile(attrProfile.Tenant, + attrProfile.ID, utils.NonTransactional, true); err != nil { + t.Error(err) + } + //check if index is removed + if _, err := dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey("cgrates.org", "con3"), + utils.MetaString, false, false); err != nil && err != utils.ErrNotFound { + t.Error(err) + } + + if _, err := dataManager.GetIndexes( + utils.CacheReverseFilterIndexes, + fp.TenantID(), + utils.EmptyString, false, false); err != utils.ErrNotFound { + t.Error(err) + } +} + +func testITTestAttributeProfileFilterIndexes2(t *testing.T) { + fp := &Filter{ + Tenant: "cgrates.org", + ID: "AttrFilter", + Rules: []*FilterRule{ + { + Element: "EventType", + Type: utils.MetaString, + Values: []string{"~*req.Event1", "~*req.Event2"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + } + if err := dataManager.SetFilter(fp, true); err != nil { + t.Error(err) + } + attrProfile := &AttributeProfile{ + Tenant: "cgrates.org", + ID: "AttrPrf", + FilterIDs: []string{"AttrFilter"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + Contexts: []string{"con1", "con2"}, + Attributes: []*Attribute{ + { + Path: "FN1", + Value: config.NewRSRParsersMustCompile("Val1", utils.INFIELD_SEP), + }, + }, + Weight: 20, + } + //Set AttributeProfile with 2 contexts ( con1 , con2) + if err := dataManager.SetAttributeProfile(attrProfile, true); err != nil { + t.Error(err) + } + eIdxes := map[string]utils.StringSet{ + "*string:*req.Event1:EventType": { + "AttrPrf": struct{}{}, + }, + "*string:*req.Event2:EventType": { "AttrPrf": struct{}{}, }, } @@ -599,6 +689,79 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) } } + //Set AttributeProfile with 1 new context (con3) + attrProfile = &AttributeProfile{ // recreate the profile because if we test on internal + Tenant: "cgrates.org", // each update on the original item will update the item from DB + ID: "AttrPrf", + FilterIDs: []string{"AttrFilter"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + Contexts: []string{"con3"}, + Attributes: []*Attribute{ + { + Path: "FN1", + Value: config.NewRSRParsersMustCompile("Val1", utils.INFIELD_SEP), + }, + }, + Weight: 20, + } + if err := dataManager.SetAttributeProfile(attrProfile, true); err != nil { + t.Error(err) + } + //check indexes with the new context (con3) + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProfile.Tenant, "con3"), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + //check if old contexts was delete + for _, ctx := range []string{"con1", "con2"} { + if _, err = dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProfile.Tenant, ctx), + utils.EmptyString, false, false); err == nil || + err != utils.ErrNotFound { + t.Error(err) + } + } + + fp = &Filter{ + Tenant: "cgrates.org", + ID: "AttrFilter", + Rules: []*FilterRule{ + { + Element: "~*req.EventType", + Type: utils.MetaString, + Values: []string{"Event3", "~*req.Event4"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + } + if err := dataManager.SetFilter(fp, true); err != nil { + t.Error(err) + } + eIdxes = map[string]utils.StringSet{ + "*string:*req.EventType:Event3": { + "AttrPrf": struct{}{}, + }, + } + for _, ctx := range attrProfile.Contexts { + if rcvIdx, err := dataManager.GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProfile.Tenant, ctx), + utils.EmptyString, false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", utils.ToJSON(eIdxes), rcvIdx) + } + } eIdxes = map[string]utils.StringSet{ "*attribute_filter_indexes": { @@ -640,7 +803,7 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { ID: "Filter1", Rules: []*FilterRule{ { - Element: "EventType", + Element: "~*req.EventType", Type: utils.MetaString, Values: []string{"Event1", "Event2"}, }, @@ -669,10 +832,10 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test": struct{}{}, }, } @@ -688,7 +851,7 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { Tenant: "cgrates.org", // each update on the original item will update the item from DB ID: "THD_Test", ActivationInterval: &utils.ActivationInterval{}, - FilterIDs: []string{"Filter1", "*string:Account:1001"}, + FilterIDs: []string{"Filter1", "*string:~*req.Account:1001"}, MaxHits: 12, MinSleep: time.Duration(0 * time.Second), Blocker: true, @@ -699,13 +862,13 @@ func testITTestThresholdInlineFilterIndexing(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "THD_Test": struct{}{}, }, - "*string:EventType:Event1": { + "*string:*req.EventType:Event1": { "THD_Test": struct{}{}, }, - "*string:EventType:Event2": { + "*string:*req.EventType:Event2": { "THD_Test": struct{}{}, }, } @@ -979,19 +1142,19 @@ func testITTestIndexingThresholds(t *testing.T) { th := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH1", - FilterIDs: []string{"*string:Account:1001", "*gt:Balance:1000"}, + FilterIDs: []string{"*string:~*req.Account:1001", "*gt:~*req.Balance:1000"}, ActionIDs: []string{}, } th2 := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH2", - FilterIDs: []string{"*string:Account:1001", "*gt:Balance:1000"}, + FilterIDs: []string{"*string:~*req.Account:1001", "*gt:~*req.Balance:1000"}, ActionIDs: []string{}, } th3 := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH3", - FilterIDs: []string{"*string:Account:1002", "*lt:Balance:1000"}, + FilterIDs: []string{"*string:~*req.Account:1002", "*lt:~*req.Balance:1000"}, ActionIDs: []string{}, } if err := dataManager.SetThresholdProfile(th, true); err != nil { @@ -1004,11 +1167,11 @@ func testITTestIndexingThresholds(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "TH1": struct{}{}, "TH2": struct{}{}, }, - "*string:Account:1002": { + "*string:*req.Account:1002": { "TH3": struct{}{}, }, } @@ -1022,14 +1185,14 @@ func testITTestIndexingThresholds(t *testing.T) { } } eMp := map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "TH1": struct{}{}, "TH2": struct{}{}, }, } if rcvMp, err := dataManager.GetIndexes( utils.CacheThresholdFilterIndexes, th.Tenant, - utils.ConcatenatedKey(utils.MetaString, utils.Account, "1001"), + utils.ConcatenatedKey(utils.MetaString, utils.MetaReq+utils.NestingSep+utils.Account, "1001"), true, true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { @@ -1041,19 +1204,19 @@ func testITTestIndexingMetaNot(t *testing.T) { th := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH1", - FilterIDs: []string{"*string:Account:1001", "*notstring:Destination:+49123"}, + FilterIDs: []string{"*string:~*req.Account:1001", "*notstring:~*req.Destination:+49123"}, ActionIDs: []string{}, } th2 := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH2", - FilterIDs: []string{"*prefix:EventName:Name", "*notprefix:Destination:10"}, + FilterIDs: []string{"*prefix:~*req.EventName:Name", "*notprefix:~*req.Destination:10"}, ActionIDs: []string{}, } th3 := &ThresholdProfile{ Tenant: "cgrates.org", ID: "TH3", - FilterIDs: []string{"*notstring:Account:1002", "*notstring:Balance:1000"}, + FilterIDs: []string{"*notstring:~*req.Account:1002", "*notstring:~*req.Balance:1000"}, ActionIDs: []string{}, } if err := dataManager.SetThresholdProfile(th, true); err != nil { @@ -1066,10 +1229,10 @@ func testITTestIndexingMetaNot(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "TH1": struct{}{}, }, - "*prefix:EventName:Name": { + "*prefix:*req.EventName:Name": { "TH2": struct{}{}, }, } @@ -1083,13 +1246,13 @@ func testITTestIndexingMetaNot(t *testing.T) { } } eMp := map[string]utils.StringSet{ - "*string:Account:1001": { + "*string:*req.Account:1001": { "TH1": struct{}{}, }, } if rcvMp, err := dataManager.GetIndexes( utils.CacheThresholdFilterIndexes, th.Tenant, - utils.ConcatenatedKey(utils.MetaString, utils.Account, "1001"), + utils.ConcatenatedKey(utils.MetaString, utils.MetaReq+utils.NestingSep+utils.Account, "1001"), true, true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { @@ -1115,7 +1278,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Category:call"}, Weight: 0, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.12, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Minute), @@ -1128,7 +1291,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Category:voice"}, Weight: 10, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.06, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Second), @@ -1142,10 +1305,10 @@ func testITIndexRateProfile(t *testing.T) { t.Error(err) } eIdxes := map[string]utils.StringSet{ - "*string:~*req.Category:call": { + "*string:*req.Category:call": { "FIRST_GI": struct{}{}, }, - "*string:~*req.Category:voice": { + "*string:*req.Category:voice": { "SECOND_GI": struct{}{}, }, } @@ -1176,7 +1339,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Category:call"}, Weight: 0, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.12, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Minute), @@ -1189,7 +1352,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Category:voice"}, Weight: 10, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.06, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Second), @@ -1202,7 +1365,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Category:custom"}, Weight: 20, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.06, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Second), @@ -1216,13 +1379,13 @@ func testITIndexRateProfile(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:~*req.Category:call": { + "*string:*req.Category:call": { "FIRST_GI": struct{}{}, }, - "*string:~*req.Category:voice": { + "*string:*req.Category:voice": { "SECOND_GI": struct{}{}, }, - "*string:~*req.Category:custom": { + "*string:*req.Category:custom": { "THIRD_GI": struct{}{}, }, } @@ -1251,7 +1414,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Subject:1001"}, Weight: 0, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.12, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Minute), @@ -1264,7 +1427,7 @@ func testITIndexRateProfile(t *testing.T) { FilterIDs: []string{"*string:~*req.Subject:1001", "*string:~*req.Category:call"}, Weight: 10, IntervalRates: []*IntervalRate{ - &IntervalRate{ + { Value: 0.6, Unit: time.Duration(1 * time.Minute), Increment: time.Duration(1 * time.Second), @@ -1278,11 +1441,11 @@ func testITIndexRateProfile(t *testing.T) { t.Error(err) } eIdxes = map[string]utils.StringSet{ - "*string:~*req.Subject:1001": { + "*string:*req.Subject:1001": { "CUSTOM_RATE1": struct{}{}, "CUSTOM_RATE2": struct{}{}, }, - "*string:~*req.Category:call": { + "*string:*req.Category:call": { "CUSTOM_RATE2": struct{}{}, }, } diff --git a/migrator/alias_it_test.go b/migrator/alias_it_test.go index a78c6efe7..5d5d00705 100644 --- a/migrator/alias_it_test.go +++ b/migrator/alias_it_test.go @@ -218,10 +218,10 @@ func testAlsITMigrateAndMove(t *testing.T) { } expAlsIdx := map[string]utils.StringSet{ - "*string:~*req.Account:1001": { + "*string:*req.Account:1001": { "*out:*any:*any:1001:call_1001:*rated": struct{}{}, }, - "*string:~*req.Subject:call_1001": { + "*string:*req.Subject:call_1001": { "*out:*any:*any:1001:call_1001:*rated": struct{}{}, }, } diff --git a/migrator/derived_chargers_it_test.go b/migrator/derived_chargers_it_test.go index f53774909..661badc47 100644 --- a/migrator/derived_chargers_it_test.go +++ b/migrator/derived_chargers_it_test.go @@ -235,7 +235,7 @@ func testDCITMigrateAndMove(t *testing.T) { t.Error("Error should be not found : ", err) } expDcIdx := map[string]utils.StringSet{ - "*string:~*req.Account:1003": { + "*string:*req.Account:1003": { "*out:cgrates.org:*any:1003:*any_0": struct{}{}, }, } @@ -248,7 +248,7 @@ func testDCITMigrateAndMove(t *testing.T) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expDcIdx), utils.ToJSON(dcidx)) } expDcIdx = map[string]utils.StringSet{ - "*string:~*req.Account:1003": { + "*string:*req.Account:1003": { "*out:cgrates.org:*any:1003:*any_0": struct{}{}, }, } diff --git a/migrator/filters_it_test.go b/migrator/filters_it_test.go index ee2a11d2c..6aa072708 100644 --- a/migrator/filters_it_test.go +++ b/migrator/filters_it_test.go @@ -201,8 +201,9 @@ func testFltrITMigrateAndMove(t *testing.T) { } // manually set the indexes because the GetFilter functions compile the value from DB that is still the old version wrongFltrIdx := map[string]utils.StringSet{ - "*prefix::1001": {"ATTR_1": struct{}{}}, - "*string:Account:1001": {"ATTR_1": struct{}{}}} + "*prefix::1001": {"ATTR_1": struct{}{}}, + // "*string:Account:1001": {"ATTR_1": struct{}{}}, + } if err := fltrMigrator.dmIN.DataManager().SetIndexes( utils.CacheAttributeFilterIndexes, @@ -261,17 +262,20 @@ func testFltrITMigrateAndMove(t *testing.T) { if !reflect.DeepEqual(*expAttrProf, *resultattr) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultattr)) } - expFltrIdx := map[string]utils.StringSet{ - "*prefix:~*req.Account:1001": {"ATTR_1": struct{}{}}, - "*string:~*req.Account:1001": {"ATTR_1": struct{}{}}} + // the Indexes should be recomputed after migration to the same DB + if inPath != outPath { + expFltrIdx := map[string]utils.StringSet{ + "*prefix:*req.Account:1001": {"ATTR_1": struct{}{}}, + "*string:*req.Account:1001": {"ATTR_1": struct{}{}}} - if fltridx, err := fltrMigrator.dmOut.DataManager().GetIndexes( - utils.CacheAttributeFilterIndexes, - utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), - "", false, false); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expFltrIdx, fltridx) { - t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) + if fltridx, err := fltrMigrator.dmOut.DataManager().GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), + "", false, false); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expFltrIdx, fltridx) { + t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) + } } case utils.Move: if err := fltrMigrator.dmIN.DataManager().SetFilter(expFilters, true); err != nil { @@ -454,18 +458,22 @@ func testFltrITMigratev2(t *testing.T) { if !reflect.DeepEqual(*expAttrProf, *resultAttr) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expAttrProf), utils.ToJSON(resultAttr)) } - expFltrIdx := map[string]utils.StringSet{ - "*string:~*req.Account:1001": {"ATTR_1": struct{}{}}, - "*string:~*req.Subject:1001": {"ATTR_1": struct{}{}}, - } - if fltridx, err := fltrMigrator.dmOut.DataManager().GetIndexes( - utils.CacheAttributeFilterIndexes, - utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), - "", false, true); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expFltrIdx, fltridx) { - t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) + // the Indexes should be recomputed after migration to the same DB + if inPath != outPath { + expFltrIdx := map[string]utils.StringSet{ + "*string:*req.Account:1001": {"ATTR_1": struct{}{}}, + "*string:*req.Subject:1001": {"ATTR_1": struct{}{}}, + } + + if fltridx, err := fltrMigrator.dmOut.DataManager().GetIndexes( + utils.CacheAttributeFilterIndexes, + utils.ConcatenatedKey(attrProf.Tenant, utils.META_ANY), + "", false, true); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expFltrIdx, fltridx) { + t.Errorf("Expected %v, recived: %v", utils.ToJSON(expFltrIdx), utils.ToJSON(fltridx)) + } } } diff --git a/migrator/user_it_test.go b/migrator/user_it_test.go index f8a20a63d..f3edf5bef 100644 --- a/migrator/user_it_test.go +++ b/migrator/user_it_test.go @@ -194,7 +194,7 @@ func testUsrITMigrateAndMove(t *testing.T) { } expUsrIdx := map[string]utils.StringSet{ - "*string:~Account:1002": { + "*string:Account:1002": { "1001": struct{}{}, }, } diff --git a/rates/rates.go b/rates/rates.go index ad6050ec6..1ab65d6d5 100644 --- a/rates/rates.go +++ b/rates/rates.go @@ -72,10 +72,11 @@ func (rS *RateS) Call(serviceMethod string, args interface{}, reply interface{}) // matchingRateProfileForEvent returns the matched RateProfile for the given event func (rS *RateS) matchingRateProfileForEvent(args *ArgsCostForEvent, rPfIDs []string) (rtPfl *engine.RateProfile, err error) { + evNm := utils.MapStorage{utils.MetaReq: args.CGREvent.Event} if len(rPfIDs) == 0 { var rPfIDMp utils.StringSet if rPfIDMp, err = engine.MatchingItemIDsForEvent( - args.CGREvent.Event, + evNm, rS.cfg.RateSCfg().StringIndexedFields, rS.cfg.RateSCfg().PrefixIndexedFields, rS.dm, @@ -88,7 +89,6 @@ func (rS *RateS) matchingRateProfileForEvent(args *ArgsCostForEvent, rPfIDs []st } rPfIDs = rPfIDMp.AsSlice() } - evNm := utils.MapStorage{utils.MetaReq: args.CGREvent.Event} var sTime time.Time if sTime, err = args.StartTime(rS.cfg.GeneralCfg().DefaultTimezone); err != nil { return @@ -157,7 +157,7 @@ func (rS *RateS) rateProfileCostForEvent(rtPfl *engine.RateProfile, args *ArgsCo } */ -// AttrArgsProcessEvent arguments used for proccess event +// ArgsCostForEvent arguments used for proccess event type ArgsCostForEvent struct { RateProfileIDs []string Opts map[string]interface{} diff --git a/utils/mapstorage.go b/utils/mapstorage.go index 627c1ec61..a18c7e423 100644 --- a/utils/mapstorage.go +++ b/utils/mapstorage.go @@ -34,7 +34,7 @@ type dataStorage interface { Set(fldPath []string, val interface{}) error Remove(fldPath []string) error - GetKeys(nesteed bool) []string + GetKeys(nested bool, nesteedLimit int, prefix string) []string } // MapStorage is the basic dataStorage @@ -207,64 +207,78 @@ func (ms MapStorage) Set(fldPath []string, val interface{}) (err error) { } // GetKeys returns all the keys from map -func (ms MapStorage) GetKeys(nesteed bool) (keys []string) { - if !nesteed { - keys = make([]string, len(ms)) - i := 0 - for k := range ms { - keys[i] = k - i++ +func (ms MapStorage) GetKeys(nested bool, nestedLimit int, prefix string) (keys []string) { + if prefix != EmptyString { + prefix += NestingSep + } + if !nested { + // this is a special case for the filter matching were we have the full map: + /* + ms:=MapStorage{ + "*req":MapStorage{ + ... + }, + "*opts":MapStorage{ + ... + }, + } + when nested is false we should stiil look inside `*req` and `*opts` but only in the first level of them + */ + if nestedLimit <= 1 { + keys = make([]string, 0, len(ms)) + for k := range ms { + keys = append(keys, prefix+k) + } + return + } + for k, v := range ms { // in case of nested on false we take in consideraton the nestedLimit + keys = append(keys, prefix+k) + switch rv := v.(type) { // and for performance we only take in consideration a limited set of types for nested false + case dataStorage: + keys = append(keys, rv.GetKeys(nested, nestedLimit-1, prefix+k)...) + case map[string]interface{}: + keys = append(keys, MapStorage(rv).GetKeys(nested, nestedLimit-1, prefix+k)...) + } } return } for k, v := range ms { - keys = append(keys, k) + keys = append(keys, prefix+k) switch rv := v.(type) { case dataStorage: - for _, dsKey := range rv.GetKeys(nesteed) { - keys = append(keys, k+NestingSep+dsKey) - } + keys = append(keys, rv.GetKeys(nested, nestedLimit, prefix+k)...) case map[string]interface{}: - for _, dsKey := range MapStorage(rv).GetKeys(nesteed) { - keys = append(keys, k+NestingSep+dsKey) - } + keys = append(keys, MapStorage(rv).GetKeys(nested, nestedLimit, prefix+k)...) case []MapStorage: for i, dp := range rv { - pref := k + fmt.Sprintf("[%v]", i) + pref := prefix + k + fmt.Sprintf("[%v]", i) keys = append(keys, pref) - for _, dsKey := range dp.GetKeys(nesteed) { - keys = append(keys, pref+NestingSep+dsKey) - } + keys = append(keys, dp.GetKeys(nested, nestedLimit, pref)...) } case []dataStorage: for i, dp := range rv { - pref := k + fmt.Sprintf("[%v]", i) + pref := prefix + k + fmt.Sprintf("[%v]", i) keys = append(keys, pref) - for _, dsKey := range dp.GetKeys(nesteed) { - keys = append(keys, pref+NestingSep+dsKey) - } + keys = append(keys, dp.GetKeys(nested, nestedLimit, pref)...) } case []map[string]interface{}: for i, dp := range rv { - pref := k + fmt.Sprintf("[%v]", i) + pref := prefix + k + fmt.Sprintf("[%v]", i) keys = append(keys, pref) - for _, dsKey := range MapStorage(dp).GetKeys(nesteed) { - keys = append(keys, pref+NestingSep+dsKey) - } + keys = append(keys, MapStorage(dp).GetKeys(nested, nestedLimit, pref)...) } case []interface{}: for i := range rv { - keys = append(keys, k+fmt.Sprintf("[%v]", i)) + keys = append(keys, prefix+k+fmt.Sprintf("[%v]", i)) } case []string: for i := range rv { - keys = append(keys, k+fmt.Sprintf("[%v]", i)) + keys = append(keys, prefix+k+fmt.Sprintf("[%v]", i)) } default: // ToDo:should not be called - keys = append(keys, getPathFromInterface(v, k+NestingSep)...) + keys = append(keys, getPathFromInterface(v, prefix+k+NestingSep)...) } - } return diff --git a/utils/mapstorage_test.go b/utils/mapstorage_test.go index c8b9e2a13..0678208e1 100644 --- a/utils/mapstorage_test.go +++ b/utils/mapstorage_test.go @@ -277,7 +277,7 @@ func TestNavMapFieldAsInterface(t *testing.T) { func TestNavMapGetKeys(t *testing.T) { navMp := MapStorage{ - "FirstLevel": map[string]interface{}{ + "FirstLevel": MapStorage{ "SecondLevel": map[string]interface{}{ "ThirdLevel": map[string]interface{}{ "Fld1": 123.123, @@ -319,7 +319,20 @@ func TestNavMapGetKeys(t *testing.T) { "Field6[0]", "Field6[1]", } - keys := navMp.GetKeys(true) + keys := navMp.GetKeys(true, 0, EmptyString) + sort.Strings(expKeys) + sort.Strings(keys) + if !reflect.DeepEqual(expKeys, keys) { + t.Errorf("Expecting: %+v, received: %+v", ToJSON(expKeys), ToJSON(keys)) + } + + expKeys = []string{ + "*req.FirstLevel", + "*req.FistLever2", + "*req.Field5", + "*req.Field6", + } + keys = navMp.GetKeys(false, 1, MetaReq) sort.Strings(expKeys) sort.Strings(keys) if !reflect.DeepEqual(expKeys, keys) { @@ -328,11 +341,15 @@ func TestNavMapGetKeys(t *testing.T) { expKeys = []string{ "FirstLevel", + "FirstLevel.SecondLevel", "FistLever2", + "FistLever2.SecondLevel2", + "FistLever2.Field3", + "FistLever2.Field4", "Field5", "Field6", } - keys = navMp.GetKeys(false) + keys = navMp.GetKeys(false, 2, EmptyString) sort.Strings(expKeys) sort.Strings(keys) if !reflect.DeepEqual(expKeys, keys) { @@ -667,7 +684,7 @@ func TestNavMapGetKeys2(t *testing.T) { "Field10[0]", "Field10[0].A", } - keys := navMp.GetKeys(true) + keys := navMp.GetKeys(true, 0, EmptyString) sort.Strings(expKeys) sort.Strings(keys) if !reflect.DeepEqual(expKeys, keys) {