From a5ee8e8398d1af9d25b60bfe8a8b7ec0bb24355c Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 1 Mar 2019 14:35:32 +0200 Subject: [PATCH] New AttributeProfile structure ( version3) --- apier/v1/attributes_it_test.go | 18 ---- apier/v1/chargers_it_test.go | 1 - apier/v1/filter_indexes_it_test.go | 2 - apier/v1/filterindexecache_it_test.go | 4 - config/config_defaults.go | 18 ++-- config/config_json_test.go | 28 ++--- config/config_test.go | 28 ++--- .../mysql/create_tariffplan_tables.sql | 5 +- .../postgres/create_tariffplan_tables.sql | 5 +- data/tariffplans/cluelrn/Attributes.csv | 16 +-- data/tariffplans/dispatchers/Attributes.csv | 22 ++-- data/tariffplans/oldtutorial/Attributes.csv | 6 +- data/tariffplans/testit/Attributes.csv | 8 +- data/tariffplans/testtp/Attributes.csv | 6 +- data/tariffplans/tutorial/Attributes.csv | 44 ++++---- dispatchers/attributes_it_test.go | 3 - engine/attributes.go | 45 ++++---- engine/attributes_test.go | 70 +----------- engine/filterindexer_it_test.go | 1 - engine/libattributes.go | 17 +-- engine/loader_csv_test.go | 12 +-- engine/model_helpers.go | 34 +++--- engine/model_helpers_test.go | 24 +---- engine/models.go | 9 +- engine/onstor_it_test.go | 8 -- engine/version.go | 2 +- general_tests/sentinel_it_test.go | 3 - loaders/loader_test.go | 43 +++----- migrator/alias.go | 7 +- migrator/alias_it_test.go | 11 +- migrator/alias_test.go | 29 ++--- migrator/attributes.go | 101 +++++++++++++++++- migrator/attributes_it_test.go | 5 +- migrator/attributes_test.go | 3 +- migrator/derived_chargers.go | 3 +- migrator/derived_chargers_it_test.go | 13 +-- migrator/derived_chargers_test.go | 22 ---- migrator/migrator_datadb.go | 3 + migrator/storage_map_datadb.go | 16 +++ migrator/storage_mongo_datadb.go | 35 ++++++ migrator/storage_redis.go | 48 +++++++++ migrator/user.go | 4 - migrator/user_it_test.go | 12 +-- migrator/user_test.go | 14 +-- utils/apitpdata.go | 3 +- 45 files changed, 387 insertions(+), 424 deletions(-) diff --git a/apier/v1/attributes_it_test.go b/apier/v1/attributes_it_test.go index 1160c46b4..bf9a1083f 100644 --- a/apier/v1/attributes_it_test.go +++ b/apier/v1/attributes_it_test.go @@ -170,13 +170,11 @@ func testAttributeSGetAttributeForEvent(t *testing.T) { FieldName: utils.Account, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10.0, @@ -224,7 +222,6 @@ func testAttributeSGetAttributeForEventNotFound(t *testing.T) { FieldName: utils.Account, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, }, Weight: 10.0, @@ -276,7 +273,6 @@ func testAttributeSGetAttributeForEventWithMetaAnyContext(t *testing.T) { FieldName: utils.Account, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, }, Weight: 10.0, @@ -424,13 +420,11 @@ func testAttributeSProcessEventWithNoneSubstitute(t *testing.T) { FieldName: utils.Account, Initial: "1008", Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile(utils.META_NONE, true, utils.INFIELD_SEP), - Append: false, }, }, Weight: 20, @@ -492,13 +486,11 @@ func testAttributeSProcessEventWithNoneSubstitute2(t *testing.T) { FieldName: utils.Account, Initial: "1008", Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile(utils.META_NONE, true, utils.INFIELD_SEP), - Append: false, }, }, Weight: 20, @@ -571,13 +563,11 @@ func testAttributeSProcessEventWithNoneSubstitute3(t *testing.T) { FieldName: utils.Account, Initial: "1008", Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: "1008", Substitute: config.NewRSRParsersMustCompile(utils.META_NONE, true, utils.INFIELD_SEP), - Append: false, }, }, Weight: 20, @@ -625,7 +615,6 @@ func testAttributeSProcessEventWithHeader(t *testing.T) { FieldName: "Field2", Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("~Field1", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: true, @@ -699,7 +688,6 @@ func testAttributeSSetAlsPrf(t *testing.T) { FieldName: "FL1", Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -728,13 +716,11 @@ func testAttributeSUpdateAlsPrf(t *testing.T) { FieldName: "FL1", Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, { FieldName: "FL2", Initial: "In2", Substitute: config.NewRSRParsersMustCompile("Al2", true, utils.INFIELD_SEP), - Append: false, }, } alsPrf.Compile() @@ -797,7 +783,6 @@ func testAttributeSSetAlsPrf2(t *testing.T) { AllFiltersMatch: true, }, }, - Append: false, }, }, Blocker: false, @@ -840,7 +825,6 @@ func testAttributeSSetAlsPrf3(t *testing.T) { Rules: "", }, }, - Append: false, }, }, Blocker: false, @@ -869,7 +853,6 @@ func testAttributeSSetAlsPrf4(t *testing.T) { Substitute: config.RSRParsers{ &config.RSRParser{}, }, - Append: false, }, }, Blocker: false, @@ -904,7 +887,6 @@ func testAttributeSProcessEventWithSearchAndReplace(t *testing.T) { FieldName: "Category", Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("~Category:s/(.*)/${1}_suffix/", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: true, diff --git a/apier/v1/chargers_it_test.go b/apier/v1/chargers_it_test.go index abb0e1004..cb999494d 100755 --- a/apier/v1/chargers_it_test.go +++ b/apier/v1/chargers_it_test.go @@ -163,7 +163,6 @@ func testChargerSLoadAddCharger(t *testing.T) { AllFiltersMatch: true, }, }, - Append: true, }, }, Blocker: false, diff --git a/apier/v1/filter_indexes_it_test.go b/apier/v1/filter_indexes_it_test.go index 3d007f449..0ca11dfdf 100644 --- a/apier/v1/filter_indexes_it_test.go +++ b/apier/v1/filter_indexes_it_test.go @@ -1196,7 +1196,6 @@ func testV1FIdxSetAttributeProfileIndexes(t *testing.T) { FieldName: "FL1", Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -1306,7 +1305,6 @@ func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) { FieldName: "FL1", Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }}, Weight: 20, } diff --git a/apier/v1/filterindexecache_it_test.go b/apier/v1/filterindexecache_it_test.go index 9f002c7cf..49a3263c4 100644 --- a/apier/v1/filterindexecache_it_test.go +++ b/apier/v1/filterindexecache_it_test.go @@ -845,13 +845,11 @@ func testV1FIdxCaSetAttributeProfile(t *testing.T) { FieldName: utils.Account, Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -938,13 +936,11 @@ func testV1FIdxCaUpdateAttributeProfile(t *testing.T) { FieldName: utils.Account, Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, { FieldName: utils.Subject, Initial: "*any", Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, diff --git a/config/config_defaults.go b/config/config_defaults.go index b109b8c62..c39ed36eb 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -523,11 +523,10 @@ const CGRATES_CFG_JSON = ` {"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "~2"}, {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~3"}, {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~4"}, - {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "~5"}, - {"tag": "Initial", "field_id": "Initial", "type": "*composed", "value": "~6"}, + {"tag": "AttributeFilterIDs", "field_id": "AttributeFilterIDs", "type": "*composed", "value": "~5"}, + {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "~6"}, {"tag": "Substitute", "field_id": "Substitute", "type": "*composed", "value": "~7"}, - {"tag": "Append", "field_id": "Append", "type": "*composed", "value": "~8"}, - {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~8"}, ], }, { @@ -570,12 +569,11 @@ const CGRATES_CFG_JSON = ` {"tag": "QueueLength", "field_id": "QueueLength", "type": "*composed", "value": "~4"}, {"tag": "TTL", "field_id": "TTL", "type": "*composed", "value": "~5"}, {"tag": "Metrics", "field_id": "Metrics", "type": "*composed", "value": "~6"}, - {"tag": "MetricParams", "field_id": "Parameters", "type": "*composed", "value": "~7"}, - {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~8"}, - {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "~9"}, - {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~10"}, - {"tag": "MinItems", "field_id": "MinItems", "type": "*composed", "value": "~11"}, - {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "~12"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~7"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "~8"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~9"}, + {"tag": "MinItems", "field_id": "MinItems", "type": "*composed", "value": "~10"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "~11"}, ], }, { diff --git a/config/config_json_test.go b/config/config_json_test.go index 4c5e2b051..4ae0286f1 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -872,26 +872,22 @@ func TestDfLoaderJsonCfg(t *testing.T) { Field_id: utils.StringPointer("ActivationInterval"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~4")}, - {Tag: utils.StringPointer("FieldName"), - Field_id: utils.StringPointer(utils.FieldName), + {Tag: utils.StringPointer("AttributeFilterIDs"), + Field_id: utils.StringPointer("AttributeFilterIDs"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~5")}, - {Tag: utils.StringPointer("Initial"), - Field_id: utils.StringPointer(utils.Initial), + {Tag: utils.StringPointer("FieldName"), + Field_id: utils.StringPointer(utils.FieldName), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~6")}, {Tag: utils.StringPointer("Substitute"), Field_id: utils.StringPointer(utils.Substitute), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~7")}, - {Tag: utils.StringPointer("Append"), - Field_id: utils.StringPointer(utils.Append), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~8")}, {Tag: utils.StringPointer("Weight"), Field_id: utils.StringPointer(utils.Weight), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~9")}, + Value: utils.StringPointer("~8")}, }, }, { @@ -1012,30 +1008,26 @@ func TestDfLoaderJsonCfg(t *testing.T) { Field_id: utils.StringPointer("Metrics"), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("~6")}, - {Tag: utils.StringPointer("MetricParams"), - Field_id: utils.StringPointer("Parameters"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~7")}, {Tag: utils.StringPointer("Blocker"), Field_id: utils.StringPointer("Blocker"), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~8")}, + Value: utils.StringPointer("~7")}, {Tag: utils.StringPointer("Stored"), Field_id: utils.StringPointer("Stored"), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~9")}, + Value: utils.StringPointer("~8")}, {Tag: utils.StringPointer("Weight"), Field_id: utils.StringPointer("Weight"), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~10")}, + Value: utils.StringPointer("~9")}, {Tag: utils.StringPointer("MinItems"), Field_id: utils.StringPointer("MinItems"), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~11")}, + Value: utils.StringPointer("~10")}, {Tag: utils.StringPointer("ThresholdIDs"), Field_id: utils.StringPointer("ThresholdIDs"), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~12")}, + Value: utils.StringPointer("~11")}, }, }, { diff --git a/config/config_test.go b/config/config_test.go index a6877b21a..de507f385 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -1059,26 +1059,22 @@ func TestCgrLoaderCfgITDefaults(t *testing.T) { FieldId: "ActivationInterval", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile("~4", true, utils.INFIELD_SEP)}, - {Tag: "FieldName", - FieldId: "FieldName", + {Tag: "AttributeFilterIDs", + FieldId: "AttributeFilterIDs", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile("~5", true, utils.INFIELD_SEP)}, - {Tag: "Initial", - FieldId: "Initial", + {Tag: "FieldName", + FieldId: "FieldName", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP)}, {Tag: "Substitute", FieldId: "Substitute", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, - {Tag: "Append", - FieldId: "Append", - Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, {Tag: "Weight", FieldId: "Weight", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, }, }, { @@ -1199,30 +1195,26 @@ func TestCgrLoaderCfgITDefaults(t *testing.T) { FieldId: "Metrics", Type: utils.META_COMPOSED, Value: NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP)}, - {Tag: "MetricParams", - FieldId: "Parameters", - Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, {Tag: "Blocker", FieldId: "Blocker", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, {Tag: "Stored", FieldId: "Stored", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, {Tag: "Weight", FieldId: "Weight", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~10", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, {Tag: "MinItems", FieldId: "MinItems", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~11", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~10", true, utils.INFIELD_SEP)}, {Tag: "ThresholdIDs", FieldId: "ThresholdIDs", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~12", true, utils.INFIELD_SEP)}, + Value: NewRSRParsersMustCompile("~11", true, utils.INFIELD_SEP)}, }, }, { diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 43966911b..58da8f677 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -384,17 +384,16 @@ CREATE TABLE tp_attributes ( `contexts` varchar(64) NOT NULL, `filter_ids` varchar(64) NOT NULL, `activation_interval` varchar(64) NOT NULL, + `attribute_filter_ids` varchar(64) NOT NULL, `field_name` varchar(64) NOT NULL, - `initial` varchar(64) NOT NULL, `substitute` varchar(64) NOT NULL, - `append` BOOLEAN NOT NULL, `blocker` BOOLEAN NOT NULL, `weight` decimal(8,2) NOT NULL, `created_at` TIMESTAMP, PRIMARY KEY (`pk`), KEY `tpid` (`tpid`), UNIQUE KEY `unique_tp_attributes` (`tpid`,`tenant`, - `id`,`filter_ids`,`field_name`,`initial`,`substitute` ) + `id`,`filter_ids`,`field_name`,`substitute` ) ); -- diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index 1dc8b3021..9d2634a15 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -375,17 +375,16 @@ CREATE INDEX tp_suppliers_unique ON tp_suppliers ("tpid", "tenant", "id", "contexts" varchar(64) NOT NULL, "filter_ids" varchar(64) NOT NULL, "activation_interval" varchar(64) NOT NULL, + "attribute_filter_ids" varchar(64) NOT NULL, "field_name" varchar(64) NOT NULL, - "initial" varchar(64) NOT NULL, "substitute" varchar(64) NOT NULL, - "append" BOOLEAN NOT NULL, "blocker" BOOLEAN NOT NULL, "weight" decimal(8,2) NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE ); CREATE INDEX tp_attributes_ids ON tp_attributes (tpid); CREATE INDEX tp_attributes_unique ON tp_attributes ("tpid", "tenant", "id", - "filter_ids","field_name","initial","substitute"); + "filter_ids","field_name","substitute"); -- -- Table structure for table `tp_chargers` diff --git a/data/tariffplans/cluelrn/Attributes.csv b/data/tariffplans/cluelrn/Attributes.csv index fae2d53b8..53be00292 100644 --- a/data/tariffplans/cluelrn/Attributes.csv +++ b/data/tariffplans/cluelrn/Attributes.csv @@ -1,8 +1,8 @@ -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,LRN_Dst3125650565,lrn,*string:Destination:3125650565,,Destination,*any,13128543000,true,false,10 -cgrates.org,LRN_Dst3125650565,,,,OriginalDestination,*any,3125650565,true,false,10 -cgrates.org,LRN_LATA_Dst13128543000,lrn,*string:Destination:13128543000;*rsr::~OriginalDestination(!^$),,DestinationLATA,*any,358,true,false,20 -cgrates.org,LRN_LATA_Cli9174269000,lrn,*string:Account:9174269000;*rsr::~DestinationLATA(!^$),,CallerLATA,*any,132,true,false,30 -cgrates.org,LRN_JURISDICTION_NY,lrn,FLTR_INTRALATA_NEWYORK,,LRNJurisdiction,*any,INTRA,true,false,50 -cgrates.org,LRN_JURISDICTION_IL,lrn,FLTR_INTRALATA_ILLINOIS,,LRNJurisdiction,*any,INTRA,true,false,50 -cgrates.org,LRN_JURISDICTION_INTER,lrn,*string:Destination:13128543000;*rsr::~CallerLATA(!^$),,LRNJurisdiction,*any,INTER,true,false,40 +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,LRN_Dst3125650565,lrn,*string:Destination:3125650565,,Destination,*any,13128543000,false,10 +cgrates.org,LRN_Dst3125650565,,,,OriginalDestination,*any,3125650565,false,10 +cgrates.org,LRN_LATA_Dst13128543000,lrn,*string:Destination:13128543000;*rsr::~OriginalDestination(!^$),,DestinationLATA,*any,358,false,20 +cgrates.org,LRN_LATA_Cli9174269000,lrn,*string:Account:9174269000;*rsr::~DestinationLATA(!^$),,CallerLATA,*any,132,false,30 +cgrates.org,LRN_JURISDICTION_NY,lrn,FLTR_INTRALATA_NEWYORK,,LRNJurisdiction,*any,INTRA,false,50 +cgrates.org,LRN_JURISDICTION_IL,lrn,FLTR_INTRALATA_ILLINOIS,,LRNJurisdiction,*any,INTRA,false,50 +cgrates.org,LRN_JURISDICTION_INTER,lrn,*string:Destination:13128543000;*rsr::~CallerLATA(!^$),,LRNJurisdiction,*any,INTER,false,40 diff --git a/data/tariffplans/dispatchers/Attributes.csv b/data/tariffplans/dispatchers/Attributes.csv index c0aa84ab8..2481c0166 100644 --- a/data/tariffplans/dispatchers/Attributes.csv +++ b/data/tariffplans/dispatchers/Attributes.csv @@ -1,12 +1,12 @@ -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ATTR_1001_SIMPLEAUTH,simpleauth,*string:Account:1001,,Password,*any,CGRateS.org,true,false,20 -cgrates.org,ATTR_API_ATTR_FAKE_AUTH,*auth,*string:APIKey:12345,,APIMethods,*any,,true,false,20 -cgrates.org,ATTR_API_ATTR_AUTH,*auth,*string:APIKey:attr12345,,APIMethods,*any,AttributeSv1.Ping&AttributeSv1.GetAttributeForEvent&AttributeSv1.ProcessEvent,true,false,20 -cgrates.org,ATTR_API_CHRG_AUTH,*auth,*string:APIKey:chrg12345,,APIMethods,*any,ChargerSv1.Ping&ChargerSv1.GetChargersForEvent&ChargerSv1.ProcessEvent,true,false,20 -cgrates.org,ATTR_API_THR_AUTH,*auth,*string:APIKey:thr12345,,APIMethods,*any,ThresholdSv1.Ping&ThresholdSv1.GetThresholdsForEvent&ThresholdSv1.ProcessEvent&ThresholdSv1.GetThreshold&ThresholdSv1.GetThresholdIDs,true,false,20 -cgrates.org,ATTR_API_SUP_AUTH,*auth,*string:APIKey:sup12345,,APIMethods,*any,SupplierSv1.Ping&SupplierSv1.GetSuppliers,true,false,20 -cgrates.org,ATTR_API_STAT_AUTH,*auth,*string:APIKey:stat12345,,APIMethods,*any,StatSv1.Ping&StatSv1.GetStatQueuesForEvent&StatSv1.GetQueueStringMetrics&StatSv1.ProcessEvent&StatSv1.GetQueueIDs&StatSv1.GetQueueFloatMetrics,true,false,20 -cgrates.org,ATTR_API_RES_AUTH,*auth,*string:APIKey:res12345,,APIMethods,*any,ResourceSv1.Ping&ResourceSv1.GetResourcesForEvent&ResourceSv1.AuthorizeResources&ResourceSv1.AllocateResources&ResourceSv1.ReleaseResources,true,false,20 -cgrates.org,ATTR_API_SES_AUTH,*auth,*string:APIKey:ses12345,,APIMethods,*any,SessionSv1.Ping&SessionSv1.AuthorizeEventWithDigest&SessionSv1.InitiateSessionWithDigest&SessionSv1.UpdateSession&SessionSv1.TerminateSession&SessionSv1.ProcessCDR&SessionSv1.ProcessEvent&SessionSv1.GetActiveSessions&SessionSv1.GetActiveSessionsCount&SessionSv1.ForceDisconnect&SessionSv1.GetPassiveSessions&SessionSv1.GetPassiveSessionsCount&SessionSv1.SetPassiveSession&SessionSv1.ReplicateSessions,true,false,20 -cgrates.org,ATTR_API_RSP_AUTH,*auth,*string:APIKey:rsp12345,,APIMethods,*any,Responder.Status,true,false,20 +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,ATTR_1001_SIMPLEAUTH,simpleauth,*string:Account:1001,,Password,*any,CGRateS.org,false,20 +cgrates.org,ATTR_API_ATTR_FAKE_AUTH,*auth,*string:APIKey:12345,,APIMethods,*any,,false,20 +cgrates.org,ATTR_API_ATTR_AUTH,*auth,*string:APIKey:attr12345,,APIMethods,*any,AttributeSv1.Ping&AttributeSv1.GetAttributeForEvent&AttributeSv1.ProcessEvent,false,20 +cgrates.org,ATTR_API_CHRG_AUTH,*auth,*string:APIKey:chrg12345,,APIMethods,*any,ChargerSv1.Ping&ChargerSv1.GetChargersForEvent&ChargerSv1.ProcessEvent,false,20 +cgrates.org,ATTR_API_THR_AUTH,*auth,*string:APIKey:thr12345,,APIMethods,*any,ThresholdSv1.Ping&ThresholdSv1.GetThresholdsForEvent&ThresholdSv1.ProcessEvent&ThresholdSv1.GetThreshold&ThresholdSv1.GetThresholdIDs,false,20 +cgrates.org,ATTR_API_SUP_AUTH,*auth,*string:APIKey:sup12345,,APIMethods,*any,SupplierSv1.Ping&SupplierSv1.GetSuppliers,false,20 +cgrates.org,ATTR_API_STAT_AUTH,*auth,*string:APIKey:stat12345,,APIMethods,*any,StatSv1.Ping&StatSv1.GetStatQueuesForEvent&StatSv1.GetQueueStringMetrics&StatSv1.ProcessEvent&StatSv1.GetQueueIDs&StatSv1.GetQueueFloatMetrics,false,20 +cgrates.org,ATTR_API_RES_AUTH,*auth,*string:APIKey:res12345,,APIMethods,*any,ResourceSv1.Ping&ResourceSv1.GetResourcesForEvent&ResourceSv1.AuthorizeResources&ResourceSv1.AllocateResources&ResourceSv1.ReleaseResources,false,20 +cgrates.org,ATTR_API_SES_AUTH,*auth,*string:APIKey:ses12345,,APIMethods,*any,SessionSv1.Ping&SessionSv1.AuthorizeEventWithDigest&SessionSv1.InitiateSessionWithDigest&SessionSv1.UpdateSession&SessionSv1.TerminateSession&SessionSv1.ProcessCDR&SessionSv1.ProcessEvent&SessionSv1.GetActiveSessions&SessionSv1.GetActiveSessionsCount&SessionSv1.ForceDisconnect&SessionSv1.GetPassiveSessions&SessionSv1.GetPassiveSessionsCount&SessionSv1.SetPassiveSession&SessionSv1.ReplicateSessions,false,20 +cgrates.org,ATTR_API_RSP_AUTH,*auth,*string:APIKey:rsp12345,,APIMethods,*any,Responder.Status,false,20 diff --git a/data/tariffplans/oldtutorial/Attributes.csv b/data/tariffplans/oldtutorial/Attributes.csv index 4bb9f90c7..b3843411f 100644 --- a/data/tariffplans/oldtutorial/Attributes.csv +++ b/data/tariffplans/oldtutorial/Attributes.csv @@ -1,3 +1,3 @@ -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ATTR_1,*sessions;*cdrs,*string:Account:1007,2014-01-14T00:00:00Z,Account,*any,1001,false,false,10 -cgrates.org,ATTR_1,,,,Subject,*any,1001,true,, +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,ATTR_1,*sessions;*cdrs,*string:Account:1007,2014-01-14T00:00:00Z,Account,*any,1001,false,10 +cgrates.org,ATTR_1,,,,Subject,*any,1001,, diff --git a/data/tariffplans/testit/Attributes.csv b/data/tariffplans/testit/Attributes.csv index a5cd98d3d..3dd7f6b53 100644 --- a/data/tariffplans/testit/Attributes.csv +++ b/data/tariffplans/testit/Attributes.csv @@ -1,4 +1,4 @@ -#Tenant,ID,Context,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ATTR_ACNT_1001,*sessions,FLTR_ACCOUNT_1001,,OfficeGroup,*any,Marketing,true,false,10 -cgrates.org,ATTR_SUPPLIER1,*chargers,,,Subject,*any,SUPPLIER1,true,false,10 -cgrates.org,ATTR_PAYPAL,*cdrs,*string:Subject:ANY2CNT,,PayPalAccount,*any,paypal@cgrates.org,true,false,10 +#Tenant,ID,Context,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,ATTR_ACNT_1001,*sessions,FLTR_ACCOUNT_1001,,OfficeGroup,*any,Marketing,false,10 +cgrates.org,ATTR_SUPPLIER1,*chargers,,,Subject,*any,SUPPLIER1,false,10 +cgrates.org,ATTR_PAYPAL,*cdrs,*string:Subject:ANY2CNT,,PayPalAccount,*any,paypal@cgrates.org,false,10 diff --git a/data/tariffplans/testtp/Attributes.csv b/data/tariffplans/testtp/Attributes.csv index c7fc49204..60f653758 100644 --- a/data/tariffplans/testtp/Attributes.csv +++ b/data/tariffplans/testtp/Attributes.csv @@ -1,3 +1,3 @@ -#,Tenant,ID,Context,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ALS1,con1,FLTR_1,2014-07-29T15:00:00Z,Field1,Initial1,Sub1,true,false,20 -cgrates.org,ALS1,,,,Field2,Initial2,Sub2,false,, +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,ALS1,con1,FLTR_1,2014-07-29T15:00:00Z,Field1,Initial1,Sub1,false,20 +cgrates.org,ALS1,,,,Field2,Initial2,Sub2,, diff --git a/data/tariffplans/tutorial/Attributes.csv b/data/tariffplans/tutorial/Attributes.csv index 02c0dc313..1d5f657a2 100644 --- a/data/tariffplans/tutorial/Attributes.csv +++ b/data/tariffplans/tutorial/Attributes.csv @@ -1,22 +1,22 @@ -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ATTR_1001_SIMPLEAUTH,simpleauth,*string:Account:1001,,Password,*any,CGRateS.org,true,false,20 -cgrates.org,ATTR_1002_SIMPLEAUTH,simpleauth,*string:Account:1002,,Password,*any,CGRateS.org,true,false,20 -cgrates.org,ATTR_1003_SIMPLEAUTH,simpleauth,*string:Account:1003,,Password,*any,CGRateS.org,true,false,20 -cgrates.org,ATTR_1001_SESSIONAUTH,*sessions,*string:Account:1001,,Password,*any,CGRateS.org,true,false,10 -cgrates.org,ATTR_1001_SESSIONAUTH,,,,RequestType,*any,*prepaid,true,, -cgrates.org,ATTR_1001_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,true,, -cgrates.org,ATTR_1001_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,true,, -cgrates.org,ATTR_1002_SESSIONAUTH,*sessions,*string:Account:1002,,Password,*any,CGRateS.org,true,false,10 -cgrates.org,ATTR_1002_SESSIONAUTH,,,,RequestType,*any,*postpaid,true,, -cgrates.org,ATTR_1002_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,true,, -cgrates.org,ATTR_1002_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,true,, -cgrates.org,ATTR_1002_SESSIONAUTH,,,,ResourceAllocation,*any,"ResGroup1",true,, -cgrates.org,ATTR_1003_SESSIONAUTH,*sessions,*string:Account:1003,,Password,*any,CGRateS.org,true,false,10 -cgrates.org,ATTR_1003_SESSIONAUTH,,,,RequestType,*any,*prepaid,true,, -cgrates.org,ATTR_1003_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,true,, -cgrates.org,ATTR_1003_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,true,, -cgrates.com,ATTR_TNT_ALIAS,*any,*string:SubscriberId:1006,,Account,*any,1001,true,false,10 -cgrates.com,ATTR_TNT_ALIAS,*any,,,RequestType,*any,*prepaid,true,, -cgrates.com,ATTR_TNT_ALIAS,*any,,,*tenant,*any,cgrates.org,true,, -cgrates.com,ATTR_TNT_1001,*any,*string:Account:1001,,*tenant,*any,cgrates.org,true,, -cgrates.com,ATTR_TNT_DISC,*any,*string:Account:testDiamInitWithSessionDisconnect,,*tenant,*any,cgrates.org,true,, +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Blocker,Weight +cgrates.org,ATTR_1001_SIMPLEAUTH,simpleauth,*string:Account:1001,,Password,*any,CGRateS.org,false,20 +cgrates.org,ATTR_1002_SIMPLEAUTH,simpleauth,*string:Account:1002,,Password,*any,CGRateS.org,false,20 +cgrates.org,ATTR_1003_SIMPLEAUTH,simpleauth,*string:Account:1003,,Password,*any,CGRateS.org,false,20 +cgrates.org,ATTR_1001_SESSIONAUTH,*sessions,*string:Account:1001,,Password,*any,CGRateS.org,false,10 +cgrates.org,ATTR_1001_SESSIONAUTH,,,,RequestType,*any,*prepaid,, +cgrates.org,ATTR_1001_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,, +cgrates.org,ATTR_1001_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,, +cgrates.org,ATTR_1002_SESSIONAUTH,*sessions,*string:Account:1002,,Password,*any,CGRateS.org,false,10 +cgrates.org,ATTR_1002_SESSIONAUTH,,,,RequestType,*any,*postpaid,, +cgrates.org,ATTR_1002_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,, +cgrates.org,ATTR_1002_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,, +cgrates.org,ATTR_1002_SESSIONAUTH,,,,ResourceAllocation,*any,"ResGroup1",, +cgrates.org,ATTR_1003_SESSIONAUTH,*sessions,*string:Account:1003,,Password,*any,CGRateS.org,false,10 +cgrates.org,ATTR_1003_SESSIONAUTH,,,,RequestType,*any,*prepaid,, +cgrates.org,ATTR_1003_SESSIONAUTH,,,,PaypalAccount,*any,cgrates@paypal.com,, +cgrates.org,ATTR_1003_SESSIONAUTH,,,,LCRProfile,*any,premium_cli,, +cgrates.com,ATTR_TNT_ALIAS,*any,*string:SubscriberId:1006,,Account,*any,1001,false,10 +cgrates.com,ATTR_TNT_ALIAS,*any,,,RequestType,*any,*prepaid,, +cgrates.com,ATTR_TNT_ALIAS,*any,,,*tenant,*any,cgrates.org,, +cgrates.com,ATTR_TNT_1001,*any,*string:Account:1001,,*tenant,*any,cgrates.org,, +cgrates.com,ATTR_TNT_DISC,*any,*string:Account:testDiamInitWithSessionDisconnect,,*tenant,*any,cgrates.org,, diff --git a/dispatchers/attributes_it_test.go b/dispatchers/attributes_it_test.go index cf33fb689..ecbae7780 100755 --- a/dispatchers/attributes_it_test.go +++ b/dispatchers/attributes_it_test.go @@ -121,7 +121,6 @@ func testDspAttrGetAttrFailover(t *testing.T) { FieldName: "Password", Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("CGRateS.org", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20.0, @@ -297,7 +296,6 @@ func testDspAttrTestAuthKey2(t *testing.T) { FieldName: "Password", Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("CGRateS.org", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20.0, @@ -389,7 +387,6 @@ func testDspAttrGetAttrRoundRobin(t *testing.T) { FieldName: "Password", Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("CGRateS.org", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20.0, diff --git a/engine/attributes.go b/engine/attributes.go index 085436c09..f61b02c94 100644 --- a/engine/attributes.go +++ b/engine/attributes.go @@ -158,36 +158,27 @@ func (alS *AttributeService) processEvent(args *AttrArgsProcessEvent) ( MatchedProfiles: []string{attrPrf.ID}, CGREvent: args.Clone(), blocker: attrPrf.Blocker} - for fldName, initialMp := range attrPrf.attributesIdx { - initEvValIf, has := args.Event[fldName] - if !has { - anyInitial, hasAny := initialMp[utils.ANY] - if hasAny && anyInitial.Append { // add field name - substitute, err := anyInitial.Substitute.ParseEvent(args.Event) - if err != nil { - return nil, err - } - rply.CGREvent.Event[fldName] = substitute - rply.AlteredFields = append(rply.AlteredFields, fldName) - } - continue - } - attrVal, has := initialMp[initEvValIf] - if !has { - attrVal, has = initialMp[utils.ANY] - } - if has { - substitute, err := attrVal.Substitute.ParseEvent(args.Event) - if err != nil { + + for _, attribute := range attrPrf.Attributes { + //in case that we have filter for attribute send them to FilterS to be processed + if len(attribute.FilterIDs) != 0 { + if pass, err := alS.filterS.Pass(args.Tenant, attribute.FilterIDs, + config.NewNavigableMap(args.Event)); err != nil { return nil, err + } else if !pass { + continue } - if substitute == utils.META_NONE { - delete(rply.CGREvent.Event, fldName) - } else { - rply.CGREvent.Event[fldName] = substitute - } - rply.AlteredFields = append(rply.AlteredFields, fldName) } + substitute, err := attribute.Substitute.ParseEvent(args.Event) + if err != nil { + return nil, err + } + if substitute == utils.META_NONE { + delete(rply.CGREvent.Event, attribute.FieldName) + } else { + rply.CGREvent.Event[attribute.FieldName] = substitute + } + rply.AlteredFields = append(rply.AlteredFields, attribute.FieldName) } return } diff --git a/engine/attributes_test.go b/engine/attributes_test.go index e6b49f67e..49cb02ba3 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -31,17 +31,7 @@ var ( expTimeAttributes = time.Now().Add(time.Duration(20 * time.Minute)) attrService *AttributeService dmAtr *DataManager - mapSubstitutes = map[string]map[interface{}]*Attribute{ - utils.Account: { - utils.META_ANY: { - FieldName: utils.Account, - Initial: utils.META_ANY, - Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), - Append: true, - }, - }, - } - attrEvs = []*AttrArgsProcessEvent{ + attrEvs = []*AttrArgsProcessEvent{ { Context: utils.StringPointer(utils.MetaSessionS), CGREvent: utils.CGREvent{ //matching AttributeProfile1 @@ -99,13 +89,10 @@ var ( Attributes: []*Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), - Append: true, }, }, - Weight: 20, - attributesIdx: mapSubstitutes, + Weight: 20, }, &AttributeProfile{ Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, @@ -119,13 +106,10 @@ var ( Attributes: []*Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), - Append: true, }, }, - Weight: 20, - attributesIdx: mapSubstitutes, + Weight: 20, }, &AttributeProfile{ Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, @@ -139,13 +123,10 @@ var ( Attributes: []*Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), - Append: true, }, }, - attributesIdx: mapSubstitutes, - Weight: 20, + Weight: 20, }, &AttributeProfile{ Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, @@ -159,13 +140,10 @@ var ( Attributes: []*Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), - Append: true, }, }, - attributesIdx: mapSubstitutes, - Weight: 20, + Weight: 20, }, } ) @@ -433,8 +411,6 @@ func TestAttributeIndexer(t *testing.T) { Attributes: []*Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, - Append: true, Substitute: config.NewRSRParsersMustCompile("1010", true, utils.INFIELD_SEP), }, }, @@ -501,9 +477,7 @@ func TestAttributeProcessWithMultipleRuns1(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10, @@ -519,9 +493,7 @@ func TestAttributeProcessWithMultipleRuns1(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -537,9 +509,7 @@ func TestAttributeProcessWithMultipleRuns1(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field3", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value3", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 30, @@ -615,9 +585,7 @@ func TestAttributeProcessWithMultipleRuns2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10, @@ -633,9 +601,7 @@ func TestAttributeProcessWithMultipleRuns2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -651,9 +617,7 @@ func TestAttributeProcessWithMultipleRuns2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field3", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value3", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 30, @@ -728,9 +692,7 @@ func TestAttributeProcessWithMultipleRuns3(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10, @@ -746,9 +708,7 @@ func TestAttributeProcessWithMultipleRuns3(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -764,9 +724,7 @@ func TestAttributeProcessWithMultipleRuns3(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field3", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value3", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 30, @@ -841,9 +799,7 @@ func TestAttributeProcessWithMultipleRuns4(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10, @@ -859,9 +815,7 @@ func TestAttributeProcessWithMultipleRuns4(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -933,9 +887,7 @@ func TestAttributeMultipleProcessWithBlocker(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 10, @@ -951,9 +903,7 @@ func TestAttributeMultipleProcessWithBlocker(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: true, @@ -970,9 +920,7 @@ func TestAttributeMultipleProcessWithBlocker(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field3", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value3", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 30, @@ -1047,9 +995,7 @@ func TestAttributeMultipleProcessWithBlocker2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field1", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value1", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: true, @@ -1066,9 +1012,7 @@ func TestAttributeMultipleProcessWithBlocker2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value2", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -1084,9 +1028,7 @@ func TestAttributeMultipleProcessWithBlocker2(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field3", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("Value3", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 30, @@ -1160,9 +1102,7 @@ func TestAttributeProcessSubstitute(t *testing.T) { Attributes: []*Attribute{ { FieldName: "Field2", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("~Field1", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: true, diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index 3ccc2f119..a1d1d9efc 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -432,7 +432,6 @@ func testITTestAttributeProfileFilterIndexes(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("Val1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, diff --git a/engine/libattributes.go b/engine/libattributes.go index 4c07333d3..dc9aeb3cf 100644 --- a/engine/libattributes.go +++ b/engine/libattributes.go @@ -26,10 +26,9 @@ import ( ) type Attribute struct { + FilterIDs []string FieldName string - Initial interface{} Substitute config.RSRParsers - Append bool } type AttributeProfile struct { @@ -41,19 +40,6 @@ type AttributeProfile struct { Attributes []*Attribute Blocker bool // blocker flag to stop processing on multiple runs Weight float64 - - attributesIdx map[string]map[interface{}]*Attribute // map[FieldName][InitialValue]*Attribute, used as event match index -} - -// computeAttributesIndex populates .attributes -func (ap *AttributeProfile) computeAttributesIndex() { - ap.attributesIdx = make(map[string]map[interface{}]*Attribute) - for _, attr := range ap.Attributes { - if _, has := ap.attributesIdx[attr.FieldName]; !has { - ap.attributesIdx[attr.FieldName] = make(map[interface{}]*Attribute) - } - ap.attributesIdx[attr.FieldName][attr.Initial] = attr - } } func (ap *AttributeProfile) compileSubstitutes() (err error) { @@ -67,7 +53,6 @@ func (ap *AttributeProfile) compileSubstitutes() (err error) { // Compile is a wrapper for convenience setting up the AttributeProfile func (ap *AttributeProfile) Compile() error { - ap.computeAttributesIndex() return ap.compileSubstitutes() } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 183d905da..23069169c 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -255,9 +255,9 @@ cgrates.org,SPP_1,,,,,supplier1,FLTR_DST_DE,Account2,RPL_3,ResGroup3,Stat2,10,,, cgrates.org,SPP_1,,,,,supplier1,,,,ResGroup4,Stat3,10,,, ` attributeProfiles = ` -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Blocker,Weight -cgrates.org,ALS1,con1,FLTR_1,2014-07-29T15:00:00Z,Field1,Initial1,Sub1,true,true,20 -cgrates.org,ALS1,con2;con3,,,Field2,Initial2,Sub2,false,, +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,FieldName,Substitute,Blocker,Weight +cgrates.org,ALS1,con1,FLTR_1,2014-07-29T15:00:00Z,*string:Field1:Initial,Field1,Sub1,true,20 +cgrates.org,ALS1,con2;con3,,,,Field2,Sub2,true,20 ` chargerProfiles = ` #Tenant,ID,FilterIDs,ActivationInterval,RunID,AttributeIDs,Weight @@ -1479,16 +1479,14 @@ func TestLoadAttributeProfiles(t *testing.T) { }, Attributes: []*utils.TPAttribute{ &utils.TPAttribute{ + FilterIDs: []string{"*string:Field1:Initial"}, FieldName: "Field1", - Initial: "Initial1", Substitute: "Sub1", - Append: true, }, &utils.TPAttribute{ + FilterIDs: []string{}, FieldName: "Field2", - Initial: "Initial2", Substitute: "Sub2", - Append: false, }, }, Blocker: true, diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 4355d8f6c..71f863c68 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2031,6 +2031,7 @@ func (tps TPAttributes) AsTPAttributes() (result []*utils.TPAttributeProfile) { filterMap := make(map[string]utils.StringMap) contextMap := make(map[string]utils.StringMap) for _, tp := range tps { + key := &utils.TenantID{Tenant: tp.Tenant, ID: tp.ID} th, found := mst[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()] if !found { th = &utils.TPAttributeProfile{ @@ -2054,32 +2055,38 @@ func (tps TPAttributes) AsTPAttributes() (result []*utils.TPAttributeProfile) { } } if tp.FilterIDs != "" { - if _, has := filterMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()]; !has { - filterMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()] = make(utils.StringMap) + if _, has := filterMap[key.TenantID()]; !has { + filterMap[key.TenantID()] = make(utils.StringMap) } filterSplit := strings.Split(tp.FilterIDs, utils.INFIELD_SEP) for _, filter := range filterSplit { - filterMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()][filter] = true + filterMap[key.TenantID()][filter] = true } } if tp.Contexts != "" { if _, has := contextMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()]; !has { - contextMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()] = make(utils.StringMap) + contextMap[key.TenantID()] = make(utils.StringMap) } contextSplit := strings.Split(tp.Contexts, utils.INFIELD_SEP) for _, context := range contextSplit { - contextMap[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()][context] = true + contextMap[key.TenantID()][context] = true } } if tp.FieldName != "" { + filterIDs := make([]string, 0) + if tp.AttributeFilterIDs != "" { + filterAttrSplit := strings.Split(tp.AttributeFilterIDs, utils.INFIELD_SEP) + for _, filterAttr := range filterAttrSplit { + filterIDs = append(filterIDs, filterAttr) + } + } th.Attributes = append(th.Attributes, &utils.TPAttribute{ + FilterIDs: filterIDs, FieldName: tp.FieldName, - Initial: tp.Initial, Substitute: tp.Substitute, - Append: tp.Append, }) } - mst[(&utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}).TenantID()] = th + mst[key.TenantID()] = th } result = make([]*utils.TPAttributeProfile, len(mst)) i := 0 @@ -2131,11 +2138,15 @@ func APItoModelTPAttribute(th *utils.TPAttributeProfile) (mdls TPAttributes) { if th.Weight != 0 { mdl.Weight = th.Weight } + for i, val := range reqAttribute.FilterIDs { + if i != 0 { + mdl.AttributeFilterIDs += utils.INFIELD_SEP + } + mdl.AttributeFilterIDs += val + } } mdl.FieldName = reqAttribute.FieldName - mdl.Initial = reqAttribute.Initial mdl.Substitute = reqAttribute.Substitute - mdl.Append = reqAttribute.Append mdls = append(mdls, mdl) } return @@ -2163,9 +2174,8 @@ func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (a return nil, err } attrPrf.Attributes[i] = &Attribute{ - Append: reqAttr.Append, + FilterIDs: reqAttr.FilterIDs, FieldName: reqAttr.FieldName, - Initial: reqAttr.Initial, Substitute: sbstPrsr, } } diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index e1df79cf8..eb8a81018 100644 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -1245,21 +1245,11 @@ func TestAPItoAttributeProfile(t *testing.T) { Attributes: []*utils.TPAttribute{ &utils.TPAttribute{ FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, }, }, Weight: 20, } - mapSubstitutes := make(map[string]map[interface{}]*Attribute) - mapSubstitutes["FL1"] = make(map[interface{}]*Attribute) - mapSubstitutes["FL1"]["In1"] = &Attribute{ - FieldName: "FL1", - Initial: "In1", - Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, - } expected := &AttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", @@ -1271,9 +1261,7 @@ func TestAPItoAttributeProfile(t *testing.T) { Attributes: []*Attribute{ &Attribute{ FieldName: "FL1", - Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -1299,9 +1287,7 @@ func TestAPItoModelTPAttribute(t *testing.T) { Attributes: []*utils.TPAttribute{ &utils.TPAttribute{ FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, }, }, Weight: 20, @@ -1314,9 +1300,7 @@ func TestAPItoModelTPAttribute(t *testing.T) { Contexts: "con1", FilterIDs: "FLTR_ACNT_dan;FLTR_DST_DE", FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, ActivationInterval: "2014-07-14T14:35:00Z", Weight: 20, }, @@ -1336,9 +1320,7 @@ func TestModelAsTPAttribute(t *testing.T) { Contexts: "con1", FilterIDs: "FLTR_ACNT_dan;FLTR_DST_DE", FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, ActivationInterval: "2014-07-14T14:35:00Z", Weight: 20, }, @@ -1355,10 +1337,9 @@ func TestModelAsTPAttribute(t *testing.T) { }, Attributes: []*utils.TPAttribute{ &utils.TPAttribute{ + FilterIDs: []string{}, FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, }, }, Weight: 20, @@ -1375,10 +1356,9 @@ func TestModelAsTPAttribute(t *testing.T) { }, Attributes: []*utils.TPAttribute{ &utils.TPAttribute{ + FilterIDs: []string{}, FieldName: "FL1", - Initial: "In1", Substitute: "Al1", - Append: true, }, }, Weight: 20, diff --git a/engine/models.go b/engine/models.go index 4d885918c..1dfd406ce 100644 --- a/engine/models.go +++ b/engine/models.go @@ -364,12 +364,11 @@ type TPAttribute struct { Contexts string `index:"2" re:""` FilterIDs string `index:"3" re:""` ActivationInterval string `index:"4" re:""` - FieldName string `index:"5" re:""` - Initial string `index:"6" re:""` + AttributeFilterIDs string `index:"5" re:""` + FieldName string `index:"6" re:""` Substitute string `index:"7" re:""` - Append bool `index:"8" re:""` - Blocker bool `index:"9" re:""` - Weight float64 `index:"10" re:"\d+\.?\d*"` + Blocker bool `index:"8" re:""` + Weight float64 `index:"9" re:"\d+\.?\d*"` CreatedAt time.Time } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 58e642289..1db4f9ff1 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -1969,7 +1969,6 @@ func testOnStorITAttributeProfile(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, } attrProfile := &AttributeProfile{ Tenant: "cgrates.org", @@ -1984,7 +1983,6 @@ func testOnStorITAttributeProfile(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -2061,7 +2059,6 @@ func testOnStorITTestAttributeSubstituteIface(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("Val1", true, utils.INFIELD_SEP), - Append: true, } attrProfile := &AttributeProfile{ Tenant: "cgrates.org", @@ -2076,7 +2073,6 @@ func testOnStorITTestAttributeSubstituteIface(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("Val1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -2108,14 +2104,12 @@ func testOnStorITTestAttributeSubstituteIface(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("123.123", true, utils.INFIELD_SEP), - Append: true, } attrProfile.Attributes = []*Attribute{ { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("123.123", true, utils.INFIELD_SEP), - Append: true, }, } if err := onStor.SetAttributeProfile(attrProfile, false); err != nil { @@ -2141,14 +2135,12 @@ func testOnStorITTestAttributeSubstituteIface(t *testing.T) { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("true", true, utils.INFIELD_SEP), - Append: true, } attrProfile.Attributes = []*Attribute{ { FieldName: "FN1", Initial: "Init1", Substitute: config.NewRSRParsersMustCompile("true", true, utils.INFIELD_SEP), - Append: true, }, } if err := onStor.SetAttributeProfile(attrProfile, false); err != nil { diff --git a/engine/version.go b/engine/version.go index 920c64ad7..153ab9b4e 100644 --- a/engine/version.go +++ b/engine/version.go @@ -144,7 +144,7 @@ func CurrentDataDBVersions() Versions { utils.SharedGroups: 2, utils.Thresholds: 3, utils.Suppliers: 1, - utils.Attributes: 2, + utils.Attributes: 3, utils.Timing: 1, utils.RQF: 1, utils.Resource: 1, diff --git a/general_tests/sentinel_it_test.go b/general_tests/sentinel_it_test.go index 6dffda6cd..34180345a 100755 --- a/general_tests/sentinel_it_test.go +++ b/general_tests/sentinel_it_test.go @@ -133,7 +133,6 @@ func testRedisSentinelSetGetAttribute(t *testing.T) { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -169,7 +168,6 @@ func testRedisSentinelInsertion(t *testing.T) { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -255,7 +253,6 @@ func testRedisSentinelGetAttrAfterFailover(t *testing.T) { FieldName: utils.Subject, Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, diff --git a/loaders/loader_test.go b/loaders/loader_test.go index 8a67929f5..a87091492 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -32,9 +32,9 @@ import ( ) func TestLoaderProcessContentSingleFile(t *testing.T) { - attrsCSV := `#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Weight -cgrates.org,TestLoader1,*sessions;*cdrs,*string:Account:1007,2014-01-14T00:00:00Z,Account,*any,1001,false,10 -cgrates.org,TestLoader1,lcr,*string:Account:1008;*string:Account:1009,,Subject,*any,1001,true, + attrsCSV := `#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,FieldName,Substitute,Weight +cgrates.org,TestLoader1,*sessions;*cdrs,*string:Account:1007,2014-01-14T00:00:00Z,*exist:Account:,Account,1001,10 +cgrates.org,TestLoader1,lcr,*string:Account:1008;*string:Account:1009,,,Subject,1001, ` data, _ := engine.NewMapStorage() ldr := &Loader{ @@ -67,26 +67,22 @@ cgrates.org,TestLoader1,lcr,*string:Account:1008;*string:Account:1009,,Subject,* FieldId: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~4", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "FieldName", - FieldId: "FieldName", + &config.FCTemplate{Tag: "AttributeFilterIDs", + FieldId: "AttributeFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~5", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Initial", - FieldId: "Initial", + &config.FCTemplate{Tag: "FieldName", + FieldId: "FieldName", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP)}, &config.FCTemplate{Tag: "Substitute", FieldId: "Substitute", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Append", - FieldId: "Append", - Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, &config.FCTemplate{Tag: "Weight", FieldId: "Weight", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, + Value: config.NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, }, } rdr := ioutil.NopCloser(strings.NewReader(attrsCSV)) @@ -110,16 +106,14 @@ cgrates.org,TestLoader1,lcr,*string:Account:1008;*string:Account:1009,,Subject,* ActivationTime: time.Date(2014, 1, 14, 0, 0, 0, 0, time.UTC)}, Attributes: []*engine.Attribute{ &engine.Attribute{ + FilterIDs: []string{"*exist:Account:"}, FieldName: "Account", - Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: false, }, &engine.Attribute{ + FilterIDs: []string{}, FieldName: "Subject", - Initial: utils.ANY, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }}, Weight: 10.0, } @@ -130,13 +124,13 @@ cgrates.org,TestLoader1,lcr,*string:Account:1008;*string:Account:1009,,Subject,* true, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(eAP.Attributes, ap.Attributes) { - t.Errorf("expecting: %s, received: %s", + t.Errorf("expecting: %s, \n received: %s", utils.ToJSON(eAP), utils.ToJSON(ap)) } } func TestLoaderProcessContentMultiFiles(t *testing.T) { - file1CSV := `ignored,ignored,ignored,ignored,ignored,Subject,*any,1001,ignored,ignored` + file1CSV := `ignored,ignored,ignored,ignored,ignored,,Subject,1001,ignored,ignored` file2CSV := `ignored,TestLoader2` data, _ := engine.NewMapStorage() ldr := &Loader{ @@ -164,19 +158,11 @@ func TestLoaderProcessContentMultiFiles(t *testing.T) { &config.FCTemplate{Tag: "FieldName", FieldId: "FieldName", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~File1.csv:5", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Initial", - FieldId: "Initial", - Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File1.csv:6", true, utils.INFIELD_SEP)}, &config.FCTemplate{Tag: "Substitute", FieldId: "Substitute", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File1.csv:7", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Append", - FieldId: "Append", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("true", true, utils.INFIELD_SEP)}, &config.FCTemplate{Tag: "Weight", FieldId: "Weight", Type: utils.MetaString, @@ -206,9 +192,8 @@ func TestLoaderProcessContentMultiFiles(t *testing.T) { Attributes: []*engine.Attribute{ &engine.Attribute{ FieldName: "Subject", - Initial: utils.ANY, + FilterIDs: []string{}, Substitute: config.NewRSRParsersMustCompile("1001", true, utils.INFIELD_SEP), - Append: true, }}, Weight: 10.0, } @@ -219,7 +204,7 @@ func TestLoaderProcessContentMultiFiles(t *testing.T) { true, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(eAP.Attributes, ap.Attributes) { - t.Errorf("expecting: %s, received: %s", + t.Errorf("expecting: %s, \n received: %s", utils.ToJSON(eAP), utils.ToJSON(ap)) } } diff --git a/migrator/alias.go b/migrator/alias.go index d777e1c9e..4a9352f85 100644 --- a/migrator/alias.go +++ b/migrator/alias.go @@ -102,11 +102,14 @@ func alias2AtttributeProfile(alias *v1Alias, defaultTenant string) *engine.Attri } for fieldname, vals := range av.Pairs { for initial, substitute := range vals { + filterIDs := make([]string, 0) + if initial != utils.META_ANY { + filterIDs = append(filterIDs, utils.MetaString+":"+fieldname+":"+initial) + } out.Attributes = append(out.Attributes, &engine.Attribute{ + FilterIDs: filterIDs, FieldName: fieldname, - Initial: initial, Substitute: config.NewRSRParsersMustCompile(substitute, true, utils.INFIELD_SEP), - Append: true, }) } } diff --git a/migrator/alias_it_test.go b/migrator/alias_it_test.go index f57485f0e..b67bca207 100644 --- a/migrator/alias_it_test.go +++ b/migrator/alias_it_test.go @@ -163,16 +163,14 @@ func testAlsITMigrateAndMove(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, { + FilterIDs: []string{"*string:Category:call_1001"}, FieldName: "Category", - Initial: "call_1001", Substitute: config.NewRSRParsersMustCompile("call_1002", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -215,7 +213,7 @@ func testAlsITMigrateAndMove(t *testing.T) { result.Compile() sort.Slice(result.Attributes, func(i, j int) bool { if result.Attributes[i].FieldName == result.Attributes[j].FieldName { - return result.Attributes[i].Initial.(string) < result.Attributes[j].Initial.(string) + return result.Attributes[i].FilterIDs[0] < result.Attributes[j].FilterIDs[0] } return result.Attributes[i].FieldName < result.Attributes[j].FieldName }) // only for test; map returns random keys @@ -235,7 +233,8 @@ func testAlsITMigrateAndMove(t *testing.T) { "*out:*any:*any:1001:call_1001:*rated": true, }, } - if alsidx, err := alsMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { + if alsidx, err := alsMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], + utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { t.Error(err) } else if !reflect.DeepEqual(expAlsIdx, alsidx) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expAlsIdx), utils.ToJSON(alsidx)) diff --git a/migrator/alias_test.go b/migrator/alias_test.go index 9685af147..ab3243c45 100644 --- a/migrator/alias_test.go +++ b/migrator/alias_test.go @@ -164,10 +164,9 @@ func TestAlias2AtttributeProfile(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -181,16 +180,14 @@ func TestAlias2AtttributeProfile(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, { + FilterIDs: []string{"*string:Account:1003"}, FieldName: "Account", - Initial: "1003", Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -204,16 +201,14 @@ func TestAlias2AtttributeProfile(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, { + FilterIDs: []string{"*string:Account:1003"}, FieldName: "Account", - Initial: "1003", Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -231,16 +226,14 @@ func TestAlias2AtttributeProfile(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, { + FilterIDs: []string{"*string:Subject:1001"}, FieldName: "Subject", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("call_1001", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -258,16 +251,14 @@ func TestAlias2AtttributeProfile(t *testing.T) { ActivationInterval: nil, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:Account:1001"}, FieldName: "Account", - Initial: "1001", Substitute: config.NewRSRParsersMustCompile("1002", true, utils.INFIELD_SEP), - Append: true, }, { + FilterIDs: []string{"*string:Category:call_1001"}, FieldName: "Category", - Initial: "call_1001", Substitute: config.NewRSRParsersMustCompile("call_1002", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -278,7 +269,7 @@ func TestAlias2AtttributeProfile(t *testing.T) { rply := alias2AtttributeProfile(aliases[i], defaultTenant) sort.Slice(rply.Attributes, func(i, j int) bool { if rply.Attributes[i].FieldName == rply.Attributes[j].FieldName { - return rply.Attributes[i].Initial.(string) < rply.Attributes[j].Initial.(string) + return rply.Attributes[i].FilterIDs[0] < rply.Attributes[j].FilterIDs[0] } return rply.Attributes[i].FieldName < rply.Attributes[j].FieldName }) // only for test; map returns random keys diff --git a/migrator/attributes.go b/migrator/attributes.go index 8189bef27..e129a7d7b 100644 --- a/migrator/attributes.go +++ b/migrator/attributes.go @@ -112,6 +112,45 @@ func (m *Migrator) migrateV1Attributes() (err error) { return } +func (m *Migrator) migrateV2Attributes() (err error) { + var v2Attr *v2AttributeProfile + for { + v2Attr, err = m.dmIN.getV2AttributeProfile() + if err != nil && err != utils.ErrNoMoreData { + return err + } + if err == utils.ErrNoMoreData { + break + } + if v2Attr != nil { + attrPrf, err := v2Attr.AsAttributeProfile() + if err != nil { + return err + } + if m.dryRun != true { + if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil { + return err + } + if err := m.dmIN.remV2AttributeProfile(v2Attr.Tenant, v2Attr.ID); err != nil { + return err + } + m.stats[utils.Attributes] += 1 + } + } + } + if m.dryRun != true { + // All done, update version wtih current one + vrs := engine.Versions{utils.Attributes: engine.CurrentDataDBVersions()[utils.Attributes]} + if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating Thresholds version into dataDB", err.Error())) + } + } + return +} + func (m *Migrator) migrateAttributeProfile() (err error) { var vrs engine.Versions current := engine.CurrentDataDBVersions() @@ -140,6 +179,10 @@ func (m *Migrator) migrateAttributeProfile() (err error) { if err := m.migrateV1Attributes(); err != nil { return err } + case 2: + if err := m.migrateV2Attributes(); err != nil { + return err + } } return } @@ -155,18 +198,70 @@ func (v1AttrPrf v1AttributeProfile) AsAttributeProfile() (attrPrf *engine.Attrib } for _, mp := range v1AttrPrf.Attributes { for _, attr := range mp { - initIface := utils.StringToInterface(attr.Initial) + filterIDs := make([]string, 0) + //append false translate to if FieldName exist do stuff + if attr.Append == false { + filterIDs = append(filterIDs, utils.MetaExists+":"+attr.FieldName+":") + } + //Initial not *any translate to if value of fieldName = initial do stuff + if attr.Initial != utils.META_ANY { + filterIDs = append(filterIDs, utils.MetaString+":"+attr.FieldName+":"+attr.Initial) + } sbstPrsr, err := config.NewRSRParsers(attr.Substitute, true, config.CgrConfig().GeneralCfg().RsrSepatarot) if err != nil { return nil, err } attrPrf.Attributes = append(attrPrf.Attributes, &engine.Attribute{ + FilterIDs: filterIDs, FieldName: attr.FieldName, - Initial: initIface, Substitute: sbstPrsr, - Append: attr.Append, }) } } return } + +type v2Attribute struct { + FieldName string + Initial interface{} + Substitute config.RSRParsers + Append bool +} + +type v2AttributeProfile struct { + Tenant string + ID string + Contexts []string // bind this AttributeProfile to multiple contexts + FilterIDs []string + ActivationInterval *utils.ActivationInterval // Activation interval + Attributes []*v2Attribute + Weight float64 +} + +func (v2AttrPrf v2AttributeProfile) AsAttributeProfile() (attrPrf *engine.AttributeProfile, err error) { + attrPrf = &engine.AttributeProfile{ + Tenant: v2AttrPrf.Tenant, + ID: v2AttrPrf.ID, + Contexts: v2AttrPrf.Contexts, + FilterIDs: v2AttrPrf.FilterIDs, + Weight: v2AttrPrf.Weight, + ActivationInterval: v2AttrPrf.ActivationInterval, + } + for _, attr := range v2AttrPrf.Attributes { + filterIDs := make([]string, 0) + //append false translate to if FieldName exist do stuff + if attr.Append == false { + filterIDs = append(filterIDs, utils.MetaExists+":"+attr.FieldName+":") + } + //Initial not *any translate to if value of fieldName = initial do stuff + if attr.Initial.(string) != utils.META_ANY { + filterIDs = append(filterIDs, utils.MetaString+":"+attr.FieldName+":"+attr.Initial.(string)) + } + attrPrf.Attributes = append(attrPrf.Attributes, &engine.Attribute{ + FilterIDs: filterIDs, + FieldName: attr.FieldName, + Substitute: attr.Substitute, + }) + } + return +} diff --git a/migrator/attributes_it_test.go b/migrator/attributes_it_test.go index 616604c08..d1cc0a10a 100755 --- a/migrator/attributes_it_test.go +++ b/migrator/attributes_it_test.go @@ -253,10 +253,9 @@ func testAttrITMigrateAndMove(t *testing.T) { }, Attributes: []*engine.Attribute{ { + FilterIDs: []string{"*string:FL1:In1"}, FieldName: "FL1", - Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, @@ -286,7 +285,7 @@ func testAttrITMigrateAndMove(t *testing.T) { if vrs, err := attrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { t.Error(err) - } else if vrs[utils.Attributes] != 2 { + } else if vrs[utils.Attributes] != 3 { t.Errorf("Unexpected version returned: %d", vrs[utils.Attributes]) } result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", diff --git a/migrator/attributes_test.go b/migrator/attributes_test.go index c2a04423a..b2c850e2c 100644 --- a/migrator/attributes_test.go +++ b/migrator/attributes_test.go @@ -64,10 +64,9 @@ func Testv1AttributeProfileAsAttributeProfile(t *testing.T) { }, Attributes: []*engine.Attribute{ &engine.Attribute{ + FilterIDs: []string{"*string:FL1:In1"}, FieldName: "FL1", - Initial: "In1", Substitute: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP), - Append: true, }, }, Weight: 20, diff --git a/migrator/derived_chargers.go b/migrator/derived_chargers.go index 97b30c9bf..fa12b9384 100644 --- a/migrator/derived_chargers.go +++ b/migrator/derived_chargers.go @@ -84,11 +84,10 @@ func fieldinfo2Attribute(attr []*engine.Attribute, fieldName, fieldInfo string) } return append(attr, &engine.Attribute{ FieldName: fieldName, - Initial: utils.META_ANY, Substitute: rp, - Append: true, }) } + func derivedChargers2AttributeProfile(dc *v1DerivedCharger, tenant, key string, filters []string) (attr *engine.AttributeProfile) { attr = &engine.AttributeProfile{ Tenant: tenant, diff --git a/migrator/derived_chargers_it_test.go b/migrator/derived_chargers_it_test.go index b3a9ca324..4fc80a871 100644 --- a/migrator/derived_chargers_it_test.go +++ b/migrator/derived_chargers_it_test.go @@ -169,15 +169,11 @@ func testDCITMigrateAndMove(t *testing.T) { Attributes: []*engine.Attribute{ { FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), - Append: true, }, { FieldName: utils.Subject, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("call_1003", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -232,9 +228,6 @@ func testDCITMigrateAndMove(t *testing.T) { } result.Compile() sort.Slice(result.Attributes, func(i, j int) bool { - if result.Attributes[i].FieldName == result.Attributes[j].FieldName { - return result.Attributes[i].Initial.(string) < result.Attributes[j].Initial.(string) - } return result.Attributes[i].FieldName < result.Attributes[j].FieldName }) // only for test; map returns random keys if !reflect.DeepEqual(*attrProf, *result) { @@ -257,7 +250,8 @@ func testDCITMigrateAndMove(t *testing.T) { "*out:cgrates.org:*any:1003:*any_0": true, }, } - if dcidx, err := dcMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { + if dcidx, err := dcMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], + utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { t.Error(err) } else if !reflect.DeepEqual(expDcIdx, dcidx) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expDcIdx), utils.ToJSON(dcidx)) @@ -267,7 +261,8 @@ func testDCITMigrateAndMove(t *testing.T) { "*out:cgrates.org:*any:1003:*any_0": true, }, } - if dcidx, err := dcMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.ChargerProfilePrefix], utils.ConcatenatedKey("cgrates.org", utils.META_ANY), + if dcidx, err := dcMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.ChargerProfilePrefix], + utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Errorf("Expected error %v, recived: %v with reply: %v", utils.ErrNotFound, err, utils.ToJSON(dcidx)) } diff --git a/migrator/derived_chargers_test.go b/migrator/derived_chargers_test.go index 1e3ce6189..94e864e4d 100644 --- a/migrator/derived_chargers_test.go +++ b/migrator/derived_chargers_test.go @@ -54,9 +54,7 @@ func TestFieldinfo2Attribute(t *testing.T) { Expected: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP), - Append: true, }, }, }, @@ -67,9 +65,7 @@ func TestFieldinfo2Attribute(t *testing.T) { Expected: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Subject, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile(`~effective_caller_id_number:s/(\d+)/+$1/`, true, utils.INFIELD_SEP), - Append: true, }, }, }, @@ -79,23 +75,17 @@ func TestFieldinfo2Attribute(t *testing.T) { Initial: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP), - Append: true, }, }, Expected: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP), - Append: true, }, &engine.Attribute{ FieldName: utils.Subject, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("call_1003", true, utils.INFIELD_SEP), - Append: true, }, }, }, @@ -134,15 +124,11 @@ func TestDerivedChargers2AttributeProfile(t *testing.T) { Attributes: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Category, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP), - Append: true, }, &engine.Attribute{ FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -169,27 +155,19 @@ func TestDerivedChargers2AttributeProfile(t *testing.T) { Attributes: []*engine.Attribute{ &engine.Attribute{ FieldName: utils.Category, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP), - Append: true, }, &engine.Attribute{ FieldName: utils.Account, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP), - Append: true, }, &engine.Attribute{ FieldName: utils.Subject, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("call_1003_to_1004", true, utils.INFIELD_SEP), - Append: true, }, &engine.Attribute{ FieldName: utils.Destination, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, diff --git a/migrator/migrator_datadb.go b/migrator/migrator_datadb.go index 6bd711493..daec48a12 100644 --- a/migrator/migrator_datadb.go +++ b/migrator/migrator_datadb.go @@ -55,6 +55,9 @@ type MigratorDataDB interface { getV1DerivedChargers() (v1d *v1DerivedChargersWithKey, err error) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err error) remV1DerivedChargers(key string) (err error) + getV2AttributeProfile() (v2attrPrf *v2AttributeProfile, err error) + setV2AttributeProfile(x *v2AttributeProfile) (err error) + remV2AttributeProfile(tenant, id string) (err error) DataManager() *engine.DataManager } diff --git a/migrator/storage_map_datadb.go b/migrator/storage_map_datadb.go index 2b8b4c6f6..56a90a4ac 100755 --- a/migrator/storage_map_datadb.go +++ b/migrator/storage_map_datadb.go @@ -214,3 +214,19 @@ func (v1ms *mapMigrator) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err func (v1ms *mapMigrator) remV1DerivedChargers(key string) (err error) { return utils.ErrNotImplemented } + +//AttributeProfile methods +//get +func (v1ms *mapMigrator) getV2AttributeProfile() (v2attrPrf *v2AttributeProfile, err error) { + return nil, utils.ErrNotImplemented +} + +//set +func (v1ms *mapMigrator) setV2AttributeProfile(x *v2AttributeProfile) (err error) { + return utils.ErrNotImplemented +} + +//rem +func (v1ms *mapMigrator) remV2AttributeProfile(tenant, id string) (err error) { + return utils.ErrNotImplemented +} diff --git a/migrator/storage_mongo_datadb.go b/migrator/storage_mongo_datadb.go index ca798c42d..bd82960e5 100644 --- a/migrator/storage_mongo_datadb.go +++ b/migrator/storage_mongo_datadb.go @@ -522,3 +522,38 @@ func (v1ms *mongoMigrator) remV1DerivedChargers(key string) (err error) { _, err = v1ms.mgoDB.DB().Collection(v1DerivedChargersCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"key": key}) return } + +//AttributeProfile methods +//get +func (v1ms *mongoMigrator) getV2AttributeProfile() (v2attrPrf *v2AttributeProfile, err error) { + if v1ms.cursor == nil { + var cursor mongo.Cursor + cursor, err = v1ms.mgoDB.DB().Collection(v1AttributeProfilesCol).Find(v1ms.mgoDB.GetContext(), bson.D{}) + if err != nil { + return nil, err + } + v1ms.cursor = &cursor + } + if !(*v1ms.cursor).Next(v1ms.mgoDB.GetContext()) { + (*v1ms.cursor).Close(v1ms.mgoDB.GetContext()) + v1ms.cursor = nil + return nil, utils.ErrNoMoreData + } + v2attrPrf = new(v2AttributeProfile) + if err := (*v1ms.cursor).Decode(v2attrPrf); err != nil { + return nil, err + } + return v2attrPrf, nil +} + +//set +func (v1ms *mongoMigrator) setV2AttributeProfile(x *v2AttributeProfile) (err error) { + _, err = v1ms.mgoDB.DB().Collection(v1AttributeProfilesCol).InsertOne(v1ms.mgoDB.GetContext(), x) + return +} + +//rem +func (v1ms *mongoMigrator) remV2AttributeProfile(tenant, id string) (err error) { + _, err = v1ms.mgoDB.DB().Collection(v1AttributeProfilesCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"tenant": tenant, "id": id}) + return +} diff --git a/migrator/storage_redis.go b/migrator/storage_redis.go index 5b60506a4..83ef48a2f 100644 --- a/migrator/storage_redis.go +++ b/migrator/storage_redis.go @@ -654,3 +654,51 @@ func (v1rs *redisMigrator) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (e func (v1rs *redisMigrator) remV1DerivedChargers(key string) (err error) { return v1rs.rds.Cmd("DEL", utils.DERIVEDCHARGERS_PREFIX+key).Err } + +//AttributeProfile methods +//get +func (v1rs *redisMigrator) getV2AttributeProfile() (v2attrPrf *v2AttributeProfile, err error) { + var v2attr *v2AttributeProfile + if v1rs.qryIdx == nil { + v1rs.dataKeys, err = v1rs.rds.GetKeysForPrefix(utils.AttributeProfilePrefix) + if err != nil { + return + } else if len(v1rs.dataKeys) == 0 { + return nil, utils.ErrNotFound + } + v1rs.qryIdx = utils.IntPointer(0) + } + if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 { + strVal, err := v1rs.rds.Cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes() + if err != nil { + return nil, err + } + if err := v1rs.rds.Marshaler().Unmarshal(strVal, &v2attr); err != nil { + return nil, err + } + *v1rs.qryIdx = *v1rs.qryIdx + 1 + } else { + v1rs.qryIdx = nil + return nil, utils.ErrNoMoreData + } + return v2attr, nil +} + +//set +func (v1rs *redisMigrator) setV2AttributeProfile(x *v2AttributeProfile) (err error) { + key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(x.Tenant, x.ID) + bit, err := v1rs.rds.Marshaler().Marshal(x) + if err != nil { + return err + } + if err = v1rs.rds.Cmd("SET", key, bit).Err; err != nil { + return err + } + return +} + +//rem +func (v1rs *redisMigrator) remV2AttributeProfile(tenant, id string) (err error) { + key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(tenant, id) + return v1rs.rds.Cmd("DEL", key).Err +} diff --git a/migrator/user.go b/migrator/user.go index 8216d31c1..c4ee51766 100644 --- a/migrator/user.go +++ b/migrator/user.go @@ -64,9 +64,7 @@ func userProfile2attributeProfile(user *v1UserProfile) (attr *engine.AttributePr if user.Tenant != attr.Tenant { attr.Attributes = append(attr.Attributes, &engine.Attribute{ FieldName: utils.MetaTenant, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile(user.Tenant, true, utils.INFIELD_SEP), - Append: true, }) } for fieldName, substitute := range user.Profile { @@ -79,9 +77,7 @@ func userProfile2attributeProfile(user *v1UserProfile) (attr *engine.AttributePr } attr.Attributes = append(attr.Attributes, &engine.Attribute{ FieldName: fieldName, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile(substitute, true, utils.INFIELD_SEP), - Append: true, }) } return diff --git a/migrator/user_it_test.go b/migrator/user_it_test.go index 346f9fe6a..a97cf9072 100644 --- a/migrator/user_it_test.go +++ b/migrator/user_it_test.go @@ -150,21 +150,15 @@ func testUsrITMigrateAndMove(t *testing.T) { Attributes: []*engine.Attribute{ { FieldName: utils.MetaTenant, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("cgrates.com", true, utils.INFIELD_SEP), - Append: true, }, { FieldName: utils.RequestType, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("*prepaid", true, utils.INFIELD_SEP), - Append: true, }, { FieldName: "msisdn", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("123423534646752", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -206,9 +200,6 @@ func testUsrITMigrateAndMove(t *testing.T) { } result.Compile() sort.Slice(result.Attributes, func(i, j int) bool { - if result.Attributes[i].FieldName == result.Attributes[j].FieldName { - return result.Attributes[i].Initial.(string) < result.Attributes[j].Initial.(string) - } return result.Attributes[i].FieldName < result.Attributes[j].FieldName }) // only for test; map returns random keys if !reflect.DeepEqual(*attrProf, *result) { @@ -224,7 +215,8 @@ func testUsrITMigrateAndMove(t *testing.T) { "1001": true, }, } - if usridx, err := usrMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { + if usridx, err := usrMigrator.dmOut.DataManager().GetFilterIndexes(utils.PrefixToIndexCache[utils.AttributeProfilePrefix], + utils.ConcatenatedKey("cgrates.org", utils.META_ANY), utils.MetaString, nil); err != nil { t.Error(err) } else if !reflect.DeepEqual(expUsrIdx, usridx) { t.Errorf("Expected %v, recived: %v", utils.ToJSON(expUsrIdx), utils.ToJSON(usridx)) diff --git a/migrator/user_test.go b/migrator/user_test.go index 292f462a9..b9f94c778 100644 --- a/migrator/user_test.go +++ b/migrator/user_test.go @@ -93,15 +93,11 @@ func TestUserProfile2attributeProfile(t *testing.T) { Attributes: []*engine.Attribute{ { FieldName: utils.MetaTenant, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile(usrTenant, true, utils.INFIELD_SEP), - Append: true, }, { FieldName: "Subject", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("call_1001", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -116,15 +112,11 @@ func TestUserProfile2attributeProfile(t *testing.T) { Attributes: []*engine.Attribute{ { FieldName: utils.RequestType, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("*prepaid", true, utils.INFIELD_SEP), - Append: true, }, { FieldName: "msisdn", - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("123423534646752", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -139,15 +131,11 @@ func TestUserProfile2attributeProfile(t *testing.T) { Attributes: []*engine.Attribute{ { FieldName: utils.MetaTenant, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile(usrTenant, true, utils.INFIELD_SEP), - Append: true, }, { FieldName: utils.RequestType, - Initial: utils.META_ANY, Substitute: config.NewRSRParsersMustCompile("*prepaid", true, utils.INFIELD_SEP), - Append: true, }, }, Blocker: false, @@ -158,7 +146,7 @@ func TestUserProfile2attributeProfile(t *testing.T) { rply := userProfile2attributeProfile(users[i]) sort.Slice(rply.Attributes, func(i, j int) bool { if rply.Attributes[i].FieldName == rply.Attributes[j].FieldName { - return rply.Attributes[i].Initial.(string) < rply.Attributes[j].Initial.(string) + return rply.Attributes[i].FilterIDs[0] < rply.Attributes[j].FilterIDs[0] } return rply.Attributes[i].FieldName < rply.Attributes[j].FieldName }) // only for test; map returns random keys diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 7864d8f9a..bcac2622c 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1131,10 +1131,9 @@ type TPSupplierProfile struct { } type TPAttribute struct { + FilterIDs []string FieldName string - Initial string Substitute string - Append bool } type TPAttributeProfile struct {