From 1fd69c1acea73bb1d74503eec4625553bc55c2b3 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 15 Mar 2019 15:01:53 +0200 Subject: [PATCH] Updated precache tests --- apier/v1/precache_it_test.go | 26 +- .../samples/precache/tutmongo/cgrates.json | 165 ++++++++++ .../samples/precache/tutmysql/cgrates.json | 292 ++++++++++++++++++ 3 files changed, 470 insertions(+), 13 deletions(-) create mode 100644 data/conf/samples/precache/tutmongo/cgrates.json create mode 100644 data/conf/samples/precache/tutmysql/cgrates.json diff --git a/apier/v1/precache_it_test.go b/apier/v1/precache_it_test.go index d06f8f173..bb22550ac 100644 --- a/apier/v1/precache_it_test.go +++ b/apier/v1/precache_it_test.go @@ -71,7 +71,7 @@ func TestPrecacheITMongo(t *testing.T) { func testPrecacheInitCfg(t *testing.T) { var err error - precacheCfgPath = path.Join(precacheDataDir, "conf", "samples", precacheConfigDIR) + precacheCfgPath = path.Join(precacheDataDir, "conf", "samples", "precache", precacheConfigDIR) precacheCfg, err = config.NewCGRConfigFromPath(precacheCfgPath) if err != nil { t.Error(err) @@ -300,7 +300,7 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "attribute_filter_indexes": { - Items: 0, + Items: 2, Groups: 0, }, "attribute_profiles": { @@ -336,19 +336,19 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "filters": { - Items: 15, // expected to have 16 items + Items: 15, Groups: 0, }, "rating_plans": { - Items: 4, // expected to have 4 items + Items: 4, Groups: 0, }, "rating_profiles": { - Items: 5, // expected to have 10 items + Items: 5, Groups: 0, }, "resource_filter_indexes": { - Items: 0, + Items: 6, Groups: 0, }, "resource_profiles": { @@ -356,7 +356,7 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "resources": { - Items: 3, //expected to have 3 items + Items: 3, Groups: 0, }, "reverse_destinations": { @@ -372,7 +372,7 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "stat_filter_indexes": { - Items: 0, + Items: 2, Groups: 0, }, "statqueue_profiles": { @@ -380,19 +380,19 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "statqueues": { - Items: 1, // expected to have 1 item + Items: 1, Groups: 0, }, "supplier_filter_indexes": { - Items: 0, + Items: 6, Groups: 0, }, "supplier_profiles": { - Items: 3, // expected to have 3 items + Items: 3, Groups: 0, }, "threshold_filter_indexes": { - Items: 0, + Items: 10, Groups: 0, }, "threshold_profiles": { @@ -400,7 +400,7 @@ func testPrecacheGetCacheStatsAfterRestart(t *testing.T) { Groups: 0, }, "thresholds": { - Items: 7, // expected to have 7 items + Items: 7, Groups: 0, }, "timings": { diff --git a/data/conf/samples/precache/tutmongo/cgrates.json b/data/conf/samples/precache/tutmongo/cgrates.json new file mode 100644 index 000000000..3d4605aca --- /dev/null +++ b/data/conf/samples/precache/tutmongo/cgrates.json @@ -0,0 +1,165 @@ +{ +// CGRateS Configuration file + + +"general": { + "log_level": 7, + "reply_timeout": "30s", +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017, +}, + + +"stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017, +}, + + +"cache":{ + "destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "rating_plans": {"limit": 10000, "ttl":"0s","precache": true}, + "rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "actions": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "account_action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_triggers": {"limit": 10000, "ttl":"0s", "precache": true}, + "shared_groups": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resources": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueues": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueue_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "thresholds": {"limit": 10000, "ttl":"0s", "precache": true}, + "threshold_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "filters": {"limit": 10000, "ttl":"0s", "precache": true}, + "supplier_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "attribute_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_filter_indexes" :{"limit": 10000, "ttl":"0s","precache": true}, + "stat_filter_indexes" : {"limit": 10000, "ttl":"0s","precache": true}, + "threshold_filter_indexes" : {"limit": 10000, "ttl":"0s","precache": true}, + "supplier_filter_indexes" : {"limit": 10000, "ttl":"0s","precache": true}, + "attribute_filter_indexes" : {"limit": 10000, "ttl":"0s","precache": true}, + "charger_filter_indexes" : {"limit": 10000, "ttl":"0s","precache": true}, +}, + + +"rals": { + "enabled": true, + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"scheduler": { + "enabled": true, + "cdrs_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"cdrs": { + "enabled": true, +}, + + +"cdre": { + "TestTutITExportCDR": { + "content_fields": [ + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "~RunID"}, + {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, + {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, + {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, + {"tag":"Category", "type": "*composed", "value": "~Category"}, + {"tag":"Account", "type": "*composed", "value": "~Account"}, + {"tag":"Destination", "type": "*composed", "value": "~Destination"}, + {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag":"Usage", "type": "*composed", "value": "~Usage"}, + {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, + {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, + ], + }, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": [ + {"address": "*internal"} + ], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"suppliers": { + "enabled": true, + "stats_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], + "resources_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"attributes": { // Attribute service + "enabled": true, // starts Attribute service: . +}, + + +"sessions": { + "enabled": true, +}, + + +"migrator": { + "out_datadb_type": "mongo", + "out_datadb_port": "27017", + "out_datadb_name": "10", + "out_stordb_type": "mongo", + "out_stordb_port": "27017", + "out_stordb_name": "cgrates", + "users_filters":["Account"], +}, + + +} diff --git a/data/conf/samples/precache/tutmysql/cgrates.json b/data/conf/samples/precache/tutmysql/cgrates.json new file mode 100644 index 000000000..45f999317 --- /dev/null +++ b/data/conf/samples/precache/tutmysql/cgrates.json @@ -0,0 +1,292 @@ +{ +// CGRateS Configuration file +// + + +"general": { + "log_level": 7, +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + +"data_db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to +}, + +"stor_db": { + "db_password": "CGRateS.org", +}, + + +"cache":{ + "destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "rating_plans": {"limit": 10000, "ttl":"0s","precache": true}, + "rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "actions": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "account_action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_triggers": {"limit": 10000, "ttl":"0s", "precache": true}, + "shared_groups": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resources": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueues": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueue_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "thresholds": {"limit": 10000, "ttl":"0s", "precache": true}, + "threshold_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "filters": {"limit": 10000, "ttl":"0s", "precache": true}, + "supplier_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "attribute_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_filter_indexes" :{"limit": 10000, "ttl":"0s", "precache": true}, + "stat_filter_indexes" : {"limit": 10000, "ttl":"0s", "precache": true}, + "threshold_filter_indexes" : {"limit": 10000, "ttl":"0s", "precache": true}, + "supplier_filter_indexes" : {"limit": 10000, "ttl":"0s", "precache": true}, + "attribute_filter_indexes" : {"limit": 10000, "ttl":"0s", "precache": true}, + "charger_filter_indexes" : {"limit": 10000, "ttl":"0s", "precache": true}, +}, + + +"rals": { + "enabled": true, + "thresholds_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"scheduler": { + "enabled": true, + "cdrs_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"cdrs": { + "enabled": true, + "chargers_conns":[ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"cdre": { + "TestTutITExportCDR": { + "content_fields": [ + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "~RunID"}, + {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, + {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, + {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, + {"tag":"Category", "type": "*composed", "value": "~Category"}, + {"tag":"Account", "type": "*composed", "value": "~Account"}, + {"tag":"Destination", "type": "*composed", "value": "~Destination"}, + {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag":"Usage", "type": "*composed", "value": "~Usage"}, + {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, + {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, + ], + }, +}, + +"loaders": [ + { + "id": "TeoLoader", // identifier of the Loader + "enabled": false, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/In", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/Out", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*resources", // data source type + "file_name": "Resources.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~3"}, + {"tag": "TTL", "field_id": "UsageTTL", "type": "*composed", "value": "~4"}, + {"tag": "Limit", "field_id": "Limit", "type": "*composed", "value": "~5"}, + {"tag": "AllocationMessage", "field_id": "AllocationMessage", "type": "*composed", "value": "~6"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~7"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "~8"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~9"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "~10"}, + ], + }, + { + "type": "*stats", // data source type + "file_name": "Stats.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~3"}, + {"tag": "QueueLength", "field_id": "QueueLength", "type": "*composed", "value": "~4"}, + {"tag": "TTL", "field_id": "TTL", "type": "*composed", "value": "~5"}, + {"tag": "Metrics", "field_id": "Metrics", "type": "*composed", "value": "~6"}, + {"tag": "MetricParams", "field_id": "Parameters", "type": "*composed", "value": "~7"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~8"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "~9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~10"}, + {"tag": "MinItems", "field_id": "MinItems", "type": "*composed", "value": "~11"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "~12"}, + ], + }, + { + "type": "*thresholds", // data source type + "file_name": "Thresholds.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~3"}, + {"tag": "MaxHits", "field_id": "MaxHits", "type": "*composed", "value": "~4"}, + {"tag": "MinHits", "field_id": "MinHits", "type": "*composed", "value": "~5"}, + {"tag": "MinSleep", "field_id": "MinSleep", "type": "*composed", "value": "~6"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~7"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~8"}, + {"tag": "ActionIDs", "field_id": "ActionIDs", "type": "*composed", "value": "~9"}, + {"tag": "Async", "field_id": "Async", "type": "*composed", "value": "~10"}, + ], + }, + { + "type": "*suppliers", // data source type + "file_name": "Suppliers.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~3"}, + {"tag": "Sorting", "field_id": "Sorting", "type": "*composed", "value": "~4"}, + {"tag": "SortingParamameters", "field_id": "SortingParamameters", "type": "*composed", "value": "~5"}, + {"tag": "SupplierID", "field_id": "SupplierID", "type": "*composed", "value": "~6"}, + {"tag": "SupplierFilterIDs", "field_id": "SupplierFilterIDs", "type": "*composed", "value": "~7"}, + {"tag": "SupplierAccountIDs", "field_id": "SupplierAccountIDs", "type": "*composed", "value": "~8"}, + {"tag": "SupplierRatingPlanIDs", "field_id": "SupplierRatingPlanIDs", "type": "*composed", "value": "~9"}, + {"tag": "SupplierResourceIDs", "field_id": "SupplierResourceIDs", "type": "*composed", "value": "~10"}, + {"tag": "SupplierStatIDs", "field_id": "SupplierStatIDs", "type": "*composed", "value": "~11"}, + {"tag": "SupplierWeight", "field_id": "SupplierWeight", "type": "*composed", "value": "~12"}, + {"tag": "SupplierBlocker", "field_id": "SupplierBlocker", "type": "*composed", "value": "~13"}, + {"tag": "SupplierParameters", "field_id": "SupplierParameters", "type": "*composed", "value": "~14"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~15"}, + ], + }, + ], + }, + { + "id": "FilterLoader", // identifier of the Loader + "enabled": false, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/FilterIn", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/FilterOut", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*filters", // data source type + "file_name": "Filters.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "FilterType", "field_id": "FilterType", "type": "*composed", "value": "~2"}, + {"tag": "FilterFieldName", "field_id": "FilterFieldName", "type": "*composed", "value": "~3"}, + {"tag": "FilterFieldValues", "field_id": "FilterFieldValues", "type": "*composed", "value": "~4"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~5"}, + ], + }, + ], + }, +], + + + +"attributes": { + "enabled": true, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"suppliers": { + "enabled": true, + "prefix_indexed_fields":["Destination"], + "stats_conns": [ + {"address": "*internal"}, + ], + "resources_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"sessions": { + "enabled": true, + "suppliers_conns": [ + {"address": "*internal"} + ], + "resources_conns": [ + {"address": "*internal"} + ], + "attributes_conns": [ + {"address": "*internal"} + ], + "rals_conns": [ + {"address": "*internal"} + ], +}, + + +"migrator":{ + "out_stordb_password": "CGRateS.org", + "users_filters":["Account"], +}, + + +}