diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json new file mode 100644 index 000000000..ae0280593 --- /dev/null +++ b/data/conf/samples/loaders/tutmongo/cgrates.json @@ -0,0 +1,182 @@ +{ +// CGRateS Configuration file + + +"general": { + "log_level": 7, + "reply_timeout": "30s", +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017, +}, + + +"stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017, +}, + + +"rals": { + "enabled": true, + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"scheduler": { + "enabled": true, + "cdrs_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"cdrs": { + "enabled": true, +}, + + +"cdre": { + "TestTutITExportCDR": { + "content_fields": [ + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "~RunID"}, + {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, + {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, + {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, + {"tag":"Category", "type": "*composed", "value": "~Category"}, + {"tag":"Account", "type": "*composed", "value": "~Account"}, + {"tag":"Destination", "type": "*composed", "value": "~Destination"}, + {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag":"Usage", "type": "*composed", "value": "~Usage"}, + {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, + {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, + ], + }, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": [ + {"address": "*internal"} + ], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"suppliers": { + "enabled": true, + "stats_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], + "resources_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"attributes": { // Attribute service + "enabled": true, // starts Attribute service: . +}, + + +"loaders": [ + { + "id": "CustomLoader", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/In", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/Out", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "~2"}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~3"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~4"}, + {"tag": "AttributeFilterIDs", "field_id": "AttributeFilterIDs", "type": "*composed", "value": "~5"}, + {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "~6"}, + {"tag": "Type", "field_id": "Type", "type": "*composed", "value": "~7"}, + {"tag": "Value", "field_id": "Value", "type": "*composed", "value": "~8"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~10"}, + ], + }, + ], + }, +], + + +"sessions": { + "enabled": true, + "rals_conns": [ + {"address": "*internal"} + ], + "cdrs_conns": [ + {"address": "*internal"} + ], + "chargers_conns": [ + {"address": "*internal"} + ], +}, + + +"migrator": { + "out_datadb_type": "mongo", + "out_datadb_port": "27017", + "out_datadb_name": "10", + "out_stordb_type": "mongo", + "out_stordb_port": "27017", + "out_stordb_name": "cgrates", + "users_filters":["Account"], +}, + + +} diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json new file mode 100644 index 000000000..cb6125280 --- /dev/null +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -0,0 +1,186 @@ +{ +// CGRateS Configuration file +// + + +"general": { + "log_level": 7, + "reply_timeout": "50s", +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + +"data_db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to +}, + +"stor_db": { + "db_password": "CGRateS.org", +}, + + +"rals": { + "enabled": true, + "thresholds_conns": [ + {"address": "*internal"}, + ], +}, + + +"scheduler": { + "enabled": true, + "cdrs_conns": [ + {"address": "*internal"}, + ], +}, + + +"cdrs": { + "enabled": true, + "chargers_conns":[ + {"address": "*internal"}, + ], +}, + + +"cdre": { + "TestTutITExportCDR": { + "content_fields": [ + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "~RunID"}, + {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, + {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, + {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, + {"tag":"Category", "type": "*composed", "value": "~Category"}, + {"tag":"Account", "type": "*composed", "value": "~Account"}, + {"tag":"Destination", "type": "*composed", "value": "~Destination"}, + {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag":"Usage", "type": "*composed", "value": "~Usage"}, + {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, + {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, + ], + }, +}, + +"loaders": [ + { + "id": "CustomLoader", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/In", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/Out", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "~0", "mandatory": true}, + {"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "~1", "mandatory": true}, + {"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "~2"}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "~3"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "~4"}, + {"tag": "AttributeFilterIDs", "field_id": "AttributeFilterIDs", "type": "*composed", "value": "~5"}, + {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "~6"}, + {"tag": "Type", "field_id": "Type", "type": "*composed", "value": "~7"}, + {"tag": "Value", "field_id": "Value", "type": "*composed", "value": "~8"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "~9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "~10"}, + ], + }, + ], + }, +], + + + +"attributes": { + "enabled": true, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": [ + {"address": "*internal"}, + ], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"suppliers": { + "enabled": true, + "prefix_indexed_fields":["Destination"], + "stats_conns": [ + {"address": "*internal"}, + ], + "resources_conns": [ + {"address": "*internal"}, + ], +}, + + +"sessions": { + "enabled": true, + "suppliers_conns": [ + {"address": "*internal"} + ], + "resources_conns": [ + {"address": "*internal"} + ], + "attributes_conns": [ + {"address": "*internal"} + ], + "rals_conns": [ + {"address": "*internal"}, + ], + "cdrs_conns": [ + {"address": "*internal"} + ], + "chargers_conns": [ + {"address": "*internal"}, + ], +}, + + +"migrator":{ + "out_stordb_password": "CGRateS.org", + "users_filters":["Account"], +}, + + +} diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index aef0c1f6a..98c8847ec 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -46,10 +46,10 @@ var ( var sTestsLoader = []func(t *testing.T){ testLoaderInitCfg, + testLoaderMakeFolders, testLoaderResetDataDB, testLoaderStartEngine, testLoaderRPCConn, - testLoaderMakeFolders, testLoaderPopulateData, testLoaderLoadAttributes, testLoaderVerifyOutDir, @@ -74,7 +74,7 @@ func TestLoaderITMongo(t *testing.T) { func testLoaderInitCfg(t *testing.T) { var err error - loaderCfgPath = path.Join(loaderDataDir, "conf", "samples", loaderConfigDIR) + loaderCfgPath = path.Join(loaderDataDir, "conf", "samples", "loaders", loaderConfigDIR) loaderCfg, err = config.NewCGRConfigFromPath(loaderCfgPath) if err != nil { t.Error(err) @@ -83,6 +83,25 @@ func testLoaderInitCfg(t *testing.T) { config.SetCgrConfig(loaderCfg) } +func testLoaderMakeFolders(t *testing.T) { + // active the loaders here + for _, ldr := range loaderCfg.LoaderCfg() { + if ldr.Id == "CustomLoader" { + for _, dir := range []string{ldr.TpInDir, ldr.TpOutDir} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal("Error creating folder: ", dir, err) + } + } + loaderPathIn = ldr.TpInDir + loaderPathOut = ldr.TpOutDir + } + } + +} + // Wipe out the cdr database func testLoaderResetDataDB(t *testing.T) { if err := engine.InitDataDb(loaderCfg); err != nil { @@ -106,19 +125,6 @@ func testLoaderRPCConn(t *testing.T) { } } -func testLoaderMakeFolders(t *testing.T) { - for _, dir := range []string{loaderCfg.LoaderCfg()[1].TpInDir, loaderCfg.LoaderCfg()[1].TpOutDir} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - loaderPathIn = loaderCfg.LoaderCfg()[1].TpInDir - loaderPathOut = loaderCfg.LoaderCfg()[1].TpOutDir -} - func testLoaderPopulateData(t *testing.T) { fileName := utils.AttributesCsv tmpFilePath := path.Join("/tmp", fileName)