From 35fe328044267a0addda0701cd8457af732778c6 Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 26 Aug 2020 14:51:37 +0300 Subject: [PATCH] Add integration test for LoaderS with *template --- apier/v1/preload_it_test.go | 2 +- .../samples/loaders/tutinternal/cgrates.json | 37 ++++++++++++++++- .../samples/loaders/tutmongo/cgrates.json | 41 ++++++++++++++++++- .../samples/loaders/tutmysql/cgrates.json | 41 ++++++++++++++++++- loaders/lib_test.go | 2 +- loaders/loader_it_test.go | 36 ++++++++++++++++ 6 files changed, 154 insertions(+), 5 deletions(-) diff --git a/apier/v1/preload_it_test.go b/apier/v1/preload_it_test.go index 22ba40658..d04b12985 100644 --- a/apier/v1/preload_it_test.go +++ b/apier/v1/preload_it_test.go @@ -65,7 +65,7 @@ func TestPreload(t *testing.T) { func testCreateDirs(t *testing.T) { for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} { + "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut"} { if err := os.RemoveAll(dir); err != nil { t.Fatal("Error removing folder: ", dir, err) } diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json index ee3733703..610fe8ecd 100644 --- a/data/conf/samples/loaders/tutinternal/cgrates.json +++ b/data/conf/samples/loaders/tutinternal/cgrates.json @@ -167,6 +167,27 @@ ], }, ], + }, + { + "id": "LoaderWithTemplate", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "tenant": "cgrates.org", + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": ["*internal"], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/templateLoaderIn", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/templateLoaderOut", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} + ], + }, + ], } ], @@ -183,7 +204,21 @@ "attributes_conns": ["*internal"], }, - +"templates": { + "attrTemplateLoader": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], +}, "apiers": { "enabled": true, diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json index 7c6da6fb9..f60099434 100644 --- a/data/conf/samples/loaders/tutmongo/cgrates.json +++ b/data/conf/samples/loaders/tutmongo/cgrates.json @@ -207,6 +207,27 @@ ], }, ], + }, + { + "id": "LoaderWithTemplate", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "tenant": "cgrates.org", + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": ["*internal"], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/templateLoaderIn", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/templateLoaderOut", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} + ], + }, + ], } ], @@ -230,7 +251,25 @@ }, -"apiers": { +"templates": { + "attrTemplateLoader": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], +}, + + + + "apiers": { "enabled": true, "scheduler_conns": ["*internal"], }, diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json index 9e7332ff7..2f35e5857 100644 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -168,6 +168,27 @@ ], }, ], + }, + { + "id": "LoaderWithTemplate", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "tenant": "cgrates.org", + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": ["*internal"], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/templateLoaderIn", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/templateLoaderOut", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} + ], + }, + ], } ], @@ -229,7 +250,25 @@ }, -"apiers": { +"templates": { + "attrTemplateLoader": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], +}, + + + + "apiers": { "enabled": true, "scheduler_conns": ["*internal"], }, diff --git a/loaders/lib_test.go b/loaders/lib_test.go index 990f1b4e1..3c0c1f488 100644 --- a/loaders/lib_test.go +++ b/loaders/lib_test.go @@ -36,7 +36,7 @@ var ( ) var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} + "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut"} func newRPCClient(cfg *config.ListenCfg) (c *rpc.Client, err error) { switch *encoding { diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index 54734ce41..2842d2d3e 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -65,6 +65,11 @@ var ( testLoaderLoadAttributesWithoutSubpathWithMove, testLoaderVerifyOutDirWithSubpathWithMove, testLoaderCheckAttributes, + testLoaderResetDataDB, + testLoaderPopulateDataForTemplateLoader, + testLoaderLoadAttributesForTemplateLoader, + testLoaderVerifyOutDirForTemplateLoader, + testLoaderCheckAttributes, testLoaderKillEngine, } ) @@ -299,6 +304,37 @@ func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) { } } +func testLoaderPopulateDataForTemplateLoader(t *testing.T) { + fileName := utils.AttributesCsv + tmpFilePath := path.Join("/tmp/", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil { + t.Fatal(err.Error()) + } + if err := os.MkdirAll("/tmp/templateLoaderIn", 0755); err != nil { + t.Fatal("Error creating folder: /tmp/templateLoaderIn", err) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/templateLoaderIn", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func testLoaderLoadAttributesForTemplateLoader(t *testing.T) { + var reply string + if err := loaderRPC.Call(utils.LoaderSv1Load, + &ArgsProcessFolder{LoaderID: "LoaderWithTemplate"}, &reply); err != nil { + t.Error(err) + } +} + +func testLoaderVerifyOutDirForTemplateLoader(t *testing.T) { + time.Sleep(100 * time.Millisecond) + if outContent1, err := ioutil.ReadFile(path.Join("/tmp/templateLoaderOut", utils.AttributesCsv)); err != nil { + t.Error(err) + } else if engine.AttributesCSVContent != string(outContent1) { + t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1)) + } +} + func testLoaderKillEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { t.Error(err)