From 04a6e94a23df25c3a144e77a0a71a6afdc7a3894 Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 5 Aug 2020 14:45:39 +0300 Subject: [PATCH] Add integration test for LoaderS for sub_path in fileName --- .../samples/loaders/tutinternal/cgrates.json | 62 +++++++++++++++ .../samples/loaders/tutmongo/cgrates.json | 62 +++++++++++++++ .../samples/loaders/tutmysql/cgrates.json | 62 +++++++++++++++ loaders/lib_test.go | 3 + loaders/loader_it_test.go | 77 ++++++++++++++++++- 5 files changed, 264 insertions(+), 2 deletions(-) diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json index a738c8c4c..ee3733703 100644 --- a/data/conf/samples/loaders/tutinternal/cgrates.json +++ b/data/conf/samples/loaders/tutinternal/cgrates.json @@ -105,6 +105,68 @@ ], }, ], + }, + { + "id": "SubpathLoaderWithoutMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathWithoutMove", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + }, + { + "id": "SubpathLoaderWithMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathLoaderWithMove", + "tp_out_dir": "/tmp/SubpathOut", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], } ], diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json index 83c046988..7c6da6fb9 100644 --- a/data/conf/samples/loaders/tutmongo/cgrates.json +++ b/data/conf/samples/loaders/tutmongo/cgrates.json @@ -145,6 +145,68 @@ ], }, ], + }, + { + "id": "SubpathLoaderWithoutMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathWithoutMove", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + }, + { + "id": "SubpathLoaderWithMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathLoaderWithMove", + "tp_out_dir": "/tmp/SubpathOut", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], } ], diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json index b030482e4..9e7332ff7 100644 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -106,6 +106,68 @@ ], }, ], + }, + { + "id": "SubpathLoaderWithoutMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathWithoutMove", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + }, + { + "id": "SubpathLoaderWithMove", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/SubpathLoaderWithMove", + "tp_out_dir": "/tmp/SubpathOut", + "data":[ + { + "type": "*attributes", + "file_name": "folder1/Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], } ], diff --git a/loaders/lib_test.go b/loaders/lib_test.go index 89229649b..990f1b4e1 100644 --- a/loaders/lib_test.go +++ b/loaders/lib_test.go @@ -35,6 +35,9 @@ var ( dbType = flag.String("dbtype", utils.MetaInternal, "The type of DataBase (Internal/Mongo/mySql)") ) +var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", + "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} + func newRPCClient(cfg *config.ListenCfg) (c *rpc.Client, err error) { switch *encoding { case utils.MetaJSON: diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index 3589011bf..54734ce41 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -55,6 +55,16 @@ var ( testLoaderLoadAttributesWithoutMoving, testLoaderVerifyOutDirWithoutMoving, testLoaderCheckAttributes, + testLoaderResetDataDB, + testLoaderPopulateDataWithSubpath, + testLoaderLoadAttributesWithSubpath, + testLoaderVerifyOutDirWithSubpath, + testLoaderCheckAttributes, + testLoaderResetDataDB, + testLoaderPopulateDataWithSubpathWithMove, + testLoaderLoadAttributesWithoutSubpathWithMove, + testLoaderVerifyOutDirWithSubpathWithMove, + testLoaderCheckAttributes, testLoaderKillEngine, } ) @@ -74,7 +84,6 @@ func TestLoaderIT(t *testing.T) { t.Fatal("Unknown Database type") } - loaderCfgDIR = "tutmysql" for _, stest := range sTestsLoader { t.Run(loaderCfgDIR, stest) } @@ -93,7 +102,7 @@ func testLoaderInitCfg(t *testing.T) { func testLoaderMakeFolders(t *testing.T) { // active the loaders here - for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn"} { + for _, dir := range loaderPaths { if err := os.RemoveAll(dir); err != nil { t.Fatal("Error removing folder: ", dir, err) } @@ -108,6 +117,7 @@ func testLoaderResetDataDB(t *testing.T) { if err := engine.InitDataDb(loaderCfg); err != nil { t.Fatal(err) } + engine.Cache.Clear(nil) } // Start CGR Engine @@ -226,6 +236,69 @@ func testLoaderVerifyOutDirWithoutMoving(t *testing.T) { } } +func testLoaderPopulateDataWithSubpath(t *testing.T) { + fileName := utils.AttributesCsv + tmpFilePath := path.Join("/tmp/", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil { + t.Fatal(err.Error()) + } + if err := os.MkdirAll("/tmp/SubpathWithoutMove/folder1", 0755); err != nil { + t.Fatal("Error creating folder: /tmp/SubpathWithoutMove/folder1", err) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathWithoutMove/folder1", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func testLoaderLoadAttributesWithSubpath(t *testing.T) { + var reply string + if err := loaderRPC.Call(utils.LoaderSv1Load, + &ArgsProcessFolder{LoaderID: "SubpathLoaderWithoutMove"}, &reply); err != nil { + t.Error(err) + } +} + +func testLoaderVerifyOutDirWithSubpath(t *testing.T) { + time.Sleep(100 * time.Millisecond) + // we expect that after the LoaderS process the file leave in in the input folder + if outContent1, err := ioutil.ReadFile(path.Join("/tmp/SubpathWithoutMove/folder1", utils.AttributesCsv)); err != nil { + t.Error(err) + } else if engine.AttributesCSVContent != string(outContent1) { + t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1)) + } +} + +func testLoaderPopulateDataWithSubpathWithMove(t *testing.T) { + fileName := utils.AttributesCsv + tmpFilePath := path.Join("/tmp/", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil { + t.Fatal(err.Error()) + } + if err := os.MkdirAll("/tmp/SubpathLoaderWithMove/folder1", 0755); err != nil { + t.Fatal("Error creating folder: /tmp/SubpathLoaderWithMove/folder1", err) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathLoaderWithMove/folder1", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func testLoaderLoadAttributesWithoutSubpathWithMove(t *testing.T) { + var reply string + if err := loaderRPC.Call(utils.LoaderSv1Load, + &ArgsProcessFolder{LoaderID: "SubpathLoaderWithMove"}, &reply); err != nil { + t.Error(err) + } +} + +func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) { + time.Sleep(100 * time.Millisecond) + if outContent1, err := ioutil.ReadFile(path.Join("/tmp/SubpathOut/folder1", utils.AttributesCsv)); err != nil { + t.Error(err) + } else if engine.AttributesCSVContent != string(outContent1) { + t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1)) + } +} + func testLoaderKillEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { t.Error(err)