[LoaderS] In case of empty output directory path don't move the processed file + integration test

2026-02-11 18:16:24 +05:00 · 2020-08-03 14:54:57 +03:00
parent e7c24c9d0f
commit fc7b2d7793
7 changed files with 146 additions and 13 deletions
--- a/config/configsanity.go
+++ b/config/configsanity.go
@@ -119,9 +119,12 @@ func (cfg *CGRConfig) checkConfigSanity() error {
 		if !ldrSCfg.Enabled {
 			continue
 		}
-		for _, dir := range []string{ldrSCfg.TpInDir, ldrSCfg.TpOutDir} {
-			if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
-				return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, dir)
+		if _, err := os.Stat(ldrSCfg.TpInDir); err != nil && os.IsNotExist(err) { // if loader is enabled tpInDir must exist
+			return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpInDir)
+		}
+		if ldrSCfg.TpOutDir != utils.EmptyString { // tpOutDir support empty string for no moving files after process
+			if _, err := os.Stat(ldrSCfg.TpOutDir); err != nil && os.IsNotExist(err) {
+				return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpOutDir)
 			}
 		}
 		for _, data := range ldrSCfg.Data {
--- a/data/conf/samples/loaders/tutinternal/cgrates.json
+++ b/data/conf/samples/loaders/tutinternal/cgrates.json
@@ -75,6 +75,37 @@
 			},
 		],
 	},
+	{
+		"id": "WithoutMoveToOut",			
+		"enabled": true,
+		"dry_run": false,
+		"tenant": "cgrates.org",
+		"run_delay": 0,
+		"lock_filename": ".cgr.lock",
+		"caches_conns": ["*internal"],
+		"field_separator": ",",
+		"tp_in_dir": "/tmp/LoaderIn",
+		"tp_out_dir": "",
+		"data":[
+			{
+				"type": "*attributes",
+				"file_name": "Attributes.csv",
+				"fields": [
+					{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+					{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+					{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+					{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+					{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+					{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+					{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+					{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+					{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+					{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+					{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+				],
+			},
+		],
+	}
 ],


--- a/data/conf/samples/loaders/tutmongo/cgrates.json
+++ b/data/conf/samples/loaders/tutmongo/cgrates.json
@@ -115,6 +115,37 @@
 			},
 		],
 	},
+	{
+		"id": "WithoutMoveToOut",
+		"enabled": true,
+		"dry_run": false,
+		"tenant": "cgrates.org",
+		"run_delay": 0,
+		"lock_filename": ".cgr.lock",
+		"caches_conns": ["*internal"],
+		"field_separator": ",",
+		"tp_in_dir": "/tmp/LoaderIn",
+		"tp_out_dir": "",
+		"data":[
+			{
+				"type": "*attributes",
+				"file_name": "Attributes.csv",
+				"fields": [
+					{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+					{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+					{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+					{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+					{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+					{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+					{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+					{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+					{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+					{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+					{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+				],
+			},
+		],
+	}
 ],


--- a/data/conf/samples/loaders/tutmysql/cgrates.json
+++ b/data/conf/samples/loaders/tutmysql/cgrates.json
@@ -76,6 +76,37 @@
 			},
 		],
 	},
+	{
+		"id": "WithoutMoveToOut",
+		"enabled": true,
+		"dry_run": false,
+		"tenant": "cgrates.org",
+		"run_delay": 0,
+		"lock_filename": ".cgr.lock",
+		"caches_conns": ["*internal"],
+		"field_separator": ",",
+		"tp_in_dir": "/tmp/LoaderIn",
+		"tp_out_dir": "",
+		"data":[
+			{
+				"type": "*attributes",
+				"file_name": "Attributes.csv",
+				"fields": [
+					{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+					{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+					{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+					{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+					{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+					{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+					{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+					{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+					{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+					{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+					{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+				],
+			},
+		],
+	}
 ],


--- a/loaders/loader.go
+++ b/loaders/loader.go
@@ -161,6 +161,9 @@ func (ldr *Loader) unreferenceFile(loaderType, fileName string) (err error) {
 }

 func (ldr *Loader) moveFiles() (err error) {
+	if ldr.tpOutDir == utils.EmptyString {
+		return
+	}
 	filesInDir, _ := ioutil.ReadDir(ldr.tpInDir)
 	for _, file := range filesInDir {
 		fName := file.Name()
--- a/loaders/loader_it_test.go
+++ b/loaders/loader_it_test.go
@@ -35,11 +35,10 @@ import (
 )

 var (
-	loaderCfgPath               string
-	loaderCfgDIR                string //run tests for specific configuration
-	loaderCfg                   *config.CGRConfig
-	loaderRPC                   *rpc.Client
-	loaderPathIn, loaderPathOut string
+	loaderCfgPath string
+	loaderCfgDIR  string //run tests for specific configuration
+	loaderCfg     *config.CGRConfig
+	loaderRPC     *rpc.Client

 	sTestsLoader = []func(t *testing.T){
 		testLoaderMakeFolders,
@@ -51,6 +50,11 @@ var (
 		testLoaderLoadAttributes,
 		testLoaderVerifyOutDir,
 		testLoaderCheckAttributes,
+		testLoaderResetDataDB,
+		testLoaderPopulateDataWithoutMoving,
+		testLoaderLoadAttributesWithoutMoving,
+		testLoaderVerifyOutDirWithoutMoving,
+		testLoaderCheckAttributes,
 		testLoaderKillEngine,
 	}
 )
@@ -89,7 +93,7 @@ func testLoaderInitCfg(t *testing.T) {

 func testLoaderMakeFolders(t *testing.T) {
 	// active the loaders here
-	for _, dir := range []string{"/tmp/In", "/tmp/Out"} {
+	for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn"} {
 		if err := os.RemoveAll(dir); err != nil {
 			t.Fatal("Error removing folder: ", dir, err)
 		}
@@ -97,8 +101,6 @@ func testLoaderMakeFolders(t *testing.T) {
 			t.Fatal("Error creating folder: ", dir, err)
 		}
 	}
-	loaderPathIn = "/tmp/In"
-	loaderPathOut = "/tmp/Out"
 }

 // Wipe out the cdr database
@@ -130,7 +132,7 @@ func testLoaderPopulateData(t *testing.T) {
 	if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
 		t.Fatal(err.Error())
 	}
-	if err := os.Rename(tmpFilePath, path.Join(loaderPathIn, fileName)); err != nil {
+	if err := os.Rename(tmpFilePath, path.Join("/tmp/In", fileName)); err != nil {
 		t.Fatal("Error moving file to processing directory: ", err)
 	}
 }
@@ -145,7 +147,7 @@ func testLoaderLoadAttributes(t *testing.T) {

 func testLoaderVerifyOutDir(t *testing.T) {
 	time.Sleep(100 * time.Millisecond)
-	if outContent1, err := ioutil.ReadFile(path.Join(loaderPathOut, utils.AttributesCsv)); err != nil {
+	if outContent1, err := ioutil.ReadFile(path.Join("/tmp/Out", utils.AttributesCsv)); err != nil {
 		t.Error(err)
 	} else if engine.AttributesCSVContent != string(outContent1) {
 		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
@@ -195,6 +197,35 @@ func testLoaderCheckAttributes(t *testing.T) {
 	}
 }

+func testLoaderPopulateDataWithoutMoving(t *testing.T) {
+	fileName := utils.AttributesCsv
+	tmpFilePath := path.Join("/tmp/", fileName)
+	if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := os.Rename(tmpFilePath, path.Join("/tmp/LoaderIn", fileName)); err != nil {
+		t.Fatal("Error moving file to processing directory: ", err)
+	}
+}
+
+func testLoaderLoadAttributesWithoutMoving(t *testing.T) {
+	var reply string
+	if err := loaderRPC.Call(utils.LoaderSv1Load,
+		&ArgsProcessFolder{LoaderID: "WithoutMoveToOut"}, &reply); err != nil {
+		t.Error(err)
+	}
+}
+
+func testLoaderVerifyOutDirWithoutMoving(t *testing.T) {
+	time.Sleep(100 * time.Millisecond)
+	// we expect that after the LoaderS process the file leave in in the input folder
+	if outContent1, err := ioutil.ReadFile(path.Join("/tmp/LoaderIn", utils.AttributesCsv)); err != nil {
+		t.Error(err)
+	} else if engine.AttributesCSVContent != string(outContent1) {
+		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
+	}
+}
+
 func testLoaderKillEngine(t *testing.T) {
 	if err := engine.KillEngine(100); err != nil {
 		t.Error(err)
--- a/packages/debian/changelog
+++ b/packages/debian/changelog
@@ -87,6 +87,9 @@ cgrates (0.11.0~dev) UNRELEASED; urgency=medium
  * [AttributeS] Updated inline AttributeProfiles to unite all consecutive inline attributes in a single profile 
  * [SessionS] Added *processRuns option to control the process runs for AttributeS
  * [DispatcherS] Removed ArgDispatcher in favor of Opts
+  * [ERs] Add support for *template type
+  * [EEs] Add support for *template type
+  * [LoaderS] In case of empty output directory path don't move the processed file

 -- DanB <danb@cgrates.org>  Wed, 19 Feb 2020 13:25:52 +0200