Add integration test for LoaderS with *template

2026-02-11 18:16:24 +05:00 · 2020-08-26 14:51:37 +03:00
parent 13dce13a10
commit 35fe328044
6 changed files with 154 additions and 5 deletions
--- a/apier/v1/preload_it_test.go
+++ b/apier/v1/preload_it_test.go
@@ -65,7 +65,7 @@ func TestPreload(t *testing.T) {

 func testCreateDirs(t *testing.T) {
 	for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
-		"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} {
+		"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut"} {
 		if err := os.RemoveAll(dir); err != nil {
 			t.Fatal("Error removing folder: ", dir, err)
 		}
--- a/data/conf/samples/loaders/tutinternal/cgrates.json
+++ b/data/conf/samples/loaders/tutinternal/cgrates.json
@@ -167,6 +167,27 @@
 				],
 			},
 		],
+	},
+	{
+		"id": "LoaderWithTemplate",									// identifier of the Loader
+		"enabled": true,									// starts as service: <true|false>.
+		"dry_run": false,									// do not send the CDRs to CDRS, just parse them
+		"tenant": "cgrates.org",
+		"run_delay": 0,										// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"lock_filename": ".cgr.lock",						// Filename containing concurrency lock in case of delayed processing
+		"caches_conns": ["*internal"],
+		"field_separator": ",",								// separator used in case of csv files
+		"tp_in_dir": "/tmp/templateLoaderIn",		// absolute path towards the directory where the CDRs are stored
+		"tp_out_dir": "/tmp/templateLoaderOut",	// absolute path towards the directory where processed CDRs will be moved
+		"data":[
+			{
+				"type": "*attributes",						// data source type
+				"file_name": "Attributes.csv",				// file name in the tp_in_dir
+				"fields": [
+					{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
+				],
+			},
+		],
 	}
 ],

@@ -183,7 +204,21 @@
 	"attributes_conns": ["*internal"],
 },

-
+"templates": {
+	"attrTemplateLoader": [
+		{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+		{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+		{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+		{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+		{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+		{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+		{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+		{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+		{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+		{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+		{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+	],
+},

 "apiers": {
 	"enabled": true,
--- a/data/conf/samples/loaders/tutmongo/cgrates.json
+++ b/data/conf/samples/loaders/tutmongo/cgrates.json
@@ -207,6 +207,27 @@
 				],
 			},
 		],
+	},
+	{
+		"id": "LoaderWithTemplate",									// identifier of the Loader
+		"enabled": true,									// starts as service: <true|false>.
+		"dry_run": false,									// do not send the CDRs to CDRS, just parse them
+		"tenant": "cgrates.org",
+		"run_delay": 0,										// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"lock_filename": ".cgr.lock",						// Filename containing concurrency lock in case of delayed processing
+		"caches_conns": ["*internal"],
+		"field_separator": ",",								// separator used in case of csv files
+		"tp_in_dir": "/tmp/templateLoaderIn",		// absolute path towards the directory where the CDRs are stored
+		"tp_out_dir": "/tmp/templateLoaderOut",	// absolute path towards the directory where processed CDRs will be moved
+		"data":[
+			{
+				"type": "*attributes",						// data source type
+				"file_name": "Attributes.csv",				// file name in the tp_in_dir
+				"fields": [
+					{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
+				],
+			},
+		],
 	}
 ],

@@ -230,7 +251,25 @@
 },


-"apiers": {
+"templates": {
+	"attrTemplateLoader": [
+		{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+		{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+		{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+		{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+		{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+		{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+		{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+		{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+		{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+		{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+		{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+	],
+},
+
+
+
+	"apiers": {
 	"enabled": true,
 	"scheduler_conns": ["*internal"],
 },
--- a/data/conf/samples/loaders/tutmysql/cgrates.json
+++ b/data/conf/samples/loaders/tutmysql/cgrates.json
@@ -168,6 +168,27 @@
 				],
 			},
 		],
+	},
+	{
+		"id": "LoaderWithTemplate",									// identifier of the Loader
+		"enabled": true,									// starts as service: <true|false>.
+		"dry_run": false,									// do not send the CDRs to CDRS, just parse them
+		"tenant": "cgrates.org",
+		"run_delay": 0,										// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"lock_filename": ".cgr.lock",						// Filename containing concurrency lock in case of delayed processing
+		"caches_conns": ["*internal"],
+		"field_separator": ",",								// separator used in case of csv files
+		"tp_in_dir": "/tmp/templateLoaderIn",		// absolute path towards the directory where the CDRs are stored
+		"tp_out_dir": "/tmp/templateLoaderOut",	// absolute path towards the directory where processed CDRs will be moved
+		"data":[
+			{
+				"type": "*attributes",						// data source type
+				"file_name": "Attributes.csv",				// file name in the tp_in_dir
+				"fields": [
+					{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
+				],
+			},
+		],
 	}
 ],

@@ -229,7 +250,25 @@
 },


-"apiers": {
+"templates": {
+	"attrTemplateLoader": [
+		{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
+		{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
+		{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
+		{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
+		{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
+		{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
+		{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
+		{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
+		{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
+		{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
+		{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
+	],
+},
+
+
+
+	"apiers": {
 	"enabled": true,
 	"scheduler_conns": ["*internal"],
 },
--- a/loaders/lib_test.go
+++ b/loaders/lib_test.go
@@ -36,7 +36,7 @@ var (
 )

 var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
-	"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"}
+	"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut"}

 func newRPCClient(cfg *config.ListenCfg) (c *rpc.Client, err error) {
 	switch *encoding {
--- a/loaders/loader_it_test.go
+++ b/loaders/loader_it_test.go
@@ -65,6 +65,11 @@ var (
 		testLoaderLoadAttributesWithoutSubpathWithMove,
 		testLoaderVerifyOutDirWithSubpathWithMove,
 		testLoaderCheckAttributes,
+		testLoaderResetDataDB,
+		testLoaderPopulateDataForTemplateLoader,
+		testLoaderLoadAttributesForTemplateLoader,
+		testLoaderVerifyOutDirForTemplateLoader,
+		testLoaderCheckAttributes,
 		testLoaderKillEngine,
 	}
 )
@@ -299,6 +304,37 @@ func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) {
 	}
 }

+func testLoaderPopulateDataForTemplateLoader(t *testing.T) {
+	fileName := utils.AttributesCsv
+	tmpFilePath := path.Join("/tmp/", fileName)
+	if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := os.MkdirAll("/tmp/templateLoaderIn", 0755); err != nil {
+		t.Fatal("Error creating folder: /tmp/templateLoaderIn", err)
+	}
+	if err := os.Rename(tmpFilePath, path.Join("/tmp/templateLoaderIn", fileName)); err != nil {
+		t.Fatal("Error moving file to processing directory: ", err)
+	}
+}
+
+func testLoaderLoadAttributesForTemplateLoader(t *testing.T) {
+	var reply string
+	if err := loaderRPC.Call(utils.LoaderSv1Load,
+		&ArgsProcessFolder{LoaderID: "LoaderWithTemplate"}, &reply); err != nil {
+		t.Error(err)
+	}
+}
+
+func testLoaderVerifyOutDirForTemplateLoader(t *testing.T) {
+	time.Sleep(100 * time.Millisecond)
+	if outContent1, err := ioutil.ReadFile(path.Join("/tmp/templateLoaderOut", utils.AttributesCsv)); err != nil {
+		t.Error(err)
+	} else if engine.AttributesCSVContent != string(outContent1) {
+		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
+	}
+}
+
 func testLoaderKillEngine(t *testing.T) {
 	if err := engine.KillEngine(100); err != nil {
 		t.Error(err)