diff --git a/config/configsanity.go b/config/configsanity.go index e6e2c78c9..04b09becb 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -119,9 +119,12 @@ func (cfg *CGRConfig) checkConfigSanity() error { if !ldrSCfg.Enabled { continue } - for _, dir := range []string{ldrSCfg.TpInDir, ldrSCfg.TpOutDir} { - if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { - return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, dir) + if _, err := os.Stat(ldrSCfg.TpInDir); err != nil && os.IsNotExist(err) { // if loader is enabled tpInDir must exist + return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpInDir) + } + if ldrSCfg.TpOutDir != utils.EmptyString { // tpOutDir support empty string for no moving files after process + if _, err := os.Stat(ldrSCfg.TpOutDir); err != nil && os.IsNotExist(err) { + return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpOutDir) } } for _, data := range ldrSCfg.Data { diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json index 69ccfd469..a738c8c4c 100644 --- a/data/conf/samples/loaders/tutinternal/cgrates.json +++ b/data/conf/samples/loaders/tutinternal/cgrates.json @@ -75,6 +75,37 @@ }, ], }, + { + "id": "WithoutMoveToOut", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/LoaderIn", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + } ], diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json index 6015be8d7..83c046988 100644 --- a/data/conf/samples/loaders/tutmongo/cgrates.json +++ b/data/conf/samples/loaders/tutmongo/cgrates.json @@ -115,6 +115,37 @@ }, ], }, + { + "id": "WithoutMoveToOut", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/LoaderIn", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + } ], diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json index 54e006c47..b030482e4 100644 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -76,6 +76,37 @@ }, ], }, + { + "id": "WithoutMoveToOut", + "enabled": true, + "dry_run": false, + "tenant": "cgrates.org", + "run_delay": 0, + "lock_filename": ".cgr.lock", + "caches_conns": ["*internal"], + "field_separator": ",", + "tp_in_dir": "/tmp/LoaderIn", + "tp_out_dir": "", + "data":[ + { + "type": "*attributes", + "file_name": "Attributes.csv", + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + } ], diff --git a/loaders/loader.go b/loaders/loader.go index a5e5cadee..e6e790ebf 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -161,6 +161,9 @@ func (ldr *Loader) unreferenceFile(loaderType, fileName string) (err error) { } func (ldr *Loader) moveFiles() (err error) { + if ldr.tpOutDir == utils.EmptyString { + return + } filesInDir, _ := ioutil.ReadDir(ldr.tpInDir) for _, file := range filesInDir { fName := file.Name() diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index c94e7fbd1..3589011bf 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -35,11 +35,10 @@ import ( ) var ( - loaderCfgPath string - loaderCfgDIR string //run tests for specific configuration - loaderCfg *config.CGRConfig - loaderRPC *rpc.Client - loaderPathIn, loaderPathOut string + loaderCfgPath string + loaderCfgDIR string //run tests for specific configuration + loaderCfg *config.CGRConfig + loaderRPC *rpc.Client sTestsLoader = []func(t *testing.T){ testLoaderMakeFolders, @@ -51,6 +50,11 @@ var ( testLoaderLoadAttributes, testLoaderVerifyOutDir, testLoaderCheckAttributes, + testLoaderResetDataDB, + testLoaderPopulateDataWithoutMoving, + testLoaderLoadAttributesWithoutMoving, + testLoaderVerifyOutDirWithoutMoving, + testLoaderCheckAttributes, testLoaderKillEngine, } ) @@ -89,7 +93,7 @@ func testLoaderInitCfg(t *testing.T) { func testLoaderMakeFolders(t *testing.T) { // active the loaders here - for _, dir := range []string{"/tmp/In", "/tmp/Out"} { + for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn"} { if err := os.RemoveAll(dir); err != nil { t.Fatal("Error removing folder: ", dir, err) } @@ -97,8 +101,6 @@ func testLoaderMakeFolders(t *testing.T) { t.Fatal("Error creating folder: ", dir, err) } } - loaderPathIn = "/tmp/In" - loaderPathOut = "/tmp/Out" } // Wipe out the cdr database @@ -130,7 +132,7 @@ func testLoaderPopulateData(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil { t.Fatal(err.Error()) } - if err := os.Rename(tmpFilePath, path.Join(loaderPathIn, fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join("/tmp/In", fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } @@ -145,7 +147,7 @@ func testLoaderLoadAttributes(t *testing.T) { func testLoaderVerifyOutDir(t *testing.T) { time.Sleep(100 * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(loaderPathOut, utils.AttributesCsv)); err != nil { + if outContent1, err := ioutil.ReadFile(path.Join("/tmp/Out", utils.AttributesCsv)); err != nil { t.Error(err) } else if engine.AttributesCSVContent != string(outContent1) { t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1)) @@ -195,6 +197,35 @@ func testLoaderCheckAttributes(t *testing.T) { } } +func testLoaderPopulateDataWithoutMoving(t *testing.T) { + fileName := utils.AttributesCsv + tmpFilePath := path.Join("/tmp/", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil { + t.Fatal(err.Error()) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/LoaderIn", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func testLoaderLoadAttributesWithoutMoving(t *testing.T) { + var reply string + if err := loaderRPC.Call(utils.LoaderSv1Load, + &ArgsProcessFolder{LoaderID: "WithoutMoveToOut"}, &reply); err != nil { + t.Error(err) + } +} + +func testLoaderVerifyOutDirWithoutMoving(t *testing.T) { + time.Sleep(100 * time.Millisecond) + // we expect that after the LoaderS process the file leave in in the input folder + if outContent1, err := ioutil.ReadFile(path.Join("/tmp/LoaderIn", utils.AttributesCsv)); err != nil { + t.Error(err) + } else if engine.AttributesCSVContent != string(outContent1) { + t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1)) + } +} + func testLoaderKillEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { t.Error(err) diff --git a/packages/debian/changelog b/packages/debian/changelog index 8dbf9a0b1..1b5194781 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -87,6 +87,9 @@ cgrates (0.11.0~dev) UNRELEASED; urgency=medium * [AttributeS] Updated inline AttributeProfiles to unite all consecutive inline attributes in a single profile * [SessionS] Added *processRuns option to control the process runs for AttributeS * [DispatcherS] Removed ArgDispatcher in favor of Opts + * [ERs] Add support for *template type + * [EEs] Add support for *template type + * [LoaderS] In case of empty output directory path don't move the processed file -- DanB Wed, 19 Feb 2020 13:25:52 +0200