Added sanity check for Loaders in config.go and added parameters: Run_delay and Lock_filename

config/config.go

@@ -345,7 +345,7 @@ type CGRConfig struct {
 	statsCfg                 *StatSCfg                // Configuration for StatS
 	thresholdSCfg            *ThresholdSCfg           // configuration for ThresholdS
 	supplierSCfg             *SupplierSCfg            // configuration for SupplierS
-	LoaderProfiles           []*LoaderSConfig         // configuration for Loader
+	loaderSCfg               []*LoaderSConfig         // configuration for Loader
 	MailerServer             string                   // The server to use when sending emails out
 	MailerAuthUser           string                   // Authenticate to email server using this user
 	MailerAuthPass           string                   // Authenticate to email server with this password
@@ -474,6 +474,23 @@ func (self *CGRConfig) checkConfigSanity() error {
 			}
 		}
 	}
+	// Loaders sanity checks
+	for _, ldrSCfg := range self.loaderSCfg {
+		if !ldrSCfg.Enabled {
+			continue
+		}
+		for _, dir := range []string{ldrSCfg.TpInDir} {
+			if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+				fmt.Errorf("Nonexistent folder: %s", dir)
+			}
+		}
+		for _, dir := range []string{ldrSCfg.TpOutDir} {
+			if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+				fmt.Errorf("Nonexistent folder: %s", dir)
+			}
+		}
+	}
+
 	// SMGeneric checks
 	if self.sessionSCfg.Enabled {
 		if len(self.sessionSCfg.RALsConns) == 0 {
@@ -1147,12 +1164,12 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) {
 	}
 
 	if jsnLoaderSCfg != nil {
-		if self.LoaderProfiles == nil {
-			self.LoaderProfiles = make([]*LoaderSConfig, len(jsnLoaderSCfg))
+		if self.loaderSCfg == nil {
+			self.loaderSCfg = make([]*LoaderSConfig, len(jsnLoaderSCfg))
 		}
 		for idx, profile := range jsnLoaderSCfg {
-			self.LoaderProfiles[idx] = NewDfltLoadersConfig()
-			self.LoaderProfiles[idx].loadFromJsonCfg(profile)
+			self.loaderSCfg[idx] = NewDfltLoadersConfig()
+			self.loaderSCfg[idx].loadFromJsonCfg(profile)
 		}
 	}
 
@@ -1392,3 +1409,7 @@ func (cfg *CGRConfig) FilterSCfg() *FilterSCfg {
 func (cfg *CGRConfig) CacheCfg() CacheConfig {
 	return cfg.cacheConfig
 }
+
+func (cfg *CGRConfig) LoaderSCfg() []*LoaderSConfig {
+	return cfg.loaderSCfg
+}

config/config_defaults.go

@@ -457,8 +457,10 @@ const CGRATES_CFG_JSON = `
 		"id": "*default",									// identifier of the Loader
 		"enabled": false,									// starts as service: <true|false>.
 		"dry_run": false,									// do not send the CDRs to CDRS, just parse them
+		"run_delay": 0,										// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"lock_filename": "cgr.lock",						// Filename containing concurrency lock in case of delayed processing
 		"caches_conns": [
-			{"address": "*internal"},                       // address where to reach the CacheS for data reload, empty for no reloads  <""|*internal|x.y.z.y:1234>
+			{"address": "*internal"},						// address where to reach the CacheS for data reload, empty for no reloads  <""|*internal|x.y.z.y:1234>
 		],
 		"field_separator": ",",								// separator used in case of csv files
 		"max_open_files": 1024,								// maximum simultaneous files to process, 0 for unlimited
@@ -468,7 +470,7 @@ const CGRATES_CFG_JSON = `
 		"data":[											// data profiles to load
 			{
 				"type": "*attributes",						// data source type
-				"file_name": "Attributes.csv", 				// file name in the tp_in_dir
+				"file_name": "Attributes.csv",				// file name in the tp_in_dir
 				"fields": [
 					{"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true},
 					{"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true},

config/config_json_test.go

@@ -811,9 +811,11 @@ func TestDfLoaderSJsonCfg(t *testing.T) {
 	}
 	eCfg := []*LoaderSJsonCfg{
 		&LoaderSJsonCfg{
-			ID:      utils.StringPointer(utils.META_DEFAULT),
-			Enabled: utils.BoolPointer(false),
-			Dry_run: utils.BoolPointer(false),
+			ID:            utils.StringPointer(utils.META_DEFAULT),
+			Enabled:       utils.BoolPointer(false),
+			Dry_run:       utils.BoolPointer(false),
+			Run_delay:     utils.IntPointer(0),
+			Lock_filename: utils.StringPointer("cgr.lock"),
 			Caches_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{
 				Address: utils.StringPointer(utils.MetaInternal),
 			}},

config/config_test.go

@@ -919,9 +919,11 @@ func TestDbDefaults(t *testing.T) {
 func TestLoaderDefaults(t *testing.T) {
 	eCfg := []*LoaderSConfig{
 		&LoaderSConfig{
-			Id:      utils.META_DEFAULT,
-			Enabled: false,
-			DryRun:  false,
+			Id:           utils.META_DEFAULT,
+			Enabled:      false,
+			DryRun:       false,
+			RunDelay:     0,
+			LockFileName: "cgr.lock",
 			CacheSConns: []*HaPoolConfig{
 				&HaPoolConfig{
 					Address: utils.MetaInternal,
@@ -983,7 +985,7 @@ func TestLoaderDefaults(t *testing.T) {
 			},
 		},
 	}
-	if !reflect.DeepEqual(eCfg, cgrCfg.LoaderProfiles) {
-		t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.LoaderProfiles)
+	if !reflect.DeepEqual(eCfg, cgrCfg.loaderSCfg) {
+		t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderSCfg)
 	}
 }

config/libconfig_json.go

@@ -439,6 +439,8 @@ type LoaderSJsonCfg struct {
 	ID              *string
 	Enabled         *bool
 	Dry_run         *bool
+	Run_delay       *int
+	Lock_filename   *string
 	Caches_conns    *[]*HaPoolJsonCfg
 	Field_separator *string
 	Max_open_files  *int

config/loadersconfig.go

@@ -18,6 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 
 package config
 
+import (
+	"time"
+)
+
 func NewDfltLoadersConfig() *LoaderSConfig {
 	if dfltLoadersConfig == nil {
 		return new(LoaderSConfig)
@@ -30,6 +34,8 @@ type LoaderSConfig struct {
 	Id             string
 	Enabled        bool
 	DryRun         bool
+	RunDelay       time.Duration
+	LockFileName   string
 	CacheSConns    []*HaPoolConfig
 	FieldSeparator string
 	MaxOpenFiles   int
@@ -84,6 +90,12 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error {
 	if jsnCfg.Dry_run != nil {
 		self.DryRun = *jsnCfg.Dry_run
 	}
+	if jsnCfg.Run_delay != nil {
+		self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second
+	}
+	if jsnCfg.Lock_filename != nil {
+		self.LockFileName = *jsnCfg.Lock_filename
+	}
 	if jsnCfg.Caches_conns != nil {
 		self.CacheSConns = make([]*HaPoolConfig, len(*jsnCfg.Caches_conns))
 		for idx, jsnHaCfg := range *jsnCfg.Caches_conns {
@@ -119,6 +131,8 @@ func (self *LoaderSConfig) Clone() *LoaderSConfig {
 	clnLoader.Id = self.Id
 	clnLoader.Enabled = self.Enabled
 	clnLoader.DryRun = self.DryRun
+	clnLoader.RunDelay = self.RunDelay
+	clnLoader.LockFileName = self.LockFileName
 	clnLoader.CacheSConns = make([]*HaPoolConfig, len(self.CacheSConns))
 	for idx, cdrConn := range self.CacheSConns {
 		clonedVal := *cdrConn