diff --git a/config/config.go b/config/config.go index 9035ed09e..aaa1a4316 100755 --- a/config/config.go +++ b/config/config.go @@ -345,7 +345,7 @@ type CGRConfig struct { statsCfg *StatSCfg // Configuration for StatS thresholdSCfg *ThresholdSCfg // configuration for ThresholdS supplierSCfg *SupplierSCfg // configuration for SupplierS - LoaderProfiles []*LoaderSConfig // configuration for Loader + loaderSCfg []*LoaderSConfig // configuration for Loader MailerServer string // The server to use when sending emails out MailerAuthUser string // Authenticate to email server using this user MailerAuthPass string // Authenticate to email server with this password @@ -474,6 +474,23 @@ func (self *CGRConfig) checkConfigSanity() error { } } } + // Loaders sanity checks + for _, ldrSCfg := range self.loaderSCfg { + if !ldrSCfg.Enabled { + continue + } + for _, dir := range []string{ldrSCfg.TpInDir} { + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + fmt.Errorf("Nonexistent folder: %s", dir) + } + } + for _, dir := range []string{ldrSCfg.TpOutDir} { + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + fmt.Errorf("Nonexistent folder: %s", dir) + } + } + } + // SMGeneric checks if self.sessionSCfg.Enabled { if len(self.sessionSCfg.RALsConns) == 0 { @@ -1147,12 +1164,12 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { } if jsnLoaderSCfg != nil { - if self.LoaderProfiles == nil { - self.LoaderProfiles = make([]*LoaderSConfig, len(jsnLoaderSCfg)) + if self.loaderSCfg == nil { + self.loaderSCfg = make([]*LoaderSConfig, len(jsnLoaderSCfg)) } for idx, profile := range jsnLoaderSCfg { - self.LoaderProfiles[idx] = NewDfltLoadersConfig() - self.LoaderProfiles[idx].loadFromJsonCfg(profile) + self.loaderSCfg[idx] = NewDfltLoadersConfig() + self.loaderSCfg[idx].loadFromJsonCfg(profile) } } @@ -1392,3 +1409,7 @@ func (cfg *CGRConfig) FilterSCfg() *FilterSCfg { func (cfg *CGRConfig) CacheCfg() CacheConfig { return cfg.cacheConfig } + +func (cfg *CGRConfig) LoaderSCfg() []*LoaderSConfig { + return cfg.loaderSCfg +} diff --git a/config/config_defaults.go b/config/config_defaults.go index 310eb872a..8398684d6 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -457,8 +457,10 @@ const CGRATES_CFG_JSON = ` "id": "*default", // identifier of the Loader "enabled": false, // starts as service: . "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": "cgr.lock", // Filename containing concurrency lock in case of delayed processing "caches_conns": [ - {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> ], "field_separator": ",", // separator used in case of csv files "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited @@ -468,7 +470,7 @@ const CGRATES_CFG_JSON = ` "data":[ // data profiles to load { "type": "*attributes", // data source type - "file_name": "Attributes.csv", // file name in the tp_in_dir + "file_name": "Attributes.csv", // file name in the tp_in_dir "fields": [ {"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, {"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, diff --git a/config/config_json_test.go b/config/config_json_test.go index 48f7bdc4e..ed14ec400 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -811,9 +811,11 @@ func TestDfLoaderSJsonCfg(t *testing.T) { } eCfg := []*LoaderSJsonCfg{ &LoaderSJsonCfg{ - ID: utils.StringPointer(utils.META_DEFAULT), - Enabled: utils.BoolPointer(false), - Dry_run: utils.BoolPointer(false), + ID: utils.StringPointer(utils.META_DEFAULT), + Enabled: utils.BoolPointer(false), + Dry_run: utils.BoolPointer(false), + Run_delay: utils.IntPointer(0), + Lock_filename: utils.StringPointer("cgr.lock"), Caches_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{ Address: utils.StringPointer(utils.MetaInternal), }}, diff --git a/config/config_test.go b/config/config_test.go index 6f57ce6cd..8beabcdaa 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -919,9 +919,11 @@ func TestDbDefaults(t *testing.T) { func TestLoaderDefaults(t *testing.T) { eCfg := []*LoaderSConfig{ &LoaderSConfig{ - Id: utils.META_DEFAULT, - Enabled: false, - DryRun: false, + Id: utils.META_DEFAULT, + Enabled: false, + DryRun: false, + RunDelay: 0, + LockFileName: "cgr.lock", CacheSConns: []*HaPoolConfig{ &HaPoolConfig{ Address: utils.MetaInternal, @@ -983,7 +985,7 @@ func TestLoaderDefaults(t *testing.T) { }, }, } - if !reflect.DeepEqual(eCfg, cgrCfg.LoaderProfiles) { - t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.LoaderProfiles) + if !reflect.DeepEqual(eCfg, cgrCfg.loaderSCfg) { + t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderSCfg) } } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 9eb4c5d4d..1eb14944a 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -439,6 +439,8 @@ type LoaderSJsonCfg struct { ID *string Enabled *bool Dry_run *bool + Run_delay *int + Lock_filename *string Caches_conns *[]*HaPoolJsonCfg Field_separator *string Max_open_files *int diff --git a/config/loadersconfig.go b/config/loadersconfig.go index 9a926708c..8e1baf6bd 100644 --- a/config/loadersconfig.go +++ b/config/loadersconfig.go @@ -18,6 +18,10 @@ along with this program. If not, see package config +import ( + "time" +) + func NewDfltLoadersConfig() *LoaderSConfig { if dfltLoadersConfig == nil { return new(LoaderSConfig) @@ -30,6 +34,8 @@ type LoaderSConfig struct { Id string Enabled bool DryRun bool + RunDelay time.Duration + LockFileName string CacheSConns []*HaPoolConfig FieldSeparator string MaxOpenFiles int @@ -84,6 +90,12 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error { if jsnCfg.Dry_run != nil { self.DryRun = *jsnCfg.Dry_run } + if jsnCfg.Run_delay != nil { + self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second + } + if jsnCfg.Lock_filename != nil { + self.LockFileName = *jsnCfg.Lock_filename + } if jsnCfg.Caches_conns != nil { self.CacheSConns = make([]*HaPoolConfig, len(*jsnCfg.Caches_conns)) for idx, jsnHaCfg := range *jsnCfg.Caches_conns { @@ -119,6 +131,8 @@ func (self *LoaderSConfig) Clone() *LoaderSConfig { clnLoader.Id = self.Id clnLoader.Enabled = self.Enabled clnLoader.DryRun = self.DryRun + clnLoader.RunDelay = self.RunDelay + clnLoader.LockFileName = self.LockFileName clnLoader.CacheSConns = make([]*HaPoolConfig, len(self.CacheSConns)) for idx, cdrConn := range self.CacheSConns { clonedVal := *cdrConn