From 4a7a00e967aa1162ef60b6de2690bb2ee6de21ed Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 26 Mar 2018 20:19:53 +0200 Subject: [PATCH] LoaderS NewLoader method --- config/config_defaults.go | 1 - config/config_json_test.go | 1 - config/config_test.go | 1 - config/libconfig_json.go | 1 - config/loadersconfig.go | 5 ----- loaders/loader.go | 35 +++++++++++++++++++++++++++++++++++ loaders/loaders.go | 20 ++++++++++++++++++++ 7 files changed, 55 insertions(+), 9 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index d116a2e6f..dd5180067 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -463,7 +463,6 @@ const CGRATES_CFG_JSON = ` {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> ], "field_separator": ",", // separator used in case of csv files - "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited "tp_in_dir": "/var/spool/cgrates/tploader/in", // absolute path towards the directory where the CDRs are stored "tp_out_dir": "/var/spool/cgrates/tploader/out", // absolute path towards the directory where processed CDRs will be moved "data":[ // data profiles to load diff --git a/config/config_json_test.go b/config/config_json_test.go index cc0d148d0..7b4667acb 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -820,7 +820,6 @@ func TestDfLoaderSJsonCfg(t *testing.T) { Address: utils.StringPointer(utils.MetaInternal), }}, Field_separator: utils.StringPointer(","), - Max_open_files: utils.IntPointer(1024), Tp_in_dir: utils.StringPointer("/var/spool/cgrates/tploader/in"), Tp_out_dir: utils.StringPointer("/var/spool/cgrates/tploader/out"), Data: &[]*LoaderSJsonDataType{dataType}, diff --git a/config/config_test.go b/config/config_test.go index 2b592d67d..6e8f0fa8e 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -930,7 +930,6 @@ func TestLoaderDefaults(t *testing.T) { }, }, FieldSeparator: ",", - MaxOpenFiles: 1024, TpInDir: "/var/spool/cgrates/tploader/in", TpOutDir: "/var/spool/cgrates/tploader/out", Data: []*LoaderSDataType{ diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 1eb14944a..747e42860 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -443,7 +443,6 @@ type LoaderSJsonCfg struct { Lock_filename *string Caches_conns *[]*HaPoolJsonCfg Field_separator *string - Max_open_files *int Tp_in_dir *string Tp_out_dir *string Data *[]*LoaderSJsonDataType diff --git a/config/loadersconfig.go b/config/loadersconfig.go index 8e1baf6bd..a264a179f 100644 --- a/config/loadersconfig.go +++ b/config/loadersconfig.go @@ -38,7 +38,6 @@ type LoaderSConfig struct { LockFileName string CacheSConns []*HaPoolConfig FieldSeparator string - MaxOpenFiles int TpInDir string TpOutDir string Data []*LoaderSDataType @@ -106,9 +105,6 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error { if jsnCfg.Field_separator != nil { self.FieldSeparator = *jsnCfg.Field_separator } - if jsnCfg.Max_open_files != nil { - self.MaxOpenFiles = *jsnCfg.Max_open_files - } if jsnCfg.Tp_in_dir != nil { self.TpInDir = *jsnCfg.Tp_in_dir } @@ -139,7 +135,6 @@ func (self *LoaderSConfig) Clone() *LoaderSConfig { clnLoader.CacheSConns[idx] = &clonedVal } clnLoader.FieldSeparator = self.FieldSeparator - clnLoader.MaxOpenFiles = self.MaxOpenFiles clnLoader.TpInDir = self.TpInDir clnLoader.TpOutDir = self.TpOutDir clnLoader.Data = make([]*LoaderSDataType, len(self.Data)) diff --git a/loaders/loader.go b/loaders/loader.go index 89546ec8c..22f4395e4 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -24,6 +24,7 @@ import ( "io" "os" "path" + "strings" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -36,8 +37,42 @@ type openedCSVFile struct { csvRdr *csv.Reader } +func NewLoader(dm *engine.DataManager, cfg *config.LoaderSConfig, + timezone string) (ldr *Loader, err error) { + ldr = &Loader{ + enabled: cfg.Enabled, + dryRun: cfg.DryRun, + tpInDir: cfg.TpInDir, + tpOutDir: cfg.TpOutDir, + lockFilename: cfg.LockFileName, + fieldSep: cfg.FieldSeparator, + dataTpls: make(map[string][]*config.CfgCdrField), + rdrs: make(map[string]map[string]*openedCSVFile), + bufLoaderData: make(map[string][]LoaderData), + dm: dm, + timezone: timezone, + } + for _, ldrData := range cfg.Data { + ldr.dataTpls[ldrData.Type] = ldrData.Fields + ldr.rdrs[ldrData.Type] = make(map[string]*openedCSVFile) + if ldrData.Filename != "" { + ldr.rdrs[ldrData.Type][ldrData.Filename] = nil + } + for _, cfgFld := range ldrData.Fields { // add all possible files to be opened + for _, cfgFldVal := range cfgFld.Value { + if idx := strings.Index(cfgFldVal.Id, utils.InInFieldSep); idx != -1 { + ldr.rdrs[ldrData.Type][cfgFldVal.Id[:idx]] = nil + } + } + } + } + return +} + // Loader is one instance loading from a folder type Loader struct { + enabled bool + dryRun bool ldrID string tpInDir string tpOutDir string diff --git a/loaders/loaders.go b/loaders/loaders.go index 3043b3668..eb10059fc 100644 --- a/loaders/loaders.go +++ b/loaders/loaders.go @@ -18,7 +18,27 @@ along with this program. If not, see package loaders +import ( + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" +) + +func NewLoaderS(dm *engine.DataManager, cfg []*config.LoaderSConfig, + timezone string) (ldrS *LoaderS, err error) { + return +} + // LoaderS is the Loader service handling independent Loaders type LoaderS struct { loaders map[string]*Loader } + +// isEnabled returns true if at least one loader is enabled +func (ldrS *LoaderS) isEnabled() bool { + for _, ldr := range ldrS.loaders { + if ldr.enabled { + return true + } + } + return false +}