From f2201f3e62df7afca3f4e5d8b51b7326e51b6c8a Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Mon, 22 May 2023 07:10:07 -0400 Subject: [PATCH] Revise err handling for CSV storage constructor NewFileCSVStorage() now returns an error besides the storage struct itself, which is logged and returned instead of calling log.Fatal() which was causing the engine to crash. Fixed compilation errors by creating the CSVStorage separately and passing it as an argument to the TpReader constructor. --- apier/v1/apier.go | 17 ++++++++++++++--- apier/v2/apier.go | 9 ++++++++- cmd/cgr-loader/cgr-loader.go | 3 +-- engine/libtest.go | 6 +++++- engine/storage_csv.go | 6 +++--- engine/tpimporter_csv.go | 8 ++++++-- engine/z_loader_it_test.go | 14 ++++++++++---- 7 files changed, 47 insertions(+), 16 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 72b1675a4..d197fea55 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1049,10 +1049,15 @@ func (apierSv1 *APIerSv1) LoadTariffPlanFromFolder(attrs *utils.AttrLoadTpFromFo return utils.ErrInvalidPath } + // initialize CSV storage + csvStorage, err := engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath) + if err != nil { + return utils.NewErrServerError(err) + } + // create the TpReader loader, err := engine.NewTpReader(apierSv1.DataManager.DataDB(), - engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath), - "", apierSv1.Config.GeneralCfg().DefaultTimezone, + csvStorage, "", apierSv1.Config.GeneralCfg().DefaultTimezone, apierSv1.Config.ApierCfg().CachesConns, apierSv1.Config.ApierCfg().SchedulerConns, apierSv1.Config.DataDbCfg().Type == utils.MetaInternal) if err != nil { @@ -1116,9 +1121,15 @@ func (apierSv1 *APIerSv1) RemoveTPFromFolder(attrs *utils.AttrLoadTpFromFolder, return utils.ErrInvalidPath } + // initialize CSV storage + csvStorage, err := engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath) + if err != nil { + return utils.NewErrServerError(err) + } + // create the TpReader loader, err := engine.NewTpReader(apierSv1.DataManager.DataDB(), - engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath), "", apierSv1.Config.GeneralCfg().DefaultTimezone, + csvStorage, "", apierSv1.Config.GeneralCfg().DefaultTimezone, apierSv1.Config.ApierCfg().CachesConns, apierSv1.Config.ApierCfg().SchedulerConns, apierSv1.Config.DataDbCfg().Type == utils.MetaInternal) if err != nil { diff --git a/apier/v2/apier.go b/apier/v2/apier.go index daa262bf7..e59786431 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -118,8 +118,15 @@ func (apiv2 *APIerSv2) LoadTariffPlanFromFolder(attrs *utils.AttrLoadTpFromFolde } else if !fi.IsDir() { return utils.ErrInvalidPath } + + // initialize CSV storage + csvStorage, err := engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath) + if err != nil { + return utils.NewErrServerError(err) + } + loader, err := engine.NewTpReader(apiv2.DataManager.DataDB(), - engine.NewFileCSVStorage(utils.CSVSep, attrs.FolderPath), "", apiv2.Config.GeneralCfg().DefaultTimezone, + csvStorage, "", apiv2.Config.GeneralCfg().DefaultTimezone, apiv2.Config.ApierCfg().CachesConns, apiv2.Config.ApierCfg().SchedulerConns, apiv2.Config.DataDbCfg().Type == utils.MetaInternal) if err != nil { diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 85965ece6..6a74ed6de 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -321,8 +321,7 @@ func getLoader(cfg *config.CGRConfig) (loader engine.LoadReader, err error) { return engine.NewGoogleCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, strings.TrimPrefix(*dataPath, gprefix)) } if !utils.IsURL(*dataPath) { - loader = engine.NewFileCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, *dataPath) - return + return engine.NewFileCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, *dataPath) } loader = engine.NewURLCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, *dataPath) return diff --git a/engine/libtest.go b/engine/libtest.go index ff2b8a11c..8f546921a 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -398,7 +398,11 @@ func StopStartEngine(cfgPath string, waitEngine int) (*exec.Cmd, error) { func LoadTariffPlanFromFolder(tpPath, timezone string, dm *DataManager, disable_reverse bool, cacheConns, schedConns []string) error { - loader, err := NewTpReader(dm.dataDB, NewFileCSVStorage(utils.CSVSep, tpPath), "", + csvStorage, err := NewFileCSVStorage(utils.CSVSep, tpPath) + if err != nil { + return utils.NewErrServerError(err) + } + loader, err := NewTpReader(dm.dataDB, csvStorage, "", timezone, cacheConns, schedConns, false) if err != nil { return utils.NewErrServerError(err) diff --git a/engine/storage_csv.go b/engine/storage_csv.go index 818468b62..5a22428ec 100644 --- a/engine/storage_csv.go +++ b/engine/storage_csv.go @@ -100,10 +100,10 @@ func NewCSVStorage(sep rune, } // NewFileCSVStorage returns a csv storage that uses all files from the folder -func NewFileCSVStorage(sep rune, dataPath string) *CSVStorage { +func NewFileCSVStorage(sep rune, dataPath string) (*CSVStorage, error) { allFoldersPath, err := getAllFolders(dataPath) if err != nil { - log.Fatal(err) + return nil, fmt.Errorf("failed to retrieve the folders from data path '%s': %s", dataPath, err.Error()) } destinationsPaths := appendName(allFoldersPath, utils.DestinationsCsv) timingsPaths := appendName(allFoldersPath, utils.TimingsCsv) @@ -146,7 +146,7 @@ func NewFileCSVStorage(sep rune, dataPath string) *CSVStorage { chargersPaths, dispatcherprofilesPaths, dispatcherhostsPaths, - ) + ), nil } // NewStringCSVStorage creates a csv storage from strings diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index a996bfc0a..b5c7c9c4e 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -63,7 +63,11 @@ var fileHandlers = map[string]func(*TPCSVImporter, string) error{ } func (tpImp *TPCSVImporter) Run() error { - tpImp.csvr = NewFileCSVStorage(tpImp.Sep, tpImp.DirPath) + var err error + tpImp.csvr, err = NewFileCSVStorage(tpImp.Sep, tpImp.DirPath) + if err != nil { + return err + } files, _ := os.ReadDir(tpImp.DirPath) var withErrors bool for _, f := range files { @@ -71,7 +75,7 @@ func (tpImp *TPCSVImporter) Run() error { if !hasName { continue } - if err := fHandler(tpImp, f.Name()); err != nil { + if err = fHandler(tpImp, f.Name()); err != nil { withErrors = true utils.Logger.Err(fmt.Sprintf(" Importing file: %s, got error: %s", f.Name(), err.Error())) } diff --git a/engine/z_loader_it_test.go b/engine/z_loader_it_test.go index 6ae5f84cf..c8da7b7b6 100644 --- a/engine/z_loader_it_test.go +++ b/engine/z_loader_it_test.go @@ -134,8 +134,11 @@ func testLoaderITRemoveLoad(t *testing.T) { t.Error("Failed validating data: ", err.Error()) } }*/ - loader, err = NewTpReader(dataDbCsv, NewFileCSVStorage(utils.CSVSep, - path.Join(*dataDir, "tariffplans", *tpCsvScenario)), "", "", + csvStorage, err := NewFileCSVStorage(utils.CSVSep, path.Join(*dataDir, "tariffplans", *tpCsvScenario)) + if err != nil { + t.Fatal(err) + } + loader, err = NewTpReader(dataDbCsv, csvStorage, "", "", []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, nil, false) if err != nil { t.Error(err) @@ -213,8 +216,11 @@ func testLoaderITLoadFromCSV(t *testing.T) { t.Error("Failed validating data: ", err.Error()) } }*/ - loader, err = NewTpReader(dataDbCsv, NewFileCSVStorage(utils.CSVSep, - path.Join(*dataDir, "tariffplans", *tpCsvScenario)), "", "", + csvStorage, err := NewFileCSVStorage(utils.CSVSep, path.Join(*dataDir, "tariffplans", *tpCsvScenario)) + if err != nil { + t.Fatal(err) + } + loader, err = NewTpReader(dataDbCsv, csvStorage, "", "", []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, nil, false) if err != nil { t.Error(err)