diff --git a/loaders/lib_test.go b/loaders/lib_test.go index 9480f9a9c..910e1f45f 100644 --- a/loaders/lib_test.go +++ b/loaders/lib_test.go @@ -938,12 +938,12 @@ type dataDBMockError struct { //For Threshold func (dbM *dataDBMockError) RemThresholdProfileDrv(tenant, id string) (err error) { - return nil + return } func (dbM *dataDBMockError) SetIndexesDrv(idxItmType, tntCtx string, indexes map[string]utils.StringSet, commit bool, transactionID string) (err error) { - return nil + return } func (dbM *dataDBMockError) RemoveThresholdDrv(string, string) error { @@ -958,6 +958,18 @@ func (dbM *dataDBMockError) GetThresholdProfileDrv(tenant string, ID string) (tp return expThresholdPrf, nil } +func (dbM *dataDBMockError) SetThresholdProfileDrv(tp *engine.ThresholdProfile) (err error) { + return +} + +func (dbM *dataDBMockError) GetThresholdDrv(string, string) (*engine.Threshold, error) { + return nil, utils.ErrNoDatabaseConn +} + +func (dbM *dataDBMockError) HasDataDrv(string, string, string) (bool, error) { + return false, nil +} + //For StatQueue func (dbM *dataDBMockError) GetStatQueueProfileDrv(tenant string, ID string) (sq *engine.StatQueueProfile, err error) { return nil, nil @@ -971,6 +983,18 @@ func (dbM *dataDBMockError) RemStatQueueDrv(tenant, id string) (err error) { return utils.ErrNoDatabaseConn } +func (dbM *dataDBMockError) GetStatQueueDrv(tenant, id string) (sq *engine.StatQueue, err error) { + return nil, utils.ErrNoDatabaseConn +} + +func (dbM *dataDBMockError) SetStatQueueDrv(ssq *engine.StoredStatQueue, sq *engine.StatQueue) (err error) { + return utils.ErrNoDatabaseConn +} + +func (dbM *dataDBMockError) SetStatQueueProfileDrv(sq *engine.StatQueueProfile) (err error) { + return nil +} + //For Resources func (dbM *dataDBMockError) GetResourceProfileDrv(string, string) (*engine.ResourceProfile, error) { return nil, nil @@ -987,3 +1011,11 @@ func (dbM *dataDBMockError) RemoveResourceDrv(tenant, id string) (err error) { func (dbM *dataDBMockError) GetIndexesDrv(idxItmType, tntCtx, idxKey string) (indexes map[string]utils.StringSet, err error) { return nil, nil } + +func (dbM *dataDBMockError) SetResourceProfileDrv(*engine.ResourceProfile) error { + return nil +} + +func (dbM *dataDBMockError) SetResourceDrv(*engine.Resource) error { + return utils.ErrNoDatabaseConn +} diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index 2d1f9f440..7f170447c 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -62,6 +62,7 @@ var ( testProcessFileUnableToOpen, testProcessFileRenameError, testAllFilesPresentEmptyCSV, + testIsFolderLocked, testLoaderLoadAttributes, testLoaderVerifyOutDir, testLoaderCheckAttributes, @@ -979,3 +980,18 @@ func testAllFilesPresentEmptyCSV(t *testing.T) { t.Errorf("Expecting false") } } + +func testIsFolderLocked(t *testing.T) { + flPath := "/tmp/testIsFolderLocked" + ldr := &Loader{ + ldrID: "TestLoadAndRemoveResources", + tpInDir: flPath, + lockFilename: utils.EmptyString, + bufLoaderData: make(map[string][]LoaderData), + timezone: "UTC", + } + expected := "stat /\x00: invalid argument" + if _, err := ldr.isFolderLocked(); err != nil { + t.Errorf("Expected %+v, received %+v", expected, err) + } +} diff --git a/loaders/loader_test.go b/loaders/loader_test.go index 4c99dc770..ceb15d15f 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -3524,7 +3524,7 @@ func TestLoadAccountProfilesAsStructErrConversion(t *testing.T) { } actPrfCsv := ` #ActivationInterval -* * * * * * +* * * * * * * ` rdr := ioutil.NopCloser(strings.NewReader(actPrfCsv)) rdrCsv := csv.NewReader(rdr) @@ -3544,6 +3544,57 @@ func TestLoadAccountProfilesAsStructErrConversion(t *testing.T) { } } +func TestProcessContentAccountProfileAsTPError(t *testing.T) { + data := engine.NewInternalDB(nil, nil, true) + ldr := &Loader{ + ldrID: "TestProcessContentAccountProfileAsTPError", + bufLoaderData: make(map[string][]LoaderData), + dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), + timezone: "UTC", + } + ldr.dataTpls = map[string][]*config.FCTemplate{ + utils.MetaAccountProfiles: { + {Tag: "Tenant", + Path: "Tenant", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP)}, + {Tag: "ID", + Path: "ID", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP)}, + {Tag: "BalanceID", + Path: "BalanceID", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, + {Tag: "BalanceUnitFactors", + Path: "BalanceUnitFactors", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, + }, + } + + accPrfCSv := ` +#Tenant,ID,BalanceID,BalanceUnitFactors +cgrates.org,1001,MonetaryBalance,fltr1&fltr2;100;fltr3 +` + rdr := ioutil.NopCloser(strings.NewReader(accPrfCSv)) + rdrCsv := csv.NewReader(rdr) + rdrCsv.Comment = '#' + ldr.rdrs = map[string]map[string]*openedCSVFile{ + utils.MetaAccountProfiles: { + utils.AccountProfilesCsv: &openedCSVFile{ + fileName: utils.AccountProfilesCsv, + rdr: rdr, + csvRdr: rdrCsv, + }, + }, + } + expectedErr := "invlid key: for BalanceUnitFactors" + if err := ldr.processContent(utils.MetaAccountProfiles, utils.EmptyString); err == nil || err.Error() != expectedErr { + t.Errorf("Expected %+v, received %+v", expectedErr, err) + } +} + func TestLoadAccountProfilesAsStructErrType(t *testing.T) { data := engine.NewInternalDB(nil, nil, true) ldr := &Loader{ @@ -4833,12 +4884,39 @@ cgrates.org,REM_RATEPROFILE_1,RT_WEEKEND Tenant: "cgrates.org", ID: "REM_RATEPROFILE_1", } - if err := ldr.dm.SetRateProfile(expRtPrf, true); err != nil { - t.Error(err) - } ldr.flagsTpls[utils.MetaRateProfiles] = utils.FlagsWithParamsFromSlice([]string{utils.MetaPartial}) ldr.dm = nil expected := "NO_DATA_BASE_CONNECTION" + if err := ldr.processContent(utils.MetaRateProfiles, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", utils.ToJSON(expected), utils.ToJSON(err)) + } + + rdr = ioutil.NopCloser(strings.NewReader(rtPrfCsv)) + csvRdr = csv.NewReader(rdr) + csvRdr.Comment = '#' + ldr.rdrs = map[string]map[string]*openedCSVFile{ + utils.MetaRateProfiles: { + utils.RateProfilesCsv: &openedCSVFile{ + fileName: utils.RateProfilesCsv, + rdr: rdr, + csvRdr: csvRdr, + }, + }, + } + ldr.flagsTpls[utils.MetaRateProfiles] = utils.FlagsWithParamsFromSlice([]string{"INVALID_FLAGS"}) + expected = "NO_DATA_BASE_CONNECTION" + if err := ldr.processContent(utils.MetaRateProfiles, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", utils.ToJSON(expected), utils.ToJSON(err)) + } + + ldr.dm = engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + if err := ldr.dm.SetRateProfile(expRtPrf, true); err != nil { + t.Error(err) + } + + ldr.dm = nil + ldr.flagsTpls[utils.MetaRateProfiles] = utils.FlagsWithParamsFromSlice([]string{utils.MetaPartial}) + expected = "NO_DATA_BASE_CONNECTION" if err := ldr.removeContent(utils.MetaRateProfiles, utils.EmptyString); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", utils.ToJSON(expected), utils.ToJSON(err)) } @@ -4895,9 +4973,12 @@ cgrates.org,REM_THRESHOLDS_1, newData := &dataDBMockError{} ldr.dm = engine.NewDataManager(newData, config.CgrConfig().CacheCfg(), nil) expected := "NO_DATA_BASE_CONNECTION" - if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err == nil || err.Error() != expected { + if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", expected, err) + } else if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } + } func TestRemoveStatQueueMockError(t *testing.T) { @@ -4942,13 +5023,83 @@ cgrates.org,REM_STATS_1 Tenant: "cgrates.org", ID: "REM_STATS_1", } + if err := ldr.dm.SetStatQueueProfile(expStats, true); err != nil { t.Error(err) } + newData := &dataDBMockError{} ldr.dm = engine.NewDataManager(newData, config.CgrConfig().CacheCfg(), nil) expected := "NO_DATA_BASE_CONNECTION" + if err := ldr.removeContent(utils.MetaStatS, utils.EmptyString); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) + } else if err := ldr.processContent(utils.MetaStatS, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", expected, err) } } + +func TestRemoveResourcesMockError(t *testing.T) { + data := engine.NewInternalDB(nil, nil, true) + ldr := &Loader{ + ldrID: "TestLoadAndRemoveResources", + bufLoaderData: make(map[string][]LoaderData), + dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), + timezone: "UTC", + } + ldr.dataTpls = map[string][]*config.FCTemplate{ + utils.MetaResources: { + {Tag: "Tenant", + Path: "Tenant", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), + Mandatory: true}, + {Tag: "ID", + Path: "ID", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), + Mandatory: true}, + }, + } + resourcesCSV := ` +#Tenant[0],ID[1] +cgrates.org,NewRes1 +` + rdr := ioutil.NopCloser(strings.NewReader(resourcesCSV)) + rdrCsv := csv.NewReader(rdr) + rdrCsv.Comment = '#' + ldr.rdrs = map[string]map[string]*openedCSVFile{ + utils.MetaResources: { + "Resources.csv": &openedCSVFile{fileName: "Resources.csv", + rdr: rdr, csvRdr: rdrCsv}}, + } + + resPrf := &engine.ResourceProfile{ + Tenant: "cgrates.org", + ID: "NewRes1", + } + + if err := ldr.dm.SetResourceProfile(resPrf, true); err != nil { + t.Error(err) + } + + newData := &dataDBMockError{} + ldr.dm = engine.NewDataManager(newData, config.CgrConfig().CacheCfg(), nil) + expected := "NO_DATA_BASE_CONNECTION" + + if err := ldr.removeContent(utils.MetaResources, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", expected, err) + } else if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err == nil || err.Error() != expected { + t.Errorf("Expected %+v, received %+v", expected, err) + } +} + +func TestLoaderHandleFolder(t *testing.T) { + stopChan := make(chan struct{}, 1) + stopChan <- struct{}{} + ldr := &Loader{ + ldrID: "TestLoaderHandleFolder", + runDelay: 1, + } + ldr.handleFolder(stopChan) +} diff --git a/loaders/loaders_it_test.go b/loaders/loaders_it_test.go new file mode 100644 index 000000000..9778e3953 --- /dev/null +++ b/loaders/loaders_it_test.go @@ -0,0 +1,293 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package loaders + +import ( + "encoding/csv" + "io/ioutil" + "os" + "path" + "reflect" + "strings" + "testing" + + "github.com/cgrates/cgrates/config" + + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + sTestItLoaders = []func(t *testing.T){ + testV1LoadResource, + testV1LoadDefaultIDError, + testV1LoadUnableToDeleteFile, + testV1LoadProcessFolderError, + } +) + +func TestITLoaders(t *testing.T) { + for _, test := range sTestItLoaders { + t.Run("Loaders_IT_Tests", test) + } +} + +func testV1LoadResource(t *testing.T) { + flPath := "/tmp/testV1LoadResource" + if err := os.MkdirAll(flPath, 0777); err != nil { + t.Error(err) + } + file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) + if err != nil { + t.Error(err) + } + file.Write([]byte(` +#Tenant[0],ID[1] +cgrates.org,NewRes1 +`)) + file.Close() + + data := engine.NewInternalDB(nil, nil, true) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() + cfgLdr[0] = &config.LoaderSCfg{ + ID: "testV1LoadResource", + Enabled: true, + FieldSeparator: utils.FIELDS_SEP, + TpInDir: flPath, + TpOutDir: "/tmp", + LockFileName: utils.ResourcesCsv, + Data: nil, + } + ldrs := NewLoaderService(dm, cfgLdr, "UTC", nil, nil) + ldrs.ldrs["testV1LoadResource"].dataTpls = map[string][]*config.FCTemplate{ + utils.MetaResources: { + {Tag: "Tenant", + Path: "Tenant", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), + Mandatory: true}, + {Tag: "ID", + Path: "ID", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), + Mandatory: true}, + }, + } + + resCsv := ` +#Tenant[0],ID[1] +cgrates.org,NewRes1 +` + rdr := ioutil.NopCloser(strings.NewReader(resCsv)) + csvRdr := csv.NewReader(rdr) + csvRdr.Comment = '#' + ldrs.ldrs["testV1LoadResource"].rdrs = map[string]map[string]*openedCSVFile{ + utils.MetaResources: { + utils.ResourcesCsv: &openedCSVFile{ + fileName: utils.ResourcesCsv, + rdr: rdr, + csvRdr: csvRdr, + }, + }, + } + + var reply string + expected := "ANOTHER_LOADER_RUNNING" + //cannot load when there is another loader running + if err := ldrs.V1Load(&ArgsProcessFolder{ + LoaderID: "testV1LoadResource", + ForceLock: false}, &reply); err == nil && reply != utils.EmptyString && err.Error() != expected { + t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply) + } + + if err := ldrs.V1Load(&ArgsProcessFolder{ + LoaderID: "testV1LoadResource", + ForceLock: true}, &reply); err != nil && reply != utils.OK { + t.Error(err) + } + + expRes := &engine.ResourceProfile{ + Tenant: "cgrates.org", + ID: "NewRes1", + } + + if rcv, err := ldrs.ldrs["testV1LoadResource"].dm.GetResourceProfile(expRes.Tenant, expRes.ID, + true, true, utils.NonTransactional); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rcv, expRes) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv)) + } + + if err := os.Remove(flPath); err != nil { + t.Error(err) + } +} + +func testV1LoadDefaultIDError(t *testing.T) { + flPath := "/tmp/testV1LoadResource" + if err := os.MkdirAll(flPath, 0777); err != nil { + t.Error(err) + } + file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) + if err != nil { + t.Error(err) + } + file.Write([]byte(` +#Tenant[0],ID[1] +cgrates.org,NewRes1 +`)) + file.Close() + + data := engine.NewInternalDB(nil, nil, true) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() + cfgLdr[0] = &config.LoaderSCfg{ + ID: "testV1LoadDefaultIDError", + Enabled: true, + FieldSeparator: utils.FIELDS_SEP, + TpInDir: flPath, + TpOutDir: "/tmp", + LockFileName: utils.ResourcesCsv, + Data: nil, + } + + var reply string + ldrs := NewLoaderService(dm, cfgLdr, "UTC", nil, nil) + if err := ldrs.V1Load(&ArgsProcessFolder{ + LoaderID: utils.EmptyString}, &reply); err == nil && reply != utils.EmptyString && err.Error() != utils.EmptyString { + t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply) + } + + if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { + t.Error(err) + } else if err := os.Remove(flPath); err != nil { + t.Error(err) + } +} + +func testV1LoadUnableToDeleteFile(t *testing.T) { + flPath := "testV1LoadUnableToDeleteFile" + if err := os.MkdirAll(flPath, 0777); err != nil { + t.Error(err) + } + _, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) + if err != nil { + t.Error(err) + } + + data := engine.NewInternalDB(nil, nil, true) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() + cfgLdr[0] = &config.LoaderSCfg{ + ID: "testV1LoadUnableToDeleteFile", + Enabled: true, + FieldSeparator: utils.FIELDS_SEP, + TpInDir: "/\x00", + TpOutDir: "/tmp", + LockFileName: utils.ResourcesCsv, + Data: nil, + } + var reply string + ldrs := NewLoaderService(dm, cfgLdr, "UTC", nil, nil) + expected := "SERVER_ERROR: stat /\x00/Resources.csv: invalid argument" + if err := ldrs.V1Load(&ArgsProcessFolder{ + LoaderID: "testV1LoadUnableToDeleteFile", + ForceLock: true}, &reply); err == nil || err.Error() != expected { + t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply) + } + + if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { + t.Error(err) + } else if err := os.Remove(flPath); err != nil { + t.Error(err) + } +} + +func testV1LoadProcessFolderError(t *testing.T) { + flPath := "testV1LoadProcessFolderError" + if err := os.MkdirAll(flPath, 0777); err != nil { + t.Error(err) + } + file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) + if err != nil { + t.Error(err) + } + file.Write([]byte(` +#ActivationInterval[0] +* * * * * * * +`)) + file.Close() + + data := engine.NewInternalDB(nil, nil, true) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() + cfgLdr[0] = &config.LoaderSCfg{ + ID: "testV1LoadResource", + Enabled: true, + FieldSeparator: utils.FIELDS_SEP, + TpInDir: flPath, + TpOutDir: "/tmp", + LockFileName: utils.ResourcesCsv, + Data: nil, + } + ldrs := NewLoaderService(dm, cfgLdr, "UTC", nil, nil) + ldrs.ldrs["testV1LoadResource"].dataTpls = map[string][]*config.FCTemplate{ + utils.MetaResources: { + {Tag: "ActivationInterval", + Path: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), + Mandatory: true}, + }, + } + + resCsv := ` +#ActivationInterval[0] +* * * * * * * +` + rdr := ioutil.NopCloser(strings.NewReader(resCsv)) + csvRdr := csv.NewReader(rdr) + csvRdr.Comment = '#' + ldrs.ldrs["testV1LoadResource"].rdrs = map[string]map[string]*openedCSVFile{ + utils.MetaResources: { + utils.ResourcesCsv: &openedCSVFile{ + fileName: utils.ResourcesCsv, + rdr: rdr, + csvRdr: csvRdr, + }, + }, + } + + var reply string + //try to load by changing the caching method + if err := ldrs.V1Load(&ArgsProcessFolder{ + LoaderID: "testV1LoadResource", + ForceLock: true, + Caching: utils.StringPointer("not_reload"), + StopOnError: true}, &reply); err != nil && reply != utils.OK { + t.Error(err) + } + + if err := os.Remove(flPath); err != nil { + t.Error(err) + } +} diff --git a/loaders/loaders_test.go b/loaders/loaders_test.go deleted file mode 100644 index c5792394b..000000000 --- a/loaders/loaders_test.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package loaders - -import "testing" - -func TestV1ReloadUnknownLoader(t *testing.T) { - ldr := &Loader{} - ldrs := &LoaderService{ldrs: map[string]*Loader{ - "INVALID_LOADER": ldr, - }} - var reply string - expected := "UNKNOWN_LOADER: LOADER1" - if err := ldrs.V1Load(&ArgsProcessFolder{LoaderID: "LOADER1"}, &reply); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -}