diff --git a/actions/accounts_test.go b/actions/accounts_test.go index 04428d290..9037dcdcf 100644 --- a/actions/accounts_test.go +++ b/actions/accounts_test.go @@ -66,7 +66,7 @@ func TestACExecuteAccountsSetBalance(t *testing.T) { } actCdrLG.config.ActionSCfg().AccountSConns = []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaAccounts)} - expected = "Closed unspilit syntax " + expected = "Closed unspilit syntax" if err := actCdrLG.execute(context.Background(), dataStorage, utils.MetaBalanceLimit); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } diff --git a/config/rsrparser.go b/config/rsrparser.go index 1a8543e48..4ba1c7d91 100644 --- a/config/rsrparser.go +++ b/config/rsrparser.go @@ -37,7 +37,7 @@ func NewRSRParsers(parsersRules string, rsrSeparator string) (prsrs RSRParsers, return } if count := strings.Count(parsersRules, utils.RSRConstSep); count%2 != 0 { // check if we have matching ` - return nil, fmt.Errorf("Closed unspilit syntax ") + return nil, fmt.Errorf("Closed unspilit syntax") } else if count != 0 { var splitedRule []string for idx := strings.IndexByte(parsersRules, utils.RSRConstChar); idx != -1; idx = strings.IndexByte(parsersRules, utils.RSRConstChar) { diff --git a/console/loader_remove.go b/console/loader_remove.go deleted file mode 100644 index 38117bc19..000000000 --- a/console/loader_remove.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/loaders" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdLoaderRemove{ - name: "loader_remove", - rpcMethod: utils.LoaderSv1Remove, - rpcParams: &loaders.ArgsProcessFolder{}, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -type CmdLoaderRemove struct { - name string - rpcMethod string - rpcParams *loaders.ArgsProcessFolder - *CommandExecuter -} - -func (self *CmdLoaderRemove) Name() string { - return self.name -} - -func (self *CmdLoaderRemove) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdLoaderRemove) RpcParams(reset bool) interface{} { - if reset || self.rpcParams == nil { - self.rpcParams = &loaders.ArgsProcessFolder{} - } - return self.rpcParams -} - -func (self *CmdLoaderRemove) PostprocessRpcParams() error { - return nil -} - -func (self *CmdLoaderRemove) RpcResult() interface{} { - var s string - return &s -} diff --git a/console/loader_load.go b/console/loader_run.go similarity index 96% rename from console/loader_load.go rename to console/loader_run.go index d8c9f3a96..fc71a5a39 100644 --- a/console/loader_load.go +++ b/console/loader_run.go @@ -25,8 +25,8 @@ import ( func init() { c := &CmdLoaderLoad{ - name: "loader_load", - rpcMethod: utils.LoaderSv1Load, + name: "loader_run", + rpcMethod: utils.LoaderSv1Run, rpcParams: &loaders.ArgsProcessFolder{}, } commands[c.Name()] = c diff --git a/engine/model_helpers.go b/engine/model_helpers.go index fac3d80a2..63272f0e4 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1322,7 +1322,7 @@ func APItoModelTPCharger(tpCPP *utils.TPChargerProfile) (mdls ChargerMdls) { return } -func APItoChargerProfile(tpCPP *utils.TPChargerProfile, timezone string) (cpp *ChargerProfile, err error) { +func APItoChargerProfile(tpCPP *utils.TPChargerProfile, timezone string) (cpp *ChargerProfile) { cpp = &ChargerProfile{ Tenant: tpCPP.Tenant, ID: tpCPP.ID, @@ -1337,7 +1337,7 @@ func APItoChargerProfile(tpCPP *utils.TPChargerProfile, timezone string) (cpp *C for i, attribute := range tpCPP.AttributeIDs { cpp.AttributeIDs[i] = attribute } - return cpp, nil + return cpp } func ChargerProfileToAPI(chargerPrf *ChargerProfile) (tpCharger *utils.TPChargerProfile) { @@ -1519,7 +1519,7 @@ func APItoModelTPDispatcherProfile(tpDPP *utils.TPDispatcherProfile) (mdls Dispa return } -func APItoDispatcherProfile(tpDPP *utils.TPDispatcherProfile, timezone string) (dpp *DispatcherProfile, err error) { +func APItoDispatcherProfile(tpDPP *utils.TPDispatcherProfile, timezone string) (dpp *DispatcherProfile) { dpp = &DispatcherProfile{ Tenant: tpDPP.Tenant, ID: tpDPP.ID, @@ -1560,7 +1560,7 @@ func APItoDispatcherProfile(tpDPP *utils.TPDispatcherProfile, timezone string) ( } } - return dpp, nil + return dpp } func DispatcherProfileToAPI(dpp *DispatcherProfile) (tpDPP *utils.TPDispatcherProfile) { diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index f281a9c65..d0581d263 100644 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -1334,9 +1334,7 @@ func TestAPItoChargerProfile(t *testing.T) { AttributeIDs: []string{"ATTR1", "ATTR2"}, Weight: 20, } - if rcv, err := APItoChargerProfile(tpCPP, "UTC"); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expected, rcv) { + if rcv := APItoChargerProfile(tpCPP, "UTC"); !reflect.DeepEqual(expected, rcv) { t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(expected), utils.ToJSON(rcv)) } } @@ -1701,9 +1699,7 @@ func TestAPItoDispatcherProfile(t *testing.T) { }, }, } - if rcv, err := APItoDispatcherProfile(tpDPP, "UTC"); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expected, rcv) { + if rcv := APItoDispatcherProfile(tpDPP, "UTC"); !reflect.DeepEqual(expected, rcv) { t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(expected), utils.ToJSON(rcv)) } } @@ -3392,7 +3388,7 @@ func TestAPItoDispatcherProfileCase2(t *testing.T) { }, }, } - result, _ := APItoDispatcherProfile(structTest, "") + result := APItoDispatcherProfile(structTest, "") if !reflect.DeepEqual(result, expStruct) { t.Errorf("\nExpecting <%+v>>,\n Received <%+v>", utils.ToJSON(expStruct), utils.ToJSON(result)) } @@ -4008,7 +4004,7 @@ func TestAPItoAttributeProfileError2(t *testing.T) { } _, err := APItoAttributeProfile(tpAlsPrf, "UTC") - expected := "Closed unspilit syntax " + expected := "Closed unspilit syntax" if err == nil || err.Error() != expected { t.Errorf("\nExpecting <%+v>,\n Received <%+v>", expected, err) } diff --git a/engine/tpreader.go b/engine/tpreader.go index d0b9713f7..0b0142720 100644 --- a/engine/tpreader.go +++ b/engine/tpreader.go @@ -486,10 +486,7 @@ func (tpr *TpReader) WriteToDatabase(verbose, disableReverse bool) (err error) { log.Print("ChargerProfiles:") } for _, tpTH := range tpr.chargerProfiles { - var th *ChargerProfile - if th, err = APItoChargerProfile(tpTH, tpr.timezone); err != nil { - return - } + th := APItoChargerProfile(tpTH, tpr.timezone) if err = tpr.dm.SetChargerProfile(context.TODO(), th, true); err != nil { return } @@ -504,10 +501,7 @@ func (tpr *TpReader) WriteToDatabase(verbose, disableReverse bool) (err error) { log.Print("DispatcherProfiles:") } for _, tpTH := range tpr.dispatcherProfiles { - var th *DispatcherProfile - if th, err = APItoDispatcherProfile(tpTH, tpr.timezone); err != nil { - return - } + th := APItoDispatcherProfile(tpTH, tpr.timezone) if err = tpr.dm.SetDispatcherProfile(context.TODO(), th, true); err != nil { return } diff --git a/engine/z_loader_it_test.go b/engine/z_loader_it_test.go index b8b04ad52..ade73535c 100644 --- a/engine/z_loader_it_test.go +++ b/engine/z_loader_it_test.go @@ -324,10 +324,7 @@ func testLoaderITWriteToDatabase(t *testing.T) { if err != nil { t.Errorf("Failed GetChargerProfile, tenant: %s, id: %s, error: %s ", cpp.Tenant, cpp.ID, err.Error()) } - cp, err := APItoChargerProfile(cpp, "UTC") - if err != nil { - t.Error(err) - } + cp := APItoChargerProfile(cpp, "UTC") if !reflect.DeepEqual(cp, rcv) { t.Errorf("Expecting: %v, received: %v", cp, rcv) } @@ -338,10 +335,7 @@ func testLoaderITWriteToDatabase(t *testing.T) { if err != nil { t.Errorf("Failed GetDispatcherProfile, tenant: %s, id: %s, error: %s ", dpp.Tenant, dpp.ID, err.Error()) } - dp, err := APItoDispatcherProfile(dpp, "UTC") - if err != nil { - t.Error(err) - } + dp := APItoDispatcherProfile(dpp, "UTC") if !reflect.DeepEqual(dp, rcv) { t.Errorf("Expecting: %v, received: %v", dp, rcv) } diff --git a/loaders/csvreader.go b/loaders/csvreader.go index 0c948ff15..46173b74f 100644 --- a/loaders/csvreader.go +++ b/loaders/csvreader.go @@ -44,14 +44,14 @@ func NewCSVReader(csvType, dPath, fn string, sep rune, nrFlds int) (CSVReader, e case utils.MetaFileCSV: return NewFileCSV(path.Join(dPath, fn), sep, nrFlds) case utils.MetaUrl: - return NewURLCSV(path.Join(dPath, fn), sep, nrFlds) + return NewURLCSV(strings.TrimSuffix(dPath, utils.Slash)+utils.Slash+fn, sep, nrFlds) case utils.MetaGoogleAPI: // TODO: Implement *gapi + return nil, nil case utils.MetaString: return NewStringCSV(fn, sep, nrFlds) default: return nil, fmt.Errorf("unsupported CSVReader type: <%q>", csvType) } - return nil, nil } func NewCSVFile(rdr io.ReadCloser, path string, sep rune, nrFlds int) CSVReader { diff --git a/loaders/csvreader_test.go b/loaders/csvreader_test.go new file mode 100644 index 000000000..5f32c2c21 --- /dev/null +++ b/loaders/csvreader_test.go @@ -0,0 +1,173 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package loaders + +import ( + "encoding/csv" + "fmt" + "io" + "math/rand" + "net/http" + "net/http/httptest" + "os" + "path" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/cgrates/cgrates/utils" +) + +func TestNewCSVStringReader(t *testing.T) { + data := `cgrates.org,ATTR_VARIABLE,,20,,*req.Category,*variable,~*req.ToR,` + + expErrMsg := `unsupported CSVReader type: <"nonValidType">` + if _, err := NewCSVReader("nonValidType", utils.EmptyString, data, utils.CSVSep, -1); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if np, err := NewCSVReader(utils.MetaGoogleAPI, utils.EmptyString, data, utils.CSVSep, -1); err != nil || np != nil { // for coverage (remove when implemented) + t.Error("This tipe should not be implemented yet") + } + csvR, err := NewCSVReader(utils.MetaString, utils.EmptyString, data, utils.CSVSep, -1) + if err != nil { + t.Fatal(err) + } + cls := io.NopCloser(strings.NewReader(data)) + exp := &CSVFile{ + path: data, + cls: cls, + csvRdr: csv.NewReader(cls), + } + exp.csvRdr.Comma = utils.CSVSep + exp.csvRdr.Comment = utils.CommentChar + exp.csvRdr.FieldsPerRecord = -1 + exp.csvRdr.TrailingComma = true + + if !reflect.DeepEqual(csvR, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, csvR) + } + if p := csvR.Path(); p != data { + t.Errorf("Expeceted: %+v, received: %+v", data, p) + } + + if p, err := csvR.Read(); err != nil { + t.Fatal(err) + } else if exp := strings.Split(data, ","); !reflect.DeepEqual(p, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, p) + } + + if err := csvR.Close(); err != nil { + t.Error(err) + } +} + +func TestNewCSVReaderErrors(t *testing.T) { + path := "TestNewCSVReaderErrors" + strconv.Itoa(rand.Int()) + utils.CSVSuffix + expErrMsg := fmt.Sprintf("open %s: no such file or directory", path) + if _, err := NewCSVReader(utils.MetaFileCSV, ".", path, utils.CSVSep, -1); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = fmt.Sprintf("parse %q: invalid URI for request", "./"+path) + if _, err := NewCSVReader(utils.MetaUrl, ".", path, utils.CSVSep, -1); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + prePath := "http:/localhost:" + strconv.Itoa(rand.Int()) + expErrMsg = fmt.Sprintf(`path:"%s/%s" is not reachable`, prePath, path) + if _, err := NewCSVReader(utils.MetaUrl, prePath, path, utils.CSVSep, -1); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %q, received: %q", expErrMsg, err.Error()) + } +} + +func TestNewCSVURLReader(t *testing.T) { + data := `cgrates.org,ATTR_VARIABLE,,20,,*req.Category,*variable,~*req.ToR,` + mux := http.NewServeMux() + mux.Handle("/ok/", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.Write([]byte(data)) })) + s := httptest.NewServer(mux) + defer s.Close() + runtime.Gosched() + + if _, err := NewCSVReader(utils.MetaUrl, s.URL+"/notFound", utils.AttributesCsv, utils.CSVSep, -1); err != utils.ErrNotFound { + t.Errorf("Expeceted: %v, received: %v", utils.ErrNotFound, err) + } + + csvR, err := NewCSVReader(utils.MetaUrl, s.URL+"/ok", utils.AttributesCsv, utils.CSVSep, -1) + if err != nil { + t.Fatal(err) + } + expPath := s.URL + "/ok/" + utils.AttributesCsv + if p := csvR.Path(); p != expPath { + t.Errorf("Expeceted: %+v, received: %+v", data, expPath) + } + + if p, err := csvR.Read(); err != nil { + t.Fatal(err) + } else if exp := strings.Split(data, ","); !reflect.DeepEqual(p, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, p) + } + + if err := csvR.Close(); err != nil { + t.Error(err) + } +} + +func TestNewCSVFileReader(t *testing.T) { + data := `cgrates.org,ATTR_VARIABLE,,20,,*req.Category,*variable,~*req.ToR,` + dir, err := os.MkdirTemp(utils.EmptyString, "TestNewCSVFileReader") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + fp := path.Join(dir, utils.AttributesCsv) + f, err := os.Create(fp) + if err != nil { + t.Fatal(err) + } + if _, err = f.WriteString(data); err != nil { + t.Fatal(err) + } + if err = f.Sync(); err != nil { + t.Fatal(err) + } + if err = f.Close(); err != nil { + t.Fatal(err) + } + + csvR, err := NewCSVReader(utils.MetaFileCSV, dir, utils.AttributesCsv, utils.CSVSep, -1) + if err != nil { + t.Fatal(err) + } + if p := csvR.Path(); p != fp { + t.Errorf("Expeceted: %+v, received: %+v", data, fp) + } + + if p, err := csvR.Read(); err != nil { + t.Fatal(err) + } else if exp := strings.Split(data, ","); !reflect.DeepEqual(p, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, p) + } + + if err := csvR.Close(); err != nil { + t.Error(err) + } + +} diff --git a/loaders/lib_test.go b/loaders/lib_test.go deleted file mode 100644 index 98f38563a..000000000 --- a/loaders/lib_test.go +++ /dev/null @@ -1,926 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "encoding/csv" - "errors" - "flag" - "io" - "strings" - "testing" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/jsonrpc" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/rpcclient" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - waitRater = flag.Int("wait_rater", 200, "Number of miliseconds to wait for rater to start and cache") - dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") - encoding = flag.String("rpc", utils.MetaJSON, "what encoding whould be used for rpc comunication") - dbType = flag.String("dbtype", utils.MetaInternal, "The type of DataBase (Internal/Mongo/mySql)") -) - -var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut", - "/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} - -func newRPCClient(cfg *config.ListenCfg) (c *birpc.Client, err error) { - switch *encoding { - case utils.MetaJSON: - return jsonrpc.Dial(utils.TCP, cfg.RPCJSONListen) - case utils.MetaGOB: - return birpc.Dial(utils.TCP, cfg.RPCGOBListen) - default: - return nil, errors.New("UNSUPPORTED_RPC") - } -} - -type testMockCacheConn struct { - calls map[string]func(_ *context.Context, _, _ interface{}) error -} - -func (s *testMockCacheConn) Call(ctx *context.Context, method string, arg, rply interface{}) error { - if call, has := s.calls[method]; !has { - return rpcclient.ErrUnsupporteServiceMethod - } else { - return call(ctx, arg, rply) - } -} - -func TestProcessContentCallsLoadCache(t *testing.T) { - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1LoadCache: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type: %T", rply) - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - internalCacheSChann := make(chan birpc.ClientConnector, 1) - internalCacheSChann <- sMock - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - ldr := &Loader{ - ldrID: "TestProcessContentCallsLoadCache", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - connMgr: engine.NewConnManager(config.CgrConfig()), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChann) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRateProfiles: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - }, - } - ratePrfCsv := ` -#Tenant[0],ID[1],Weight[2] -cgrates.org,MOCK_RELOAD_ID,20 -` - rdr := io.NopCloser(strings.NewReader(ratePrfCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRateProfiles: { - utils.RatesCsv: &openedCSVFile{ - fileName: utils.RatesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(context.Background(), utils.MetaRateProfiles, utils.MetaLoad); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(ratePrfCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRateProfiles: { - utils.RatesCsv: &openedCSVFile{ - fileName: utils.RatesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(context.Background(), utils.MetaRateProfiles, utils.MetaLoad); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestProcessContentCallsReloadCache(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock2 := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1ReloadCache: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock2 - ldr := &Loader{ - ldrID: "TestProcessContentCalls", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRateProfiles: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - }, - } - ratePrfCsv := ` -#Tenant[0],ID[1],Weight[2] -cgrates.org,MOCK_RELOAD_ID,20 -` - rdr := io.NopCloser(strings.NewReader(ratePrfCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRateProfiles: { - utils.RatesCsv: &openedCSVFile{ - fileName: utils.RatesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(context.Background(), utils.MetaRateProfiles, utils.MetaReload); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(ratePrfCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRateProfiles: { - utils.RatesCsv: &openedCSVFile{ - fileName: utils.RatesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(context.Background(), utils.MetaRateProfiles, utils.MetaReload); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestProcessContentCallsRemoveItems(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1RemoveItems: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestProcessContentCallsRemoveItems", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_ID -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(context.Background(), utils.MetaAttributes, utils.MetaRemove); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(context.Background(), utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - // Calling the method again while caching method is invalid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected = "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(context.Background(), utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestProcessContentCallsClear(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestProcessContentCallsClear", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_ID -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(context.Background(), utils.MetaAttributes, utils.MetaClear); err != nil { - t.Error(err) - } - - //inexisting method(*none) of cache and reinitialized the reader will do nothing - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(context.Background(), utils.MetaAttributes, utils.MetaNone); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(context.Background(), utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsReload(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1ReloadCache: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_2 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_2", - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaReload); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaReload); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsLoad(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1LoadCache: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_3 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_3", - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaLoad); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaLoad); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsRemove(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1RemoveItems: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_4 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_4", - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaRemove); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - //inexisting method(*none) of cache and reinitialized the reader will do nothing - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaNone); err != nil { - t.Error(err) - } -} - -func TestRemoveContentCallsClear(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(_ *context.Context, _, _ interface{}) error{ - utils.CacheSv1Clear: func(_ *context.Context, _, rply interface{}) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data := engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig()), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.connMgr.AddInternalConn(utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "", internalCacheSChan) - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_3 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_3", - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaClear); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - // Calling the method again while caching method is invalid - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.dm.SetAttributeProfile(context.Background(), attrPrf, true); err != nil { - t.Error(err) - } - expected = "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(context.Background(), utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} diff --git a/loaders/libloader.go b/loaders/libloader.go index 7f3530797..8382c15a1 100644 --- a/loaders/libloader.go +++ b/loaders/libloader.go @@ -39,31 +39,8 @@ func newRecord(ctx *context.Context, req utils.DataProvider, tmpls []*config.FCT cfg: cfg.GetDataProvider(), cache: cache, } - for _, fld := range tmpls { - // Make sure filters are matching - if len(fld.Filters) != 0 { - if pass, err := filterS.Pass(ctx, tnt, - fld.Filters, r); err != nil { - return nil, err - } else if !pass { - continue // Not passes filters, ignore this CDR - } - } - - var out interface{} - if out, err = engine.ParseAttribute(r, utils.FirstNonEmpty(fld.Type, utils.MetaVariable), utils.DynamicDataPrefix+fld.Path, fld.Value, - cfg.GeneralCfg().RoundingDecimals, utils.FirstNonEmpty(fld.Timezone, cfg.GeneralCfg().DefaultTimezone), fld.Layout, cfg.GeneralCfg().RSRSep); err != nil { - return - } - ps := fld.GetPathSlice() - if fld.Type == utils.MetaComposed { - if val, err := r.FieldAsString(ps); err == nil { - out = utils.IfaceAsString(out) + val - } - } - if err = r.Set(ps, out); err != nil { - return - } + if err = r.parseTemplates(ctx, tmpls, tnt, filterS, cfg.GeneralCfg().RoundingDecimals, cfg.GeneralCfg().DefaultTimezone, cfg.GeneralCfg().RSRSep); err != nil { + return } return r.data, nil } @@ -76,16 +53,46 @@ type record struct { tntID *string } +func (r *record) parseTemplates(ctx *context.Context, tmpls []*config.FCTemplate, tnt string, filterS *engine.FilterS, rndDec int, dftTmz, rsrSep string) (err error) { + for _, fld := range tmpls { + // Make sure filters are matching + if len(fld.Filters) != 0 { + if pass, err := filterS.Pass(ctx, tnt, + fld.Filters, r); err != nil { + return err + } else if !pass { + continue // Not passes filters, ignore this CDR + } + } + var out interface{} + if out, err = engine.ParseAttribute(r, utils.FirstNonEmpty(fld.Type, utils.MetaVariable), utils.DynamicDataPrefix+fld.Path, fld.Value, + rndDec, utils.FirstNonEmpty(fld.Timezone, dftTmz), fld.Layout, rsrSep); err != nil { + return + } + ps := fld.GetPathSlice() + if fld.Type == utils.MetaComposed { + if val, err := r.FieldAsString(ps); err == nil { + out = utils.IfaceAsString(out) + val + } + } + if err = r.Set(ps, out); err != nil { + return + } + } + return +} + func (r *record) String() string { return r.req.String() } func (r *record) FieldAsInterface(path []string) (val interface{}, err error) { switch path[0] { - case utils.MetaCache: - if path[1] == "*id" { - path[1] = r.tenatID() + case utils.MetaUCH: + cp := strings.Join(path[1:], utils.NestingSep) + if path[1] == utils.MetaTntID { + cp = r.tenatID() + strings.TrimPrefix(cp, utils.MetaTntID) } var ok bool - if val, ok = r.cache.Get(strings.Join(path[1:], utils.NestingSep)); !ok || val == nil { + if val, ok = r.cache.Get(cp); !ok || val == nil { err = utils.ErrNotFound } return @@ -108,11 +115,12 @@ func (r *record) FieldAsString(path []string) (str string, err error) { func (r *record) Set(path []string, val interface{}) (err error) { switch path[0] { - case utils.MetaCache: - if path[1] == "*id" { - path[1] = r.tenatID() + case utils.MetaUCH: + cp := strings.Join(path[1:], utils.NestingSep) + if path[1] == utils.MetaTntID { + cp = r.tenatID() + strings.TrimPrefix(cp, utils.MetaTntID) } - r.cache.Set(strings.Join(path[1:], utils.NestingSep), val, nil) + r.cache.Set(cp, val, nil) return default: return r.data.Set(path, val) diff --git a/loaders/libloader_test.go b/loaders/libloader_test.go index 08899ebe1..f92adc4d3 100644 --- a/loaders/libloader_test.go +++ b/loaders/libloader_test.go @@ -23,358 +23,116 @@ import ( "testing" "github.com/cgrates/birpc/context" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/ltcache" ) -func TestDataUpdateFromCSVOneFile(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - { - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - { - Path: "Contexts", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - { - Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - { - Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - { - Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - { - Path: "Initial", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - { - Path: "Substitute", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - { - Path: "Append", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - } - - rows := [][]string{ - {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "10", "Account", "*any", "1001", "false"}, - {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true"}, - } - lData := make(LoaderData) - if err := lData.UpdateFromCSV(context.Background(), "Attributes.csv", rows[0], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*sessions;*cdrs", - "FilterIDs": "*string:Account:1007", - "Weight": "10", - "Path": "Account", - "Initial": "*any", - "Substitute": "1001", - "Append": "false", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } - lData = make(LoaderData) - if err := lData.UpdateFromCSV(context.Background(), "Attributes.csv", rows[1], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData = LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "", - "FilterIDs": "", - "Weight": "", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) +func TestTenantIDFromMap(t *testing.T) { + exp := utils.NewTenantID("cgrates.org:ATTR1") + r := TenantIDFromMap(utils.MapStorage{ + utils.Tenant: exp.Tenant, + utils.ID: exp.ID, + }) + if !reflect.DeepEqual(r, exp) { + t.Errorf("Expected %+v, received %+q", exp, r) } } -func TestDataUpdateFromCSVOneFile2(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - { - Path: "ID", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - { - Path: "Contexts", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - { - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - { - Path: "Weight", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - { - Path: "Path", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - { - Path: "Initial", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - { - Path: "Substitute", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - { - Path: "Append", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, +func TestRateIDsFromMap(t *testing.T) { + expErrMsg := "cannot find RateIDs in map" + if _, err := RateIDsFromMap(utils.MapStorage{}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) } - - rows := [][]string{ - {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "10", "Account", "*any", "1001", "false"}, - {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true"}, + exp := []string{"RT1", "RT2"} + r, err := RateIDsFromMap(utils.MapStorage{ + utils.RateIDs: "RT1;RT2", + }) + if err != nil { + t.Fatal(err) } - lData := make(LoaderData) - if err := lData.UpdateFromCSV(context.Background(), "Attributes.csv", rows[0], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*sessions;*cdrs", - "FilterIDs": "*string:Account:1007", - "Weight": "10", - "Path": "Account", - "Initial": "*any", - "Substitute": "1001", - "Append": "false", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } - lData = make(LoaderData) - if err := lData.UpdateFromCSV(context.Background(), "Attributes.csv", rows[1], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData = LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "", - "FilterIDs": "", - "Weight": "", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) + if !reflect.DeepEqual(r, exp) { + t.Errorf("Expected %+v, received %+q", exp, r) } } -func TestDataUpdateFromCSVMultiFiles(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Mandatory: true}, - { - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Mandatory: true}, - { - Path: "Contexts", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("*any", utils.InfieldSep)}, - { - Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).5", utils.InfieldSep)}, - { - Path: "Initial", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.InfieldSep)}, - { - Path: "Substitute", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.InfieldSep)}, - { - Path: "Append", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("true", utils.InfieldSep)}, - { - Path: "Weight", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("10", utils.InfieldSep)}, - } - - loadRun1 := map[string][]string{ - "File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, - "File2.csv": {"ignored", "ATTR_1"}, - } - lData := make(LoaderData) - for fName, record := range loadRun1 { - if err := lData.UpdateFromCSV(context.Background(), fName, record, attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - } - - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*any", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - "Weight": "10", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } -} - -func TestGetRateIDsLoaderData(t *testing.T) { - ldrData := LoaderData{ - "File1.csv": []string{"Subject", "*any", "1001"}, - } - expected := "cannot find RateIDs in " - if _, err := ldrData.GetRateIDs(); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -// func TestUpdateFromCsvParseValueError(t *testing.T) { -// ldrData := LoaderData{ -// "File1.csv": []string{"Subject", "*any", "1001"}, -// } -// tnt := config.NewRSRParsersMustCompile("asd{*duration_seconds}", utils.InfieldSep) -// // fmt.Println(utils.IfaceAsString(tnt)) -// expected := "time: invalid duration \"asd\"" -// if err := ldrData.UpdateFromCSV(context.Background(), "File1.csv", nil, nil, utils.IfaceAsString(tnt), nil); err == nil || err.Error() != expected { -// t.Errorf("Expected %+v, received %+v", expected, err) -// } -// } - -func TestUpdateFromCsvWithFiltersError(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Filters: []string{"*string:~*req.Account:10"}, - Mandatory: true}, - { - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Filters: []string{"*string:~*req.Account:10"}, - Mandatory: true}, - } - loadRunStr := map[string][]string{ - "File1.csv": {"cgrates.org", "TEST_1"}, - } - lData := make(LoaderData) - +func TestNewRecord(t *testing.T) { cfg := config.NewDefaultCGRConfig() - data := engine.NewInternalDB(nil, nil, cfg.DataDbCfg().Items) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - filterS := engine.NewFilterS(cfg, nil, dm) - - for fName, record := range loadRunStr { - expected := "Ignoring record: [\"cgrates.org\" \"TEST_1\"] with error : strconv.Atoi: parsing \"Account\": invalid syntax" - if err := lData.UpdateFromCSV(context.Background(), fName, record, attrSFlds, - "cgrates.org", filterS); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } + fs := engine.NewFilterS(cfg, nil, nil) + expErrMsg := "inline parse error for string: <*string>" + if _, err := newRecord(context.Background(), utils.MapStorage{}, + []*config.FCTemplate{ + {Filters: []string{"*exists:~*req.NoField:"}}, + {Filters: []string{"*string"}}, + }, + "cgrates.org", fs, cfg, ltcache.NewCache(-1, 0, false, nil)); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %q, received: %v", expErrMsg, err) } + r, err := newRecord(context.Background(), utils.MapStorage{}, []*config.FCTemplate{}, "cgrates.org", fs, cfg, ltcache.NewCache(-1, 0, false, nil)) + if err != nil { + t.Fatal(err) + } + exp := utils.MapStorage{} + if !reflect.DeepEqual(r, exp) { + t.Errorf("Expected %+v, received %+q", exp, r) + } + } -func TestUpdateFromCsvWithFiltersContinue(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Filters: []string{`*string:~*req.2:10`}, - Mandatory: true}, - { - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Filters: []string{`*string:~*req.2:10`}, - Mandatory: true}, - } - loadRunStr := map[string][]string{ - "File1.csv": {"Subject", "*any", "1001"}, - } - lData := make(LoaderData) - +func TestNewRecordErrors(t *testing.T) { cfg := config.NewDefaultCGRConfig() - data := engine.NewInternalDB(nil, nil, cfg.DataDbCfg().Items) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - filterS := engine.NewFilterS(cfg, nil, dm) + fs := engine.NewFilterS(cfg, nil, nil) + expErrMsg := "unsupported type: " + if _, err := newRecord(context.Background(), utils.MapStorage{}, + []*config.FCTemplate{ + {Type: "notSupported"}, + }, + "cgrates.org", fs, cfg, ltcache.NewCache(-1, 0, false, nil)); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %q, received: %v", expErrMsg, err) + } - for fName, record := range loadRunStr { - if err := lData.UpdateFromCSV(context.Background(), fName, record, attrSFlds, - "cgrates.org", filterS); err != nil { - t.Error(err) - } + r := &record{ + data: utils.MapStorage{ + "Tenant": []string{}, + }, + req: utils.MapStorage{}, + cfg: cfg.GetDataProvider(), + cache: ltcache.NewCache(-1, 0, false, nil), + } + if exp, rply := "{}", r.String(); exp != rply { + t.Errorf("Expected %q, received %q", exp, rply) + } + fc := []*config.FCTemplate{ + {Type: utils.MetaComposed, Path: "Tenant", Value: config.NewRSRParsersMustCompile("~*cfg.general.node_id", ";")}, + {Type: utils.MetaComposed, Path: "Tenant.NewID", Value: config.NewRSRParsersMustCompile("10", ";")}, + } + for _, f := range fc { + f.ComputePath() + } + if err := r.parseTemplates(context.Background(), fc, + "cgrates.org", fs, 0, "", ";"); err != utils.ErrWrongPath { + t.Errorf("Expeceted: %q, received: %v", utils.ErrWrongPath, err) } } -func TestLoadersFieldAsInterfaceError(t *testing.T) { - loadRun1 := map[string][]string{ - "File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, +func TestNewRecordWithCahe(t *testing.T) { + cfg := config.NewDefaultCGRConfig() + fs := engine.NewFilterS(cfg, nil, nil) + fc := []*config.FCTemplate{ + {Type: utils.MetaVariable, Path: "Tenant", Value: config.NewRSRParsersMustCompile("~*req.0", ";")}, + {Type: utils.MetaVariable, Path: "ID", Value: config.NewRSRParsersMustCompile("~*req.1", ";")}, + {Type: utils.MetaComposed, Path: "*uch.*tntID.Value", Value: config.NewRSRParsersMustCompile("0", ";")}, + {Type: utils.MetaVariable, Path: "Value", Value: config.NewRSRParsersMustCompile("~*uch.*tntID.Value", ";")}, } - csvProv := newCsvProvider(loadRun1["File1.csv"], "File1.csv") - - csvString := csvProv.String() - expected := "{}" - if csvString != expected { - t.Errorf("Expected %+v, received %+v", expected, csvString) + for _, f := range fc { + f.ComputePath() } - - expected = "invalid prefix for : [File2.csv]" - if _, err := csvProv.FieldAsInterface([]string{"File2.csv"}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - expected = "filter rule <[*file() ]> needs to end in )" - if _, err := csvProv.FieldAsInterface([]string{"*file()", ""}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+q", expected, err) - } - - expected = "filter rule <[*file() File1.csv]> needs to end in )" - if _, err := csvProv.FieldAsInterface([]string{"*file()", "File1.csv"}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+q", expected, err) + exp := utils.MapStorage{"ID": "Attr1", "Tenant": "cgrates.org", "Value": "0"} + if r, err := newRecord(context.Background(), config.NewSliceDP([]string{"cgrates.org", "Attr1"}, nil), + fc, "cgrates.org", fs, cfg, ltcache.NewCache(-1, 0, false, nil)); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(r, exp) { + t.Errorf("Expected %+v, received %+q", exp, r) } } diff --git a/loaders/loader.go b/loaders/loader.go index 6bbf2128a..c5f64c49d 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -190,11 +190,7 @@ func setToDB(ctx *context.Context, dm *engine.DataManager, lType, tmz string, lD } for _, tpCPP := range cppModels.AsTPChargers() { - var cpp *engine.ChargerProfile - if cpp, err = engine.APItoChargerProfile(tpCPP, tmz); err != nil { - return - } - if err = dm.SetChargerProfile(ctx, cpp, withIndex); err != nil { + if err = dm.SetChargerProfile(ctx, engine.APItoChargerProfile(tpCPP, tmz), withIndex); err != nil { return } } @@ -207,11 +203,7 @@ func setToDB(ctx *context.Context, dm *engine.DataManager, lType, tmz string, lD } } for _, tpDsp := range dispModels.AsTPDispatcherProfiles() { - var dsp *engine.DispatcherProfile - if dsp, err = engine.APItoDispatcherProfile(tpDsp, tmz); err != nil { - return - } - if err = dm.SetDispatcherProfile(ctx, dsp, withIndex); err != nil { + if err = dm.SetDispatcherProfile(ctx, engine.APItoDispatcherProfile(tpDsp, tmz), withIndex); err != nil { return } } @@ -228,8 +220,7 @@ func setToDB(ctx *context.Context, dm *engine.DataManager, lType, tmz string, lD return } for _, tpDsp := range tpDsps { - dsp := engine.APItoDispatcherHost(tpDsp) - if err = dm.SetDispatcherHost(ctx, dsp); err != nil { + if err = dm.SetDispatcherHost(ctx, engine.APItoDispatcherHost(tpDsp)); err != nil { return } } @@ -298,7 +289,7 @@ func setToDB(ctx *context.Context, dm *engine.DataManager, lType, tmz string, lD return } -func dryRun(ctx *context.Context, dm *engine.DataManager, lType, tmz, ldrID string, lDataSet []utils.MapStorage) (err error) { +func dryRun(ctx *context.Context, lType, tmz, ldrID string, lDataSet []utils.MapStorage) (err error) { switch lType { case utils.MetaAttributes: attrModels := make(engine.AttributeMdls, len(lDataSet)) @@ -414,13 +405,9 @@ func dryRun(ctx *context.Context, dm *engine.DataManager, lType, tmz, ldrID stri } for _, tpCPP := range cppModels.AsTPChargers() { - var cpp *engine.ChargerProfile - if cpp, err = engine.APItoChargerProfile(tpCPP, tmz); err != nil { - return - } utils.Logger.Info( fmt.Sprintf("<%s-%s> DRY_RUN: ChargerProfile: %s", - utils.LoaderS, ldrID, utils.ToJSON(cpp))) + utils.LoaderS, ldrID, utils.ToJSON(engine.APItoChargerProfile(tpCPP, tmz)))) } case utils.MetaDispatchers: dispModels := make(engine.DispatcherProfileMdls, len(lDataSet)) @@ -431,13 +418,9 @@ func dryRun(ctx *context.Context, dm *engine.DataManager, lType, tmz, ldrID stri } } for _, tpDsp := range dispModels.AsTPDispatcherProfiles() { - var dsp *engine.DispatcherProfile - if dsp, err = engine.APItoDispatcherProfile(tpDsp, tmz); err != nil { - return - } utils.Logger.Info( fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherProfile: %s", - utils.LoaderS, ldrID, utils.ToJSON(dsp))) + utils.LoaderS, ldrID, utils.ToJSON(engine.APItoDispatcherProfile(tpDsp, tmz)))) } case utils.MetaDispatcherHosts: dispModels := make(engine.DispatcherHostMdls, len(lDataSet)) @@ -452,10 +435,9 @@ func dryRun(ctx *context.Context, dm *engine.DataManager, lType, tmz, ldrID stri return } for _, tpDsp := range tpDsps { - dsp := engine.APItoDispatcherHost(tpDsp) utils.Logger.Info( fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherHost: %s", - utils.LoaderS, ldrID, utils.ToJSON(dsp))) + utils.LoaderS, ldrID, utils.ToJSON(engine.APItoDispatcherHost(tpDsp)))) } case utils.MetaRateProfiles: rpMdls := make(engine.RateProfileMdls, len(lDataSet)) @@ -551,7 +533,7 @@ func (l *loader) process(ctx *context.Context, tntID *utils.TenantID, lDataSet [ case utils.MetaParse: return case utils.MetaDryRun: - return dryRun(ctx, l.dm, lType, l.timezone, l.ldrCfg.ID, lDataSet) + return dryRun(ctx, lType, l.timezone, l.ldrCfg.ID, lDataSet) case utils.MetaStore: err = setToDB(ctx, l.dm, lType, l.timezone, lDataSet, withIndex, partialRates) case utils.MetaRemove: diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go index dbce28672..b89e9f42d 100644 --- a/loaders/loader_it_test.go +++ b/loaders/loader_it_test.go @@ -20,27 +20,7 @@ along with this program. If not, see */ package loaders -import ( - "encoding/csv" - "fmt" - "sort" - - "io" - "os" - "path" - "reflect" - "strings" - "testing" - "time" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" -) - +/* var ( loaderCfgPath string loaderCfgDIR string //run tests for specific configuration @@ -1163,3 +1143,4 @@ func testNewIsFolderUnlock(t *testing.T) { t.Error("Expected the file to be unlocked") } } +*/ diff --git a/loaders/loader_test.go b/loaders/loader_test.go index 88b72ee2e..0b28c2f0b 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -19,23 +19,20 @@ along with this program. If not, see package loaders import ( - "encoding/csv" - "io" + "bytes" + "log" + "os" "reflect" - "sort" "strings" "testing" - "time" - "github.com/cgrates/birpc" "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/rpcclient" - "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) +/* var ( loaderDB = engine.NewInternalDB(nil, nil, config.CgrConfig().DataDbCfg().Items) ) @@ -5193,3 +5190,458 @@ func TestLockFolderEmptyString(t *testing.T) { // t.Errorf("Expected %v \n but received \n %v", expPath, ldr.lockFilepath) // } // } +*/ + +func TestRemoveFromDB(t *testing.T) { + cfg := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, cfg.DataDbCfg().Items), cfg.CacheCfg(), nil) + for _, lType := range []string{utils.MetaAttributes, utils.MetaResources, utils.MetaFilters, utils.MetaStats, + utils.MetaThresholds, utils.MetaRoutes, utils.MetaChargers, utils.MetaDispatchers, utils.MetaDispatcherHosts, + utils.MetaRateProfiles, utils.MetaActionProfiles, utils.MetaAccounts} { + if err := removeFromDB(context.Background(), dm, lType, "cgrates.org", "ID", true, false, utils.MapStorage{}); err != utils.ErrNotFound { + t.Error(err) + } + } + expErrMsg := "cannot find RateIDs in map" + if err := removeFromDB(context.Background(), dm, utils.MetaRateProfiles, "cgrates.org", "ID", true, true, utils.MapStorage{}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := removeFromDB(context.Background(), dm, utils.MetaRateProfiles, "cgrates.org", "ID", true, true, utils.MapStorage{utils.RateIDs: "RT1"}); err != utils.ErrNotFound { + t.Error(err) + } + if err := removeFromDB(context.Background(), dm, utils.EmptyString, "cgrates.org", "ID", true, false, utils.MapStorage{}); err != nil { + t.Error(err) + } +} + +func testDryRunWithData(lType string, data []utils.MapStorage) (string, error) { + buf := bytes.NewBuffer([]byte{}) + log.SetOutput(buf) + lgr := utils.Logger + defer func() { utils.Logger = lgr; log.SetOutput(os.Stderr) }() + utils.Logger, _ = utils.Newlogger(utils.MetaStdLog, utils.EmptyString) + utils.Logger.SetLogLevel(7) + + err := dryRun(context.Background(), lType, "", "test", data) + return buf.String(), err +} + +func testDryRun(t *testing.T, lType string) string { + buf, err := testDryRunWithData(lType, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}) + if err != nil { + t.Fatal(err) + } + return buf +} + +func TestDryRun(t *testing.T) { + if expLog, rplyLog := "[INFO] DRY_RUN: AttributeProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Attributes\":[],\"Blocker\":false,\"Weight\":0}", + testDryRun(t, utils.MetaAttributes); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: ResourceProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"UsageTTL\":0,\"Limit\":0,\"AllocationMessage\":\"\",\"Blocker\":false,\"Stored\":false,\"Weight\":0,\"ThresholdIDs\":[]}", + testDryRun(t, utils.MetaResources); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: Filter: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"Rules\":[]}", + testDryRun(t, utils.MetaFilters); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: StatsQueueProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"QueueLength\":0,\"TTL\":0,\"MinItems\":0,\"Metrics\":[],\"Stored\":false,\"Blocker\":false,\"Weight\":0,\"ThresholdIDs\":[]}", + testDryRun(t, utils.MetaStats); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: ThresholdProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"MaxHits\":0,\"MinHits\":0,\"MinSleep\":0,\"Blocker\":false,\"Weight\":0,\"ActionProfileIDs\":[],\"Async\":false}", + testDryRun(t, utils.MetaThresholds); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: RouteProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Sorting\":\"\",\"SortingParameters\":[],\"Routes\":[],\"Weights\":null}", + testDryRun(t, utils.MetaRoutes); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: ChargerProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"RunID\":\"\",\"AttributeIDs\":[],\"Weight\":0}", + testDryRun(t, utils.MetaChargers); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: DispatcherProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Strategy\":\"\",\"StrategyParams\":{},\"Weight\":0,\"Hosts\":[]}", + testDryRun(t, utils.MetaDispatchers); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + + if expLog, rplyLog := "[INFO] DRY_RUN: RateProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Weights\":null,\"MinCost\":0,\"MaxCost\":0,\"MaxCostStrategy\":\"\",\"Rates\":{}}", + testDryRun(t, utils.MetaRateProfiles); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: ActionProfile: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Weight\":0,\"Schedule\":\"\",\"Targets\":{},\"Actions\":[]}", + testDryRun(t, utils.MetaActionProfiles); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + if expLog, rplyLog := "[INFO] DRY_RUN: Accounts: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"FilterIDs\":[],\"Weights\":null,\"Opts\":null,\"Balances\":{},\"ThresholdIDs\":[]}", + testDryRun(t, utils.MetaAccounts); !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } + + rplyLog, err := testDryRunWithData(utils.MetaDispatcherHosts, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", utils.Address: "127.0.0.1"}}) + if err != nil { + t.Fatal(err) + } + if expLog := "[INFO] DRY_RUN: DispatcherHost: {\"Tenant\":\"cgrates.org\",\"ID\":\"ID\",\"Address\":\"127.0.0.1\",\"Transport\":\"*json\",\"ConnectAttempts\":0,\"Reconnects\":0,\"ConnectTimeout\":0,\"ReplyTimeout\":0,\"TLS\":false,\"ClientKey\":\"\",\"ClientCertificate\":\"\",\"CaCertificate\":\"\"}"; !strings.Contains(rplyLog, expLog) { + t.Errorf("Expected %+q, received %+q", expLog, rplyLog) + } +} + +func TestDryRunWithUpdateStructErrors(t *testing.T) { + expErrMsg := `strconv.ParseFloat: parsing "notWeight": invalid syntax` + if _, err := testDryRunWithData(utils.MetaAttributes, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if _, err := testDryRunWithData(utils.MetaResources, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaStats, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaThresholds, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if _, err := testDryRunWithData(utils.MetaChargers, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaDispatchers, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if _, err := testDryRunWithData(utils.MetaActionProfiles, []utils.MapStorage{{utils.Weight: "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `cannot update unsupported struct field: 0` + if _, err := testDryRunWithData(utils.MetaFilters, []utils.MapStorage{{"PK": "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaRoutes, []utils.MapStorage{{"PK": "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaDispatcherHosts, []utils.MapStorage{{"PK": "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaRateProfiles, []utils.MapStorage{{"PK": "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaAccounts, []utils.MapStorage{{"PK": "notWeight"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } +} + +func TestDryRunWithModelsErrors(t *testing.T) { + expErrMsg := `Closed unspilit syntax` + if _, err := testDryRunWithData(utils.MetaAttributes, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Value": "`", "Path": "Test"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `strconv.ParseFloat: parsing "float": invalid syntax` + if _, err := testDryRunWithData(utils.MetaResources, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Limit": "float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `emtpy RSRParser in rule: <>` + if _, err := testDryRunWithData(utils.MetaFilters, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Values": "`;", "Type": utils.MetaRSR}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `time: invalid duration "float"` + if _, err := testDryRunWithData(utils.MetaStats, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "TTL": "float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaThresholds, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "MinSleep": "float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + expErrMsg = `invalid Weight in string: <;float>` + if _, err := testDryRunWithData(utils.MetaRoutes, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaRateProfiles, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if _, err := testDryRunWithData(utils.MetaAccounts, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `time: invalid duration "float"` + if _, err := testDryRunWithData(utils.MetaDispatcherHosts, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "ReplyTimeout": "float", "Address": "127.0.0.1"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if _, err := testDryRunWithData(utils.MetaActionProfiles, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "ActionTTL": "float", "ActionID": "ACCT"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + expErrMsg = `invalid key: <1> for BalanceCostIncrements` + if _, err := testDryRunWithData(utils.MetaAccounts, []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "BalanceCostIncrements": "1", "BalanceID": "BalID"}}); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } +} + +func TestSetToDBWithUpdateStructErrors(t *testing.T) { + expErrMsg := `strconv.ParseFloat: parsing "notWeight": invalid syntax` + if err := setToDB(context.Background(), nil, utils.MetaAttributes, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if err := setToDB(context.Background(), nil, utils.MetaResources, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaStats, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaThresholds, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if err := setToDB(context.Background(), nil, utils.MetaChargers, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaDispatchers, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if err := setToDB(context.Background(), nil, utils.MetaActionProfiles, "", []utils.MapStorage{{utils.Weight: "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `cannot update unsupported struct field: 0` + if err := setToDB(context.Background(), nil, utils.MetaFilters, "", []utils.MapStorage{{"PK": "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaRoutes, "", []utils.MapStorage{{"PK": "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaDispatcherHosts, "", []utils.MapStorage{{"PK": "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaRateProfiles, "", []utils.MapStorage{{"PK": "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaAccounts, "", []utils.MapStorage{{"PK": "notWeight"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } +} + +func TestSetToDBWithModelsErrors(t *testing.T) { + expErrMsg := `Closed unspilit syntax` + if err := setToDB(context.Background(), nil, utils.MetaAttributes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Value": "`", "Path": "Test"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `strconv.ParseFloat: parsing "float": invalid syntax` + if err := setToDB(context.Background(), nil, utils.MetaResources, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Limit": "float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `emtpy RSRParser in rule: <>` + if err := setToDB(context.Background(), nil, utils.MetaFilters, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Values": "`;", "Type": utils.MetaRSR}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `time: invalid duration "float"` + if err := setToDB(context.Background(), nil, utils.MetaStats, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "TTL": "float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaThresholds, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "MinSleep": "float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + expErrMsg = `invalid Weight in string: <;float>` + if err := setToDB(context.Background(), nil, utils.MetaRoutes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaRateProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + if err := setToDB(context.Background(), nil, utils.MetaAccounts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Weights": ";float"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + expErrMsg = `time: invalid duration "float"` + if err := setToDB(context.Background(), nil, utils.MetaDispatcherHosts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "ReplyTimeout": "float", "Address": "127.0.0.1"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + + if err := setToDB(context.Background(), nil, utils.MetaActionProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "ActionTTL": "float", "ActionID": "ACCT"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } + expErrMsg = `invalid key: <1> for BalanceCostIncrements` + if err := setToDB(context.Background(), nil, utils.MetaAccounts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "BalanceCostIncrements": "1", "BalanceID": "BalID"}}, true, false); err == nil || err.Error() != expErrMsg { + t.Errorf("Expeceted: %v, received: %v", expErrMsg, err) + } +} + +func TestSetToDBWithDBError(t *testing.T) { + if err := setToDB(context.Background(), nil, utils.MetaAttributes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + + if err := setToDB(context.Background(), nil, utils.MetaResources, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaStats, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaThresholds, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + + if err := setToDB(context.Background(), nil, utils.MetaChargers, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaDispatchers, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + + if err := setToDB(context.Background(), nil, utils.MetaActionProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + + if err := setToDB(context.Background(), nil, utils.MetaFilters, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaRoutes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaDispatcherHosts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Address": "127.0.0.1"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaRateProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaRateProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", utils.RateIDs: "RT1;RT2"}}, true, true); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } + if err := setToDB(context.Background(), nil, utils.MetaAccounts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != utils.ErrNoDatabaseConn { + t.Fatal(err) + } +} + +func TestSetToDB(t *testing.T) { + cfg := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, cfg.DataDbCfg().Items), cfg.CacheCfg(), nil) + if err := setToDB(context.Background(), dm, utils.MetaAttributes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v1 := &engine.AttributeProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, Attributes: []*engine.Attribute{}} + if prf, err := dm.GetAttributeProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v1, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v1), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaResources, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v2 := &engine.ResourceProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, ThresholdIDs: []string{}} + if prf, err := dm.GetResourceProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v2, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v2), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaStats, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v3 := &engine.StatQueueProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, ThresholdIDs: []string{}, Metrics: []*engine.MetricWithFilters{}} + if prf, err := dm.GetStatQueueProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v3, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v3), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaThresholds, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v4 := &engine.ThresholdProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, ActionProfileIDs: []string{}} + if prf, err := dm.GetThresholdProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v4, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v4), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaChargers, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v5 := &engine.ChargerProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, AttributeIDs: []string{}} + if prf, err := dm.GetChargerProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v5, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v5), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaDispatchers, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v6 := &engine.DispatcherProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, StrategyParams: make(map[string]interface{}), Hosts: engine.DispatcherHostProfiles{}} + if prf, err := dm.GetDispatcherProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v6, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v6), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaActionProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v7 := &engine.ActionProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, Targets: map[string]utils.StringSet{}, Actions: []*engine.APAction{}} + if prf, err := dm.GetActionProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v7, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v7), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaFilters, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v8 := &engine.Filter{Tenant: "cgrates.org", ID: "ID", Rules: []*engine.FilterRule{}} + if prf, err := dm.GetFilter(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v8, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v8), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaRoutes, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v9 := &engine.RouteProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, SortingParameters: []string{}, Routes: []*engine.Route{}} + if prf, err := dm.GetRouteProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v9, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v9), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaDispatcherHosts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID", "Address": "127.0.0.1"}}, true, false); err != nil { + t.Fatal(err) + } + v10 := &engine.DispatcherHost{Tenant: "cgrates.org", RemoteHost: &config.RemoteHost{ID: "ID", Address: "127.0.0.1", Transport: utils.MetaJSON}} + if prf, err := dm.GetDispatcherHost(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v10, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v10), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaRateProfiles, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v11 := &utils.RateProfile{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, Rates: map[string]*utils.Rate{}, MinCost: utils.NewDecimal(0, 0), MaxCost: utils.NewDecimal(0, 0)} + if prf, err := dm.GetRateProfile(context.Background(), "cgrates.org", "ID", true, true, utils.NonTransactional); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v11, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v11), utils.ToJSON(prf)) + } + + if err := setToDB(context.Background(), dm, utils.MetaAccounts, "", []utils.MapStorage{{utils.Tenant: "cgrates.org", utils.ID: "ID"}}, true, false); err != nil { + t.Fatal(err) + } + v12 := &utils.Account{Tenant: "cgrates.org", ID: "ID", FilterIDs: []string{}, Balances: map[string]*utils.Balance{}, ThresholdIDs: []string{}} + if prf, err := dm.GetAccount(context.Background(), "cgrates.org", "ID"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v12, prf) { + t.Errorf("Expeceted: %v, received: %v", utils.ToJSON(v12), utils.ToJSON(prf)) + } +} diff --git a/loaders/loaders_it_test.go b/loaders/loaders_it_test.go index d847103e9..5dd49151c 100644 --- a/loaders/loaders_it_test.go +++ b/loaders/loaders_it_test.go @@ -21,21 +21,7 @@ along with this program. If not, see package loaders -import ( - "encoding/csv" - "io" - "os" - "path" - "reflect" - "strings" - "testing" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) +/* var ( sTestItLoaders = []func(t *testing.T){ @@ -651,3 +637,4 @@ func testLoaderServiceReload(t *testing.T) { t.Error("Expected to be populated") } } +*/ diff --git a/loaders/locker.go b/loaders/locker.go index c48b30e71..c5eac978a 100644 --- a/loaders/locker.go +++ b/loaders/locker.go @@ -21,8 +21,6 @@ package loaders import ( "io" "os" - - "github.com/cgrates/cgrates/utils" ) type Locker interface { @@ -33,7 +31,7 @@ type Locker interface { } func newLocker(path string) Locker { - if path != utils.EmptyString { + if len(path) != 0 { return folderLock(path) } return new(nopLock) diff --git a/loaders/locker_test.go b/loaders/locker_test.go new file mode 100644 index 000000000..7a6f00078 --- /dev/null +++ b/loaders/locker_test.go @@ -0,0 +1,86 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package loaders + +import ( + "os" + "path" + "reflect" + "testing" + + "github.com/cgrates/cgrates/utils" +) + +func TestNopLocker(t *testing.T) { + np := newLocker(utils.EmptyString) + if err := np.Lock(); err != nil { + t.Error(err) + } + exp := new(nopLock) + if !reflect.DeepEqual(np, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, np) + } + if lk, err := np.Locked(); err != nil { + t.Error(err) + } else if lk { + t.Error("Expected no lock") + } + if err := np.Unlock(); err != nil { + t.Error(err) + } + if np.IsLockFile(utils.EmptyString) { + t.Error("Expected to not be lock file") + } +} + +func TestFolderLocker(t *testing.T) { + dir, err := os.MkdirTemp(utils.EmptyString, "TestFolderLocker") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + fp := path.Join(dir, ".lkr") + np := newLocker(fp) + exp := folderLock(fp) + if !reflect.DeepEqual(np, exp) { + t.Errorf("Expeceted: %+v, received: %+v", exp, np) + } + if err := np.Lock(); err != nil { + t.Error(err) + } + if lk, err := np.Locked(); err != nil { + t.Error(err) + } else if !lk { + t.Error("Expected lock") + } + if err := np.Unlock(); err != nil { + t.Error(err) + } + if np.IsLockFile(utils.EmptyString) { + t.Error("Expected to not be lock file") + } + if !np.IsLockFile(fp) { + t.Error("Expected to be lock file") + } + if lk, err := np.Locked(); err != nil { + t.Error(err) + } else if lk { + t.Error("Expected no lock") + } +} diff --git a/utils/consts.go b/utils/consts.go index e654f8d3d..3205ae7e5 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -832,6 +832,7 @@ const ( DNSTtl = "Ttl" DNSRdlength = "Rdlength" DNSData = "Data" + MetaTntID = "*tntID" ) // Migrator Action @@ -1442,10 +1443,9 @@ const ( // LoaderS APIs const ( - LoaderSv1 = "LoaderSv1" - LoaderSv1Load = "LoaderSv1.Load" - LoaderSv1Remove = "LoaderSv1.Remove" - LoaderSv1Ping = "LoaderSv1.Ping" + LoaderSv1 = "LoaderSv1" + LoaderSv1Run = "LoaderSv1.Run" + LoaderSv1Ping = "LoaderSv1.Ping" ) // CacheS APIs