From a513cd24fd77e7ee5e54d50482f7b1e4852b4822 Mon Sep 17 00:00:00 2001 From: TeoV Date: Sun, 30 Aug 2020 14:53:13 +0300 Subject: [PATCH] Add back APIerSv1.ExportCDRs with integration tests --- apier/v1/apier.go | 4 +- apier/v1/ees_it_test.go | 232 ++++++++++++++++++++ data/conf/samples/ees_internal/cgrates.json | 100 +++++++++ data/conf/samples/ees_mongo/cgrates.json | 104 +++++++++ data/conf/samples/ees_mysql/cgrates.json | 102 +++++++++ data/conf/samples/ers_mysql/cgrates.json | 8 +- utils/apitpdata.go | 3 +- 7 files changed, 546 insertions(+), 7 deletions(-) create mode 100644 apier/v1/ees_it_test.go create mode 100644 data/conf/samples/ees_internal/cgrates.json create mode 100644 data/conf/samples/ees_mongo/cgrates.json create mode 100644 data/conf/samples/ees_mysql/cgrates.json diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 98406c881..0eab7a65f 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1895,10 +1895,10 @@ func (apierSv1 *APIerSv1) ExportCDRs(args *utils.ArgExportCDRs, reply *string) e withErros := false for _, cdr := range cdrs { if err := apierSv1.ConnMgr.Call(apierSv1.Config.ApierCfg().EEsConns, nil, utils.EventExporterSv1ProcessEvent, - utils.CGREventWithIDs{ + &utils.CGREventWithIDs{ IDs: args.ExporterIDs, CGREventWithOpts: &utils.CGREventWithOpts{CGREvent: cdr.AsCGREvent()}, - }, &reply); err != nil { + }, reply); err != nil { utils.Logger.Warning(fmt.Sprintf("<%s> error: <%s> processing event: <%s> with <%s>", utils.ApierS, err.Error(), utils.ToJSON(cdr.AsCGREvent()), utils.EventExporterS)) withErros = true diff --git a/apier/v1/ees_it_test.go b/apier/v1/ees_it_test.go new file mode 100644 index 000000000..49988d5d4 --- /dev/null +++ b/apier/v1/ees_it_test.go @@ -0,0 +1,232 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package v1 + +import ( + "io/ioutil" + "net/rpc" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + eeSCfgPath string + eeSCfg *config.CGRConfig + eeSRPC *rpc.Client + eeSConfigDIR string //run tests for specific configuration + + sTestsEEs = []func(t *testing.T){ + testEEsPrepareFolder, + testEEsInitCfg, + testEEsInitDataDb, + testEEsResetStorDb, + testEEsStartEngine, + testEEsRPCConn, + testEEsAddCDRs, + testEEsExportCDRs, + testEEsVerifyExports, + testEEsKillEngine, + testEEsCleanFolder, + } +) + +//Test start here +func TestExportCDRs(t *testing.T) { + switch *dbType { + case utils.MetaInternal: + eeSConfigDIR = "ees_internal" + case utils.MetaMySQL: + eeSConfigDIR = "ees_mysql" + case utils.MetaMongo: + eeSConfigDIR = "ees_mongo" + case utils.MetaPostgres: + t.SkipNow() + default: + t.Fatal("Unknown Database type") + } + for _, stest := range sTestsEEs { + t.Run(eeSConfigDIR, stest) + } +} + +func testEEsPrepareFolder(t *testing.T) { + for _, dir := range []string{"/tmp/testCSV"} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + t.Fatal("Error creating folder: ", dir, err) + } + } +} + +func testEEsInitCfg(t *testing.T) { + var err error + eeSCfgPath = path.Join(*dataDir, "conf", "samples", eeSConfigDIR) + eeSCfg, err = config.NewCGRConfigFromPath(eeSCfgPath) + if err != nil { + t.Fatal(err) + } + eeSCfg.DataFolderPath = alsPrfDataDir // Share DataFolderPath through config towards StoreDb for Flush() + config.SetCgrConfig(eeSCfg) +} + +func testEEsInitDataDb(t *testing.T) { + if err := engine.InitDataDb(eeSCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func testEEsResetStorDb(t *testing.T) { + if err := engine.InitStorDb(eeSCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func testEEsStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(eeSCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func testEEsRPCConn(t *testing.T) { + var err error + eeSRPC, err = newRPCClient(eeSCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +func testEEsAddCDRs(t *testing.T) { + //add a default charger + chargerProfile := &ChargerWithCache{ + ChargerProfile: &engine.ChargerProfile{ + Tenant: "cgrates.org", + ID: "Default", + RunID: utils.MetaRaw, + AttributeIDs: []string{"*none"}, + Weight: 20, + }, + } + var result string + if err := eeSRPC.Call(utils.APIerSv1SetChargerProfile, chargerProfile, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } + storedCdrs := []*engine.CDR{ + {CGRID: "Cdr1", + OrderID: 1, ToR: utils.VOICE, OriginID: "OriginCDR1", OriginHost: "192.168.1.1", Source: "test", + RequestType: utils.META_RATED, Tenant: "cgrates.org", + Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), + AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(10) * time.Second, + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + }, + {CGRID: "Cdr2", + OrderID: 2, ToR: utils.VOICE, OriginID: "OriginCDR2", OriginHost: "192.168.1.1", Source: "test2", + RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", + Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), + AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(5) * time.Second, + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + }, + {CGRID: "Cdr3", + OrderID: 3, ToR: utils.VOICE, OriginID: "OriginCDR3", OriginHost: "192.168.1.1", Source: "test2", + RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", + Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), + AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(30) * time.Second, + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + }, + {CGRID: "Cdr4", + OrderID: 4, ToR: utils.VOICE, OriginID: "OriginCDR4", OriginHost: "192.168.1.1", Source: "test3", + RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", + Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), + AnswerTime: time.Time{}, RunID: utils.MetaDefault, Usage: time.Duration(0) * time.Second, + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + }, + } + for _, cdr := range storedCdrs { + var reply string + if err := eeSRPC.Call(utils.CDRsV1ProcessCDR, &engine.CDRWithOpts{CDR: cdr}, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + } + time.Sleep(100 * time.Millisecond) +} + +func testEEsExportCDRs(t *testing.T) { + attr := &utils.ArgExportCDRs{ + ExporterIDs: []string{"CSVExporter"}, + } + var rply string + if err := eeSRPC.Call(utils.APIerSv1ExportCDRs, &attr, &rply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } + time.Sleep(1 * time.Second) +} + +func testEEsVerifyExports(t *testing.T) { + var files []string + err := filepath.Walk("/tmp/testCSV/", func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, utils.CSVSuffix) { + files = append(files, path) + } + return nil + }) + if err != nil { + t.Error(err) + } + if len(files) != 1 { + t.Errorf("Expected %+v, received: %+v", 1, len(files)) + } + eCnt := "Cdr3,*raw,*voice,OriginCDR3,*rated,cgrates.org,call,1001,1001,+4986517174963,2020-08-30T14:40:32+03:00,2020-08-30T14:40:32+03:00,30s,-1\n" + + "Cdr4,*raw,*voice,OriginCDR4,*rated,cgrates.org,call,1001,1001,+4986517174963,2020-08-30T14:40:32+03:00,0001-01-01T00:00:00Z,0s,0\n" + + "Cdr1,*raw,*voice,OriginCDR1,*rated,cgrates.org,call,1001,1001,+4986517174963,2020-08-30T14:40:32+03:00,2020-08-30T14:40:32+03:00,10s,-1\n" + + "Cdr2,*raw,*voice,OriginCDR2,*rated,cgrates.org,call,1001,1001,+4986517174963,2020-08-30T14:40:32+03:00,2020-08-30T14:40:32+03:00,5s,-1\n" + if outContent1, err := ioutil.ReadFile(files[0]); err != nil { + t.Error(err) + } else if len(eCnt) != len(string(outContent1)) { + t.Errorf("Expecting: \n<%q>, \nreceived: \n<%q>", eCnt, string(outContent1)) + } +} + +func testEEsKillEngine(t *testing.T) { + if err := engine.KillEngine(100); err != nil { + t.Error(err) + } +} + +func testEEsCleanFolder(t *testing.T) { + for _, dir := range []string{"/tmp/testCSV"} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + } +} diff --git a/data/conf/samples/ees_internal/cgrates.json b/data/conf/samples/ees_internal/cgrates.json new file mode 100644 index 000000000..6f3b65bc2 --- /dev/null +++ b/data/conf/samples/ees_internal/cgrates.json @@ -0,0 +1,100 @@ +{ +// Sample CGRateS Configuration file for EEs +// +// Copyright (C) ITsysCOM GmbH + +"general": { + "log_level": 7, +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "*internal", +}, + + +"stor_db": { + "db_type": "*internal", +}, + + +"rals": { + "enabled": true, +}, + + +"schedulers": { + "enabled": true, +}, + + +"cdrs": { + "enabled": true, + "chargers_conns": ["*localhost"], + "rals_conns": ["*internal"], + "session_cost_retries": 0, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + +"attributes": { + "enabled": true, +}, + + +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "cache": { + "*file_csv": {"limit": -1, "ttl": "500ms", "static_ttl": false}, + }, + "exporters": [ + { + "id": "CSVExporter", + "type": "*file_csv", + "export_path": "/tmp/testCSV", + "tenant": "cgrates.org", + "flags": ["*attributes"], + "attribute_context": "customContext", + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, + + +"apiers": { + "enabled": true, + "scheduler_conns": ["*internal"], + "ees_conns": ["*internal"], +}, + + +} diff --git a/data/conf/samples/ees_mongo/cgrates.json b/data/conf/samples/ees_mongo/cgrates.json new file mode 100644 index 000000000..80af20a80 --- /dev/null +++ b/data/conf/samples/ees_mongo/cgrates.json @@ -0,0 +1,104 @@ +{ +// Sample CGRateS Configuration file for EEs +// +// Copyright (C) ITsysCOM GmbH + +"general": { + "log_level": 7, +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017 +}, + + +"stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017 +}, + + +"rals": { + "enabled": true, +}, + + +"schedulers": { + "enabled": true, +}, + + +"cdrs": { + "enabled": true, + "chargers_conns": ["*localhost"], + "rals_conns": ["*internal"], + "session_cost_retries": 0, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + +"attributes": { + "enabled": true, +}, + + +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "cache": { + "*file_csv": {"limit": -1, "ttl": "500ms", "static_ttl": false}, + }, + "exporters": [ + { + "id": "CSVExporter", + "type": "*file_csv", + "export_path": "/tmp/testCSV", + "tenant": "cgrates.org", + "flags": ["*attributes"], + "attribute_context": "customContext", + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, + + +"apiers": { + "enabled": true, + "scheduler_conns": ["*internal"], + "ees_conns": ["*internal"], +}, + + +} diff --git a/data/conf/samples/ees_mysql/cgrates.json b/data/conf/samples/ees_mysql/cgrates.json new file mode 100644 index 000000000..afcf6d6ff --- /dev/null +++ b/data/conf/samples/ees_mysql/cgrates.json @@ -0,0 +1,102 @@ +{ +// Sample CGRateS Configuration file for EEs +// +// Copyright (C) ITsysCOM GmbH + +"general": { + "log_level": 7, +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "redis", + "db_port": 6379, + "db_name": "10" +}, + + +"stor_db": { + "db_password": "CGRateS.org" +}, + + +"rals": { + "enabled": true, +}, + + +"schedulers": { + "enabled": true, +}, + + +"cdrs": { + "enabled": true, + "chargers_conns": ["*localhost"], + "rals_conns": ["*internal"], + "session_cost_retries": 0, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + +"attributes": { + "enabled": true, +}, + + +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "cache": { + "*file_csv": {"limit": -1, "ttl": "500ms", "static_ttl": false}, + }, + "exporters": [ + { + "id": "CSVExporter", + "type": "*file_csv", + "export_path": "/tmp/testCSV", + "tenant": "cgrates.org", + "flags": ["*attributes"], + "attribute_context": "customContext", + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, + + +"apiers": { + "enabled": true, + "scheduler_conns": ["*internal"], + "ees_conns": ["*internal"], +}, + + +} diff --git a/data/conf/samples/ers_mysql/cgrates.json b/data/conf/samples/ers_mysql/cgrates.json index 73e9d40c9..e6913f900 100644 --- a/data/conf/samples/ers_mysql/cgrates.json +++ b/data/conf/samples/ers_mysql/cgrates.json @@ -16,10 +16,10 @@ "http": ":2080", }, -"data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: - "db_port": 6379, // data_db port to reach the database - "db_name": "10", // data_db database name to connect to +"data_db": { + "db_type": "redis", + "db_port": 6379, + "db_name": "10", }, diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 21ddb9c70..5148150f6 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1513,6 +1513,7 @@ type TPIntervalRate struct { } type ArgExportCDRs struct { - ExporterIDs []string + ExporterIDs []string // exporterIDs is used to said which exporter are using to export the cdrs + Verbose bool // verbose is used to inform the user about the positive and negative exported cdrs RPCCDRsFilter }