From 61c2fccd3427a8430c4c289c5ee07ad0ceb591a3 Mon Sep 17 00:00:00 2001 From: TeoV Date: Tue, 14 Jul 2020 14:35:36 +0300 Subject: [PATCH] Add integration test for cgr-engine -preload flag --- apier/v1/preload_it_test.go | 171 ++++++++++++++++++ .../samples/loaders/tutinternal/cgrates.json | 101 +++++++++++ migrator/rateprofiles.go | 19 ++ 3 files changed, 291 insertions(+) create mode 100644 apier/v1/preload_it_test.go create mode 100644 data/conf/samples/loaders/tutinternal/cgrates.json create mode 100644 migrator/rateprofiles.go diff --git a/apier/v1/preload_it_test.go b/apier/v1/preload_it_test.go new file mode 100644 index 000000000..92cd852a6 --- /dev/null +++ b/apier/v1/preload_it_test.go @@ -0,0 +1,171 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package v1 + +import ( + "fmt" + "io/ioutil" + "net/rpc" + "net/rpc/jsonrpc" + "os" + "os/exec" + "path" + "reflect" + "testing" + "time" + + "github.com/cgrates/cgrates/utils" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" +) + +var ( + preloadCfgPath string + preloadCfgDIR string + preloadCfg *config.CGRConfig + preloadRPC *rpc.Client + + preloadTests = []func(t *testing.T){ + testCreateDirs, + testPreloadITInitConfig, + testPreloadITStartEngine, + testPreloadITRpcConn, + testPreloadITVerifyAttributes, + testCleanupFiles, + testPreloadITKillEngine, + } +) + +func TestPreload(t *testing.T) { + preloadCfgDIR = "tutinternal" + for _, test := range preloadTests { + t.Run(preloadCfgDIR, test) + } +} + +func testCreateDirs(t *testing.T) { + for _, dir := range []string{"/tmp/In", "/tmp/Out"} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal("Error creating folder: ", dir, err) + } + } + if err := ioutil.WriteFile(path.Join("/tmp/In", utils.AttributesCsv), []byte(engine.AttributesCSVContent), 0644); err != nil { + t.Fatal(err.Error()) + } +} + +func testPreloadITInitConfig(t *testing.T) { + var err error + preloadCfgPath = path.Join(*dataDir, "conf", "samples", "loaders", preloadCfgDIR) + if preloadCfg, err = config.NewCGRConfigFromPath(preloadCfgPath); err != nil { + t.Fatal("Got config error: ", err.Error()) + } +} + +func testPreloadITStartEngine(t *testing.T) { + enginePath, err := exec.LookPath("cgr-engine") + if err != nil { + t.Error(err) + } + engine := exec.Command(enginePath, "-config_path", preloadCfgPath, "-preload", "CustomLoader") + if err := engine.Start(); err != nil { + t.Error(err) + } + fib := utils.Fib() + var connected bool + for i := 0; i < 200; i++ { + time.Sleep(time.Duration(fib()) * time.Millisecond) + if _, err := jsonrpc.Dial(utils.TCP, preloadCfg.ListenCfg().RPCJSONListen); err != nil { + utils.Logger.Warning(fmt.Sprintf("Error <%s> when opening test connection to: <%s>", + err.Error(), preloadCfg.ListenCfg().RPCJSONListen)) + } else { + connected = true + break + } + } + if !connected { + t.Errorf("engine did not open port <%s>", preloadCfg.ListenCfg().RPCJSONListen) + } + time.Sleep(time.Duration(500) * time.Millisecond) +} + +func testPreloadITRpcConn(t *testing.T) { + var err error + preloadRPC, err = newRPCClient(preloadCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal("Could not connect to rater: ", err.Error()) + } +} + +func testPreloadITVerifyAttributes(t *testing.T) { + eAttrPrf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "ALS1", + FilterIDs: []string{"*string:~*req.Account:1001"}, + Contexts: []string{"con1", "con2", "con3"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, + Attributes: []*engine.Attribute{ + { + FilterIDs: []string{"*string:~*req.Field1:Initial"}, + Path: "*req.Field1", + Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("Sub1", utils.INFIELD_SEP), + }, + { + FilterIDs: []string{}, + Path: "*req.Field2", + Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("Sub2", utils.INFIELD_SEP), + }, + }, + Blocker: true, + Weight: 20.0, + } + + var reply *engine.AttributeProfile + if err := preloadRPC.Call(utils.APIerSv1GetAttributeProfile, + utils.TenantIDWithArgDispatcher{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"}}, &reply); err != nil { + t.Fatal(err) + } + reply.Compile() + if !reflect.DeepEqual(eAttrPrf, reply) { + t.Errorf("Expecting : %+v,\n received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply)) + } +} + +func testCleanupFiles(t *testing.T) { + for _, dir := range []string{"/tmp/In", "/tmp/Out"} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + } +} + +func testPreloadITKillEngine(t *testing.T) { + if err := engine.KillEngine(*waitRater); err != nil { + t.Error(err) + } +} diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json new file mode 100644 index 000000000..08556d511 --- /dev/null +++ b/data/conf/samples/loaders/tutinternal/cgrates.json @@ -0,0 +1,101 @@ +{ +// CGRateS Configuration file +// + + +"general": { + "log_level": 7, + "reply_timeout": "50s", +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "*internal", +}, + + +"stor_db": { + "db_type": "*internal" +}, + + +"rals": { + "enabled": true, +}, + + +"schedulers": { + "enabled": true, + "cdrs_conns": ["*internal"], +}, + + +"cdrs": { + "enabled": true, + "chargers_conns":["*internal"], +}, + + +"loaders": [ + { + "id": "CustomLoader", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "tenant": "cgrates.org", + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": ["*internal"], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/In", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/Out", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, + {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, + {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, + {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, + {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, + {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, + {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, + {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, + {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, + {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, + {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, + ], + }, + ], + }, +], + + + +"attributes": { + "enabled": true, + "prefix_indexed_fields":["Destination"], +}, + + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + + +"apiers": { + "enabled": true, + "scheduler_conns": ["*internal"], +}, + + +} diff --git a/migrator/rateprofiles.go b/migrator/rateprofiles.go new file mode 100644 index 000000000..52e99ee94 --- /dev/null +++ b/migrator/rateprofiles.go @@ -0,0 +1,19 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator