From 65041752c12c9261d788b6bdec9418d8957d8ce1 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 15 Feb 2019 14:31:01 +0200 Subject: [PATCH] Added new config option to keep expired balances. fixes #1408 --- apier/v1/accounts_it_test.go | 182 ++++++++++++++++++ config/config_defaults.go | 1 + config/config_json_test.go | 1 + config/libconfig_json.go | 1 + config/ralscfg.go | 4 + .../samples/acc_balance_keep/cgrates.json | 162 ++++++++++++++++ engine/account.go | 14 +- 7 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 apier/v1/accounts_it_test.go create mode 100644 data/conf/samples/acc_balance_keep/cgrates.json diff --git a/apier/v1/accounts_it_test.go b/apier/v1/accounts_it_test.go new file mode 100644 index 000000000..5bae710cc --- /dev/null +++ b/apier/v1/accounts_it_test.go @@ -0,0 +1,182 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package v1 + +import ( + "net/rpc" + "net/rpc/jsonrpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + accExist bool + accCfgPath string + accCfg *config.CGRConfig + accRPC *rpc.Client + accAcount = "refundAcc" + accTenant = "cgrates.org" + accBallID = "Balance1" + + accTests = []func(t *testing.T){ + testAccITLoadConfig, + testAccITResetDataDB, + testAccITResetStorDb, + testAccITStartEngine, + testAccITRPCConn, + testAccITAddVoiceBalance, + testAccITDebitBalance, + testAccITStopCgrEngine, + } +) + +func TestAccITWithRemove(t *testing.T) { + accCfgPath = path.Join(*dataDir, "conf", "samples", "tutmongo") + for _, test := range accTests { + t.Run("TestAccIT", test) + } +} + +func TestAccITWithoutRemove(t *testing.T) { + accCfgPath = path.Join(*dataDir, "conf", "samples", "acc_balance_keep") + accExist = true + for _, test := range accTests { + t.Run("TestAccIT", test) + } +} + +func testAccITLoadConfig(t *testing.T) { + var err error + if accCfg, err = config.NewCGRConfigFromFolder(accCfgPath); err != nil { + t.Error(err) + } +} + +func testAccITResetDataDB(t *testing.T) { + if err := engine.InitDataDb(accCfg); err != nil { + t.Fatal(err) + } +} + +func testAccITResetStorDb(t *testing.T) { + if err := engine.InitStorDb(accCfg); err != nil { + t.Fatal(err) + } +} + +func testAccITStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(accCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +func testAccITRPCConn(t *testing.T) { + var err error + accRPC, err = jsonrpc.Dial("tcp", accCfg.ListenCfg().RPCJSONListen) + if err != nil { + t.Fatal(err) + } +} + +func testAccountBalance(t *testing.T, sracc, srten, balType string, expected float64) { + var acnt *engine.Account + attrs := &utils.AttrGetAccount{ + Tenant: srten, + Account: sracc, + } + if err := accRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if rply := acnt.BalanceMap[balType].GetTotalValue(); rply != expected { + t.Errorf("Expecting: %v, received: %v", + expected, rply) + } +} + +func testBalanceIfExists(t *testing.T, acc, ten, balType, balID string) (has bool) { + var acnt *engine.Account + attrs := &utils.AttrGetAccount{ + Tenant: ten, + Account: acc, + } + if err := accRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + return false + } + for _, bal := range acnt.BalanceMap[balType] { + if bal.ID == balID { + return true + } + } + return false +} + +func testAccITAddVoiceBalance(t *testing.T) { + attrSetBalance := utils.AttrSetBalance{ + Tenant: accTenant, + Account: accAcount, + BalanceType: utils.VOICE, + BalanceID: utils.StringPointer(accBallID), + Value: utils.Float64Pointer(2 * float64(time.Second)), + RatingSubject: utils.StringPointer("*zero5ms"), + ExpiryTime: utils.StringPointer(time.Now().Add(5 * time.Second).Format("2006-01-02 15:04:05")), + } + var reply string + if err := accRPC.Call("ApierV2.SetBalance", attrSetBalance, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("Received: %s", reply) + } + t.Run("TestAddVoiceBalance", func(t *testing.T) { testAccountBalance(t, accAcount, accTenant, utils.VOICE, 2*float64(time.Second)) }) + +} + +func testAccITDebitBalance(t *testing.T) { + time.Sleep(5 * time.Second) + var reply string + if err := accRPC.Call("ApierV1.DebitBalance", &AttrAddBalance{ + Tenant: accTenant, + Account: accAcount, + BalanceType: utils.VOICE, + Value: 0, + }, &reply); err != nil { + t.Error(err) + } else if reply != utils.OK { + t.Errorf("Received: %s", reply) + } + if has := testBalanceIfExists(t, accAcount, accTenant, utils.VOICE, accBallID); accExist != has { + var exstr string + if !accExist { + exstr = "not " + } + t.Fatalf("Balance with ID %s should %sexist", accBallID, exstr) + } + t.Run("TestAddVoiceBalance", func(t *testing.T) { testAccountBalance(t, accAcount, accTenant, utils.VOICE, 0) }) +} + +func testAccITStopCgrEngine(t *testing.T) { + if err := engine.KillEngine(100); err != nil { + t.Error(err) + } +} diff --git a/config/config_defaults.go b/config/config_defaults.go index 55880088e..2aea5b622 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -168,6 +168,7 @@ const CGRATES_CFG_JSON = ` "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject + "remove_expired":true, // enables remove of expired balances "max_computed_usage": { // do not compute usage higher than this, prevents memory overload "*any": "189h", "*voice": "72h", diff --git a/config/config_json_test.go b/config/config_json_test.go index 06149d847..5fd0e9a07 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -234,6 +234,7 @@ func TestDfRalsJsonCfg(t *testing.T) { Users_conns: &[]*HaPoolJsonCfg{}, Aliases_conns: &[]*HaPoolJsonCfg{}, Rp_subject_prefix_matching: utils.BoolPointer(false), + Remove_expired: utils.BoolPointer(true), Max_computed_usage: &map[string]string{ utils.ANY: "189h", utils.VOICE: "72h", diff --git a/config/libconfig_json.go b/config/libconfig_json.go index f1eab9352..6e83897bf 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -105,6 +105,7 @@ type RalsJsonCfg struct { Aliases_conns *[]*HaPoolJsonCfg Users_conns *[]*HaPoolJsonCfg Rp_subject_prefix_matching *bool + Remove_expired *bool Max_computed_usage *map[string]string } diff --git a/config/ralscfg.go b/config/ralscfg.go index 538ee62b0..ce6bb6b70 100644 --- a/config/ralscfg.go +++ b/config/ralscfg.go @@ -33,6 +33,7 @@ type RalsCfg struct { RALsUserSConns []*HaPoolConfig RALsAliasSConns []*HaPoolConfig RpSubjectPrefixMatching bool // enables prefix matching for the rating profile subject + RemoveExpired bool RALsMaxComputedUsage map[string]time.Duration } @@ -82,6 +83,9 @@ func (ralsCfg *RalsCfg) loadFromJsonCfg(jsnRALsCfg *RalsJsonCfg) (err error) { if jsnRALsCfg.Rp_subject_prefix_matching != nil { ralsCfg.RpSubjectPrefixMatching = *jsnRALsCfg.Rp_subject_prefix_matching } + if jsnRALsCfg.Remove_expired != nil { + ralsCfg.RemoveExpired = *jsnRALsCfg.Remove_expired + } if jsnRALsCfg.Max_computed_usage != nil { for k, v := range *jsnRALsCfg.Max_computed_usage { if ralsCfg.RALsMaxComputedUsage[k], err = utils.ParseDurationWithNanosecs(v); err != nil { diff --git a/data/conf/samples/acc_balance_keep/cgrates.json b/data/conf/samples/acc_balance_keep/cgrates.json new file mode 100644 index 000000000..5f8704ffb --- /dev/null +++ b/data/conf/samples/acc_balance_keep/cgrates.json @@ -0,0 +1,162 @@ +{ +// CGRateS Configuration file + + +"general": { + "log_level": 7, + "reply_timeout": "30s", +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017, +}, + + +"stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017, +}, + + +"cache":{ + "destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true}, + "rating_plans": {"limit": 10000, "ttl":"0s","precache": true}, + "rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "actions": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "account_action_plans": {"limit": 10000, "ttl":"0s", "precache": true}, + "action_triggers": {"limit": 10000, "ttl":"0s", "precache": true}, + "shared_groups": {"limit": 10000, "ttl":"0s", "precache": true}, + "aliases": {"limit": 10000, "ttl":"0s", "precache": true}, + "reverse_aliases": {"limit": 10000, "ttl":"0s", "precache": true}, + "derived_chargers": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resources": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueues": {"limit": 10000, "ttl":"0s", "precache": true}, + "statqueue_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "thresholds": {"limit": 10000, "ttl":"0s", "precache": true}, + "threshold_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "filters": {"limit": 10000, "ttl":"0s", "precache": true}, + "supplier_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "attribute_profiles": {"limit": 10000, "ttl":"0s", "precache": true}, + "resource_filter_indexes" :{"limit": 10000, "ttl":"0s"}, + "stat_filter_indexes" : {"limit": 10000, "ttl":"0s"}, + "threshold_filter_indexes" : {"limit": 10000, "ttl":"0s"}, + "supplier_filter_indexes" : {"limit": 10000, "ttl":"0s"}, + "attribute_filter_indexes" : {"limit": 10000, "ttl":"0s"}, + "charger_filter_indexes" : {"limit": 10000, "ttl":"0s"}, +}, + + +"rals": { + "enabled": true, + "thresholds_conns": [ + {"address": "*internal"} + ], + "remove_expired":false, +}, + + +"scheduler": { + "enabled": true, +}, + + +"cdrs": { + "enabled": true, +}, + + +"cdre": { + "TestTutITExportCDR": { + "content_fields": [ + {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, + {"tag": "RunID", "type": "*composed", "value": "~RunID"}, + {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, + {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, + {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, + {"tag":"Category", "type": "*composed", "value": "~Category"}, + {"tag":"Account", "type": "*composed", "value": "~Account"}, + {"tag":"Destination", "type": "*composed", "value": "~Destination"}, + {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, + {"tag":"Usage", "type": "*composed", "value": "~Usage"}, + {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, + {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, + ], + }, +}, + + +"chargers": { + "enabled": true, + "attributes_conns": [ + {"address": "*internal"} + ], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": [ + {"address": "*internal"} + ], +}, + + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"suppliers": { + "enabled": true, + "stats_conns": [ + {"address": "*internal"}, + ], +}, + + +"attributes": { // Attribute service + "enabled": true, // starts Alias service: . +}, + + +"sessions": { + "enabled": true, +}, + + +"migrator": { + "out_datadb_type": "mongo", + "out_datadb_port": "27017", + "out_datadb_name": "10", + "out_stordb_type": "mongo", + "out_stordb_port": "27017", + "out_stordb_name": "cgrates", +}, + + +} diff --git a/engine/account.go b/engine/account.go index 9d57b3675..ad4bf6857 100644 --- a/engine/account.go +++ b/engine/account.go @@ -740,14 +740,16 @@ func (acc *Account) InitCounters() { } func (acc *Account) CleanExpiredStuff() { - for key, bm := range acc.BalanceMap { - for i := 0; i < len(bm); i++ { - if bm[i].IsExpired() { - // delete it - bm = append(bm[:i], bm[i+1:]...) + if config.CgrConfig().RalsCfg().RemoveExpired { + for key, bm := range acc.BalanceMap { + for i := 0; i < len(bm); i++ { + if bm[i].IsExpired() { + // delete it + bm = append(bm[:i], bm[i+1:]...) + } } + acc.BalanceMap[key] = bm } - acc.BalanceMap[key] = bm } for i := 0; i < len(acc.ActionTriggers); i++ {