Added new config option to keep expired balances. fixes #1408

apier/v1/accounts_it_test.go

@@ -0,0 +1,182 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package v1
+
+import (
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	accExist   bool
+	accCfgPath string
+	accCfg     *config.CGRConfig
+	accRPC     *rpc.Client
+	accAcount  = "refundAcc"
+	accTenant  = "cgrates.org"
+	accBallID  = "Balance1"
+
+	accTests = []func(t *testing.T){
+		testAccITLoadConfig,
+		testAccITResetDataDB,
+		testAccITResetStorDb,
+		testAccITStartEngine,
+		testAccITRPCConn,
+		testAccITAddVoiceBalance,
+		testAccITDebitBalance,
+		testAccITStopCgrEngine,
+	}
+)
+
+func TestAccITWithRemove(t *testing.T) {
+	accCfgPath = path.Join(*dataDir, "conf", "samples", "tutmongo")
+	for _, test := range accTests {
+		t.Run("TestAccIT", test)
+	}
+}
+
+func TestAccITWithoutRemove(t *testing.T) {
+	accCfgPath = path.Join(*dataDir, "conf", "samples", "acc_balance_keep")
+	accExist = true
+	for _, test := range accTests {
+		t.Run("TestAccIT", test)
+	}
+}
+
+func testAccITLoadConfig(t *testing.T) {
+	var err error
+	if accCfg, err = config.NewCGRConfigFromFolder(accCfgPath); err != nil {
+		t.Error(err)
+	}
+}
+
+func testAccITResetDataDB(t *testing.T) {
+	if err := engine.InitDataDb(accCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testAccITResetStorDb(t *testing.T) {
+	if err := engine.InitStorDb(accCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testAccITStartEngine(t *testing.T) {
+	if _, err := engine.StopStartEngine(accCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testAccITRPCConn(t *testing.T) {
+	var err error
+	accRPC, err = jsonrpc.Dial("tcp", accCfg.ListenCfg().RPCJSONListen)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testAccountBalance(t *testing.T, sracc, srten, balType string, expected float64) {
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{
+		Tenant:  srten,
+		Account: sracc,
+	}
+	if err := accRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil {
+		t.Error(err)
+	} else if rply := acnt.BalanceMap[balType].GetTotalValue(); rply != expected {
+		t.Errorf("Expecting: %v, received: %v",
+			expected, rply)
+	}
+}
+
+func testBalanceIfExists(t *testing.T, acc, ten, balType, balID string) (has bool) {
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{
+		Tenant:  ten,
+		Account: acc,
+	}
+	if err := accRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil {
+		t.Error(err)
+		return false
+	}
+	for _, bal := range acnt.BalanceMap[balType] {
+		if bal.ID == balID {
+			return true
+		}
+	}
+	return false
+}
+
+func testAccITAddVoiceBalance(t *testing.T) {
+	attrSetBalance := utils.AttrSetBalance{
+		Tenant:        accTenant,
+		Account:       accAcount,
+		BalanceType:   utils.VOICE,
+		BalanceID:     utils.StringPointer(accBallID),
+		Value:         utils.Float64Pointer(2 * float64(time.Second)),
+		RatingSubject: utils.StringPointer("*zero5ms"),
+		ExpiryTime:    utils.StringPointer(time.Now().Add(5 * time.Second).Format("2006-01-02 15:04:05")),
+	}
+	var reply string
+	if err := accRPC.Call("ApierV2.SetBalance", attrSetBalance, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("Received: %s", reply)
+	}
+	t.Run("TestAddVoiceBalance", func(t *testing.T) { testAccountBalance(t, accAcount, accTenant, utils.VOICE, 2*float64(time.Second)) })
+
+}
+
+func testAccITDebitBalance(t *testing.T) {
+	time.Sleep(5 * time.Second)
+	var reply string
+	if err := accRPC.Call("ApierV1.DebitBalance", &AttrAddBalance{
+		Tenant:      accTenant,
+		Account:     accAcount,
+		BalanceType: utils.VOICE,
+		Value:       0,
+	}, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Errorf("Received: %s", reply)
+	}
+	if has := testBalanceIfExists(t, accAcount, accTenant, utils.VOICE, accBallID); accExist != has {
+		var exstr string
+		if !accExist {
+			exstr = "not "
+		}
+		t.Fatalf("Balance with ID %s should %sexist", accBallID, exstr)
+	}
+	t.Run("TestAddVoiceBalance", func(t *testing.T) { testAccountBalance(t, accAcount, accTenant, utils.VOICE, 0) })
+}
+
+func testAccITStopCgrEngine(t *testing.T) {
+	if err := engine.KillEngine(100); err != nil {
+		t.Error(err)
+	}
+}

config/config_defaults.go

@@ -168,6 +168,7 @@ const CGRATES_CFG_JSON = `
 	"users_conns": [],						// address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234>
 	"aliases_conns": [],					// address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234>
 	"rp_subject_prefix_matching": false,	// enables prefix matching for the rating profile subject
+	"remove_expired":true,					// enables remove of expired balances
 	"max_computed_usage": {					// do not compute usage higher than this, prevents memory overload
 		"*any": "189h",
 		"*voice": "72h",

config/config_json_test.go

@@ -234,6 +234,7 @@ func TestDfRalsJsonCfg(t *testing.T) {
 		Users_conns:                &[]*HaPoolJsonCfg{},
 		Aliases_conns:              &[]*HaPoolJsonCfg{},
 		Rp_subject_prefix_matching: utils.BoolPointer(false),
+		Remove_expired:             utils.BoolPointer(true),
 		Max_computed_usage: &map[string]string{
 			utils.ANY:   "189h",
 			utils.VOICE: "72h",

config/libconfig_json.go

@@ -105,6 +105,7 @@ type RalsJsonCfg struct {
 	Aliases_conns              *[]*HaPoolJsonCfg
 	Users_conns                *[]*HaPoolJsonCfg
 	Rp_subject_prefix_matching *bool
+	Remove_expired             *bool
 	Max_computed_usage         *map[string]string
 }
 

config/ralscfg.go

@@ -33,6 +33,7 @@ type RalsCfg struct {
 	RALsUserSConns          []*HaPoolConfig
 	RALsAliasSConns         []*HaPoolConfig
 	RpSubjectPrefixMatching bool // enables prefix matching for the rating profile subject
+	RemoveExpired           bool
 	RALsMaxComputedUsage    map[string]time.Duration
 }
 
@@ -82,6 +83,9 @@ func (ralsCfg *RalsCfg) loadFromJsonCfg(jsnRALsCfg *RalsJsonCfg) (err error) {
 	if jsnRALsCfg.Rp_subject_prefix_matching != nil {
 		ralsCfg.RpSubjectPrefixMatching = *jsnRALsCfg.Rp_subject_prefix_matching
 	}
+	if jsnRALsCfg.Remove_expired != nil {
+		ralsCfg.RemoveExpired = *jsnRALsCfg.Remove_expired
+	}
 	if jsnRALsCfg.Max_computed_usage != nil {
 		for k, v := range *jsnRALsCfg.Max_computed_usage {
 			if ralsCfg.RALsMaxComputedUsage[k], err = utils.ParseDurationWithNanosecs(v); err != nil {

data/conf/samples/acc_balance_keep/cgrates.json

@@ -0,0 +1,162 @@
+{
+// CGRateS Configuration file
+
+
+"general": {
+	"log_level": 7,
+	"reply_timeout": "30s",
+},
+
+
+"listen": {
+	"rpc_json": ":2012",
+	"rpc_gob": ":2013",
+	"http": ":2080",
+},
+
+
+"data_db": {
+	"db_type": "mongo",
+	"db_name": "10",
+	"db_port": 27017,
+},
+
+
+"stor_db": {
+	"db_type": "mongo",
+	"db_name": "cgrates",
+	"db_port": 27017,
+},
+
+
+"cache":{
+	"destinations": {"limit": 10000, "ttl":"0s", "precache": true},
+	"reverse_destinations": {"limit": 10000, "ttl":"0s", "precache": true},
+	"rating_plans": {"limit": 10000, "ttl":"0s","precache": true},
+	"rating_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"actions": {"limit": 10000, "ttl":"0s", "precache": true},
+	"action_plans": {"limit": 10000, "ttl":"0s", "precache": true},
+	"account_action_plans": {"limit": 10000, "ttl":"0s", "precache": true},
+	"action_triggers": {"limit": 10000, "ttl":"0s", "precache": true},
+	"shared_groups": {"limit": 10000, "ttl":"0s", "precache": true},
+	"aliases": {"limit": 10000, "ttl":"0s", "precache": true},
+	"reverse_aliases": {"limit": 10000, "ttl":"0s", "precache": true},
+	"derived_chargers": {"limit": 10000, "ttl":"0s", "precache": true},
+	"resource_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"resources": {"limit": 10000, "ttl":"0s", "precache": true},
+	"statqueues": {"limit": 10000, "ttl":"0s", "precache": true},
+	"statqueue_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"thresholds": {"limit": 10000, "ttl":"0s", "precache": true},
+	"threshold_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"filters": {"limit": 10000, "ttl":"0s", "precache": true},
+	"supplier_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"attribute_profiles": {"limit": 10000, "ttl":"0s", "precache": true},
+	"resource_filter_indexes" :{"limit": 10000, "ttl":"0s"},
+	"stat_filter_indexes" : {"limit": 10000, "ttl":"0s"},
+	"threshold_filter_indexes" : {"limit": 10000, "ttl":"0s"},
+	"supplier_filter_indexes" : {"limit": 10000, "ttl":"0s"},
+	"attribute_filter_indexes" : {"limit": 10000, "ttl":"0s"},
+	"charger_filter_indexes" : {"limit": 10000, "ttl":"0s"},
+},
+
+
+"rals": {
+	"enabled": true,
+	"thresholds_conns": [
+		{"address": "*internal"}
+	],
+	"remove_expired":false,
+},
+
+
+"scheduler": {
+	"enabled": true,
+},
+
+
+"cdrs": {
+	"enabled": true,
+},
+
+
+"cdre": {
+	"TestTutITExportCDR": {
+		"content_fields": [
+			{"tag": "CGRID", "type": "*composed", "value": "~CGRID"},
+			{"tag": "RunID", "type": "*composed", "value": "~RunID"},
+			{"tag":"OriginID", "type": "*composed", "value": "~OriginID"},
+			{"tag":"RequestType", "type": "*composed", "value": "~RequestType"},
+			{"tag":"Tenant", "type": "*composed", "value": "~Tenant"},
+			{"tag":"Category", "type": "*composed", "value": "~Category"},
+			{"tag":"Account", "type": "*composed", "value": "~Account"},
+			{"tag":"Destination", "type": "*composed", "value": "~Destination"},
+			{"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"Usage", "type": "*composed", "value": "~Usage"},
+			{"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4},
+			{"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"},
+		],
+	},
+},
+
+
+"chargers": {
+	"enabled": true,
+	"attributes_conns": [
+		{"address": "*internal"}
+	],
+},
+
+
+"resources": {
+	"enabled": true,
+	"store_interval": "1s",
+	"thresholds_conns": [
+		{"address": "*internal"}
+	],
+},
+
+
+"stats": {
+	"enabled": true,
+	"store_interval": "1s",
+	"thresholds_conns": [
+		{"address": "*internal"}
+	],
+},
+
+
+"thresholds": {
+	"enabled": true,
+	"store_interval": "1s",
+},
+
+
+"suppliers": {
+	"enabled": true,
+	"stats_conns": [
+		{"address": "*internal"},
+	],
+},
+
+
+"attributes": {							// Attribute service
+	"enabled": true,				// starts Alias service: <true|false>.
+},
+
+
+"sessions": {
+	"enabled": true,
+},
+
+
+"migrator": {
+	"out_datadb_type": "mongo",
+	"out_datadb_port": "27017",
+	"out_datadb_name": "10",
+	"out_stordb_type": "mongo",
+	"out_stordb_port": "27017",
+	"out_stordb_name": "cgrates",
+},
+
+
+}

engine/account.go

@@ -740,14 +740,16 @@ func (acc *Account) InitCounters() {
 }
 
 func (acc *Account) CleanExpiredStuff() {
-	for key, bm := range acc.BalanceMap {
-		for i := 0; i < len(bm); i++ {
-			if bm[i].IsExpired() {
-				// delete it
-				bm = append(bm[:i], bm[i+1:]...)
+	if config.CgrConfig().RalsCfg().RemoveExpired {
+		for key, bm := range acc.BalanceMap {
+			for i := 0; i < len(bm); i++ {
+				if bm[i].IsExpired() {
+					// delete it
+					bm = append(bm[:i], bm[i+1:]...)
+				}
 			}
+			acc.BalanceMap[key] = bm
 		}
-		acc.BalanceMap[key] = bm
 	}
 
 	for i := 0; i < len(acc.ActionTriggers); i++ {