From 8276ea73b79bf75342d0bbff2786c94f5ff8d216 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 31 Mar 2014 14:06:46 +0300 Subject: [PATCH 01/10] removed amount related stuff from calldesc --- engine/account.go | 19 +++------ engine/account_test.go | 87 ++--------------------------------------- engine/calldesc.go | 86 ++++++++-------------------------------- engine/calldesc_test.go | 12 +++--- engine/responder.go | 63 ++--------------------------- engine/storage_test.go | 2 +- 6 files changed, 35 insertions(+), 234 deletions(-) diff --git a/engine/account.go b/engine/account.go index cce870291..8a433ef30 100644 --- a/engine/account.go +++ b/engine/account.go @@ -34,11 +34,11 @@ const ( INBOUND = "*in" OUTBOUND = "*out" // Balance types - CREDIT = "*monetary" - SMS = "*sms" - TRAFFIC = "*internet" - TRAFFIC_TIME = "*internet_time" - MINUTES = "*minutes" + CREDIT = "*monetary" + SMS = "*sms" + DATA = "*data" + DATA_TIME = "*data_time" + MINUTES = "*minutes" // action price type PRICE_PERCENT = "*percent" PRICE_ABSOLUTE = "*absolute" @@ -310,15 +310,6 @@ func (ub *Account) refundIncrement(increment *Increment, direction string, count } } -// Debits some amount of user's specified balance. Returns the remaining credit in user's balance. -func (ub *Account) debitGenericBalance(balanceId string, direction string, amount float64, count bool) float64 { - if count { - ub.countUnits(&Action{BalanceType: balanceId, Direction: direction, Balance: &Balance{Value: amount}}) - } - ub.BalanceMap[balanceId+direction].Debit(amount) - return ub.BalanceMap[balanceId+direction].GetTotalValue() -} - // Scans the action trigers and execute the actions for which trigger is met func (ub *Account) executeActionTriggers(a *Action) { ub.ActionTriggers.Sort() diff --git a/engine/account_test.go b/engine/account_test.go index ca82447d3..f09cff846 100644 --- a/engine/account_test.go +++ b/engine/account_test.go @@ -157,47 +157,6 @@ func TestAccountStorageStore(t *testing.T) { } } -func TestDebitMoneyBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}}} - result := rifsBalance.debitGenericBalance(CREDIT, OUTBOUND, 6, false) - if rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value != 15 || result != rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 15, rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value) - } -} - -func TestDebitAllMoneyBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}}} - rifsBalance.debitGenericBalance(CREDIT, OUTBOUND, 21, false) - result := rifsBalance.debitGenericBalance(CREDIT, OUTBOUND, 0, false) - if rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value != 0 || result != rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 0, rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value) - } -} - -func TestDebitMoreMoneyBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}}} - result := rifsBalance.debitGenericBalance(CREDIT, OUTBOUND, 22, false) - if rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value != -1 || result != rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", -1, rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value) - } -} - -func TestDebitNegativeMoneyBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}}} - result := rifsBalance.debitGenericBalance(CREDIT, OUTBOUND, -15, false) - if rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value != 36 || result != rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 36, rifsBalance.BalanceMap[CREDIT+OUTBOUND][0].Value) - } -} - func TestDebitCreditZeroSecond(t *testing.T) { b1 := &Balance{Uuid: "testb", Value: 10, Weight: 10, DestinationId: "NAT", RateSubject: "*zero1s"} cc := &CallCost{ @@ -791,51 +750,11 @@ func TestDebitCreditSubjectMixedPartPay(t *testing.T) { } } -func TestDebitSMSBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}, SMS + OUTBOUND: BalanceChain{&Balance{Value: 100}}}} - result := rifsBalance.debitGenericBalance(SMS, OUTBOUND, 12, false) - if rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value != 88 || result != rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 88, rifsBalance.BalanceMap[SMS+OUTBOUND]) - } -} - -func TestDebitAllSMSBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}, SMS + OUTBOUND: BalanceChain{&Balance{Value: 100}}}} - result := rifsBalance.debitGenericBalance(SMS, OUTBOUND, 100, false) - if rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value != 0 || result != rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 0, rifsBalance.BalanceMap[SMS+OUTBOUND]) - } -} - -func TestDebitMoreSMSBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}, SMS + OUTBOUND: BalanceChain{&Balance{Value: 100}}}} - result := rifsBalance.debitGenericBalance(SMS, OUTBOUND, 110, false) - if rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value != -10 || result != rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", -10, rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value) - } -} - -func TestDebitNegativeSMSBalance(t *testing.T) { - b1 := &Balance{Value: 10, Weight: 10, DestinationId: "NAT"} - b2 := &Balance{Value: 100, Weight: 20, DestinationId: "RET"} - rifsBalance := &Account{Id: "other", BalanceMap: map[string]BalanceChain{MINUTES + OUTBOUND: BalanceChain{b1, b2}, CREDIT + OUTBOUND: BalanceChain{&Balance{Value: 21}}, SMS + OUTBOUND: BalanceChain{&Balance{Value: 100}}}} - result := rifsBalance.debitGenericBalance(SMS, OUTBOUND, -15, false) - if rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value != 115 || result != rifsBalance.BalanceMap[SMS+OUTBOUND][0].Value { - t.Errorf("Expected %v was %v", 115, rifsBalance.BalanceMap[SMS+OUTBOUND]) - } -} - func TestAccountdebitBalance(t *testing.T) { ub := &Account{ Id: "rif", AllowNegative: true, - BalanceMap: map[string]BalanceChain{SMS: BalanceChain{&Balance{Value: 14}}, TRAFFIC: BalanceChain{&Balance{Value: 1204}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, + BalanceMap: map[string]BalanceChain{SMS: BalanceChain{&Balance{Value: 14}}, DATA: BalanceChain{&Balance{Value: 1204}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, } newMb := &Balance{Weight: 20, DestinationId: "NEW"} a := &Action{BalanceType: MINUTES, Direction: OUTBOUND, Balance: newMb} @@ -850,7 +769,7 @@ func TestAccountdebitBalanceExists(t *testing.T) { ub := &Account{ Id: "rif", AllowNegative: true, - BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14}}, TRAFFIC + OUTBOUND: BalanceChain{&Balance{Value: 1024}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Value: 15, Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, + BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14}}, DATA + OUTBOUND: BalanceChain{&Balance{Value: 1024}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Value: 15, Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, } newMb := &Balance{Value: -10, Weight: 20, DestinationId: "NAT"} a := &Action{BalanceType: MINUTES, Direction: OUTBOUND, Balance: newMb} @@ -864,7 +783,7 @@ func TestAccountAddMinuteNil(t *testing.T) { ub := &Account{ Id: "rif", AllowNegative: true, - BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14}}, TRAFFIC + OUTBOUND: BalanceChain{&Balance{Value: 1024}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, + BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14}}, DATA + OUTBOUND: BalanceChain{&Balance{Value: 1024}}, MINUTES + OUTBOUND: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, } ub.debitBalanceAction(nil) if len(ub.BalanceMap[MINUTES+OUTBOUND]) != 2 { diff --git a/engine/calldesc.go b/engine/calldesc.go index c1497a303..ce4031bc8 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -112,11 +112,11 @@ type CallDescriptor struct { TimeStart, TimeEnd time.Time LoopIndex float64 // indicates the position of this segment in a cost request loop CallDuration time.Duration // the call duration so far (till TimeEnd) - Amount float64 - FallbackSubject string // the subject to check for destination if not found on primary subject - RatingInfos RatingInfos - Increments Increments - account *Account + //Amount float64 + FallbackSubject string // the subject to check for destination if not found on primary subject + RatingInfos RatingInfos + Increments Increments + account *Account } func (cd *CallDescriptor) ValidateCallData() error { @@ -609,60 +609,6 @@ func (cd *CallDescriptor) RefundIncrements() (left float64, err error) { return 0.0, err } -/* -Interface method used to add/substract an amount of cents from user's money balance. -The amount filed has to be filled in call descriptor. -*/ -func (cd *CallDescriptor) DebitCents() (left float64, err error) { - if userBalance, err := cd.getAccount(); err == nil && userBalance != nil { - defer accountingStorage.SetAccount(userBalance) - return userBalance.debitGenericBalance(CREDIT, cd.Direction, cd.Amount, true), nil - } - return 0.0, err -} - -/* -Interface method used to add/substract an amount of units from user's sms balance. -The amount filed has to be filled in call descriptor. -*/ -func (cd *CallDescriptor) DebitSMS() (left float64, err error) { - if userBalance, err := cd.getAccount(); err == nil && userBalance != nil { - defer accountingStorage.SetAccount(userBalance) - return userBalance.debitGenericBalance(SMS, cd.Direction, cd.Amount, true), nil - } - return 0, err -} - -/* -Interface method used to add/substract an amount of seconds from user's minutes balance. -The amount filed has to be filled in call descriptor. -*/ -func (cd *CallDescriptor) DebitSeconds() (err error) { - if userBalance, err := cd.getAccount(); err == nil && userBalance != nil { - defer accountingStorage.SetAccount(userBalance) - return userBalance.debitCreditBalance(cd.CreateCallCost(), true) - } - return err -} - -/* -Adds the specified amount of seconds to the received call seconds. When the threshold specified -in the user's tariff plan is reached then the received call balance is reseted and the bonus -specified in the tariff plan is applied. -The amount filed has to be filled in call descriptor. -*/ -func (cd *CallDescriptor) AddRecievedCallSeconds() (err error) { - if userBalance, err := cd.getAccount(); err == nil && userBalance != nil { - a := &Action{ - Direction: INBOUND, - Balance: &Balance{Value: cd.Amount, DestinationId: cd.Destination}, - } - userBalance.countUnits(a) - return nil - } - return err -} - func (cd *CallDescriptor) FlushCache() (err error) { cache2go.XFlush() cache2go.Flush() @@ -686,17 +632,17 @@ func (cd *CallDescriptor) CreateCallCost() *CallCost { func (cd *CallDescriptor) Clone() *CallDescriptor { return &CallDescriptor{ - Direction: cd.Direction, - TOR: cd.TOR, - Tenant: cd.Tenant, - Subject: cd.Subject, - Account: cd.Account, - Destination: cd.Destination, - TimeStart: cd.TimeStart, - TimeEnd: cd.TimeEnd, - LoopIndex: cd.LoopIndex, - CallDuration: cd.CallDuration, - Amount: cd.Amount, + Direction: cd.Direction, + TOR: cd.TOR, + Tenant: cd.Tenant, + Subject: cd.Subject, + Account: cd.Account, + Destination: cd.Destination, + TimeStart: cd.TimeStart, + TimeEnd: cd.TimeEnd, + LoopIndex: cd.LoopIndex, + CallDuration: cd.CallDuration, + // Amount: cd.Amount, FallbackSubject: cd.FallbackSubject, //RatingInfos: cd.RatingInfos, //Increments: cd.Increments, diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index af96ca572..df4b3a303 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -425,7 +425,7 @@ func TestMaxSessionTimeWithAccountAccount(t *testing.T) { Subject: "minu_from_tm", Account: "minu", Destination: "0723", - Amount: 1000} + } result, err := cd.GetMaxSessionDuration() expected := time.Minute if result != expected || err != nil { @@ -442,7 +442,7 @@ func TestMaxSessionTimeNoCredit(t *testing.T) { Tenant: "vdf", Subject: "broker", Destination: "0723", - Amount: 5400} + } result, err := cd.GetMaxSessionDuration() if result != time.Minute || err != nil { t.Errorf("Expected %v was %v", time.Minute, result) @@ -462,7 +462,7 @@ func TestMaxSessionModifiesCallDesc(t *testing.T) { Account: "minu", Destination: "0723", CallDuration: t2.Sub(t1), - Amount: 5400} + } initial := cd.Clone() cd.GetMaxSessionDuration() cd.account = nil // it's OK to cache the account @@ -481,7 +481,7 @@ func TestMaxDebitDurationNoGreatherThanInitialDuration(t *testing.T) { Subject: "minu_from_tm", Account: "minu", Destination: "0723", - Amount: 1000} + } initialDuration := cd.TimeEnd.Sub(cd.TimeStart) result, _ := cd.GetMaxSessionDuration() if result > initialDuration { @@ -671,7 +671,7 @@ func BenchmarkSplitting(b *testing.B) { func BenchmarkStorageSingleGetSessionTime(b *testing.B) { b.StopTimer() - cd := &CallDescriptor{Tenant: "vdf", Subject: "minutosu", Destination: "0723", Amount: 100} + cd := &CallDescriptor{Tenant: "vdf", Subject: "minutosu", Destination: "0723"} b.StartTimer() for i := 0; i < b.N; i++ { cd.GetMaxSessionDuration() @@ -680,7 +680,7 @@ func BenchmarkStorageSingleGetSessionTime(b *testing.B) { func BenchmarkStorageMultipleGetSessionTime(b *testing.B) { b.StopTimer() - cd := &CallDescriptor{Direction: "*out", TOR: "0", Tenant: "vdf", Subject: "minutosu", Destination: "0723", Amount: 5400} + cd := &CallDescriptor{Direction: "*out", TOR: "0", Tenant: "vdf", Subject: "minutosu", Destination: "0723"} b.StartTimer() for i := 0; i < b.N; i++ { cd.GetMaxSessionDuration() diff --git a/engine/responder.go b/engine/responder.go index f135b8378..84096ab6a 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -89,42 +89,6 @@ func (rs *Responder) RefundIncrements(arg CallDescriptor, reply *float64) (err e return } -func (rs *Responder) DebitCents(arg CallDescriptor, reply *float64) (err error) { - if rs.Bal != nil { - *reply, err = rs.callMethod(&arg, "Responder.DebitCents") - } else { - r, e := AccLock.Guard(arg.GetAccountKey(), func() (float64, error) { - return arg.DebitCents() - }) - *reply, err = r, e - } - return -} - -func (rs *Responder) DebitSMS(arg CallDescriptor, reply *float64) (err error) { - if rs.Bal != nil { - *reply, err = rs.callMethod(&arg, "Responder.DebitSMS") - } else { - r, e := AccLock.Guard(arg.GetAccountKey(), func() (float64, error) { - return arg.DebitSMS() - }) - *reply, err = r, e - } - return -} - -func (rs *Responder) DebitSeconds(arg CallDescriptor, reply *float64) (err error) { - if rs.Bal != nil { - *reply, err = rs.callMethod(&arg, "Responder.DebitSeconds") - } else { - r, e := AccLock.Guard(arg.GetAccountKey(), func() (float64, error) { - return 0, arg.DebitSeconds() - }) - *reply, err = r, e - } - return -} - func (rs *Responder) GetMaxSessionTime(arg CallDescriptor, reply *float64) (err error) { if rs.Bal != nil { *reply, err = rs.callMethod(&arg, "Responder.GetMaxSessionTime") @@ -138,19 +102,6 @@ func (rs *Responder) GetMaxSessionTime(arg CallDescriptor, reply *float64) (err return } -func (rs *Responder) AddRecievedCallSeconds(arg CallDescriptor, reply *float64) (err error) { - if rs.Bal != nil { - *reply, err = rs.callMethod(&arg, "Responder.AddRecievedCallSeconds") - } else { - - r, e := AccLock.Guard(arg.GetAccountKey(), func() (float64, error) { - return 0, arg.AddRecievedCallSeconds() - }) - *reply, err = r, e - } - return -} - func (rs *Responder) FlushCache(arg CallDescriptor, reply *float64) (err error) { if rs.Bal != nil { *reply, err = rs.callMethod(&arg, "Responder.FlushCache") @@ -200,12 +151,12 @@ func (rs *Responder) GetSMS(arg CallDescriptor, reply *CallCost) (err error) { } func (rs *Responder) GetInternet(arg CallDescriptor, reply *CallCost) (err error) { - err = rs.getBalance(&arg, TRAFFIC, reply) + err = rs.getBalance(&arg, DATA, reply) return err } func (rs *Responder) GetInternetTime(arg CallDescriptor, reply *CallCost) (err error) { - err = rs.getBalance(&arg, TRAFFIC_TIME, reply) + err = rs.getBalance(&arg, DATA_TIME, reply) return err } @@ -355,8 +306,6 @@ type Connector interface { Debit(CallDescriptor, *CallCost) error MaxDebit(CallDescriptor, *CallCost) error RefundIncrements(CallDescriptor, *float64) error - DebitCents(CallDescriptor, *float64) error - DebitSeconds(CallDescriptor, *float64) error GetMaxSessionTime(CallDescriptor, *float64) error } @@ -375,15 +324,11 @@ func (rcc *RPCClientConnector) Debit(cd CallDescriptor, cc *CallCost) error { func (rcc *RPCClientConnector) MaxDebit(cd CallDescriptor, cc *CallCost) error { return rcc.Client.Call("Responder.MaxDebit", cd, cc) } + func (rcc *RPCClientConnector) RefundIncrements(cd CallDescriptor, resp *float64) error { return rcc.Client.Call("Responder.RefundIncrements", cd, resp) } -func (rcc *RPCClientConnector) DebitCents(cd CallDescriptor, resp *float64) error { - return rcc.Client.Call("Responder.DebitCents", cd, resp) -} -func (rcc *RPCClientConnector) DebitSeconds(cd CallDescriptor, resp *float64) error { - return rcc.Client.Call("Responder.DebitSeconds", cd, resp) -} + func (rcc *RPCClientConnector) GetMaxSessionTime(cd CallDescriptor, resp *float64) error { return rcc.Client.Call("Responder.GetMaxSessionTime", cd, resp) } diff --git a/engine/storage_test.go b/engine/storage_test.go index e67d9e437..b34cdb618 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -147,7 +147,7 @@ func GetUB() *Account { ub := &Account{ Id: "rif", AllowNegative: true, - BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14, ExpirationDate: zeroTime}}, TRAFFIC + OUTBOUND: BalanceChain{&Balance{Value: 1024, ExpirationDate: zeroTime}}, MINUTES: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, + BalanceMap: map[string]BalanceChain{SMS + OUTBOUND: BalanceChain{&Balance{Value: 14, ExpirationDate: zeroTime}}, DATA + OUTBOUND: BalanceChain{&Balance{Value: 1024, ExpirationDate: zeroTime}}, MINUTES: BalanceChain{&Balance{Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, UnitCounters: []*UnitsCounter{uc, uc}, ActionTriggers: ActionTriggerPriotityList{at, at, at}, } From a8e4dad5f0adea33fd7f6fb1cd1e8c0759974820 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 31 Mar 2014 14:18:02 +0300 Subject: [PATCH 02/10] removed rateing id from callcost and loader --- engine/loader_csv_test.go | 3 --- engine/loader_helpers.go | 1 - engine/rateinterval.go | 1 - 3 files changed, 5 deletions(-) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index df45fddce..46e51ecf4 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -495,7 +495,6 @@ func TestLoadDestinationRateTimings(t *testing.T) { }, Ratings: map[string]*RIRate{ "d54545c1": &RIRate{ - Id: "R1", ConnectFee: 0, Rates: []*Rate{ &Rate{ @@ -509,7 +508,6 @@ func TestLoadDestinationRateTimings(t *testing.T) { RoundingDecimals: 2, }, "4bb00b9c": &RIRate{ - Id: "R2", ConnectFee: 0, Rates: []*Rate{ &Rate{ @@ -523,7 +521,6 @@ func TestLoadDestinationRateTimings(t *testing.T) { RoundingDecimals: 2, }, "e06c337f": &RIRate{ - Id: "R3", ConnectFee: 0, Rates: []*Rate{ &Rate{ diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 0b712e29d..16c5f52e1 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -126,7 +126,6 @@ func GetRateInterval(rpl *utils.TPRatingPlanBinding, dr *utils.DestinationRate) }, Weight: rpl.Weight, Rating: &RIRate{ - Id: dr.Rate.RateId, ConnectFee: dr.Rate.RateSlots[0].ConnectFee, RoundingMethod: dr.Rate.RateSlots[0].RoundingMethod, RoundingDecimals: dr.Rate.RateSlots[0].RoundingDecimals, diff --git a/engine/rateinterval.go b/engine/rateinterval.go index 190c310c7..a7522c1a6 100644 --- a/engine/rateinterval.go +++ b/engine/rateinterval.go @@ -53,7 +53,6 @@ func (rit *RITiming) Stringify() string { // Separate structure used for rating plan size optimization type RIRate struct { - Id string // informational role only ConnectFee float64 Rates RateGroups // GroupRateInterval (start time): Rate RoundingMethod string //ROUNDING_UP, ROUNDING_DOWN, ROUNDING_MIDDLE From 62c56ae26b0c5cc90f831a8c672e7e8452524ec4 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 31 Mar 2014 21:01:52 +0300 Subject: [PATCH 03/10] stale destination prefix cleaning --- cache2go/cache.go | 41 ++++++++++++++++++++++++++++--------- engine/destinations.go | 27 +++++++++++++++++++++++- engine/destinations_test.go | 16 +++++++++++++++ engine/storage_map.go | 2 ++ engine/storage_redis.go | 1 + utils/slice.go | 12 ++++++++++- 6 files changed, 87 insertions(+), 12 deletions(-) diff --git a/cache2go/cache.go b/cache2go/cache.go index b7ab7c47e..5f63c6be7 100644 --- a/cache2go/cache.go +++ b/cache2go/cache.go @@ -135,16 +135,6 @@ func RemKey(key string) { delete(cache, key) } -func RemPrefixKey(prefix string) { - mux.Lock() - defer mux.Unlock() - for key, _ := range cache { - if strings.HasPrefix(key, prefix) { - delete(cache, key) - } - } -} - func XRemKey(key string) { xMux.Lock() defer xMux.Unlock() @@ -155,6 +145,17 @@ func XRemKey(key string) { } delete(xcache, key) } + +func RemPrefixKey(prefix string) { + mux.Lock() + defer mux.Unlock() + for key, _ := range cache { + if strings.HasPrefix(key, prefix) { + delete(cache, key) + } + } +} + func XRemPrefixKey(prefix string) { xMux.Lock() defer xMux.Unlock() @@ -170,6 +171,26 @@ func XRemPrefixKey(prefix string) { } } +func GetAllEntries(prefix string) map[string]interface{} { + result := make(map[string]interface{}) + for key, timestampedValue := range cache { + if strings.HasPrefix(key, prefix) { + result[key] = timestampedValue.value + } + } + return result +} + +func XGetAllEntries(prefix string) map[string]interface{} { + result := make(map[string]interface{}) + for key, value := range xcache { + if strings.HasPrefix(key, prefix) { + result[key] = value + } + } + return result +} + // Delete all keys from expiraton cache func XFlush() { xMux.Lock() diff --git a/engine/destinations.go b/engine/destinations.go index 8d4e1df4c..caa42abbd 100644 --- a/engine/destinations.go +++ b/engine/destinations.go @@ -20,9 +20,11 @@ package engine import ( "encoding/json" - "github.com/cgrates/cgrates/cache2go" "strings" + "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/history" ) @@ -81,3 +83,26 @@ func CachedDestHasPrefix(destId, prefix string) bool { } return false } + +func CleanStalePrefixes(destIds []string) { + prefixMap := cache2go.GetAllEntries(DESTINATION_PREFIX) + for prefix, idIDs := range prefixMap { + dIDs := idIDs.([]string) + changed := false + for _, searchedDID := range destIds { + if found, i := utils.GetSliceMemberIndex(dIDs, searchedDID); found { + if len(dIDs) == 1 { + // remove de prefix from cache + cache2go.RemKey(prefix) + } else { + // delte the testination from list and put the new list in chache + dIDs[i], dIDs = dIDs[len(dIDs)-1], dIDs[:len(dIDs)-1] + changed = true + } + } + } + if changed { + cache2go.Cache(prefix, dIDs) + } + } +} diff --git a/engine/destinations_test.go b/engine/destinations_test.go index 95cd1bd5e..2ae708c9e 100644 --- a/engine/destinations_test.go +++ b/engine/destinations_test.go @@ -125,6 +125,22 @@ func TestNonCachedDestWrongPrefix(t *testing.T) { } } +func TestCleanStalePrefixes(t *testing.T) { + cache2go.Cache(DESTINATION_PREFIX+"1", []string{"D1", "D2"}) + cache2go.Cache(DESTINATION_PREFIX+"2", []string{"D1"}) + cache2go.Cache(DESTINATION_PREFIX+"3", []string{"D2"}) + CleanStalePrefixes([]string{"D1"}) + if r, err := cache2go.GetCached(DESTINATION_PREFIX + "1"); err != nil || len(r.([]string)) != 1 { + t.Error("Error cleaning stale destination ids", r) + } + if r, err := cache2go.GetCached(DESTINATION_PREFIX + "2"); err == nil { + t.Error("Error removing stale prefix: ", r) + } + if r, err := cache2go.GetCached(DESTINATION_PREFIX + "3"); err != nil || len(r.([]string)) != 1 { + t.Error("Error performing stale cleaning: ", r) + } +} + /********************************* Benchmarks **********************************/ func BenchmarkDestinationStorageStoreRestore(b *testing.B) { diff --git a/engine/storage_map.go b/engine/storage_map.go index 844859f13..949101744 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -51,6 +51,8 @@ func (ms *MapStorage) Flush() error { func (ms *MapStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys []string) error { if dKeys == nil { cache2go.RemPrefixKey(DESTINATION_PREFIX) + } else { + CleanStalePrefixes(dKeys) } if rpKeys == nil { cache2go.RemPrefixKey(RATING_PLAN_PREFIX) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index a8ac83288..82dd1ddc5 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -76,6 +76,7 @@ func (rs *RedisStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys []string) (e cache2go.RemPrefixKey(DESTINATION_PREFIX) } else if len(dKeys) != 0 { Logger.Info(fmt.Sprintf("Caching destinations: %v", dKeys)) + CleanStalePrefixes(dKeys) } for _, key := range dKeys { if _, err = rs.GetDestination(key[len(DESTINATION_PREFIX):]); err != nil { diff --git a/utils/slice.go b/utils/slice.go index d31026767..5465adb98 100644 --- a/utils/slice.go +++ b/utils/slice.go @@ -32,7 +32,17 @@ func IsSliceMember(ss []string, s string) bool { return false } -//Iterates over slice members and returns true of one starts with prefix +// Binary string search in slice +// returns true if found and the index +func GetSliceMemberIndex(ss []string, s string) (bool, int) { + sort.Strings(ss) + if i := sort.SearchStrings(ss, s); i < len(ss) && ss[i] == s { + return true, i + } + return false, len(ss) +} + +//Iterates over slice members and returns true if one starts with prefix func SliceMemberHasPrefix(ss []string, prfx string) bool { for _, mbr := range ss { if strings.HasPrefix(mbr, prfx) { From cc0534aa8b9cc6e875fd7c0b690bdcce5674d334 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 31 Mar 2014 20:27:12 +0200 Subject: [PATCH 04/10] Fixup masking in the fwv exporter --- cdre/fixedwidth.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdre/fixedwidth.go b/cdre/fixedwidth.go index 5ee748586..b9a750978 100644 --- a/cdre/fixedwidth.go +++ b/cdre/fixedwidth.go @@ -255,7 +255,11 @@ func (fwv *FixedWidthCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { } } case METATAG: - outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout) + if cfgFld.Value == META_MASKDESTINATION { + outVal, err = fwv.metaHandler(cfgFld.Value, cdr.ExportFieldValue(utils.DESTINATION)) + } else { + outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout) + } } if err != nil { engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error())) From 3313e7c41b32e7855b3074fe247f24618933dad7 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 31 Mar 2014 20:34:28 +0200 Subject: [PATCH 05/10] Fixup local test --- apier/apier_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apier/apier_local_test.go b/apier/apier_local_test.go index 90762f4b9..160c3c52b 100644 --- a/apier/apier_local_test.go +++ b/apier/apier_local_test.go @@ -910,7 +910,7 @@ func TestApierGetRatingPlan(t *testing.T) { } } */ - riRate := &engine.RIRate{Id: "RT_FS_USERS", ConnectFee: 0, RoundingMethod: "*up", RoundingDecimals: 0, Rates: []*engine.Rate{ + riRate := &engine.RIRate{ConnectFee: 0, RoundingMethod: "*up", RoundingDecimals: 0, Rates: []*engine.Rate{ &engine.Rate{GroupIntervalStart: 0, Value: 0, RateIncrement: time.Duration(60) * time.Second, RateUnit: time.Duration(60) * time.Second}, }} for _, rating := range reply.Ratings { From eb333fcfc0db25bd19f0e2d086a5d3bd75146e3a Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 1 Apr 2014 11:56:04 +0200 Subject: [PATCH 06/10] storage/GetStoredCdrs takes a higher number of filters for exports --- engine/storage_interface.go | 3 +- engine/storage_sql.go | 212 +++++++++++++++++++++---------- engine/storage_sql_local_test.go | 103 +++++++++++---- mediator/mediator.go | 2 +- mediator/mediator_local_test.go | 14 +- utils/apitpdata.go | 20 +-- 6 files changed, 246 insertions(+), 108 deletions(-) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 8a7f4c0a4..08084003a 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -106,7 +106,8 @@ type CdrStorage interface { Storage SetCdr(utils.RawCDR) error SetRatedCdr(*utils.StoredCdr, string) error - GetStoredCdrs([]string, string, string, string, string, string, string, string, string, string, string, time.Time, time.Time, bool, bool) ([]*utils.StoredCdr, error) + GetStoredCdrs([]string, []string, []string, []string, []string, []string, []string, []string, []string, []string, []string, + time.Time, time.Time, bool, bool) ([]*utils.StoredCdr, error) RemStoredCdrs([]string) error } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 2accf8dcc..e90210a6a 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -577,116 +577,196 @@ func (self *SQLStorage) SetRatedCdr(storedCdr *utils.StoredCdr, extraInfo string // Return a slice of CDRs from storDb using optional filters.a // ignoreErr - do not consider cdrs with rating errors // ignoreRated - do not consider cdrs which were already rated, including here the ones with errors -func (self *SQLStorage) GetStoredCdrs(cgrIds []string, runId string, cdrHost, cdrSource, reqType, direction, tenant, tor, account, subject, destPrefix string, +func (self *SQLStorage) GetStoredCdrs(cgrIds, runIds, cdrHosts, cdrSources, reqTypes, directions, tenants, tors, accounts, subjects, destPrefixes []string, timeStart, timeEnd time.Time, ignoreErr, ignoreRated bool) ([]*utils.StoredCdr, error) { var cdrs []*utils.StoredCdr - q := fmt.Sprintf("SELECT %s.cgrid,accid,cdrhost,cdrsource,reqtype,direction,tenant,tor,account,%s.subject,destination,setup_time,answer_time,duration,extra_fields,runid,cost FROM %s LEFT JOIN %s ON %s.cgrid=%s.cgrid LEFT JOIN %s ON %s.cgrid=%s.cgrid", utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA, utils.TBL_RATED_CDRS, utils.TBL_CDRS_PRIMARY, utils.TBL_RATED_CDRS) - fltr := "" + q := bytes.NewBufferString(fmt.Sprintf("SELECT %s.cgrid,accid,cdrhost,cdrsource,reqtype,direction,tenant,tor,account,%s.subject,destination,setup_time,answer_time,duration,extra_fields,runid,cost FROM %s LEFT JOIN %s ON %s.cgrid=%s.cgrid LEFT JOIN %s ON %s.cgrid=%s.cgrid", utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA, utils.TBL_RATED_CDRS, utils.TBL_CDRS_PRIMARY, utils.TBL_RATED_CDRS)) + fltr := new(bytes.Buffer) if len(cgrIds) != 0 { - qIds := " (" + qIds := bytes.NewBufferString(" (") for idxId, cgrId := range cgrIds { if idxId != 0 { - qIds += " OR" + qIds.WriteString(" OR") } - qIds += fmt.Sprintf(" %s.cgrid='%s'", utils.TBL_CDRS_PRIMARY, cgrId) + qIds.WriteString(fmt.Sprintf(" %s.cgrid='%s'", utils.TBL_CDRS_PRIMARY, cgrId)) } - qIds += " )" - if len(fltr) != 0 { - fltr += " AND" + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") } - fltr += qIds + fltr.Write(qIds.Bytes()) } - if len(runId) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(runIds) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, runId := range runIds { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" runid='%s'", runId)) } - fltr += fmt.Sprintf(" runid='%s'", runId) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(cdrHost) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(cdrHosts) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, host := range cdrHosts { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" cdrhost='%s'", host)) } - fltr += fmt.Sprintf(" cdrhost='%s'", cdrHost) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(cdrSource) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(cdrSources) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, src := range cdrSources { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" cdrsource='%s'", src)) } - fltr += fmt.Sprintf(" cdrsource='%s'", cdrSource) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(reqType) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(reqTypes) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, reqType := range reqTypes { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" reqtype='%s'", reqType)) } - fltr += fmt.Sprintf(" reqtype='%s'", reqType) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(direction) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(directions) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, direction := range directions { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" direction='%s'", direction)) } - fltr += fmt.Sprintf(" direction='%s'", direction) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(tenant) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(tenants) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, tenant := range tenants { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" tenant='%s'", tenant)) } - fltr += fmt.Sprintf(" tenant='%s'", tenant) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(tor) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(tors) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, tor := range tors { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" tor='%s'", tor)) } - fltr += fmt.Sprintf(" tor='%s'", tor) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(account) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(accounts) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, account := range accounts { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" account='%s'", account)) } - fltr += fmt.Sprintf(" account='%s'", account) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(subject) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(subjects) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, subject := range subjects { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" %s.subject='%s'", utils.TBL_CDRS_PRIMARY, subject)) } - fltr += fmt.Sprintf(" %s.subject='%s'", utils.TBL_CDRS_PRIMARY, subject) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } - if len(destPrefix) != 0 { - if len(fltr) != 0 { - fltr += " AND" + if len(destPrefixes) != 0 { + qIds := bytes.NewBufferString(" (") + for idx, destPrefix := range destPrefixes { + if idx != 0 { + qIds.WriteString(" OR") + } + qIds.WriteString(fmt.Sprintf(" destination LIKE '%s%%'", destPrefix)) } - fltr += fmt.Sprintf(" destination LIKE '%s%%'", destPrefix) + qIds.WriteString(" )") + if fltr.Len() != 0 { + fltr.WriteString(" AND") + } + fltr.Write(qIds.Bytes()) } if !timeStart.IsZero() { - if len(fltr) != 0 { - fltr += " AND" + if fltr.Len() != 0 { + fltr.WriteString(" AND") } - fltr += fmt.Sprintf(" answer_time>='%s'", timeStart) + fltr.WriteString(fmt.Sprintf(" answer_time>='%s'", timeStart)) } if !timeEnd.IsZero() { - if len(fltr) != 0 { - fltr += " AND" + if fltr.Len() != 0 { + fltr.WriteString(" AND") } - fltr += fmt.Sprintf(" answer_time<'%s'", timeEnd) + fltr.WriteString(fmt.Sprintf(" answer_time<'%s'", timeEnd)) } if ignoreRated { - if len(fltr) != 0 { - fltr += " AND" + if fltr.Len() != 0 { + fltr.WriteString(" AND") } if ignoreErr { - fltr += " cost IS NULL" + fltr.WriteString(" cost IS NULL") } else { - fltr += " (cost=-1 OR cost IS NULL)" + fltr.WriteString(" (cost=-1 OR cost IS NULL)") } } else if ignoreErr { - if len(fltr) != 0 { - fltr += " AND" + if fltr.Len() != 0 { + fltr.WriteString(" AND") } - fltr += " (cost!=-1 OR cost IS NULL)" + fltr.WriteString(" (cost!=-1 OR cost IS NULL)") } - if len(fltr) != 0 { - q += fmt.Sprintf(" WHERE %s", fltr) + if fltr.Len() != 0 { + q.WriteString(fmt.Sprintf(" WHERE %s", fltr.String())) } - rows, err := self.Db.Query(q) + rows, err := self.Db.Query(q.String()) if err != nil { return nil, err } diff --git a/engine/storage_sql_local_test.go b/engine/storage_sql_local_test.go index 9fed858ce..d225c2433 100644 --- a/engine/storage_sql_local_test.go +++ b/engine/storage_sql_local_test.go @@ -162,7 +162,7 @@ func TestSetCdr(t *testing.T) { SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} - strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: TEST_SQL, ReqType: "prepaid", + strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, @@ -189,13 +189,13 @@ func TestSetRatedCdr(t *testing.T) { SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} - strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: TEST_SQL, ReqType: "prepaid", + strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} strCdr3 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb3"), AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: "rated", - Direction: "*out", Tenant: "itsyscom.com", TOR: "call", Account: "1002", Subject: "1002", Destination: "+4986517174963", + Direction: "*out", Tenant: "itsyscom.com", TOR: "call", Account: "1002", Subject: "1002", Destination: "+4986517174964", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: "wholesale_run", Cost: 1.201} @@ -213,107 +213,164 @@ func TestGetStoredCdrs(t *testing.T) { } var timeStart, timeEnd time.Time // All CDRs, no filter - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 8 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on cgrids if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.FSCgrId("bbb1"), utils.FSCgrId("bbb2")}, - "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 2 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on cgrids plus reqType if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.FSCgrId("bbb1"), utils.FSCgrId("bbb2")}, - "", "", "", "prepaid", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + nil, nil, nil, []string{"prepaid"}, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on runId - if storedCdrs, err := mysql.GetStoredCdrs(nil, utils.DEFAULT_RUNID, "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, []string{utils.DEFAULT_RUNID}, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 2 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on cdrHost - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "192.168.1.2", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, []string{"192.168.1.2"}, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 3 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on multiple cdrHost + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, []string{"192.168.1.1", "192.168.1.2"}, nil, nil, nil, nil, nil, nil, nil, nil, + timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on cdrSource - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "UNKNOWN", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, []string{"UNKNOWN"}, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } - // Filter on reqType - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "prepaid", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + // Filter on multiple cdrSource + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, []string{"UNKNOWN", "UNKNOWN2"}, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 2 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on reqType + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, []string{"prepaid"}, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple reqType + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, []string{"prepaid", "pseudoprepaid"}, nil, nil, nil, nil, nil, nil, + timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on direction - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "*out", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, []string{"*out"}, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 8 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on tenant - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "itsyscom.com", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, []string{"itsyscom.com"}, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 3 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on multiple tenants + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, []string{"itsyscom.com", "cgrates.org"}, nil, nil, nil, nil, + timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on tor - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "premium_call", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, []string{"premium_call"}, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on multiple tor + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, []string{"premium_call", "call"}, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on account - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "1002", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, []string{"1002"}, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 3 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on multiple account + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, []string{"1001", "1002"}, nil, nil, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on subject - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "1000", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, []string{"1000"}, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } + // Filter on multiple subject + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, []string{"1000", "1002"}, nil, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on destPrefix + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []string{"+498651"}, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple destPrefixes + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []string{"1001", "+498651"}, timeStart, timeEnd, false, false); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 4 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } // Filter on ignoreErr - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, true, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, true, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 8 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on ignoreRated - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, true); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, true); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 5 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on timeStart timeStart = time.Date(2013, 11, 8, 8, 0, 0, 0, time.UTC) - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 5 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on timeStart and timeEnd timeEnd = time.Date(2013, 12, 1, 8, 0, 0, 0, time.UTC) - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 2 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Combined filter - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "rated", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, []string{"rated"}, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) @@ -355,7 +412,7 @@ func TestRemStoredCdrs(t *testing.T) { if err := mysql.RemStoredCdrs([]string{utils.FSCgrId("bbb1")}); err != nil { t.Error(err.Error()) } - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 7 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) @@ -364,7 +421,7 @@ func TestRemStoredCdrs(t *testing.T) { utils.FSCgrId("bbb2"), utils.FSCgrId("bbb3")}); err != nil { t.Error(err.Error()) } - if storedCdrs, err := mysql.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, false, false); err != nil { + if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 0 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) diff --git a/mediator/mediator.go b/mediator/mediator.go index 229ae6b46..a191853d4 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -161,7 +161,7 @@ func (self *Mediator) RateCdr(dbcdr utils.RawCDR) error { } func (self *Mediator) RateCdrs(timeStart, timeEnd time.Time, rerateErrors, rerateRated bool) error { - cdrs, err := self.cdrDb.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", timeStart, timeEnd, !rerateErrors, !rerateRated) + cdrs, err := self.cdrDb.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, timeStart, timeEnd, !rerateErrors, !rerateRated) if err != nil { return err } diff --git a/mediator/mediator_local_test.go b/mediator/mediator_local_test.go index 88d3c2c52..069253fcf 100644 --- a/mediator/mediator_local_test.go +++ b/mediator/mediator_local_test.go @@ -150,12 +150,12 @@ func TestPostCdrs(t *testing.T) { } } time.Sleep(10 * time.Millisecond) // Give time for CDRs to reach database - if storedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, false, false); err != nil { + if storedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, false, false); err != nil { t.Error(err) } else if len(storedCdrs) != 2 { // Make sure CDRs made it into StorDb t.Error(fmt.Sprintf("Unexpected number of CDRs stored: %d", len(storedCdrs))) } - if nonErrorCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, true, false); err != nil { + if nonErrorCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, true, false); err != nil { t.Error(err) } else if len(nonErrorCdrs) != 0 { // Just two of them should be without errors t.Error(fmt.Sprintf("Unexpected number of CDRs stored: %d", len(nonErrorCdrs))) @@ -178,12 +178,12 @@ func TestInjectCdrs(t *testing.T) { t.Error(err) } } - if storedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, false, false); err != nil { + if storedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, false, false); err != nil { t.Error(err) } else if len(storedCdrs) != 4 { // Make sure CDRs made it into StorDb t.Error(fmt.Sprintf("Unexpected number of CDRs stored: %d", len(storedCdrs))) } - if nonRatedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, true, true); err != nil { + if nonRatedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, true, true); err != nil { t.Error(err) } else if len(nonRatedCdrs) != 2 { // Just two of them should be non-rated t.Error(fmt.Sprintf("Unexpected number of CDRs non-rated: %d", len(nonRatedCdrs))) @@ -215,12 +215,12 @@ func TestRateCdrs(t *testing.T) { } else if reply != utils.OK { t.Errorf("Unexpected reply: %s", reply) } - if nonRatedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, true, true); err != nil { + if nonRatedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, true, true); err != nil { t.Error(err) } else if len(nonRatedCdrs) != 0 { // Just two of them should be non-rated t.Error(fmt.Sprintf("Unexpected number of CDRs non-rated: %d", len(nonRatedCdrs))) } - if errRatedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, false, true); err != nil { + if errRatedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, false, true); err != nil { t.Error(err) } else if len(errRatedCdrs) != 2 { // The first 2 with errors should be still there before rerating t.Error(fmt.Sprintf("Unexpected number of CDRs with errors: %d", len(errRatedCdrs))) @@ -230,7 +230,7 @@ func TestRateCdrs(t *testing.T) { } else if reply != utils.OK { t.Errorf("Unexpected reply: %s", reply) } - if errRatedCdrs, err := cdrStor.GetStoredCdrs(nil, "", "", "", "", "", "", "", "", "", "", time.Time{}, time.Time{}, false, true); err != nil { + if errRatedCdrs, err := cdrStor.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, time.Time{}, time.Time{}, false, true); err != nil { t.Error(err) } else if len(errRatedCdrs) != 1 { // One CDR with errors should be fixed now by rerating t.Error(fmt.Sprintf("Unexpected number of CDRs with errors: %d", len(errRatedCdrs))) diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 2b9f50ec4..7ea5395ef 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -324,16 +324,16 @@ type AttrExpFileCdrs struct { MaskDestinationId string // Overwrite configured MaskDestId MaskLength int // Overwrite configured MaskLength CgrIds []string // If provided, it will filter based on the cgrids present in list - MediationRunId string // If provided, it will filter on mediation runid - CdrHost string // If provided, it will filter cdrhost - CdrSource string // If provided, it will filter cdrsource - ReqType string // If provided, it will fiter reqtype - Direction string // If provided, it will fiter direction - Tenant string // If provided, it will filter tenant - Tor string // If provided, it will filter tor - Account string // If provided, it will filter account - Subject string // If provided, it will filter the rating subject - DestinationPrefix string // If provided, it will filter on destination prefix + MediationRunId []string // If provided, it will filter on mediation runid + CdrHost []string // If provided, it will filter cdrhost + CdrSource []string // If provided, it will filter cdrsource + ReqType []string // If provided, it will fiter reqtype + Direction []string // If provided, it will fiter direction + Tenant []string // If provided, it will filter tenant + Tor []string // If provided, it will filter tor + Account []string // If provided, it will filter account + Subject []string // If provided, it will filter the rating subject + DestinationPrefix []string // If provided, it will filter on destination prefix TimeStart string // If provided, it will represent the starting of the CDRs interval (>=) TimeEnd string // If provided, it will represent the end of the CDRs interval (<) SkipErrors bool // Do not export errored CDRs From cd29a3360bee366203a362f36331e8792f483f70 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 1 Apr 2014 16:08:34 +0200 Subject: [PATCH 07/10] Adding costDigitsShift formatting --- apier/cdre.go | 10 ++++++--- cdre/csv.go | 16 ++++++------- cdre/csv_test.go | 2 +- cdre/fixedwidth.go | 50 +++++++++++++++++++++-------------------- config/config.go | 5 +++++ config/config_test.go | 2 ++ config/test_data.txt | 1 + data/conf/cgrates.cfg | 1 + engine/storage_sql.go | 2 +- utils/apitpdata.go | 3 ++- utils/storedcdr.go | 9 ++++++-- utils/storedcdr_test.go | 12 ++++++---- 12 files changed, 69 insertions(+), 44 deletions(-) diff --git a/apier/cdre.go b/apier/cdre.go index fe39c8a1d..41e29ec7c 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -53,7 +53,11 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E if len(exportId) == 0 { exportId = strconv.FormatInt(time.Now().Unix(), 10) } - roundDecimals := attr.RoundingDecimals + costShiftDigits := attr.CostShiftDigits + if costShiftDigits != 0 { + costShiftDigits = self.Config.CdreCostShiftDigits + } + roundDecimals := attr.RoundDecimals if roundDecimals == 0 { roundDecimals = self.Config.RoundingDecimals } @@ -99,7 +103,7 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E return err } defer fileOut.Close() - csvWriter := cdre.NewCsvCdrWriter(fileOut, roundDecimals, maskDestId, maskLen, exportedFields) + csvWriter := cdre.NewCsvCdrWriter(fileOut, costShiftDigits, roundDecimals, maskDestId, maskLen, exportedFields) exportedIds := make([]string, 0) unexportedIds := make(map[string]string) for _, cdr := range cdrs { @@ -132,7 +136,7 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E return err } defer fileOut.Close() - fww, _ := cdre.NewFWCdrWriter(self.LogDb, fileOut, exportTemplate, exportId, roundDecimals, maskDestId, maskLen) + fww, _ := cdre.NewFWCdrWriter(self.LogDb, fileOut, exportTemplate, exportId, costShiftDigits, roundDecimals, maskDestId, maskLen) exportedIds := make([]string, 0) unexportedIds := make(map[string]string) for _, cdr := range cdrs { diff --git a/cdre/csv.go b/cdre/csv.go index 64597e1e2..b2c3f7cb0 100644 --- a/cdre/csv.go +++ b/cdre/csv.go @@ -26,15 +26,15 @@ import ( ) type CsvCdrWriter struct { - writer *csv.Writer - roundDecimals int // Round floats like Cost using this number of decimals - maskDestId string - maskLen int - exportedFields []*utils.RSRField // The fields exported, order important + writer *csv.Writer + costShiftDigits, roundDecimals int // Round floats like Cost using this number of decimals + maskDestId string + maskLen int + exportedFields []*utils.RSRField // The fields exported, order important } -func NewCsvCdrWriter(writer io.Writer, roundDecimals int, maskDestId string, maskLen int, exportedFields []*utils.RSRField) *CsvCdrWriter { - return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, maskDestId, maskLen, exportedFields} +func NewCsvCdrWriter(writer io.Writer, costShiftDigits, roundDecimals int, maskDestId string, maskLen int, exportedFields []*utils.RSRField) *CsvCdrWriter { + return &CsvCdrWriter{csv.NewWriter(writer), costShiftDigits, roundDecimals, maskDestId, maskLen, exportedFields} } func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { @@ -42,7 +42,7 @@ func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { for idx, fld := range csvwr.exportedFields { var fldVal string if fld.Id == utils.COST { - fldVal = cdr.FormatCost(csvwr.roundDecimals) + fldVal = cdr.FormatCost(csvwr.costShiftDigits, csvwr.roundDecimals) } else if fld.Id == utils.DESTINATION { fldVal = cdr.ExportFieldValue(utils.DESTINATION) if len(csvwr.maskDestId) != 0 && csvwr.maskLen > 0 && engine.CachedDestHasPrefix(csvwr.maskDestId, fldVal) { diff --git a/cdre/csv_test.go b/cdre/csv_test.go index e64627eb6..16d66163d 100644 --- a/cdre/csv_test.go +++ b/cdre/csv_test.go @@ -31,7 +31,7 @@ func TestCsvCdrWriter(t *testing.T) { writer := &bytes.Buffer{} cfg, _ := config.NewDefaultCGRConfig() exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"}) - csvCdrWriter := NewCsvCdrWriter(writer, 4, "", -1, exportedFields) + csvCdrWriter := NewCsvCdrWriter(writer, 0, 4, "", -1, exportedFields) ratedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, diff --git a/cdre/fixedwidth.go b/cdre/fixedwidth.go index b9a750978..aaecf5b03 100644 --- a/cdre/fixedwidth.go +++ b/cdre/fixedwidth.go @@ -47,38 +47,40 @@ const ( META_DURCDRS = "cdrs_duration" META_COSTCDRS = "cdrs_cost" META_MASKDESTINATION = "mask_destination" + META_FORMATCOST = "format_cost" ) var err error func NewFWCdrWriter(logDb engine.LogStorage, outFile *os.File, exportTpl *config.CgrXmlCdreFwCfg, exportId string, - roundDecimals int, maskDestId string, maskLen int) (*FixedWidthCdrWriter, error) { + costShiftDigits, roundDecimals int, maskDestId string, maskLen int) (*FixedWidthCdrWriter, error) { return &FixedWidthCdrWriter{ - logDb: logDb, - writer: outFile, - exportTemplate: exportTpl, - exportId: exportId, - roundDecimals: roundDecimals, - maskDestId: maskDestId, - maskLen: maskLen, - header: &bytes.Buffer{}, - content: &bytes.Buffer{}, - trailer: &bytes.Buffer{}}, nil + logDb: logDb, + writer: outFile, + exportTemplate: exportTpl, + exportId: exportId, + costShiftDigits: costShiftDigits, + roundDecimals: roundDecimals, + maskDestId: maskDestId, + maskLen: maskLen, + header: &bytes.Buffer{}, + content: &bytes.Buffer{}, + trailer: &bytes.Buffer{}}, nil } type FixedWidthCdrWriter struct { - logDb engine.LogStorage // Used to extract cost_details if these are requested - writer io.Writer - exportTemplate *config.CgrXmlCdreFwCfg - exportId string // Unique identifier or this export - roundDecimals int - maskDestId string - maskLen int - header, content, trailer *bytes.Buffer - firstCdrATime, lastCdrATime time.Time - numberOfRecords int - totalDuration time.Duration - totalCost float64 + logDb engine.LogStorage // Used to extract cost_details if these are requested + writer io.Writer + exportTemplate *config.CgrXmlCdreFwCfg + exportId string // Unique identifier or this export + costShiftDigits, roundDecimals int + maskDestId string + maskLen int + header, content, trailer *bytes.Buffer + firstCdrATime, lastCdrATime time.Time + numberOfRecords int + totalDuration time.Duration + totalCost float64 } // Return Json marshaled callCost attached to @@ -116,7 +118,7 @@ func (fwv *FixedWidthCdrWriter) cdrFieldValue(cdr *utils.StoredCdr, cfgHdr, layo return "", err } case utils.COST: - cdrVal = cdr.FormatCost(fwv.roundDecimals) + cdrVal = cdr.FormatCost(fwv.costShiftDigits, fwv.roundDecimals) case utils.SETUP_TIME: cdrVal = cdr.SetupTime.Format(layout) case utils.ANSWER_TIME: // Format time based on layout diff --git a/config/config.go b/config/config.go index bc88048c5..8dd5d5c16 100644 --- a/config/config.go +++ b/config/config.go @@ -93,6 +93,7 @@ type CGRConfig struct { CdreCdrFormat string // Format of the exported CDRs. CdreMaskDestId string // Id of the destination list to be masked in CDRs CdreMaskLength int // Number of digits to mask in the destination suffix if destination is in the MaskDestinationdsId + CdreCostShiftDigits int // Shift digits in the cost on export (eg: convert from EUR to cents) CdreDir string // Path towards exported cdrs directory CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs CdreFWXmlTemplate *CgrXmlCdreFwCfg // Use this configuration as export template in case of fixed fields length @@ -201,6 +202,7 @@ func (self *CGRConfig) setDefaults() error { self.CdreCdrFormat = "csv" self.CdreMaskDestId = "" self.CdreMaskLength = 0 + self.CdreCostShiftDigits = 0 self.CdreDir = "/var/log/cgrates/cdr/cdre" self.CdrcEnabled = false self.CdrcCdrs = utils.INTERNAL @@ -498,6 +500,9 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("cdre", "mask_length"); hasOpt { cfg.CdreMaskLength, _ = c.GetInt("cdre", "mask_length") } + if hasOpt = c.HasOption("cdre", "cost_shift_digits"); hasOpt { + cfg.CdreCostShiftDigits, _ = c.GetInt("cdre", "cost_shift_digits") + } if hasOpt = c.HasOption("cdre", "export_template"); hasOpt { // Load configs for csv normally from template, fixed_width from xml file exportTemplate, _ := c.GetString("cdre", "export_template") if cfg.CdreCdrFormat != utils.CDRE_FIXED_WIDTH { // Csv most likely diff --git a/config/config_test.go b/config/config_test.go index 942d16c15..a992e89c0 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -84,6 +84,7 @@ func TestDefaults(t *testing.T) { eCfg.CdreCdrFormat = "csv" eCfg.CdreMaskDestId = "" eCfg.CdreMaskLength = 0 + eCfg.CdreCostShiftDigits = 0 eCfg.CdreDir = "/var/log/cgrates/cdr/cdre" eCfg.CdrcEnabled = false eCfg.CdrcCdrs = utils.INTERNAL @@ -240,6 +241,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.CdreCdrFormat = "test" eCfg.CdreMaskDestId = "test" eCfg.CdreMaskLength = 99 + eCfg.CdreCostShiftDigits = 99 eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}} eCfg.CdreDir = "test" eCfg.CdrcEnabled = true diff --git a/config/test_data.txt b/config/test_data.txt index 515e2d664..21ad69960 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -51,6 +51,7 @@ mediator = test # Address where to reach the Mediator. Empty for disabling me cdr_format = test # Exported CDRs format mask_destination_id = test # Destination id containing called addresses to be masked on export mask_length = 99 # Length of the destination suffix to be masked +cost_shift_digits = 99 # Shift the number of cost export_dir = test # Path where the exported CDRs will be placed export_template = test # List of fields in the exported CDRs diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 2af875892..1333a6892 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -54,6 +54,7 @@ # cdr_format = csv # Exported CDRs format # mask_destination_id = # Destination id containing called addresses to be masked on export # mask_length = 0 # Length of the destination suffix to be masked +# cost_shift_digits = 0 # Shift cost on export with the number of digits digits defined here (eg: convert from Eur to cent). # export_dir = /var/log/cgrates/cdr/cdrexport/csv # Path where the exported CDRs will be placed # export_template = cgrid,mediation_runid,accid,cdrhost,reqtype,direction,tenant,tor,account,subject,destination,setup_time,answer_time,duration,cost # Exported fields template <""|fld1,fld2|*xml:instance_name> diff --git a/engine/storage_sql.go b/engine/storage_sql.go index e90210a6a..aa1aa30d3 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -784,7 +784,7 @@ func (self *SQLStorage) GetStoredCdrs(cgrIds, runIds, cdrHosts, cdrSources, reqT return nil, err } if err := json.Unmarshal(extraFields, &extraFieldsMp); err != nil { - return nil, err + return nil, fmt.Errorf("JSON unmarshal error for cgrid: %s, runid: %s, error: %s", cgrid, runid, err.Error()) } storCdr := &utils.StoredCdr{ CgrId: cgrid, AccId: accid, CdrHost: cdrhost, CdrSource: cdrsrc, ReqType: reqtype, Direction: direction, Tenant: tenant, diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 7ea5395ef..13ce692c2 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -320,7 +320,8 @@ type AttrExpFileCdrs struct { ExportId string // Optional exportid ExportFileName string // If provided the output filename will be set to this ExportTemplate string // Exported fields template <""|fld1,fld2|*xml:instance_name> - RoundingDecimals int // Overwrite configured roundDecimals with this dynamically + CostShiftDigits int // If defined it will shift cost digits before applying rouding (eg: convert from Eur->cents) + RoundDecimals int // Overwrite configured roundDecimals with this dynamically MaskDestinationId string // Overwrite configured MaskDestId MaskLength int // Overwrite configured MaskLength CgrIds []string // If provided, it will filter based on the cgrids present in list diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 60dd46a09..fa841b728 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -19,6 +19,7 @@ along with this program. If not, see package utils import ( + "math" "net/url" "strconv" "time" @@ -135,8 +136,12 @@ func (storedCdr *StoredCdr) GetExtraFields() map[string]string { } // Return cost as string, formated with number of decimals configured -func (storedCdr *StoredCdr) FormatCost(roundDecimals int) string { - return strconv.FormatFloat(storedCdr.Cost, 'f', roundDecimals, 64) +func (storedCdr *StoredCdr) FormatCost(shiftDecimals, roundDecimals int) string { + cost := storedCdr.Cost + if shiftDecimals != 0 { + cost = cost * math.Pow10(shiftDecimals) + } + return strconv.FormatFloat(cost, 'f', roundDecimals, 64) } func (storedCdr *StoredCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld string, extraFlds []string, fieldsMandatory bool) (*StoredCdr, error) { diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 6fad3e290..edfa8e3b0 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -179,11 +179,15 @@ func TestExportFieldValue(t *testing.T) { func TestFormatCost(t *testing.T) { cdr := StoredCdr{Cost: 1.01} - if cdr.FormatCost(4) != "1.0100" { - t.Error("Unexpected format of the cost: ", cdr.FormatCost(4)) + if cdr.FormatCost(0, 4) != "1.0100" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(0, 4)) } cdr = StoredCdr{Cost: 1.01001} - if cdr.FormatCost(4) != "1.0100" { - t.Error("Unexpected format of the cost: ", cdr.FormatCost(4)) + if cdr.FormatCost(0, 4) != "1.0100" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(0, 4)) + } + cdr = StoredCdr{Cost: 1.01001} + if cdr.FormatCost(2, 0) != "101" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(2, 0)) } } From ac86c6b60c59792a5750489c82d44ac2956b3f32 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 1 Apr 2014 16:13:11 +0200 Subject: [PATCH 08/10] Fix test of costShiftDigits --- utils/storedcdr_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index edfa8e3b0..db25b9103 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -186,8 +186,10 @@ func TestFormatCost(t *testing.T) { if cdr.FormatCost(0, 4) != "1.0100" { t.Error("Unexpected format of the cost: ", cdr.FormatCost(0, 4)) } - cdr = StoredCdr{Cost: 1.01001} if cdr.FormatCost(2, 0) != "101" { t.Error("Unexpected format of the cost: ", cdr.FormatCost(2, 0)) } + if cdr.FormatCost(2, 3) != "101.001" { + t.Error("Unexpected format of the cost: ", cdr.FormatCost(2, 3)) + } } From e5dc9a2e560646221258e9a3aefd9504bec48be2 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 1 Apr 2014 16:53:04 +0200 Subject: [PATCH 09/10] Work around in exporter api to keep config defaults --- apier/cdre.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apier/cdre.go b/apier/cdre.go index 41e29ec7c..02a9a6c6f 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -53,21 +53,21 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E if len(exportId) == 0 { exportId = strconv.FormatInt(time.Now().Unix(), 10) } - costShiftDigits := attr.CostShiftDigits - if costShiftDigits != 0 { - costShiftDigits = self.Config.CdreCostShiftDigits + costShiftDigits := self.Config.CdreCostShiftDigits + if attr.CostShiftDigits != -1 { // -1 enables system general config + costShiftDigits = attr.CostShiftDigits } - roundDecimals := attr.RoundDecimals - if roundDecimals == 0 { - roundDecimals = self.Config.RoundingDecimals + roundDecimals := self.Config.RoundingDecimals + if attr.RoundDecimals != -1 { // -1 enables system default config + roundDecimals = attr.RoundDecimals } maskDestId := attr.MaskDestinationId if len(maskDestId) == 0 { maskDestId = self.Config.CdreMaskDestId } - maskLen := attr.MaskLength - if maskLen == 0 { - maskLen = self.Config.CdreMaskLength + maskLen := self.Config.CdreMaskLength + if attr.MaskLength != -1 { + maskLen = attr.MaskLength } cdrs, err := self.CdrDb.GetStoredCdrs(attr.CgrIds, attr.MediationRunId, attr.CdrHost, attr.CdrSource, attr.ReqType, attr.Direction, attr.Tenant, attr.Tor, attr.Account, attr.Subject, attr.DestinationPrefix, tStart, tEnd, attr.SkipErrors, attr.SkipRated) From a03b25cc45deaf1063516d8706eb502166846adb Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 1 Apr 2014 20:49:34 +0300 Subject: [PATCH 10/10] cleaned unused fields --- engine/account.go | 7 ++----- engine/balances.go | 13 ++++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/engine/account.go b/engine/account.go index 8a433ef30..705e21ebe 100644 --- a/engine/account.go +++ b/engine/account.go @@ -62,11 +62,8 @@ type Account struct { BalanceMap map[string]BalanceChain UnitCounters []*UnitsCounter ActionTriggers ActionTriggerPriotityList - Groups GroupLinks // user info about groups - // group information - UserIds []string // group info about users - AllowNegative bool - Disabled bool + AllowNegative bool + Disabled bool } // Returns user's available minutes for the specified destination diff --git a/engine/balances.go b/engine/balances.go index 78bbf9e26..59cdfe84b 100644 --- a/engine/balances.go +++ b/engine/balances.go @@ -32,13 +32,12 @@ type Balance struct { Value float64 ExpirationDate time.Time Weight float64 - //GroupIds []string - DestinationId string - RateSubject string - SharedGroup string - precision int - account *Account // used to store ub reference for shared balances - dirty bool + DestinationId string + RateSubject string + SharedGroup string + precision int + account *Account // used to store ub reference for shared balances + dirty bool } func (b *Balance) Equal(o *Balance) bool {