From f68dc3fcfdcd38ed282b6539e7e7b0c7198168df Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 25 Nov 2015 19:27:01 +0200 Subject: [PATCH 001/227] using rpc client instead of connector --- cdrc/cdrc.go | 7 +- cmd/cgr-engine/cgr-engine.go | 32 +-- engine/cdrs.go | 15 +- engine/responder.go | 351 ++--------------------------- sessionmanager/fssessionmanager.go | 24 +- sessionmanager/kamailiosm.go | 17 +- sessionmanager/osipssm.go | 13 +- sessionmanager/session.go | 8 +- sessionmanager/session_test.go | 28 +-- sessionmanager/sessionmanager.go | 5 +- sessionmanager/smg_session.go | 11 +- sessionmanager/smgeneric.go | 15 +- 12 files changed, 104 insertions(+), 422 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index f91663678..9d77c941b 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -32,6 +32,7 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" "gopkg.in/fsnotify.v1" ) @@ -54,7 +55,7 @@ Common parameters within configs processed: Parameters specific per config instance: * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields */ -func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs engine.Connector, closeChan chan struct{}, dfltTimezone string) (*Cdrc, error) { +func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs rpcclient.RpcClientConnection, closeChan chan struct{}, dfltTimezone string) (*Cdrc, error) { var cdrcCfg *config.CdrcConfig for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break @@ -85,7 +86,7 @@ type Cdrc struct { cdrcCfgs map[string]*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name) dfltCdrcCfg *config.CdrcConfig timezone string - cdrs engine.Connector + cdrs rpcclient.RpcClientConnection httpClient *http.Client closeChan chan struct{} // Used to signal config reloads when we need to span different CDRC-Client maxOpenFiles chan struct{} // Maximum number of simultaneous files processed @@ -201,7 +202,7 @@ func (self *Cdrc) processFile(filePath string) error { utils.Logger.Info(fmt.Sprintf(" DryRun CDR: %+v", storedCdr)) continue } - if err := self.cdrs.ProcessCdr(storedCdr, &reply); err != nil { + if err := self.cdrs.Call("Responder.ProcessCdr", storedCdr, &reply); err != nil { utils.Logger.Err(fmt.Sprintf(" Failed sending CDR, %+v, error: %s", storedCdr, err.Error())) } else if reply != "OK" { utils.Logger.Err(fmt.Sprintf(" Received unexpected reply for CDR, %+v, reply: %s", storedCdr, reply)) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index d519c7c9c..cc6a85f2f 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -104,7 +104,7 @@ func startCdrcs(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan // Fires up a cdrc instance func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan *engine.Responder, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, closeChan chan struct{}, exitChan chan bool) { - var cdrsConn engine.Connector + var cdrsConn rpcclient.RpcClientConnection var cdrcCfg *config.CdrcConfig for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break @@ -122,7 +122,7 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * exitChan <- true return } - cdrsConn = &engine.RPCClientConnector{Client: conn} + cdrsConn = conn } cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan, cfg.DefaultTimezone) if err != nil { @@ -138,7 +138,7 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") - var raterConn, cdrsConn engine.Connector + var raterConn, cdrsConn rpcclient.RpcClientConnection var client *rpcclient.RpcClient var err error // Connect to rater @@ -154,7 +154,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal exitChan <- true return } - raterConn = &engine.RPCClientConnector{Client: client} + raterConn = client } } // Connect to CDRS @@ -173,7 +173,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal exitChan <- true return } - cdrsConn = &engine.RPCClientConnector{Client: client} + cdrsConn = client } } } @@ -226,7 +226,7 @@ func startDiameterAgent(internalSMGChan chan rpcclient.RpcClientConnection, exit func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-FreeSWITCH service.") - var raterConn, cdrsConn engine.Connector + var raterConn, cdrsConn rpcclient.RpcClientConnection var client *rpcclient.RpcClient var err error // Connect to rater @@ -242,7 +242,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd exitChan <- true return } - raterConn = &engine.RPCClientConnector{Client: client} + raterConn = client } } // Connect to CDRS @@ -261,7 +261,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd exitChan <- true return } - cdrsConn = &engine.RPCClientConnector{Client: client} + cdrsConn = client } } } @@ -275,7 +275,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Kamailio service.") - var raterConn, cdrsConn engine.Connector + var raterConn, cdrsConn rpcclient.RpcClientConnection var client *rpcclient.RpcClient var err error // Connect to rater @@ -291,7 +291,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - raterConn = &engine.RPCClientConnector{Client: client} + raterConn = client } } // Connect to CDRS @@ -310,7 +310,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - cdrsConn = &engine.RPCClientConnector{Client: client} + cdrsConn = client } } } @@ -324,7 +324,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-OpenSIPS service.") - var raterConn, cdrsConn engine.Connector + var raterConn, cdrsConn rpcclient.RpcClientConnection var client *rpcclient.RpcClient var err error // Connect to rater @@ -340,7 +340,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - raterConn = &engine.RPCClientConnector{Client: client} + raterConn = client } } // Connect to CDRS @@ -359,7 +359,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - cdrsConn = &engine.RPCClientConnector{Client: client} + cdrsConn = client } } } @@ -379,7 +379,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, var err error var client *rpcclient.RpcClient // Rater connection init - var raterConn engine.Connector + var raterConn rpcclient.RpcClientConnection if cfg.CDRSRater == utils.INTERNAL { responder := <-internalRaterChan // Wait for rater to come up before start querying raterConn = responder @@ -391,7 +391,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, exitChan <- true return } - raterConn = &engine.RPCClientConnector{Client: client} + raterConn = client } // Pubsub connection init var pubSubConn engine.PublisherSubscriber diff --git a/engine/cdrs.go b/engine/cdrs.go index 99d47226c..34b78110d 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -27,6 +27,7 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" "github.com/jinzhu/gorm" ) @@ -65,14 +66,14 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } } -func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater Connector, pubsub PublisherSubscriber, users UserService, aliases AliasService, stats StatsInterface) (*CdrServer, error) { - return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, rater: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{queue: make(map[string]chan bool)}}, nil +func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, client rpcclient.RpcClientConnection, pubsub PublisherSubscriber, users UserService, aliases AliasService, stats StatsInterface) (*CdrServer, error) { + return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, client: client, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{queue: make(map[string]chan bool)}}, nil } type CdrServer struct { cgrCfg *config.CGRConfig cdrDb CdrStorage - rater Connector + client rpcclient.RpcClientConnection pubsub PublisherSubscriber users UserService aliases AliasService @@ -228,7 +229,7 @@ func (self *CdrServer) rateStoreStatsReplicate(cdr *StoredCdr) error { } } // Rate CDR - if self.rater != nil && !cdr.Rated { + if self.client != nil && !cdr.Rated { if err := self.rateCDR(cdr); err != nil { cdr.Cost = -1.0 // If there was an error, mark the CDR cdr.ExtraInfo = err.Error() @@ -275,7 +276,7 @@ func (self *CdrServer) deriveCdrs(storedCdr *StoredCdr) ([]*StoredCdr, error) { attrsDC := &utils.AttrDerivedChargers{Tenant: storedCdr.Tenant, Category: storedCdr.Category, Direction: storedCdr.Direction, Account: storedCdr.Account, Subject: storedCdr.Subject} var dcs utils.DerivedChargers - if err := self.rater.GetDerivedChargers(attrsDC, &dcs); err != nil { + if err := self.client.Call("Responder.GetDerivedChargers", attrsDC, &dcs); err != nil { utils.Logger.Err(fmt.Sprintf("Could not get derived charging for cgrid %s, error: %s", storedCdr.CgrId, err.Error())) return nil, err } @@ -337,11 +338,11 @@ func (self *CdrServer) getCostFromRater(storedCdr *StoredCdr) (*CallCost, error) DurationIndex: storedCdr.Usage, } if utils.IsSliceMember([]string{utils.META_PSEUDOPREPAID, utils.META_POSTPAID, utils.META_PREPAID, utils.PSEUDOPREPAID, utils.POSTPAID, utils.PREPAID}, storedCdr.ReqType) { // Prepaid - Cost can be recalculated in case of missing records from SM - if err = self.rater.Debit(cd, cc); err == nil { // Debit has occured, we are forced to write the log, even if CDR store is disabled + if err = self.client.Call("Responder.Debit", cd, cc); err == nil { // Debit has occured, we are forced to write the log, even if CDR store is disabled self.cdrDb.LogCallCost(storedCdr.CgrId, utils.CDRS_SOURCE, storedCdr.MediationRunId, cc) } } else { - err = self.rater.GetCost(cd, cc) + err = self.client.Call("Responder.GetCost", cd, cc) } if err != nil { return cc, err diff --git a/engine/responder.go b/engine/responder.go index 531df85da..a54756683 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -31,7 +31,6 @@ import ( "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" ) // Individual session run @@ -51,17 +50,15 @@ type Responder struct { ExitChan chan bool CdrSrv *CdrServer Stats StatsInterface - Timeout time.Duration Timezone string cnt int64 responseCache *cache2go.ResponseCache } -func NewResponder(exitChan chan bool, cdrSrv *CdrServer, stats StatsInterface, timeout, timeToLive time.Duration) *Responder { +func NewResponder(exitChan chan bool, cdrSrv *CdrServer, stats StatsInterface, timeToLive time.Duration) *Responder { return &Responder{ ExitChan: exitChan, Stats: stats, - Timeout: timeToLive, responseCache: cache2go.NewResponseCache(timeToLive), } } @@ -617,339 +614,27 @@ func (rs *Responder) UnRegisterRater(clientAddress string, replay *int) error { return nil } -func (rs *Responder) GetTimeout(i int, d *time.Duration) error { - *d = rs.Timeout - return nil -} - -// Reflection worker type for not standalone balancer -type ResponderWorker struct{} - -func (rw *ResponderWorker) Call(serviceMethod string, args interface{}, reply interface{}) error { +func (rs *Responder) Call(serviceMethod string, args interface{}, reply interface{}) error { + if !strings.HasPrefix(serviceMethod, "Responder.") { + return utils.ErrNotImplemented + } methodName := strings.TrimLeft(serviceMethod, "Responder.") - switch args.(type) { - case CallDescriptor: - cd := args.(CallDescriptor) - switch reply.(type) { - case *CallCost: - rep := reply.(*CallCost) - method := reflect.ValueOf(&cd).MethodByName(methodName) - ret := method.Call([]reflect.Value{}) - *rep = *(ret[0].Interface().(*CallCost)) - case *float64: - rep := reply.(*float64) - method := reflect.ValueOf(&cd).MethodByName(methodName) - ret := method.Call([]reflect.Value{}) - *rep = *(ret[0].Interface().(*float64)) - } - case string: - switch methodName { - case "Status": - *(reply.(*string)) = "Local!" - case "Shutdown": - *(reply.(*string)) = "Done!" - } - + // get method + method := reflect.ValueOf(rs).MethodByName(methodName) + if !method.IsValid() { + return utils.ErrNotImplemented } - return nil -} -func (rw *ResponderWorker) Close() error { - return nil -} + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} -type Connector interface { - GetCost(*CallDescriptor, *CallCost) error - Debit(*CallDescriptor, *CallCost) error - MaxDebit(*CallDescriptor, *CallCost) error - RefundIncrements(*CallDescriptor, *float64) error - GetMaxSessionTime(*CallDescriptor, *float64) error - GetDerivedChargers(*utils.AttrDerivedChargers, *utils.DerivedChargers) error - GetDerivedMaxSessionTime(*StoredCdr, *float64) error - GetSessionRuns(*StoredCdr, *[]*SessionRun) error - ProcessCdr(*StoredCdr, *string) error - LogCallCost(*CallCostLog, *string) error - GetLCR(*AttrGetLcr, *LCRCost) error - GetTimeout(int, *time.Duration) error -} - -type RPCClientConnector struct { - Client *rpcclient.RpcClient - Timeout time.Duration -} - -func (rcc *RPCClientConnector) GetCost(cd *CallDescriptor, cc *CallCost) error { - return rcc.Client.Call("Responder.GetCost", cd, cc) -} - -func (rcc *RPCClientConnector) Debit(cd *CallDescriptor, cc *CallCost) error { - return rcc.Client.Call("Responder.Debit", cd, cc) -} - -func (rcc *RPCClientConnector) MaxDebit(cd *CallDescriptor, cc *CallCost) error { - return rcc.Client.Call("Responder.MaxDebit", cd, cc) -} - -func (rcc *RPCClientConnector) RefundIncrements(cd *CallDescriptor, resp *float64) error { - return rcc.Client.Call("Responder.RefundIncrements", cd, resp) -} - -func (rcc *RPCClientConnector) GetMaxSessionTime(cd *CallDescriptor, resp *float64) error { - return rcc.Client.Call("Responder.GetMaxSessionTime", cd, resp) -} - -func (rcc *RPCClientConnector) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) error { - return rcc.Client.Call("Responder.GetDerivedMaxSessionTime", ev, reply) -} - -func (rcc *RPCClientConnector) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { - return rcc.Client.Call("Responder.GetSessionRuns", ev, sRuns) -} - -func (rcc *RPCClientConnector) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *utils.DerivedChargers) error { - return rcc.Client.Call("ApierV1.GetDerivedChargers", attrs, dcs) -} - -func (rcc *RPCClientConnector) ProcessCdr(cdr *StoredCdr, reply *string) error { - return rcc.Client.Call("CdrsV1.ProcessCdr", cdr, reply) -} - -func (rcc *RPCClientConnector) LogCallCost(ccl *CallCostLog, reply *string) error { - return rcc.Client.Call("CdrsV1.LogCallCost", ccl, reply) -} - -func (rcc *RPCClientConnector) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { - return rcc.Client.Call("Responder.GetLCR", attrs, reply) -} - -func (rcc *RPCClientConnector) GetTimeout(i int, d *time.Duration) error { - *d = rcc.Timeout - return nil -} - -type ConnectorPool []Connector - -func (cp ConnectorPool) GetCost(cd *CallDescriptor, cc *CallCost) error { - for _, con := range cp { - c := make(chan error, 1) - callCost := &CallCost{} - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetCost(cd, callCost) }() - select { - case err := <-c: - *cc = *callCost - return err - case <-time.After(timeout): - // call timed out, continue - } + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) Debit(cd *CallDescriptor, cc *CallCost) error { - for _, con := range cp { - c := make(chan error, 1) - callCost := &CallCost{} - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.Debit(cd, callCost) }() - select { - case err := <-c: - *cc = *callCost - return err - case <-time.After(timeout): - // call timed out, continue - } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) MaxDebit(cd *CallDescriptor, cc *CallCost) error { - for _, con := range cp { - c := make(chan error, 1) - callCost := &CallCost{} - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.MaxDebit(cd, callCost) }() - select { - case err := <-c: - *cc = *callCost - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) RefundIncrements(cd *CallDescriptor, resp *float64) error { - for _, con := range cp { - c := make(chan error, 1) - var r float64 - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.RefundIncrements(cd, &r) }() - select { - case err := <-c: - *resp = r - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetMaxSessionTime(cd *CallDescriptor, resp *float64) error { - for _, con := range cp { - c := make(chan error, 1) - var r float64 - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetMaxSessionTime(cd, &r) }() - select { - case err := <-c: - *resp = r - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) error { - for _, con := range cp { - c := make(chan error, 1) - var r float64 - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetDerivedMaxSessionTime(ev, &r) }() - select { - case err := <-c: - *reply = r - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { - for _, con := range cp { - c := make(chan error, 1) - sr := make([]*SessionRun, 0) - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetSessionRuns(ev, &sr) }() - select { - case err := <-c: - *sRuns = sr - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *utils.DerivedChargers) error { - for _, con := range cp { - c := make(chan error, 1) - derivedChargers := utils.DerivedChargers{} - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetDerivedChargers(attrs, &derivedChargers) }() - select { - case err := <-c: - *dcs = derivedChargers - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) ProcessCdr(cdr *StoredCdr, reply *string) error { - for _, con := range cp { - c := make(chan error, 1) - var r string - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.ProcessCdr(cdr, &r) }() - select { - case err := <-c: - *reply = r - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) LogCallCost(ccl *CallCostLog, reply *string) error { - for _, con := range cp { - c := make(chan error, 1) - var r string - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.LogCallCost(ccl, &r) }() - select { - case err := <-c: - *reply = r - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetLCR(attr *AttrGetLcr, reply *LCRCost) error { - for _, con := range cp { - c := make(chan error, 1) - lcrCost := &LCRCost{} - - var timeout time.Duration - con.GetTimeout(0, &timeout) - - go func() { c <- con.GetLCR(attr, lcrCost) }() - select { - case err := <-c: - *reply = *lcrCost - return err - case <-time.After(timeout): - // call timed out, continue - } - } - return utils.ErrTimedOut -} - -func (cp ConnectorPool) GetTimeout(i int, d *time.Duration) error { - *d = 0 - return nil + return err } diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 47a728fbd..969dacf4e 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -30,9 +30,10 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "github.com/cgrates/fsock" + "github.com/cgrates/rpcclient" ) -func NewFSSessionManager(smFsConfig *config.SmFsConfig, rater, cdrs engine.Connector, timezone string) *FSSessionManager { +func NewFSSessionManager(smFsConfig *config.SmFsConfig, rater, cdrs rpcclient.RpcClientConnection, timezone string) *FSSessionManager { return &FSSessionManager{ cfg: smFsConfig, conns: make(map[string]*fsock.FSock), @@ -50,10 +51,11 @@ type FSSessionManager struct { cfg *config.SmFsConfig conns map[string]*fsock.FSock // Keep the list here for connection management purposes senderPools map[string]*fsock.FSockPool // Keep sender pools here - rater engine.Connector - cdrsrv engine.Connector - sessions *Sessions - timezone string + rater rpcclient.RpcClientConnection + cdrsrv rpcclient.RpcClientConnection + + sessions *Sessions + timezone string } func (sm *FSSessionManager) createHandlers() map[string][]func(string, string) { @@ -107,7 +109,7 @@ func (sm *FSSessionManager) setCgrLcr(ev engine.Event, connId string) error { TimeStart: startTime, TimeEnd: startTime.Add(config.CgrConfig().MaxCallDuration), } - if err := sm.rater.GetLCR(&engine.AttrGetLcr{CallDescriptor: cd}, &lcrCost); err != nil { + if err := sm.rater.Call("Responder.GetLCR", &engine.AttrGetLcr{CallDescriptor: cd}, &lcrCost); err != nil { return err } supps := []string{} @@ -131,7 +133,7 @@ func (sm *FSSessionManager) onChannelPark(ev engine.Event, connId string) { return } var maxCallDuration float64 // This will be the maximum duration this channel will be allowed to last - if err := sm.rater.GetDerivedMaxSessionTime(ev.AsStoredCdr(config.CgrConfig().DefaultTimezone), &maxCallDuration); err != nil { + if err := sm.rater.Call("Responder.GetDerivedMaxSessionTime", ev.AsStoredCdr(config.CgrConfig().DefaultTimezone), &maxCallDuration); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not get max session time for %s, error: %s", ev.GetUUID(), err.Error())) } if maxCallDuration != -1 { // For calls different than unlimited, set limits @@ -152,7 +154,7 @@ func (sm *FSSessionManager) onChannelPark(ev engine.Event, connId string) { return } var lcr engine.LCRCost - if err = sm.Rater().GetLCR(&engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { + if err = sm.Rater().Call("Responder.GetLCR", &engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { utils.Logger.Info(fmt.Sprintf(" LCR_API_ERROR: %s", err.Error())) sm.unparkCall(ev.GetUUID(), connId, ev.GetCallDestNr(utils.META_DEFAULT), SYSTEM_ERROR) } @@ -294,7 +296,7 @@ func (sm *FSSessionManager) DisconnectSession(ev engine.Event, connId, notify st func (sm *FSSessionManager) ProcessCdr(storedCdr *engine.StoredCdr) error { var reply string - if err := sm.cdrsrv.ProcessCdr(storedCdr, &reply); err != nil { + if err := sm.cdrsrv.Call("Responder.ProcessCdr", storedCdr, &reply); err != nil { utils.Logger.Err(fmt.Sprintf(" Failed processing CDR, cgrid: %s, accid: %s, error: <%s>", storedCdr.CgrId, storedCdr.AccId, err.Error())) } return nil @@ -304,11 +306,11 @@ func (sm *FSSessionManager) DebitInterval() time.Duration { return sm.cfg.DebitInterval } -func (sm *FSSessionManager) CdrSrv() engine.Connector { +func (sm *FSSessionManager) CdrSrv() rpcclient.RpcClientConnection { return sm.cdrsrv } -func (sm *FSSessionManager) Rater() engine.Connector { +func (sm *FSSessionManager) Rater() rpcclient.RpcClientConnection { return sm.rater } diff --git a/sessionmanager/kamailiosm.go b/sessionmanager/kamailiosm.go index f6ed04f85..e195bfe8c 100644 --- a/sessionmanager/kamailiosm.go +++ b/sessionmanager/kamailiosm.go @@ -29,17 +29,18 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "github.com/cgrates/kamevapi" + "github.com/cgrates/rpcclient" ) -func NewKamailioSessionManager(smKamCfg *config.SmKamConfig, rater, cdrsrv engine.Connector, timezone string) (*KamailioSessionManager, error) { +func NewKamailioSessionManager(smKamCfg *config.SmKamConfig, rater, cdrsrv rpcclient.RpcClientConnection, timezone string) (*KamailioSessionManager, error) { ksm := &KamailioSessionManager{cfg: smKamCfg, rater: rater, cdrsrv: cdrsrv, timezone: timezone, conns: make(map[string]*kamevapi.KamEvapi), sessions: NewSessions()} return ksm, nil } type KamailioSessionManager struct { cfg *config.SmKamConfig - rater engine.Connector - cdrsrv engine.Connector + rater rpcclient.RpcClientConnection + cdrsrv rpcclient.RpcClientConnection timezone string conns map[string]*kamevapi.KamEvapi sessions *Sessions @@ -64,7 +65,7 @@ func (self *KamailioSessionManager) onCgrAuth(evData []byte, connId string) { } var remainingDuration float64 var errMaxSession error - if errMaxSession = self.rater.GetDerivedMaxSessionTime(kev.AsStoredCdr(self.Timezone()), &remainingDuration); errMaxSession != nil { + if errMaxSession = self.rater.Call("Responder.GetDerivedMaxSessionTime", kev.AsStoredCdr(self.Timezone()), &remainingDuration); errMaxSession != nil { utils.Logger.Err(fmt.Sprintf(" Could not get max session time, error: %s", errMaxSession.Error())) } var supplStr string @@ -107,7 +108,7 @@ func (self *KamailioSessionManager) getSuppliers(kev KamEvent) (string, error) { return "", errors.New("LCR_PREPROCESS_ERROR") } var lcr engine.LCRCost - if err = self.Rater().GetLCR(&engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { + if err = self.Rater().Call("Responder.GetLCR", &engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { utils.Logger.Info(fmt.Sprintf(" LCR_API_ERROR error: %s", err.Error())) return "", errors.New("LCR_API_ERROR") } @@ -196,10 +197,10 @@ func (self *KamailioSessionManager) DisconnectSession(ev engine.Event, connId, n func (self *KamailioSessionManager) DebitInterval() time.Duration { return self.cfg.DebitInterval } -func (self *KamailioSessionManager) CdrSrv() engine.Connector { +func (self *KamailioSessionManager) CdrSrv() rpcclient.RpcClientConnection { return self.cdrsrv } -func (self *KamailioSessionManager) Rater() engine.Connector { +func (self *KamailioSessionManager) Rater() rpcclient.RpcClientConnection { return self.rater } @@ -208,7 +209,7 @@ func (self *KamailioSessionManager) ProcessCdr(cdr *engine.StoredCdr) error { return nil } var reply string - if err := self.cdrsrv.ProcessCdr(cdr, &reply); err != nil { + if err := self.cdrsrv.Call("Responder.ProcessCdr", cdr, &reply); err != nil { utils.Logger.Err(fmt.Sprintf(" Failed processing CDR, cgrid: %s, accid: %s, error: <%s>", cdr.CgrId, cdr.AccId, err.Error())) } return nil diff --git a/sessionmanager/osipssm.go b/sessionmanager/osipssm.go index 315cf86fc..f3a91e4a5 100644 --- a/sessionmanager/osipssm.go +++ b/sessionmanager/osipssm.go @@ -29,6 +29,7 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "github.com/cgrates/osipsdagram" + "github.com/cgrates/rpcclient" ) /* @@ -80,7 +81,7 @@ duration:: */ -func NewOSipsSessionManager(smOsipsCfg *config.SmOsipsConfig, reconnects int, rater, cdrsrv engine.Connector, timezone string) (*OsipsSessionManager, error) { +func NewOSipsSessionManager(smOsipsCfg *config.SmOsipsConfig, reconnects int, rater, cdrsrv rpcclient.RpcClientConnection, timezone string) (*OsipsSessionManager, error) { osm := &OsipsSessionManager{cfg: smOsipsCfg, reconnects: reconnects, rater: rater, cdrsrv: cdrsrv, timezone: timezone, cdrStartEvents: make(map[string]*OsipsEvent), sessions: NewSessions()} osm.eventHandlers = map[string][]func(*osipsdagram.OsipsEvent){ "E_OPENSIPS_START": []func(*osipsdagram.OsipsEvent){osm.onOpensipsStart}, // Raised when OpenSIPS starts so we can register our event handlers @@ -94,8 +95,8 @@ func NewOSipsSessionManager(smOsipsCfg *config.SmOsipsConfig, reconnects int, ra type OsipsSessionManager struct { cfg *config.SmOsipsConfig reconnects int - rater engine.Connector - cdrsrv engine.Connector + rater rpcclient.RpcClientConnection + cdrsrv rpcclient.RpcClientConnection timezone string eventHandlers map[string][]func(*osipsdagram.OsipsEvent) evSubscribeStop chan struct{} // Reference towards the channel controlling subscriptions, keep it as reference so we do not need to copy it @@ -130,12 +131,12 @@ func (osm *OsipsSessionManager) DebitInterval() time.Duration { } // Returns the connection to local cdr database, used by session to log it's final costs -func (osm *OsipsSessionManager) CdrSrv() engine.Connector { +func (osm *OsipsSessionManager) CdrSrv() rpcclient.RpcClientConnection { return osm.cdrsrv } // Returns connection to rater/controller -func (osm *OsipsSessionManager) Rater() engine.Connector { +func (osm *OsipsSessionManager) Rater() rpcclient.RpcClientConnection { return osm.rater } @@ -152,7 +153,7 @@ func (osm *OsipsSessionManager) Shutdown() error { // Process the CDR with CDRS component func (osm *OsipsSessionManager) ProcessCdr(storedCdr *engine.StoredCdr) error { var reply string - return osm.cdrsrv.ProcessCdr(storedCdr, &reply) + return osm.cdrsrv.Call("Responder.ProcessCdr", storedCdr, &reply) } // Disconnects the session diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 9fde47fb7..82347807c 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -58,7 +58,7 @@ func NewSession(ev engine.Event, connId string, sm SessionManager) *Session { sessionManager: sm, connId: connId, } - if err := sm.Rater().GetSessionRuns(ev.AsStoredCdr(s.sessionManager.Timezone()), &s.sessionRuns); err != nil || len(s.sessionRuns) == 0 { + if err := sm.Rater().Call("Responder.GetSessionRuns", ev.AsStoredCdr(s.sessionManager.Timezone()), &s.sessionRuns); err != nil || len(s.sessionRuns) == 0 { return nil } for runIdx := range s.sessionRuns { @@ -86,7 +86,7 @@ func (s *Session) debitLoop(runIdx int) { nextCd.LoopIndex = index nextCd.DurationIndex += debitPeriod // first presumed duration cc := new(engine.CallCost) - if err := s.sessionManager.Rater().MaxDebit(nextCd, cc); err != nil { + if err := s.sessionManager.Rater().Call("Responder.MaxDebit", nextCd, cc); err != nil { utils.Logger.Err(fmt.Sprintf("Could not complete debit opperation: %v", err)) if err.Error() == utils.ErrUnauthorizedDestination.Error() { s.sessionManager.DisconnectSession(s.eventStart, s.connId, UNAUTHORIZED_DESTINATION) @@ -204,7 +204,7 @@ func (s *Session) Refund(lastCC *engine.CallCost, hangupTime time.Time) error { Increments: refundIncrements, } var response float64 - err := s.sessionManager.Rater().RefundIncrements(cd, &response) + err := s.sessionManager.Rater().Call("Responder.RefundIncrements", cd, &response) if err != nil { return err } @@ -233,7 +233,7 @@ func (s *Session) SaveOperations() { } var reply string - err := s.sessionManager.CdrSrv().LogCallCost(&engine.CallCostLog{ + err := s.sessionManager.CdrSrv().Call("Responder.LogCallCost", &engine.CallCostLog{ CgrId: s.eventStart.GetCgrId(s.sessionManager.Timezone()), Source: utils.SESSION_MANAGER_SOURCE, RunId: sr.DerivedCharger.RunId, diff --git a/sessionmanager/session_test.go b/sessionmanager/session_test.go index 61fd636d3..7b1c10292 100644 --- a/sessionmanager/session_test.go +++ b/sessionmanager/session_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" ) //"github.com/cgrates/cgrates/config" @@ -80,30 +79,19 @@ func TestSessionNilSession(t *testing.T) { } */ -type MockConnector struct { +type MockRpcClient struct { refundCd *engine.CallDescriptor } -func (mc *MockConnector) GetCost(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) Debit(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) MaxDebit(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) RefundIncrements(cd *engine.CallDescriptor, reply *float64) error { - mc.refundCd = cd +func (mc *MockRpcClient) Call(methodName string, arg interface{}, reply interface{}) error { + if cd, ok := arg.(*engine.CallDescriptor); ok { + mc.refundCd = cd + } return nil } -func (mc *MockConnector) GetMaxSessionTime(*engine.CallDescriptor, *float64) error { return nil } -func (mc *MockConnector) GetDerivedChargers(*utils.AttrDerivedChargers, *utils.DerivedChargers) error { - return nil -} -func (mc *MockConnector) GetDerivedMaxSessionTime(*engine.StoredCdr, *float64) error { return nil } -func (mc *MockConnector) GetSessionRuns(*engine.StoredCdr, *[]*engine.SessionRun) error { return nil } -func (mc *MockConnector) ProcessCdr(*engine.StoredCdr, *string) error { return nil } -func (mc *MockConnector) LogCallCost(*engine.CallCostLog, *string) error { return nil } -func (mc *MockConnector) GetLCR(*engine.AttrGetLcr, *engine.LCRCost) error { return nil } -func (mc *MockConnector) GetTimeout(int, *time.Duration) error { return nil } func TestSessionRefund(t *testing.T) { - mc := &MockConnector{} + mc := &MockRpcClient{} s := &Session{sessionManager: &FSSessionManager{rater: mc}} ts := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), @@ -123,7 +111,7 @@ func TestSessionRefund(t *testing.T) { } func TestSessionRefundAll(t *testing.T) { - mc := &MockConnector{} + mc := &MockRpcClient{} s := &Session{sessionManager: &FSSessionManager{rater: mc}} ts := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), @@ -143,7 +131,7 @@ func TestSessionRefundAll(t *testing.T) { } func TestSessionRefundManyAll(t *testing.T) { - mc := &MockConnector{} + mc := &MockRpcClient{} s := &Session{sessionManager: &FSSessionManager{rater: mc}} ts1 := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), diff --git a/sessionmanager/sessionmanager.go b/sessionmanager/sessionmanager.go index 8d2155f11..7a9b5696b 100644 --- a/sessionmanager/sessionmanager.go +++ b/sessionmanager/sessionmanager.go @@ -22,11 +22,12 @@ import ( "time" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/rpcclient" ) type SessionManager interface { - Rater() engine.Connector - CdrSrv() engine.Connector + Rater() rpcclient.RpcClientConnection + CdrSrv() rpcclient.RpcClientConnection DebitInterval() time.Duration DisconnectSession(engine.Event, string, string) error WarnSessionMinDuration(string, string) diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 2ee36f58e..97f3226e9 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -25,6 +25,7 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) // One session handled by SM @@ -34,8 +35,8 @@ type SMGSession struct { connId string // Reference towards connection id on the session manager side. runId string // Keep a reference for the derived run timezone string - rater engine.Connector // Connector to Rater service - cdrsrv engine.Connector // Connector to CDRS service + rater rpcclient.RpcClientConnection // Connector to Rater service + cdrsrv rpcclient.RpcClientConnection // Connector to CDRS service extconns *SMGExternalConnections cd *engine.CallDescriptor sessionCds []*engine.CallDescriptor @@ -86,7 +87,7 @@ func (self *SMGSession) debit(dur time.Duration) (time.Duration, error) { self.cd.TimeEnd = self.cd.TimeStart.Add(dur) self.cd.DurationIndex += dur cc := &engine.CallCost{} - if err := self.rater.MaxDebit(self.cd, cc); err != nil { + if err := self.rater.Call("Responder.MaxDebit", self.cd, cc); err != nil { return 0, err } // cd corrections @@ -155,7 +156,7 @@ func (self *SMGSession) refund(refundDuration time.Duration) error { Increments: refundIncrements, } var response float64 - err := self.rater.RefundIncrements(cd, &response) + err := self.rater.Call("Responder.RefundIncrements", cd, &response) if err != nil { return err } @@ -204,7 +205,7 @@ func (self *SMGSession) saveOperations() error { firstCC.Merge(cc) } var reply string - err := self.cdrsrv.LogCallCost(&engine.CallCostLog{ + err := self.cdrsrv.Call("Responder.LogCallCost", &engine.CallCostLog{ CgrId: self.eventStart.GetCgrId(self.timezone), Source: utils.SESSION_MANAGER_SOURCE, RunId: self.runId, diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 9d0490fca..6546991a3 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -28,9 +28,10 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) -func NewSMGeneric(cgrCfg *config.CGRConfig, rater engine.Connector, cdrsrv engine.Connector, timezone string, extconns *SMGExternalConnections) *SMGeneric { +func NewSMGeneric(cgrCfg *config.CGRConfig, rater rpcclient.RpcClientConnection, cdrsrv rpcclient.RpcClientConnection, timezone string, extconns *SMGExternalConnections) *SMGeneric { gsm := &SMGeneric{cgrCfg: cgrCfg, rater: rater, cdrsrv: cdrsrv, extconns: extconns, timezone: timezone, sessions: make(map[string][]*SMGSession), sessionsMux: new(sync.Mutex), guard: engine.NewGuardianLock()} return gsm @@ -38,8 +39,8 @@ func NewSMGeneric(cgrCfg *config.CGRConfig, rater engine.Connector, cdrsrv engin type SMGeneric struct { cgrCfg *config.CGRConfig // Separate from smCfg since there can be multiple - rater engine.Connector - cdrsrv engine.Connector + rater rpcclient.RpcClientConnection + cdrsrv rpcclient.RpcClientConnection timezone string sessions map[string][]*SMGSession //Group sessions per sessionId, multiple runs based on derived charging extconns *SMGExternalConnections // Reference towards external connections manager @@ -83,7 +84,7 @@ func (self *SMGeneric) sessionStart(evStart SMGenericEvent, connId string) error sessionId := evStart.GetUUID() _, err := self.guard.Guard(func() (interface{}, error) { // Lock it on UUID level var sessionRuns []*engine.SessionRun - if err := self.rater.GetSessionRuns(evStart.AsStoredCdr(self.cgrCfg, self.timezone), &sessionRuns); err != nil { + if err := self.rater.Call("Responder.GetSessionRuns", evStart.AsStoredCdr(self.cgrCfg, self.timezone), &sessionRuns); err != nil { return nil, err } else if len(sessionRuns) == 0 { return nil, nil @@ -135,7 +136,7 @@ func (self *SMGeneric) GetMaxUsage(gev SMGenericEvent, clnt *rpc2.Client) (time. gev[utils.EVENT_NAME] = utils.CGR_AUTHORIZATION storedCdr := gev.AsStoredCdr(config.CgrConfig(), self.timezone) var maxDur float64 - if err := self.rater.GetDerivedMaxSessionTime(storedCdr, &maxDur); err != nil { + if err := self.rater.Call("Responder.GetDerivedMaxSessionTime", storedCdr, &maxDur); err != nil { return time.Duration(0), err } return time.Duration(maxDur), nil @@ -148,7 +149,7 @@ func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([ return nil, err } var lcr engine.LCRCost - if err = self.rater.GetLCR(&engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { + if err = self.rater.Call("Responder.GetLCR", &engine.AttrGetLcr{CallDescriptor: cd}, &lcr); err != nil { return nil, err } if lcr.HasErrors() { @@ -200,7 +201,7 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { var reply string - if err := self.cdrsrv.ProcessCdr(gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { + if err := self.cdrsrv.Call("Responder.ProcessCdr", gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err } return nil From 16e0eba5bd61138b36183112c995dda58ef65b43 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 26 Nov 2015 23:02:03 +0200 Subject: [PATCH 002/227] removed all proxy classes --- apier/v1/aliases.go | 17 +++--- apier/v1/apier.go | 17 +++--- apier/v1/cdrstatsv1.go | 15 +++--- apier/v2/apier.go | 4 +- cmd/cgr-engine/cgr-engine.go | 62 +++++++++++----------- cmd/cgr-engine/rater.go | 26 ++++----- cmd/cgr-engine/registration.go | 9 ++-- engine/aliases.go | 96 +++++++++++++--------------------- engine/aliases_test.go | 6 +-- engine/calldesc.go | 26 ++++----- engine/cdrs.go | 12 ++--- engine/pubsub.go | 54 +++++++++---------- engine/responder.go | 14 +++-- engine/responder_test.go | 12 +++-- engine/stats.go | 81 +++++++++------------------- engine/storage_map.go | 8 +-- engine/storage_mongo.go | 8 +-- engine/storage_redis.go | 8 +-- engine/users.go | 86 ++++++++++++------------------ engine/users_test.go | 64 +++++++++++------------ history/file_scribe.go | 32 ++++++++++++ history/mock_scribe.go | 32 ++++++++++++ history/proxy_scribe.go | 40 -------------- history/scribe.go | 4 -- 24 files changed, 340 insertions(+), 393 deletions(-) delete mode 100644 history/proxy_scribe.go diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index e17798433..72753b357 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -20,6 +20,7 @@ package v1 import ( "errors" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -48,8 +49,8 @@ func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, } var ignr string for _, alias := range attrs.Aliases { - if err := aliases.SetAlias( - engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, + if err := aliases.Call("AliasesV1.SetAlias", + &engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Subject": map[string]string{alias: attrs.Subject}}, Weight: 10.0}}}, &ignr); err != nil { return utils.NewErrServerError(err) @@ -69,7 +70,7 @@ func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRa return errors.New("ALIASES_NOT_ENABLED") } var reverseAliases map[string][]*engine.Alias - if err := aliases.GetReverseAlias(engine.AttrReverseAlias{Target: "Subject", Alias: tenantRatingSubject.Subject, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil { + if err := aliases.Call("AliasesV1.GetReverseAlias", &engine.AttrReverseAlias{Target: "Subject", Alias: tenantRatingSubject.Subject, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil { return utils.NewErrServerError(err) } var ignr string @@ -78,7 +79,7 @@ func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRa if alias.Tenant != tenantRatingSubject.Tenant { continue // From another tenant } - if err := aliases.RemoveAlias(*alias, &ignr); err != nil { + if err := aliases.Call("AliasesV1.RemoveAlias", alias, &ignr); err != nil { return utils.NewErrServerError(err) } } @@ -100,8 +101,8 @@ func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *strin } var ignr string for _, alias := range attrs.Aliases { - if err := aliases.SetAlias( - engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, + if err := aliases.Call("AliasesV1.SetAlias", + &engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Account}}, Weight: 10.0}}}, &ignr); err != nil { return utils.NewErrServerError(err) @@ -121,7 +122,7 @@ func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply return errors.New("ALIASES_NOT_ENABLED") } var reverseAliases map[string][]*engine.Alias - if err := aliases.GetReverseAlias(engine.AttrReverseAlias{Target: "Account", Alias: tenantAccount.Account, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil { + if err := aliases.Call("AliasesV1.GetReverseAlias", &engine.AttrReverseAlias{Target: "Account", Alias: tenantAccount.Account, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil { return utils.NewErrServerError(err) } var ignr string @@ -130,7 +131,7 @@ func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply if alias.Tenant != tenantAccount.Tenant { continue // From another tenant } - if err := aliases.RemoveAlias(*alias, &ignr); err != nil { + if err := aliases.Call("AliasesV1.RemoveAlias", alias, &ignr); err != nil { return utils.NewErrServerError(err) } } diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 38157f5a8..53b56fb46 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -33,6 +33,7 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) const ( @@ -48,8 +49,8 @@ type ApierV1 struct { Sched *scheduler.Scheduler Config *config.CGRConfig Responder *engine.Responder - CdrStatsSrv engine.StatsInterface - Users engine.UserService + CdrStatsSrv rpcclient.RpcClientConnection + Users rpcclient.RpcClientConnection } func (self *ApierV1) GetDestination(dstId string, reply *engine.Destination) error { @@ -524,13 +525,13 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { - if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil { + if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, nil); err != nil { return err } } if len(userKeys) != 0 && self.Users != nil { var r string - if err := self.Users.ReloadUsers("", &r); err != nil { + if err := self.Users.Call("AliasV1.ReloadUsers", "", &r); err != nil { return err } } @@ -1036,14 +1037,14 @@ func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.Cach cs.Aliases = cache2go.CountEntries(utils.ALIASES_PREFIX) if self.CdrStatsSrv != nil && self.Config.CDRStatsEnabled { var queueIds []string - if err := self.CdrStatsSrv.GetQueueIds(0, &queueIds); err != nil { + if err := self.CdrStatsSrv.Call("CDRStatsV1.GetQueueIds", 0, &queueIds); err != nil { return utils.NewErrServerError(err) } cs.CdrStats = len(queueIds) } if self.Config.RaterUserServer == utils.INTERNAL { var ups engine.UserProfiles - if err := self.Users.GetUsers(engine.UserProfile{}, &ups); err != nil { + if err := self.Users.Call("UsersV1.GetUsers", &engine.UserProfile{}, &ups); err != nil { return utils.NewErrServerError(err) } cs.Users = len(ups) @@ -1187,13 +1188,13 @@ func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, self.Sched.Restart() } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { - if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil { + if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, nil); err != nil { return err } } if len(userKeys) != 0 && self.Users != nil { var r string - if err := self.Users.ReloadUsers("", &r); err != nil { + if err := self.Users.Call("UsersV1.ReloadUsers", "", &r); err != nil { return err } } diff --git a/apier/v1/cdrstatsv1.go b/apier/v1/cdrstatsv1.go index 1677021b0..a9bef703f 100644 --- a/apier/v1/cdrstatsv1.go +++ b/apier/v1/cdrstatsv1.go @@ -23,11 +23,12 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) // Interact with Stats server type CDRStatsV1 struct { - CdrStats engine.StatsInterface + CdrStats rpcclient.RpcClientConnection } type AttrGetMetrics struct { @@ -38,23 +39,23 @@ func (sts *CDRStatsV1) GetMetrics(attr AttrGetMetrics, reply *map[string]float64 if len(attr.StatsQueueId) == 0 { return fmt.Errorf("%s:StatsQueueId", utils.ErrMandatoryIeMissing.Error()) } - return sts.CdrStats.GetValues(attr.StatsQueueId, reply) + return sts.CdrStats.Call("CDRStatsV1.GetValues", attr.StatsQueueId, reply) } func (sts *CDRStatsV1) GetQueueIds(empty string, reply *[]string) error { - return sts.CdrStats.GetQueueIds(0, reply) + return sts.CdrStats.Call("CDRStatsV1.GetQueueIds", 0, reply) } func (sts *CDRStatsV1) GetQueue(id string, sq *engine.StatsQueue) error { - return sts.CdrStats.GetQueue(id, sq) + return sts.CdrStats.Call("CDRStatsV1.GetQueue", id, sq) } func (sts *CDRStatsV1) GetQueueTriggers(id string, ats *engine.ActionTriggers) error { - return sts.CdrStats.GetQueueTriggers(id, ats) + return sts.CdrStats.Call("CDRStatsV1.GetQueueTriggers", id, ats) } func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error { - if err := sts.CdrStats.ReloadQueues(attr.StatsQueueIds, nil); err != nil { + if err := sts.CdrStats.Call("CDRStatsV1.ReloadQueues", attr.StatsQueueIds, nil); err != nil { return err } *reply = utils.OK @@ -62,7 +63,7 @@ func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply * } func (sts *CDRStatsV1) ResetQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error { - if err := sts.CdrStats.ResetQueues(attr.StatsQueueIds, nil); err != nil { + if err := sts.CdrStats.Call("CDRStatsV1.ResetQueues", attr.StatsQueueIds, nil); err != nil { return err } *reply = utils.OK diff --git a/apier/v2/apier.go b/apier/v2/apier.go index e35c48a7d..bb773ece8 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -253,14 +253,14 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, self.Sched.Restart() } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { - if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil { + if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, nil); err != nil { return err } } if len(userKeys) != 0 && self.Users != nil { var r string - if err := self.Users.ReloadUsers("", &r); err != nil { + if err := self.Users.Call("UsersV1.ReloadUsers", "", &r); err != nil { return err } } diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index cc6a85f2f..13c28408d 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -372,9 +372,9 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS } func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, cdrDb engine.CdrStorage, - internalRaterChan chan *engine.Responder, internalPubSubSChan chan engine.PublisherSubscriber, - internalUserSChan chan engine.UserService, internalAliaseSChan chan engine.AliasService, - internalCdrStatSChan chan engine.StatsInterface, server *utils.Server, exitChan chan bool) { + internalRaterChan chan *engine.Responder, internalPubSubSChan chan rpcclient.RpcClientConnection, + internalUserSChan chan rpcclient.RpcClientConnection, internalAliaseSChan chan rpcclient.RpcClientConnection, + internalCdrStatSChan chan rpcclient.RpcClientConnection, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS CDRS service.") var err error var client *rpcclient.RpcClient @@ -394,14 +394,14 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, raterConn = client } // Pubsub connection init - var pubSubConn engine.PublisherSubscriber + var pubSubConn rpcclient.RpcClientConnection if cfg.CDRSPubSub == utils.INTERNAL { pubSubs := <-internalPubSubSChan pubSubConn = pubSubs internalPubSubSChan <- pubSubs } else if len(cfg.CDRSPubSub) != 0 { if cfg.CDRSRater == cfg.CDRSPubSub { - pubSubConn = &engine.ProxyPubSub{Client: client} + pubSubConn = client } else { client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSPubSub, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -409,18 +409,18 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, exitChan <- true return } - pubSubConn = &engine.ProxyPubSub{Client: client} + pubSubConn = client } } // Users connection init - var usersConn engine.UserService + var usersConn rpcclient.RpcClientConnection if cfg.CDRSUsers == utils.INTERNAL { userS := <-internalUserSChan usersConn = userS internalUserSChan <- userS } else if len(cfg.CDRSUsers) != 0 { if cfg.CDRSRater == cfg.CDRSUsers { - usersConn = &engine.ProxyUserService{Client: client} + usersConn = client } else { client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSUsers, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -428,18 +428,18 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, exitChan <- true return } - usersConn = &engine.ProxyUserService{Client: client} + usersConn = client } } // Aliases connection init - var aliasesConn engine.AliasService + var aliasesConn rpcclient.RpcClientConnection if cfg.CDRSAliases == utils.INTERNAL { aliaseS := <-internalAliaseSChan aliasesConn = aliaseS internalAliaseSChan <- aliaseS } else if len(cfg.CDRSAliases) != 0 { if cfg.CDRSRater == cfg.CDRSAliases { - aliasesConn = &engine.ProxyAliasService{Client: client} + aliasesConn = client } else { client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSAliases, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -447,18 +447,18 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, exitChan <- true return } - aliasesConn = &engine.ProxyAliasService{Client: client} + aliasesConn = client } } // Stats connection init - var statsConn engine.StatsInterface + var statsConn rpcclient.RpcClientConnection if cfg.CDRSStats == utils.INTERNAL { statS := <-internalCdrStatSChan statsConn = statS internalCdrStatSChan <- statS } else if len(cfg.CDRSStats) != 0 { if cfg.CDRSRater == cfg.CDRSStats { - statsConn = &engine.ProxyStats{Client: client} + statsConn = client } else { client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -466,7 +466,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, exitChan <- true return } - statsConn = &engine.ProxyStats{Client: client} + statsConn = client } } @@ -495,31 +495,31 @@ func startScheduler(internalSchedulerChan chan *scheduler.Scheduler, ratingDb en exitChan <- true // Should not get out of loop though } -func startCdrStats(internalCdrStatSChan chan engine.StatsInterface, ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, server *utils.Server) { +func startCdrStats(internalCdrStatSChan chan rpcclient.RpcClientConnection, ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, server *utils.Server) { cdrStats := engine.NewStats(ratingDb, accountDb, cfg.CDRStatsSaveInterval) server.RpcRegister(cdrStats) server.RpcRegister(&v1.CDRStatsV1{CdrStats: cdrStats}) // Public APIs internalCdrStatSChan <- cdrStats } -func startHistoryServer(internalHistorySChan chan history.Scribe, server *utils.Server, exitChan chan bool) { +func startHistoryServer(internalHistorySChan chan rpcclient.RpcClientConnection, server *utils.Server, exitChan chan bool) { scribeServer, err := history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not start, error: %s", err.Error())) exitChan <- true } - server.RpcRegisterName("ScribeV1", scribeServer) + server.RpcRegisterName("HistoryV1", scribeServer) internalHistorySChan <- scribeServer } -func startPubSubServer(internalPubSubSChan chan engine.PublisherSubscriber, accountDb engine.AccountingStorage, server *utils.Server) { +func startPubSubServer(internalPubSubSChan chan rpcclient.RpcClientConnection, accountDb engine.AccountingStorage, server *utils.Server) { pubSubServer := engine.NewPubSub(accountDb, cfg.HttpSkipTlsVerify) server.RpcRegisterName("PubSubV1", pubSubServer) internalPubSubSChan <- pubSubServer } // ToDo: Make sure we are caching before starting this one -func startAliasesServer(internalAliaseSChan chan engine.AliasService, accountDb engine.AccountingStorage, server *utils.Server, exitChan chan bool) { +func startAliasesServer(internalAliaseSChan chan rpcclient.RpcClientConnection, accountDb engine.AccountingStorage, server *utils.Server, exitChan chan bool) { aliasesServer := engine.NewAliasHandler(accountDb) server.RpcRegisterName("AliasesV1", aliasesServer) if err := accountDb.CacheAccountingPrefixes(utils.ALIASES_PREFIX); err != nil { @@ -530,7 +530,7 @@ func startAliasesServer(internalAliaseSChan chan engine.AliasService, accountDb internalAliaseSChan <- aliasesServer } -func startUsersServer(internalUserSChan chan engine.UserService, accountDb engine.AccountingStorage, server *utils.Server, exitChan chan bool) { +func startUsersServer(internalUserSChan chan rpcclient.RpcClientConnection, accountDb engine.AccountingStorage, server *utils.Server, exitChan chan bool) { userServer, err := engine.NewUserMap(accountDb, cfg.UserServerIndexes) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not start, error: %s", err.Error())) @@ -543,11 +543,11 @@ func startUsersServer(internalUserSChan chan engine.UserService, accountDb engin func startRpc(server *utils.Server, internalRaterChan chan *engine.Responder, internalCdrSChan chan *engine.CdrServer, - internalCdrStatSChan chan engine.StatsInterface, - internalHistorySChan chan history.Scribe, - internalPubSubSChan chan engine.PublisherSubscriber, - internalUserSChan chan engine.UserService, - internalAliaseSChan chan engine.AliasService) { + internalCdrStatSChan chan rpcclient.RpcClientConnection, + internalHistorySChan chan rpcclient.RpcClientConnection, + internalPubSubSChan chan rpcclient.RpcClientConnection, + internalUserSChan chan rpcclient.RpcClientConnection, + internalAliaseSChan chan rpcclient.RpcClientConnection) { select { // Any of the rpc methods will unlock listening to rpc requests case resp := <-internalRaterChan: internalRaterChan <- resp @@ -668,11 +668,11 @@ func main() { internalRaterChan := make(chan *engine.Responder, 1) internalSchedulerChan := make(chan *scheduler.Scheduler, 1) internalCdrSChan := make(chan *engine.CdrServer, 1) - internalCdrStatSChan := make(chan engine.StatsInterface, 1) - internalHistorySChan := make(chan history.Scribe, 1) - internalPubSubSChan := make(chan engine.PublisherSubscriber, 1) - internalUserSChan := make(chan engine.UserService, 1) - internalAliaseSChan := make(chan engine.AliasService, 1) + internalCdrStatSChan := make(chan rpcclient.RpcClientConnection, 1) + internalHistorySChan := make(chan rpcclient.RpcClientConnection, 1) + internalPubSubSChan := make(chan rpcclient.RpcClientConnection, 1) + internalUserSChan := make(chan rpcclient.RpcClientConnection, 1) + internalAliaseSChan := make(chan rpcclient.RpcClientConnection, 1) internalSMGChan := make(chan rpcclient.RpcClientConnection, 1) // Start balancer service if cfg.BalancerEnabled { diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index f5a5a655f..edef08c05 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -26,9 +26,9 @@ import ( "github.com/cgrates/cgrates/apier/v2" "github.com/cgrates/cgrates/balancer2go" "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/history" "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) func startBalancer(internalBalancerChan chan *balancer2go.Balancer, stopHandled *bool, exitChan chan bool) { @@ -40,8 +40,8 @@ func startBalancer(internalBalancerChan chan *balancer2go.Balancer, stopHandled // Starts rater and reports on chan func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan chan *balancer2go.Balancer, internalSchedulerChan chan *scheduler.Scheduler, - internalCdrStatSChan chan engine.StatsInterface, internalHistorySChan chan history.Scribe, - internalPubSubSChan chan engine.PublisherSubscriber, internalUserSChan chan engine.UserService, internalAliaseSChan chan engine.AliasService, + internalCdrStatSChan chan rpcclient.RpcClientConnection, internalHistorySChan chan rpcclient.RpcClientConnection, + internalPubSubSChan chan rpcclient.RpcClientConnection, internalUserSChan chan rpcclient.RpcClientConnection, internalAliaseSChan chan rpcclient.RpcClientConnection, server *utils.Server, ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, loadDb engine.LoadStorage, cdrDb engine.CdrStorage, logDb engine.LogStorage, stopHandled *bool, exitChan chan bool) { @@ -109,7 +109,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c } // Connection to CDRStats - var cdrStats engine.StatsInterface + var cdrStats rpcclient.RpcClientConnection if cfg.RaterCdrStats != "" { cdrstatTaskChan := make(chan struct{}) waitTasks = append(waitTasks, cdrstatTaskChan) @@ -124,7 +124,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - } else if cdrStats, err = engine.NewProxyStats(cfg.RaterCdrStats, cfg.ConnectAttempts, -1); err != nil { + } else if cdrStats, err = rpcclient.NewRpcClient("tcp", cfg.RaterCdrStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to cdrstats, error: %s", err.Error())) exitChan <- true return @@ -138,7 +138,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c waitTasks = append(waitTasks, histTaskChan) go func() { defer close(histTaskChan) - var scribeServer history.Scribe + var scribeServer rpcclient.RpcClientConnection if cfg.RaterHistoryServer == utils.INTERNAL { select { case scribeServer = <-internalHistorySChan: @@ -148,7 +148,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - } else if scribeServer, err = history.NewProxyScribe(cfg.RaterHistoryServer, cfg.ConnectAttempts, -1); err != nil { + } else if scribeServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterHistoryServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect historys, error: %s", err.Error())) exitChan <- true return @@ -163,7 +163,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c waitTasks = append(waitTasks, pubsubTaskChan) go func() { defer close(pubsubTaskChan) - var pubSubServer engine.PublisherSubscriber + var pubSubServer rpcclient.RpcClientConnection if cfg.RaterPubSubServer == utils.INTERNAL { select { case pubSubServer = <-internalPubSubSChan: @@ -173,7 +173,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - } else if pubSubServer, err = engine.NewProxyPubSub(cfg.RaterPubSubServer, cfg.ConnectAttempts, -1); err != nil { + } else if pubSubServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterPubSubServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to pubsubs: %s", err.Error())) exitChan <- true return @@ -188,7 +188,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c waitTasks = append(waitTasks, aliasesTaskChan) go func() { defer close(aliasesTaskChan) - var aliasesServer engine.AliasService + var aliasesServer rpcclient.RpcClientConnection if cfg.RaterAliasesServer == utils.INTERNAL { select { case aliasesServer = <-internalAliaseSChan: @@ -198,7 +198,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - } else if aliasesServer, err = engine.NewProxyAliasService(cfg.RaterAliasesServer, cfg.ConnectAttempts, -1); err != nil { + } else if aliasesServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterAliasesServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to aliases, error: %s", err.Error())) exitChan <- true return @@ -208,7 +208,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c } // Connection to UserService - var userServer engine.UserService + var userServer rpcclient.RpcClientConnection if cfg.RaterUserServer != "" { usersTaskChan := make(chan struct{}) waitTasks = append(waitTasks, usersTaskChan) @@ -223,7 +223,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - } else if userServer, err = engine.NewProxyUserService(cfg.RaterUserServer, cfg.ConnectAttempts, -1); err != nil { + } else if userServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterUserServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect users, error: %s", err.Error())) exitChan <- true return diff --git a/cmd/cgr-engine/registration.go b/cmd/cgr-engine/registration.go index ec856d6f5..0b3a45ea0 100644 --- a/cmd/cgr-engine/registration.go +++ b/cmd/cgr-engine/registration.go @@ -29,6 +29,7 @@ import ( "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) /* @@ -43,7 +44,7 @@ func stopBalancerSignalHandler(bal *balancer2go.Balancer, exitChan chan bool) { exitChan <- true } -func generalSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitChan chan bool) { +func generalSignalHandler(internalCdrStatSChan chan rpcclient.RpcClientConnection, exitChan chan bool) { c := make(chan os.Signal) signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) @@ -52,7 +53,7 @@ func generalSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitC var dummyInt int select { case cdrStats := <-internalCdrStatSChan: - cdrStats.Stop(dummyInt, &dummyInt) + cdrStats.Call("CDRStatsV1.Stop", dummyInt, &dummyInt) default: } @@ -62,7 +63,7 @@ func generalSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitC /* Listens for the SIGTERM, SIGINT, SIGQUIT system signals and gracefuly unregister from balancer and closes the storage before exiting. */ -func stopRaterSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitChan chan bool) { +func stopRaterSignalHandler(internalCdrStatSChan chan rpcclient.RpcClientConnection, exitChan chan bool) { c := make(chan os.Signal) signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) sig := <-c @@ -72,7 +73,7 @@ func stopRaterSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exi var dummyInt int select { case cdrStats := <-internalCdrStatSChan: - cdrStats.Stop(dummyInt, &dummyInt) + cdrStats.Call("CDRStatsV1.Stop", dummyInt, &dummyInt) default: } exitChan <- true diff --git a/engine/aliases.go b/engine/aliases.go index eda219baa..701e8d6b3 100644 --- a/engine/aliases.go +++ b/engine/aliases.go @@ -12,7 +12,7 @@ import ( ) // Temporary export AliasService for the ApierV1 to be able to emulate old APIs -func GetAliasService() AliasService { +func GetAliasService() rpcclient.RpcClientConnection { return aliasService } @@ -156,16 +156,6 @@ type AttrReverseAlias struct { Context string } -type AliasService interface { - SetAlias(Alias, *string) error - UpdateAlias(Alias, *string) error - RemoveAlias(Alias, *string) error - GetAlias(Alias, *Alias) error - GetMatchingAlias(AttrMatchingAlias, *string) error - GetReverseAlias(AttrReverseAlias, *map[string][]*Alias) error - RemoveReverseAlias(AttrReverseAlias, *string) error -} - type AliasHandler struct { accountingDb AccountingStorage mu sync.RWMutex @@ -177,11 +167,11 @@ func NewAliasHandler(accountingDb AccountingStorage) *AliasHandler { } } -func (am *AliasHandler) SetAlias(al Alias, reply *string) error { +func (am *AliasHandler) SetAlias(al *Alias, reply *string) error { am.mu.Lock() defer am.mu.Unlock() - if err := am.accountingDb.SetAlias(&al); err != nil { + if err := am.accountingDb.SetAlias(al); err != nil { *reply = err.Error() return err } //add to cache @@ -194,7 +184,7 @@ func (am *AliasHandler) SetAlias(al Alias, reply *string) error { return nil } -func (am *AliasHandler) UpdateAlias(al Alias, reply *string) error { +func (am *AliasHandler) UpdateAlias(al *Alias, reply *string) error { am.mu.Lock() defer am.mu.Unlock() // get previous value @@ -216,7 +206,7 @@ func (am *AliasHandler) UpdateAlias(al Alias, reply *string) error { } } - if err := am.accountingDb.SetAlias(&al); err != nil { + if err := am.accountingDb.SetAlias(al); err != nil { *reply = err.Error() return err } //add to cache @@ -229,7 +219,7 @@ func (am *AliasHandler) UpdateAlias(al Alias, reply *string) error { return nil } -func (am *AliasHandler) RemoveAlias(al Alias, reply *string) error { +func (am *AliasHandler) RemoveAlias(al *Alias, reply *string) error { am.mu.Lock() defer am.mu.Unlock() if err := am.accountingDb.RemoveAlias(al.GetId()); err != nil { @@ -240,7 +230,7 @@ func (am *AliasHandler) RemoveAlias(al Alias, reply *string) error { return nil } -func (am *AliasHandler) RemoveReverseAlias(attr AttrReverseAlias, reply *string) error { +func (am *AliasHandler) RemoveReverseAlias(attr *AttrReverseAlias, reply *string) error { am.mu.Lock() defer am.mu.Unlock() rKey := utils.REVERSE_ALIASES_PREFIX + attr.Alias + attr.Target + attr.Context @@ -263,7 +253,7 @@ func (am *AliasHandler) RemoveReverseAlias(attr AttrReverseAlias, reply *string) return nil } -func (am *AliasHandler) GetAlias(al Alias, result *Alias) error { +func (am *AliasHandler) GetAlias(al *Alias, result *Alias) error { am.mu.RLock() defer am.mu.RUnlock() variants := al.GenerateIds() @@ -276,7 +266,7 @@ func (am *AliasHandler) GetAlias(al Alias, result *Alias) error { return utils.ErrNotFound } -func (am *AliasHandler) GetReverseAlias(attr AttrReverseAlias, result *map[string][]*Alias) error { +func (am *AliasHandler) GetReverseAlias(attr *AttrReverseAlias, result *map[string][]*Alias) error { am.mu.Lock() defer am.mu.Unlock() aliases := make(map[string][]*Alias) @@ -303,9 +293,9 @@ func (am *AliasHandler) GetReverseAlias(attr AttrReverseAlias, result *map[strin return nil } -func (am *AliasHandler) GetMatchingAlias(attr AttrMatchingAlias, result *string) error { +func (am *AliasHandler) GetMatchingAlias(attr *AttrMatchingAlias, result *string) error { response := Alias{} - if err := am.GetAlias(Alias{ + if err := am.GetAlias(&Alias{ Direction: attr.Direction, Tenant: attr.Tenant, Category: attr.Category, @@ -357,48 +347,32 @@ func (am *AliasHandler) GetMatchingAlias(attr AttrMatchingAlias, result *string) return utils.ErrNotFound } -type ProxyAliasService struct { - Client *rpcclient.RpcClient -} - -func NewProxyAliasService(addr string, attempts, reconnects int) (*ProxyAliasService, error) { - client, err := rpcclient.NewRpcClient("tcp", addr, attempts, reconnects, utils.GOB, nil) - if err != nil { - return nil, err +func (am *AliasHandler) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(am).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented } - return &ProxyAliasService{Client: client}, nil -} -func (ps *ProxyAliasService) SetAlias(al Alias, reply *string) error { - return ps.Client.Call("AliasesV1.SetAlias", al, reply) -} + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} -func (ps *ProxyAliasService) UpdateAlias(al Alias, reply *string) error { - return ps.Client.Call("AliasesV1.UpdateAlias", al, reply) -} - -func (ps *ProxyAliasService) RemoveAlias(al Alias, reply *string) error { - return ps.Client.Call("AliasesV1.RemoveAlias", al, reply) -} - -func (ps *ProxyAliasService) GetAlias(al Alias, alias *Alias) error { - return ps.Client.Call("AliasesV1.GetAlias", al, alias) -} - -func (ps *ProxyAliasService) GetMatchingAlias(attr AttrMatchingAlias, alias *string) error { - return ps.Client.Call("AliasesV1.GetMatchingAlias", attr, alias) -} - -func (ps *ProxyAliasService) GetReverseAlias(attr AttrReverseAlias, alias *map[string][]*Alias) error { - return ps.Client.Call("AliasesV1.GetReverseAlias", attr, alias) -} - -func (ps *ProxyAliasService) RemoveReverseAlias(attr AttrReverseAlias, reply *string) error { - return ps.Client.Call("AliasesV1.RemoveReverseAlias", attr, reply) -} - -func (ps *ProxyAliasService) ReloadAliases(in string, reply *string) error { - return ps.Client.Call("AliasesV1.ReloadAliases", in, reply) + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err } func LoadAlias(attr *AttrMatchingAlias, in interface{}, extraFields string) error { @@ -406,7 +380,7 @@ func LoadAlias(attr *AttrMatchingAlias, in interface{}, extraFields string) erro return nil } response := Alias{} - if err := aliasService.GetAlias(Alias{ + if err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: attr.Direction, Tenant: attr.Tenant, Category: attr.Category, diff --git a/engine/aliases_test.go b/engine/aliases_test.go index 67d89dd9a..2e87db3d0 100644 --- a/engine/aliases_test.go +++ b/engine/aliases_test.go @@ -12,7 +12,7 @@ func init() { } func TestAliasesGetAlias(t *testing.T) { alias := Alias{} - err := aliasService.GetAlias(Alias{ + err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -23,13 +23,13 @@ func TestAliasesGetAlias(t *testing.T) { if err != nil || len(alias.Values) != 2 || len(alias.Values[0].Pairs) != 2 { - t.Error("Error getting alias: ", err, alias) + t.Error("Error getting alias: ", err, alias, alias.Values[0]) } } func TestAliasesGetMatchingAlias(t *testing.T) { var response string - err := aliasService.GetMatchingAlias(AttrMatchingAlias{ + err := aliasService.Call("AliasesV1.GetMatchingAlias", &AttrMatchingAlias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", diff --git a/engine/calldesc.go b/engine/calldesc.go index a83cb997d..ab12be895 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -28,8 +28,8 @@ import ( "time" "github.com/cgrates/cgrates/cache2go" - "github.com/cgrates/cgrates/history" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) const ( @@ -75,10 +75,10 @@ var ( cdrStorage CdrStorage debitPeriod = 10 * time.Second globalRoundingDecimals = 5 - historyScribe history.Scribe - pubSubServer PublisherSubscriber - userService UserService - aliasService AliasService + historyScribe rpcclient.RpcClientConnection + pubSubServer rpcclient.RpcClientConnection + userService rpcclient.RpcClientConnection + aliasService rpcclient.RpcClientConnection ) // Exported method to set the storage getter. @@ -110,26 +110,26 @@ func SetCdrStorage(cStorage CdrStorage) { } // Exported method to set the history scribe. -func SetHistoryScribe(scribe history.Scribe) { +func SetHistoryScribe(scribe rpcclient.RpcClientConnection) { historyScribe = scribe } -func SetPubSub(ps PublisherSubscriber) { +func SetPubSub(ps rpcclient.RpcClientConnection) { pubSubServer = ps } -func SetUserService(us UserService) { +func SetUserService(us rpcclient.RpcClientConnection) { userService = us } -func SetAliasService(as AliasService) { +func SetAliasService(as rpcclient.RpcClientConnection) { aliasService = as } func Publish(event CgrEvent) { if pubSubServer != nil { var s string - pubSubServer.Publish(event, &s) + pubSubServer.Call("PubSubV1.Publish", event, &s) } } @@ -820,7 +820,7 @@ func (cd *CallDescriptor) GetLCRFromStorage() (*LCR, error) { return nil, utils.ErrNotFound } -func (cd *CallDescriptor) GetLCR(stats StatsInterface, p *utils.Paginator) (*LCRCost, error) { +func (cd *CallDescriptor) GetLCR(stats rpcclient.RpcClientConnection, p *utils.Paginator) (*LCRCost, error) { cd.account = nil // make sure it's not cached lcr, err := cd.GetLCRFromStorage() if err != nil { @@ -951,7 +951,7 @@ func (cd *CallDescriptor) GetLCR(stats StatsInterface, p *utils.Paginator) (*LCR if lcrCost.Entry.Strategy == LCR_STRATEGY_LOAD { for _, qId := range cdrStatsQueueIds { sq := &StatsQueue{} - if err := stats.GetQueue(qId, sq); err == nil { + if err := stats.Call("CDRStatsV1.GetQueue", qId, sq); err == nil { if sq.conf.QueueLength == 0 { //only add qeues that don't have fixed length supplierQueues = append(supplierQueues, sq) } @@ -959,7 +959,7 @@ func (cd *CallDescriptor) GetLCR(stats StatsInterface, p *utils.Paginator) (*LCR } } else { statValues := make(map[string]float64) - if err := stats.GetValues(qId, &statValues); err != nil { + if err := stats.Call("CDRStatsV1.GetValues", qId, &statValues); err != nil { lcrCost.SupplierCosts = append(lcrCost.SupplierCosts, &LCRSupplierCost{ Supplier: fullSupplier, Error: fmt.Sprintf("Get stats values for queue id %s, error %s", qId, err.Error()), diff --git a/engine/cdrs.go b/engine/cdrs.go index 34b78110d..2d02a7fd1 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -66,7 +66,7 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } } -func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, client rpcclient.RpcClientConnection, pubsub PublisherSubscriber, users UserService, aliases AliasService, stats StatsInterface) (*CdrServer, error) { +func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, client rpcclient.RpcClientConnection, pubsub rpcclient.RpcClientConnection, users rpcclient.RpcClientConnection, aliases rpcclient.RpcClientConnection, stats rpcclient.RpcClientConnection) (*CdrServer, error) { return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, client: client, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{queue: make(map[string]chan bool)}}, nil } @@ -74,10 +74,10 @@ type CdrServer struct { cgrCfg *config.CGRConfig cdrDb CdrStorage client rpcclient.RpcClientConnection - pubsub PublisherSubscriber - users UserService - aliases AliasService - stats StatsInterface + pubsub rpcclient.RpcClientConnection + users rpcclient.RpcClientConnection + aliases rpcclient.RpcClientConnection + stats rpcclient.RpcClientConnection guard *GuardianLock } @@ -255,7 +255,7 @@ func (self *CdrServer) rateStoreStatsReplicate(cdr *StoredCdr) error { } // Attach CDR to stats if self.stats != nil { // Send CDR to stats - if err := self.stats.AppendCDR(cdr, nil); err != nil { + if err := self.stats.Call("Stats.AppendCDR", cdr, nil); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not append cdr to stats: %s", err.Error())) } } diff --git a/engine/pubsub.go b/engine/pubsub.go index 61e609020..59678d472 100644 --- a/engine/pubsub.go +++ b/engine/pubsub.go @@ -3,11 +3,12 @@ package engine import ( "errors" "fmt" + "reflect" + "strings" "sync" "time" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" ) type SubscribeInfo struct { @@ -28,13 +29,6 @@ func (ce CgrEvent) PassFilters(rsrFields utils.RSRFields) bool { return true } -type PublisherSubscriber interface { - Subscribe(SubscribeInfo, *string) error - Unsubscribe(SubscribeInfo, *string) error - Publish(CgrEvent, *string) error - ShowSubscribers(string, *map[string]*SubscriberData) error -} - type SubscriberData struct { ExpTime time.Time Filters utils.RSRFields @@ -165,28 +159,30 @@ func (ps *PubSub) ShowSubscribers(in string, out *map[string]*SubscriberData) er return nil } -type ProxyPubSub struct { - Client *rpcclient.RpcClient -} - -func NewProxyPubSub(addr string, attempts, reconnects int) (*ProxyPubSub, error) { - client, err := rpcclient.NewRpcClient("tcp", addr, attempts, reconnects, utils.GOB, nil) - if err != nil { - return nil, err +func (ps *PubSub) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(ps).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented } - return &ProxyPubSub{Client: client}, nil -} -func (ps *ProxyPubSub) Subscribe(si SubscribeInfo, reply *string) error { - return ps.Client.Call("PubSubV1.Subscribe", si, reply) -} -func (ps *ProxyPubSub) Unsubscribe(si SubscribeInfo, reply *string) error { - return ps.Client.Call("PubSubV1.Unsubscribe", si, reply) -} -func (ps *ProxyPubSub) Publish(evt CgrEvent, reply *string) error { - return ps.Client.Call("PubSubV1.Publish", evt, reply) -} + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} -func (ps *ProxyPubSub) ShowSubscribers(in string, reply *map[string]*SubscriberData) error { - return ps.Client.Call("PubSubV1.ShowSubscribers", in, reply) + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err } diff --git a/engine/responder.go b/engine/responder.go index a54756683..76b9a9b9d 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -31,6 +31,7 @@ import ( "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) // Individual session run @@ -49,13 +50,13 @@ type Responder struct { Bal *balancer2go.Balancer ExitChan chan bool CdrSrv *CdrServer - Stats StatsInterface + Stats rpcclient.RpcClientConnection Timezone string cnt int64 responseCache *cache2go.ResponseCache } -func NewResponder(exitChan chan bool, cdrSrv *CdrServer, stats StatsInterface, timeToLive time.Duration) *Responder { +func NewResponder(exitChan chan bool, cdrSrv *CdrServer, stats rpcclient.RpcClientConnection, timeToLive time.Duration) *Responder { return &Responder{ ExitChan: exitChan, Stats: stats, @@ -615,12 +616,12 @@ func (rs *Responder) UnRegisterRater(clientAddress string, replay *int) error { } func (rs *Responder) Call(serviceMethod string, args interface{}, reply interface{}) error { - if !strings.HasPrefix(serviceMethod, "Responder.") { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { return utils.ErrNotImplemented } - methodName := strings.TrimLeft(serviceMethod, "Responder.") // get method - method := reflect.ValueOf(rs).MethodByName(methodName) + method := reflect.ValueOf(rs).MethodByName(parts[1]) if !method.IsValid() { return utils.ErrNotImplemented } @@ -632,6 +633,9 @@ func (rs *Responder) Call(serviceMethod string, args interface{}, reply interfac if len(ret) != 1 { return utils.ErrServerError } + if ret[0].Interface() == nil { + return nil + } err, ok := ret[0].Interface().(error) if !ok { return utils.ErrServerError diff --git a/engine/responder_test.go b/engine/responder_test.go index 80801bfad..74e87ccdd 100644 --- a/engine/responder_test.go +++ b/engine/responder_test.go @@ -282,7 +282,8 @@ func TestResponderGetLCR(t *testing.T) { } } danStatsId := "dan12_stats" - rsponder.Stats.AddQueue(&CdrStats{Id: danStatsId, Supplier: []string{"dan12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, nil) + var r int + rsponder.Stats.Call("CDRStatsV1.AddQueue", &CdrStats{Id: danStatsId, Supplier: []string{"dan12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, &r) danRpfl := &RatingProfile{Id: "*out:tenant12:call:dan12", RatingPlanActivations: RatingPlanActivations{&RatingPlanActivation{ ActivationTime: time.Date(2015, 01, 01, 8, 0, 0, 0, time.UTC), @@ -292,7 +293,7 @@ func TestResponderGetLCR(t *testing.T) { }}, } rifStatsId := "rif12_stats" - rsponder.Stats.AddQueue(&CdrStats{Id: rifStatsId, Supplier: []string{"rif12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, nil) + rsponder.Stats.Call("CDRStatsV1.AddQueue", &CdrStats{Id: rifStatsId, Supplier: []string{"rif12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, &r) rifRpfl := &RatingProfile{Id: "*out:tenant12:call:rif12", RatingPlanActivations: RatingPlanActivations{&RatingPlanActivation{ ActivationTime: time.Date(2015, 01, 01, 8, 0, 0, 0, time.UTC), @@ -302,7 +303,7 @@ func TestResponderGetLCR(t *testing.T) { }}, } ivoStatsId := "ivo12_stats" - rsponder.Stats.AddQueue(&CdrStats{Id: ivoStatsId, Supplier: []string{"ivo12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, nil) + rsponder.Stats.Call("CDRStatsV1.AddQueue", &CdrStats{Id: ivoStatsId, Supplier: []string{"ivo12"}, Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}}, &r) ivoRpfl := &RatingProfile{Id: "*out:tenant12:call:ivo12", RatingPlanActivations: RatingPlanActivations{&RatingPlanActivation{ ActivationTime: time.Date(2015, 01, 01, 8, 0, 0, 0, time.UTC), @@ -483,9 +484,10 @@ func TestResponderGetLCR(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eQTLcr.SupplierCosts, lcrQT.SupplierCosts) } cdr := &StoredCdr{Supplier: "rif12", AnswerTime: time.Now(), Usage: 3 * time.Minute, Cost: 1} - rsponder.Stats.AppendCDR(cdr, nil) + rsponder.Stats.Call("CDRStatsV1.AppendCDR", cdr, &r) cdr = &StoredCdr{Supplier: "dan12", AnswerTime: time.Now(), Usage: 5 * time.Minute, Cost: 2} - rsponder.Stats.AppendCDR(cdr, nil) + rsponder.Stats.Call("CDRStatsV1.AppendCDR", cdr, &r) + eQTLcr = &LCRCost{ Entry: &LCREntry{DestinationId: utils.ANY, RPCategory: "call", Strategy: LCR_STRATEGY_QOS_THRESHOLD, StrategyParams: "35;;;;4m;;;;;;;;;", Weight: 10.0}, SupplierCosts: []*LCRSupplierCost{ diff --git a/engine/stats.go b/engine/stats.go index 4c9599463..62ff95c48 100644 --- a/engine/stats.go +++ b/engine/stats.go @@ -20,25 +20,14 @@ package engine import ( "fmt" + "reflect" + "strings" "sync" "time" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" ) -type StatsInterface interface { - GetValues(string, *map[string]float64) error - GetQueueIds(int, *[]string) error - GetQueue(string, *StatsQueue) error - GetQueueTriggers(string, *ActionTriggers) error - AppendCDR(*StoredCdr, *int) error - AddQueue(*CdrStats, *int) error - ReloadQueues([]string, *int) error - ResetQueues([]string, *int) error - Stop(int, *int) error -} - type Stats struct { queues map[string]*StatsQueue queueSavers map[string]*queueSaver @@ -286,50 +275,30 @@ func (s *Stats) Stop(int, *int) error { return nil } -type ProxyStats struct { - Client *rpcclient.RpcClient -} - -func NewProxyStats(addr string, attempts, reconnects int) (*ProxyStats, error) { - client, err := rpcclient.NewRpcClient("tcp", addr, attempts, reconnects, utils.GOB, nil) - if err != nil { - return nil, err +func (s *Stats) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(s).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented } - return &ProxyStats{Client: client}, nil -} -func (ps *ProxyStats) GetValues(sqID string, values *map[string]float64) error { - return ps.Client.Call("Stats.GetValues", sqID, values) -} + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} -func (ps *ProxyStats) AppendCDR(cdr *StoredCdr, out *int) error { - return ps.Client.Call("Stats.AppendCDR", cdr, out) -} - -func (ps *ProxyStats) GetQueueIds(in int, ids *[]string) error { - return ps.Client.Call("Stats.GetQueueIds", in, ids) -} - -func (ps *ProxyStats) GetQueue(id string, sq *StatsQueue) error { - return ps.Client.Call("Stats.GetQueue", id, sq) -} - -func (ps *ProxyStats) GetQueueTriggers(id string, ats *ActionTriggers) error { - return ps.Client.Call("Stats.GetQueueTriggers", id, ats) -} - -func (ps *ProxyStats) AddQueue(cs *CdrStats, out *int) error { - return ps.Client.Call("Stats.AddQueue", cs, out) -} - -func (ps *ProxyStats) ReloadQueues(ids []string, out *int) error { - return ps.Client.Call("Stats.ReloadQueues", ids, out) -} - -func (ps *ProxyStats) ResetQueues(ids []string, out *int) error { - return ps.Client.Call("Stats.ResetQueues", ids, out) -} - -func (ps *ProxyStats) Stop(i int, r *int) error { - return ps.Client.Call("Stats.Stop", 0, i) + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err } diff --git a/engine/storage_map.go b/engine/storage_map.go index adf785a69..f2f3dc133 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -298,7 +298,7 @@ func (ms *MapStorage) SetRatingPlan(rp *RatingPlan) (err error) { ms.dict[utils.RATING_PLAN_PREFIX+rp.Id] = b.Bytes() response := 0 if historyScribe != nil { - go historyScribe.Record(rp.GetHistoryRecord(), &response) + go historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(), &response) } return } @@ -328,7 +328,7 @@ func (ms *MapStorage) SetRatingProfile(rpf *RatingProfile) (err error) { ms.dict[utils.RATING_PROFILE_PREFIX+rpf.Id] = result response := 0 if historyScribe != nil { - go historyScribe.Record(rpf.GetHistoryRecord(false), &response) + go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(false), &response) } return } @@ -341,7 +341,7 @@ func (ms *MapStorage) RemoveRatingProfile(key string) (err error) { response := 0 rpf := &RatingProfile{Id: key} if historyScribe != nil { - go historyScribe.Record(rpf.GetHistoryRecord(true), &response) + go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(true), &response) } } } @@ -406,7 +406,7 @@ func (ms *MapStorage) SetDestination(dest *Destination) (err error) { ms.dict[utils.DESTINATION_PREFIX+dest.Id] = b.Bytes() response := 0 if historyScribe != nil { - go historyScribe.Record(dest.GetHistoryRecord(), &response) + go historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(), &response) } return } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index aa171baa3..4f01bb942 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -612,7 +612,7 @@ func (ms *MongoStorage) SetRatingPlan(rp *RatingPlan) error { _, err := ms.db.C(colRpl).Upsert(bson.M{"id": rp.Id}, rp) if err == nil && historyScribe != nil { var response int - historyScribe.Record(rp.GetHistoryRecord(), &response) + historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(), &response) } return err } @@ -637,7 +637,7 @@ func (ms *MongoStorage) SetRatingProfile(rp *RatingProfile) error { _, err := ms.db.C(colRpf).Upsert(bson.M{"id": rp.Id}, rp) if err == nil && historyScribe != nil { var response int - historyScribe.Record(rp.GetHistoryRecord(false), &response) + historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(false), &response) } return err } @@ -653,7 +653,7 @@ func (ms *MongoStorage) RemoveRatingProfile(key string) error { rpf := &RatingProfile{Id: result.Id} if historyScribe != nil { var response int - go historyScribe.Record(rpf.GetHistoryRecord(true), &response) + go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(true), &response) } } return iter.Close() @@ -705,7 +705,7 @@ func (ms *MongoStorage) SetDestination(dest *Destination) (err error) { _, err = ms.db.C(colDst).Upsert(bson.M{"id": dest.Id}, dest) if err == nil && historyScribe != nil { var response int - historyScribe.Record(dest.GetHistoryRecord(), &response) + historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(), &response) } return } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index fe05e1c86..fbeac4032 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -436,7 +436,7 @@ func (rs *RedisStorage) SetRatingPlan(rp *RatingPlan) (err error) { err = rs.db.Cmd("SET", utils.RATING_PLAN_PREFIX+rp.Id, b.Bytes()).Err if err == nil && historyScribe != nil { response := 0 - go historyScribe.Record(rp.GetHistoryRecord(), &response) + go historyScribe.Call("HistoryV1.Record", rp.GetHistoryRecord(), &response) } return } @@ -465,7 +465,7 @@ func (rs *RedisStorage) SetRatingProfile(rpf *RatingProfile) (err error) { err = rs.db.Cmd("SET", utils.RATING_PROFILE_PREFIX+rpf.Id, result).Err if err == nil && historyScribe != nil { response := 0 - go historyScribe.Record(rpf.GetHistoryRecord(false), &response) + go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(false), &response) } return } @@ -488,7 +488,7 @@ func (rs *RedisStorage) RemoveRatingProfile(key string) error { rpf := &RatingProfile{Id: key} if historyScribe != nil { response := 0 - go historyScribe.Record(rpf.GetHistoryRecord(true), &response) + go historyScribe.Call("HistoryV1.Record", rpf.GetHistoryRecord(true), &response) } } return nil @@ -556,7 +556,7 @@ func (rs *RedisStorage) SetDestination(dest *Destination) (err error) { err = rs.db.Cmd("SET", utils.DESTINATION_PREFIX+dest.Id, b.Bytes()).Err if err == nil && historyScribe != nil { response := 0 - go historyScribe.Record(dest.GetHistoryRecord(), &response) + go historyScribe.Call("HistoryV1.Record", dest.GetHistoryRecord(), &response) } return } diff --git a/engine/users.go b/engine/users.go index 69dcc2ef4..5149b7c64 100644 --- a/engine/users.go +++ b/engine/users.go @@ -2,12 +2,12 @@ package engine import ( "fmt" + "reflect" "sort" "strings" "sync" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" ) type UserProfile struct { @@ -52,16 +52,6 @@ func (ud *UserProfile) SetId(id string) error { return nil } -type UserService interface { - SetUser(UserProfile, *string) error - RemoveUser(UserProfile, *string) error - UpdateUser(UserProfile, *string) error - GetUsers(UserProfile, *UserProfiles) error - AddIndex([]string, *string) error - GetIndexes(string, *map[string][]string) error - ReloadUsers(string, *string) error -} - type prop struct { masked bool weight float64 @@ -139,21 +129,21 @@ func (um *UserMap) ReloadUsers(in string, reply *string) error { return nil } -func (um *UserMap) SetUser(up UserProfile, reply *string) error { +func (um *UserMap) SetUser(up *UserProfile, reply *string) error { um.mu.Lock() defer um.mu.Unlock() - if err := um.accountingDb.SetUser(&up); err != nil { + if err := um.accountingDb.SetUser(up); err != nil { *reply = err.Error() return err } um.table[up.GetId()] = up.Profile um.properties[up.GetId()] = &prop{weight: up.Weight, masked: up.Masked} - um.addIndex(&up, um.indexKeys) + um.addIndex(up, um.indexKeys) *reply = utils.OK return nil } -func (um *UserMap) RemoveUser(up UserProfile, reply *string) error { +func (um *UserMap) RemoveUser(up *UserProfile, reply *string) error { um.mu.Lock() defer um.mu.Unlock() if err := um.accountingDb.RemoveUser(up.GetId()); err != nil { @@ -162,12 +152,12 @@ func (um *UserMap) RemoveUser(up UserProfile, reply *string) error { } delete(um.table, up.GetId()) delete(um.properties, up.GetId()) - um.deleteIndex(&up) + um.deleteIndex(up) *reply = utils.OK return nil } -func (um *UserMap) UpdateUser(up UserProfile, reply *string) error { +func (um *UserMap) UpdateUser(up *UserProfile, reply *string) error { um.mu.Lock() defer um.mu.Unlock() m, found := um.table[up.GetId()] @@ -212,7 +202,7 @@ func (um *UserMap) UpdateUser(up UserProfile, reply *string) error { return nil } -func (um *UserMap) GetUsers(up UserProfile, results *UserProfiles) error { +func (um *UserMap) GetUsers(up *UserProfile, results *UserProfiles) error { um.mu.RLock() defer um.mu.RUnlock() table := um.table // no index @@ -402,44 +392,32 @@ func (um *UserMap) GetIndexes(in string, reply *map[string][]string) error { return nil } -type ProxyUserService struct { - Client *rpcclient.RpcClient -} - -func NewProxyUserService(addr string, attempts, reconnects int) (*ProxyUserService, error) { - client, err := rpcclient.NewRpcClient("tcp", addr, attempts, reconnects, utils.GOB, nil) - if err != nil { - return nil, err +func (um *UserMap) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(um).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented } - return &ProxyUserService{Client: client}, nil -} -func (ps *ProxyUserService) SetUser(ud UserProfile, reply *string) error { - return ps.Client.Call("UsersV1.SetUser", ud, reply) -} + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} -func (ps *ProxyUserService) RemoveUser(ud UserProfile, reply *string) error { - return ps.Client.Call("UsersV1.RemoveUser", ud, reply) -} - -func (ps *ProxyUserService) UpdateUser(ud UserProfile, reply *string) error { - return ps.Client.Call("UsersV1.UpdateUser", ud, reply) -} - -func (ps *ProxyUserService) GetUsers(ud UserProfile, users *UserProfiles) error { - return ps.Client.Call("UsersV1.GetUsers", ud, users) -} - -func (ps *ProxyUserService) AddIndex(indexes []string, reply *string) error { - return ps.Client.Call("UsersV1.AddIndex", indexes, reply) -} - -func (ps *ProxyUserService) GetIndexes(in string, reply *map[string][]string) error { - return ps.Client.Call("UsersV1.AddIndex", in, reply) -} - -func (ps *ProxyUserService) ReloadUsers(in string, reply *string) error { - return ps.Client.Call("UsersV1.ReloadUsers", in, reply) + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err } // extraFields - Field name in the interface containing extraFields information @@ -484,7 +462,7 @@ func LoadUserProfile(in interface{}, extraFields string) error { } } ups := UserProfiles{} - if err := userService.GetUsers(*up, &ups); err != nil { + if err := userService.Call("UsersV1.GetUsers", up, &ups); err != nil { return err } if len(ups) > 0 { diff --git a/engine/users_test.go b/engine/users_test.go index 0862d96fe..a9e640351 100644 --- a/engine/users_test.go +++ b/engine/users_test.go @@ -36,7 +36,7 @@ var testMap2 = UserMap{ func TestUsersAdd(t *testing.T) { tm := newUserMap(accountingStorage, nil) var r string - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -57,7 +57,7 @@ func TestUsersAdd(t *testing.T) { func TestUsersUpdate(t *testing.T) { tm := newUserMap(accountingStorage, nil) var r string - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -88,7 +88,7 @@ func TestUsersUpdate(t *testing.T) { func TestUsersUpdateNotFound(t *testing.T) { tm := newUserMap(accountingStorage, nil) var r string - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -106,12 +106,12 @@ func TestUsersUpdateNotFound(t *testing.T) { func TestUsersUpdateInit(t *testing.T) { tm := newUserMap(accountingStorage, nil) var r string - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", } tm.SetUser(up, &r) - up = UserProfile{ + up = &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -132,7 +132,7 @@ func TestUsersUpdateInit(t *testing.T) { func TestUsersRemove(t *testing.T) { tm := newUserMap(accountingStorage, nil) var r string - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -158,7 +158,7 @@ func TestUsersRemove(t *testing.T) { } func TestUsersGetFull(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -173,7 +173,7 @@ func TestUsersGetFull(t *testing.T) { } func TestUsersGetFullMasked(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", } results := UserProfiles{} @@ -184,7 +184,7 @@ func TestUsersGetFullMasked(t *testing.T) { } func TestUsersGetFullUnMasked(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", Masked: true, } @@ -199,7 +199,7 @@ func TestUsersGetFullUnMasked(t *testing.T) { } func TestUsersGetTenant(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "testX", UserName: "user", Profile: map[string]string{ @@ -214,7 +214,7 @@ func TestUsersGetTenant(t *testing.T) { } func TestUsersGetUserName(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "userX", Profile: map[string]string{ @@ -229,7 +229,7 @@ func TestUsersGetUserName(t *testing.T) { } func TestUsersGetNotFoundProfile(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -244,7 +244,7 @@ func TestUsersGetNotFoundProfile(t *testing.T) { } func TestUsersGetMissingTenant(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ UserName: "user", Profile: map[string]string{ "t": "v", @@ -258,7 +258,7 @@ func TestUsersGetMissingTenant(t *testing.T) { } func TestUsersGetMissingUserName(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Tenant: "test", Profile: map[string]string{ "t": "v", @@ -272,7 +272,7 @@ func TestUsersGetMissingUserName(t *testing.T) { } func TestUsersGetMissingId(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "t": "v", }, @@ -285,7 +285,7 @@ func TestUsersGetMissingId(t *testing.T) { } func TestUsersGetMissingIdTwo(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "t": "v", "x": "y", @@ -299,7 +299,7 @@ func TestUsersGetMissingIdTwo(t *testing.T) { } func TestUsersGetMissingIdTwoSort(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "t": "v", "x": "y", @@ -316,7 +316,7 @@ func TestUsersGetMissingIdTwoSort(t *testing.T) { } func TestUsersGetMissingIdTwoSortWeight(t *testing.T) { - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "a": "b", "c": "d", @@ -367,7 +367,7 @@ func TestUsersGetFullindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -385,7 +385,7 @@ func TestUsersGetTenantindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Tenant: "testX", UserName: "user", Profile: map[string]string{ @@ -403,7 +403,7 @@ func TestUsersGetUserNameindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "userX", Profile: map[string]string{ @@ -421,7 +421,7 @@ func TestUsersGetNotFoundProfileindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -439,7 +439,7 @@ func TestUsersGetMissingTenantindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ UserName: "user", Profile: map[string]string{ "t": "v", @@ -456,7 +456,7 @@ func TestUsersGetMissingUserNameindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Tenant: "test", Profile: map[string]string{ "t": "v", @@ -473,7 +473,7 @@ func TestUsersGetMissingIdindex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "t": "v", }, @@ -489,7 +489,7 @@ func TestUsersGetMissingIdTwoINdex(t *testing.T) { var r string testMap.index = make(map[string]map[string]bool) // reset index testMap.AddIndex([]string{"t", "x", "UserName", "Tenant"}, &r) - up := UserProfile{ + up := &UserProfile{ Profile: map[string]string{ "t": "v", "x": "y", @@ -509,7 +509,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { if len(tm.index) != 0 { t.Error("error adding indexes: ", tm.index) } - tm.SetUser(UserProfile{ + tm.SetUser(&UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ @@ -519,7 +519,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { if len(tm.index) != 1 || !tm.index["t:v"]["test:user"] { t.Error("error adding indexes: ", tm.index) } - tm.SetUser(UserProfile{ + tm.SetUser(&UserProfile{ Tenant: "test", UserName: "best", Profile: map[string]string{ @@ -531,7 +531,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { !tm.index["t:v"]["test:best"] { t.Error("error adding indexes: ", tm.index) } - tm.UpdateUser(UserProfile{ + tm.UpdateUser(&UserProfile{ Tenant: "test", UserName: "best", Profile: map[string]string{ @@ -543,7 +543,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { !tm.index["t:v1"]["test:best"] { t.Error("error adding indexes: ", tm.index) } - tm.UpdateUser(UserProfile{ + tm.UpdateUser(&UserProfile{ Tenant: "test", UserName: "best", Profile: map[string]string{ @@ -555,7 +555,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { !tm.index["t:v"]["test:best"] { t.Error("error adding indexes: ", tm.index) } - tm.RemoveUser(UserProfile{ + tm.RemoveUser(&UserProfile{ Tenant: "test", UserName: "best", Profile: map[string]string{ @@ -567,7 +567,7 @@ func TestUsersAddUpdateRemoveIndexes(t *testing.T) { tm.index["t:v"]["test:best"] { t.Error("error adding indexes: ", tm.index) } - tm.RemoveUser(UserProfile{ + tm.RemoveUser(&UserProfile{ Tenant: "test", UserName: "user", Profile: map[string]string{ diff --git a/history/file_scribe.go b/history/file_scribe.go index 8d023ba09..e6b3d31c7 100644 --- a/history/file_scribe.go +++ b/history/file_scribe.go @@ -28,8 +28,12 @@ import ( "os" "os/exec" "path/filepath" + "reflect" + "strings" "sync" "time" + + "github.com/cgrates/cgrates/utils" ) type FileScribe struct { @@ -170,3 +174,31 @@ func (s *FileScribe) save(filename string) error { f.Close() return s.gitCommit() } + +func (s *FileScribe) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(s).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented + } + + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} + + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err +} diff --git a/history/mock_scribe.go b/history/mock_scribe.go index 1192f9cb4..a529bef51 100644 --- a/history/mock_scribe.go +++ b/history/mock_scribe.go @@ -21,7 +21,11 @@ package history import ( "bufio" "bytes" + "reflect" + "strings" "sync" + + "github.com/cgrates/cgrates/utils" ) type MockScribe struct { @@ -64,3 +68,31 @@ func (s *MockScribe) GetBuffer(fn string) *bytes.Buffer { defer s.mu.Unlock() return s.BufMap[fn] } + +func (s *MockScribe) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(s).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented + } + + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} + + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err +} diff --git a/history/proxy_scribe.go b/history/proxy_scribe.go deleted file mode 100644 index 3b00ebc0a..000000000 --- a/history/proxy_scribe.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Rating system designed to be used in VoIP Carriers World -Copyright (C) 2012-2015 ITsysCOM - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package history - -import ( - "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" -) - -type ProxyScribe struct { - Client *rpcclient.RpcClient -} - -func NewProxyScribe(addr string, attempts, reconnects int) (*ProxyScribe, error) { - client, err := rpcclient.NewRpcClient("tcp", addr, attempts, reconnects, utils.GOB, nil) - if err != nil { - return nil, err - } - return &ProxyScribe{Client: client}, nil -} - -func (ps *ProxyScribe) Record(rec Record, out *int) error { - return ps.Client.Call("Scribe.Record", rec, out) -} diff --git a/history/scribe.go b/history/scribe.go index 819bada44..2a6a1f3ba 100644 --- a/history/scribe.go +++ b/history/scribe.go @@ -30,10 +30,6 @@ const ( RATING_PROFILES_FN = "rating_profiles.json" ) -type Scribe interface { - Record(Record, *int) error -} - type Record struct { Id string Filename string From 83dcafaa66e40bff42f5beba581a77369d2f605b Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 26 Nov 2015 23:12:50 +0200 Subject: [PATCH 003/227] fix loader --- cmd/cgr-loader/cgr-loader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index ac359032f..7fce5754d 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -29,8 +29,8 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/history" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) var ( @@ -236,7 +236,7 @@ func main() { return } if *historyServer != "" { // Init scribeAgent so we can store the differences - if scribeAgent, err := history.NewProxyScribe(*historyServer, 3, 3); err != nil { + if scribeAgent, err := rpcclient.NewRpcClient("tcp", *historyServer, 3, 3, utils.GOB, nil); err != nil { log.Fatalf("Could not connect to history server, error: %s. Make sure you have properly configured it via -history_server flag.", err.Error()) return } else { From e590329321dc3c527bee5203cf5bfeb3a91a94b1 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 27 Nov 2015 10:56:19 +0200 Subject: [PATCH 004/227] fixed stats calls --- apier/v1/apier.go | 3 ++- apier/v1/cdrstatsv1.go | 6 ++++-- engine/cdrs.go | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 53b56fb46..78d3bf10f 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1188,7 +1188,8 @@ func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, self.Sched.Restart() } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { - if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, nil); err != nil { + var out int + if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, &out); err != nil { return err } } diff --git a/apier/v1/cdrstatsv1.go b/apier/v1/cdrstatsv1.go index a9bef703f..9820cfa68 100644 --- a/apier/v1/cdrstatsv1.go +++ b/apier/v1/cdrstatsv1.go @@ -55,7 +55,8 @@ func (sts *CDRStatsV1) GetQueueTriggers(id string, ats *engine.ActionTriggers) e } func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error { - if err := sts.CdrStats.Call("CDRStatsV1.ReloadQueues", attr.StatsQueueIds, nil); err != nil { + var out int + if err := sts.CdrStats.Call("CDRStatsV1.ReloadQueues", attr.StatsQueueIds, &out); err != nil { return err } *reply = utils.OK @@ -63,7 +64,8 @@ func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply * } func (sts *CDRStatsV1) ResetQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error { - if err := sts.CdrStats.Call("CDRStatsV1.ResetQueues", attr.StatsQueueIds, nil); err != nil { + var out int + if err := sts.CdrStats.Call("CDRStatsV1.ResetQueues", attr.StatsQueueIds, &out); err != nil { return err } *reply = utils.OK diff --git a/engine/cdrs.go b/engine/cdrs.go index 2d02a7fd1..39f576c35 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -255,7 +255,8 @@ func (self *CdrServer) rateStoreStatsReplicate(cdr *StoredCdr) error { } // Attach CDR to stats if self.stats != nil { // Send CDR to stats - if err := self.stats.Call("Stats.AppendCDR", cdr, nil); err != nil { + var out int + if err := self.stats.Call("CDRStatsV1.AppendCDR", cdr, &out); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not append cdr to stats: %s", err.Error())) } } From 5cd3165036adf06800b838f7dc081440e1797405 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 27 Nov 2015 11:05:44 +0200 Subject: [PATCH 005/227] one more stats call fix --- apier/v2/apier.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apier/v2/apier.go b/apier/v2/apier.go index bb773ece8..f7bcb62f7 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -253,7 +253,8 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, self.Sched.Restart() } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { - if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, nil); err != nil { + var out int + if err := self.CdrStatsSrv.Call("CDRStatsV1.ReloadQueues", cstKeys, &out); err != nil { return err } } From ffdd960d397266344826b1ad7c7faf56100f0c3e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 27 Nov 2015 14:46:51 +0200 Subject: [PATCH 006/227] cache all calls from session manager --- cache2go/response_cache.go | 2 +- engine/responder.go | 105 ++++++++++++++++++----------- sessionmanager/fsevent.go | 1 + sessionmanager/fssessionmanager.go | 2 + sessionmanager/kamailiosm.go | 1 + sessionmanager/session.go | 3 +- sessionmanager/smgeneric.go | 1 + utils/consts.go | 4 -- 8 files changed, 75 insertions(+), 44 deletions(-) diff --git a/cache2go/response_cache.go b/cache2go/response_cache.go index 4ac57e3f7..c27fdecbf 100644 --- a/cache2go/response_cache.go +++ b/cache2go/response_cache.go @@ -18,7 +18,7 @@ type CacheItem struct { type ResponseCache struct { ttl time.Duration cache map[string]*CacheItem - semaphore map[string]chan bool + semaphore map[string]chan bool // used for waiting till the first goroutine processes the response mu sync.RWMutex } diff --git a/engine/responder.go b/engine/responder.go index 76b9a9b9d..e769a0058 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -24,6 +24,7 @@ import ( "net/rpc" "reflect" "runtime" + "strconv" "strings" "time" @@ -149,7 +150,8 @@ func (rs *Responder) Debit(arg *CallDescriptor, reply *CallCost) (err error) { } func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) { - if item, err := rs.getCache().Get(utils.MAX_DEBIT_CACHE_PREFIX + arg.CgrId); err == nil && item != nil { + cacheKey := "MaxDebit" + arg.CgrId + strconv.FormatFloat(arg.LoopIndex, 'f', -1, 64) + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = *(item.Value.(*CallCost)) return item.Err } @@ -167,10 +169,12 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) Subject: arg.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, arg, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace user profile fields if err := LoadUserProfile(arg, utils.EXTRA_FIELDS); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if rs.Bal != nil { @@ -179,23 +183,19 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) } else { r, e := arg.MaxDebit() if e != nil { - rs.getCache().Cache(utils.MAX_DEBIT_CACHE_PREFIX+arg.CgrId, &cache2go.CacheItem{ - Err: e, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: e}) return e } else if r != nil { *reply = *r } } - rs.getCache().Cache(utils.MAX_DEBIT_CACHE_PREFIX+arg.CgrId, &cache2go.CacheItem{ - Value: reply, - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: reply, Err: err}) return } func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err error) { - if item, err := rs.getCache().Get(utils.REFUND_INCR_CACHE_PREFIX + arg.CgrId); err == nil && item != nil { + cacheKey := "RefundIncrements" + arg.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = *(item.Value.(*float64)) return item.Err } @@ -213,10 +213,12 @@ func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err Subject: arg.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, arg, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace user profile fields if err := LoadUserProfile(arg, utils.EXTRA_FIELDS); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if rs.Bal != nil { @@ -227,10 +229,7 @@ func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err }, 0, arg.GetAccountKey()) *reply, err = r.(float64), e } - rs.getCache().Cache(utils.REFUND_INCR_CACHE_PREFIX+arg.CgrId, &cache2go.CacheItem{ - Value: reply, - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: reply, Err: err}) return } @@ -266,8 +265,15 @@ func (rs *Responder) GetMaxSessionTime(arg *CallDescriptor, reply *float64) (err // Returns MaxSessionTime for an event received in SessionManager, considering DerivedCharging for it func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) error { + cacheKey := "GetDerivedMaxSessionTime" + ev.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *reply = *(item.Value.(*float64)) + return item.Err + } if rs.Bal != nil { - return errors.New("unsupported method on the balancer") + err := errors.New("unsupported method on the balancer") + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return err } if ev.Subject == "" { ev.Subject = ev.Account @@ -283,10 +289,12 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err Subject: ev.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, ev, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace user profile fields if err := LoadUserProfile(ev, utils.EXTRA_FIELDS); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } maxCallDuration := -1.0 @@ -294,6 +302,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err Account: ev.GetAccount(utils.META_DEFAULT), Subject: ev.GetSubject(utils.META_DEFAULT)} dcs := &utils.DerivedChargers{} if err := rs.GetDerivedChargers(attrsDC, dcs); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } dcs, _ = dcs.AppendDefaultRun() @@ -314,10 +323,12 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err } startTime, err := ev.GetSetupTime(utils.META_DEFAULT, rs.Timezone) if err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } usage, err := ev.GetDuration(utils.META_DEFAULT) if err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if usage == 0 { @@ -337,6 +348,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err err = rs.GetMaxSessionTime(cd, &remainingDuration) if err != nil { *reply = 0 + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if utils.IsSliceMember([]string{utils.META_POSTPAID, utils.POSTPAID}, ev.GetReqType(dc.ReqTypeField)) { // Only consider prepaid and pseudoprepaid for MaxSessionTime @@ -349,14 +361,22 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err maxCallDuration = remainingDuration } } + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: maxCallDuration}) *reply = maxCallDuration return nil } // Used by SM to get all the prepaid CallDescriptors attached to a session func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { + cacheKey := "GetSessionRuns" + ev.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *sRuns = *(item.Value.(*[]*SessionRun)) + return item.Err + } if rs.Bal != nil { - return errors.New("Unsupported method on the balancer") + err := errors.New("Unsupported method on the balancer") + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return err } if ev.Subject == "" { ev.Subject = ev.Account @@ -372,19 +392,19 @@ func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { Subject: ev.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, ev, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace user profile fields if err := LoadUserProfile(ev, utils.EXTRA_FIELDS); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } attrsDC := &utils.AttrDerivedChargers{Tenant: ev.GetTenant(utils.META_DEFAULT), Category: ev.GetCategory(utils.META_DEFAULT), Direction: ev.GetDirection(utils.META_DEFAULT), Account: ev.GetAccount(utils.META_DEFAULT), Subject: ev.GetSubject(utils.META_DEFAULT)} dcs := &utils.DerivedChargers{} if err := rs.GetDerivedChargers(attrsDC, dcs); err != nil { - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CgrId, &cache2go.CacheItem{ - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } dcs, _ = dcs.AppendDefaultRun() @@ -395,10 +415,9 @@ func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { } startTime, err := ev.GetAnswerTime(dc.AnswerTimeField, rs.Timezone) if err != nil { - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CgrId, &cache2go.CacheItem{ - Err: err, - }) - return errors.New("Error parsing answer event start time") + err := errors.New("Error parsing answer event start time") + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return err } cd := &CallDescriptor{ Direction: ev.GetDirection(dc.DirectionField), @@ -411,9 +430,7 @@ func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { sesRuns = append(sesRuns, &SessionRun{DerivedCharger: dc, CallDescriptor: cd}) } *sRuns = sesRuns - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CgrId, &cache2go.CacheItem{ - Value: sRuns, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: sRuns}) return nil } @@ -430,42 +447,51 @@ func (rs *Responder) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *u } func (rs *Responder) ProcessCdr(cdr *StoredCdr, reply *string) error { - if rs.CdrSrv == nil { - return errors.New("CDR_SERVER_NOT_RUNNING") + cacheKey := "ProcessCdr" + cdr.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *reply = *(item.Value.(*string)) + return item.Err } - if err := rs.CdrSrv.ProcessCdr(cdr); err != nil { + if rs.CdrSrv == nil { + err := errors.New("CDR_SERVER_NOT_RUNNING") + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } + if err := rs.CdrSrv.ProcessCdr(cdr); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return err + } + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) *reply = utils.OK return nil } func (rs *Responder) LogCallCost(ccl *CallCostLog, reply *string) error { - if item, err := rs.getCache().Get(utils.LOG_CALL_COST_CACHE_PREFIX + ccl.CgrId); err == nil && item != nil { + cacheKey := "LogCallCost" + ccl.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = item.Value.(string) return item.Err } if rs.CdrSrv == nil { err := errors.New("CDR_SERVER_NOT_RUNNING") - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if err := rs.CdrSrv.LogCallCost(ccl); err != nil { - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } *reply = utils.OK - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ - Value: utils.OK, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) return nil } func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { + cacheKey := "GetLCR" + attrs.CgrId + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *reply = *(item.Value.(*LCRCost)) + return item.Err + } if attrs.CallDescriptor.Subject == "" { attrs.CallDescriptor.Subject = attrs.CallDescriptor.Account } @@ -481,6 +507,7 @@ func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { Subject: cd.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, cd, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace user profile fields @@ -489,6 +516,7 @@ func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { } lcrCost, err := attrs.CallDescriptor.GetLCR(rs.Stats, attrs.Paginator) if err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if lcrCost.Entry.Strategy == LCR_STRATEGY_LOAD { @@ -496,6 +524,7 @@ func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { suppl.Cost = -1 // In case of load distribution we don't calculate costs } } + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: lcrCost}) *reply = *lcrCost return nil } diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index 2b46a8129..905239d70 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -373,6 +373,7 @@ func (fsev FSEvent) ComputeLcr() bool { // Converts into CallDescriptor due to responder interface needs func (fsev FSEvent) AsCallDescriptor() (*engine.CallDescriptor, error) { lcrReq := &engine.LcrRequest{ + Direction: fsev.GetDirection(utils.META_DEFAULT), Tenant: fsev.GetTenant(utils.META_DEFAULT), Category: fsev.GetCategory(utils.META_DEFAULT), diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 969dacf4e..11af53e00 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -100,6 +100,7 @@ func (sm *FSSessionManager) setCgrLcr(ev engine.Event, connId string) error { return err } cd := &engine.CallDescriptor{ + CgrId: ev.GetCgrId(sm.Timezone()), Direction: ev.GetDirection(utils.META_DEFAULT), Tenant: ev.GetTenant(utils.META_DEFAULT), Category: ev.GetCategory(utils.META_DEFAULT), @@ -148,6 +149,7 @@ func (sm *FSSessionManager) onChannelPark(ev engine.Event, connId string) { // ComputeLcr if ev.ComputeLcr() { cd, err := fsev.AsCallDescriptor() + cd.CgrId = fsev.GetCgrId(sm.Timezone()) if err != nil { utils.Logger.Info(fmt.Sprintf(" LCR_PREPROCESS_ERROR: %s", err.Error())) sm.unparkCall(ev.GetUUID(), connId, ev.GetCallDestNr(utils.META_DEFAULT), SYSTEM_ERROR) diff --git a/sessionmanager/kamailiosm.go b/sessionmanager/kamailiosm.go index e195bfe8c..26754a200 100644 --- a/sessionmanager/kamailiosm.go +++ b/sessionmanager/kamailiosm.go @@ -103,6 +103,7 @@ func (self *KamailioSessionManager) onCgrLcrReq(evData []byte, connId string) { func (self *KamailioSessionManager) getSuppliers(kev KamEvent) (string, error) { cd, err := kev.AsCallDescriptor() + cd.CgrId = kev.GetCgrId(self.timezone) if err != nil { utils.Logger.Info(fmt.Sprintf(" LCR_PREPROCESS_ERROR error: %s", err.Error())) return "", errors.New("LCR_PREPROCESS_ERROR") diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 82347807c..31ba88eeb 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -70,7 +70,7 @@ func NewSession(ev engine.Event, connId string, sm SessionManager) *Session { // the debit loop method (to be stoped by sending somenthing on stopDebit channel) func (s *Session) debitLoop(runIdx int) { nextCd := s.sessionRuns[runIdx].CallDescriptor - nextCd.CgrId = s.eventStart.GetCgrId("") + nextCd.CgrId = s.eventStart.GetCgrId(s.sessionManager.Timezone()) index := 0.0 debitPeriod := s.sessionManager.DebitInterval() for { @@ -195,6 +195,7 @@ func (s *Session) Refund(lastCC *engine.CallCost, hangupTime time.Time) error { // utils.Logger.Info(fmt.Sprintf("Refund duration: %v", initialRefundDuration-refundDuration)) if len(refundIncrements) > 0 { cd := &engine.CallDescriptor{ + CgrId: s.eventStart.GetCgrId(s.sessionManager.Timezone()), Direction: lastCC.Direction, Tenant: lastCC.Tenant, Category: lastCC.Category, diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 6546991a3..7d019dcef 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -145,6 +145,7 @@ func (self *SMGeneric) GetMaxUsage(gev SMGenericEvent, clnt *rpc2.Client) (time. func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([]string, error) { gev[utils.EVENT_NAME] = utils.CGR_LCR_REQUEST cd, err := gev.AsLcrRequest().AsCallDescriptor(self.timezone) + cd.CgrId = gev.GetCgrId(self.timezone) if err != nil { return nil, err } diff --git a/utils/consts.go b/utils/consts.go index 7024a73e5..8293abda4 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -231,10 +231,6 @@ const ( CGR_SUPPLIERS = "cgr_suppliers" KAM_FLATSTORE = "kamailio_flatstore" OSIPS_FLATSTORE = "opensips_flatstore" - MAX_DEBIT_CACHE_PREFIX = "MAX_DEBIT_" - REFUND_INCR_CACHE_PREFIX = "REFUND_INCR_" - GET_SESS_RUNS_CACHE_PREFIX = "GET_SESS_RUNS_" - LOG_CALL_COST_CACHE_PREFIX = "LOG_CALL_COSTS_" ALIAS_CONTEXT_RATING = "*rating" NOT_AVAILABLE = "N/A" CALL = "call" From 28853d935d1d8aebe9624b97914dff2e1c648621 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 27 Nov 2015 15:12:16 +0200 Subject: [PATCH 007/227] fixed session manager tests --- cmd/cgr-engine/rater.go | 6 +++--- sessionmanager/session_test.go | 6 +++--- test.sh | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index edef08c05..83aa71506 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -153,7 +153,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - engine.SetHistoryScribe(scribeServer) // ToDo: replace package sharing with connection based one + engine.SetHistoryScribe(scribeServer) }() } @@ -178,7 +178,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - engine.SetPubSub(pubSubServer) // ToDo: replace package sharing with connection based one + engine.SetPubSub(pubSubServer) }() } @@ -203,7 +203,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c exitChan <- true return } - engine.SetAliasService(aliasesServer) // ToDo: replace package sharing with connection based one + engine.SetAliasService(aliasesServer) }() } diff --git a/sessionmanager/session_test.go b/sessionmanager/session_test.go index 7b1c10292..e01d8c74d 100644 --- a/sessionmanager/session_test.go +++ b/sessionmanager/session_test.go @@ -92,7 +92,7 @@ func (mc *MockRpcClient) Call(methodName string, arg interface{}, reply interfac func TestSessionRefund(t *testing.T) { mc := &MockRpcClient{} - s := &Session{sessionManager: &FSSessionManager{rater: mc}} + s := &Session{sessionManager: &FSSessionManager{rater: mc, timezone: time.UTC.String()}, eventStart: FSEvent{SETUP_TIME: time.Now().Format(time.RFC3339)}} ts := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), TimeEnd: time.Date(2015, 6, 10, 14, 7, 30, 0, time.UTC), @@ -112,7 +112,7 @@ func TestSessionRefund(t *testing.T) { func TestSessionRefundAll(t *testing.T) { mc := &MockRpcClient{} - s := &Session{sessionManager: &FSSessionManager{rater: mc}} + s := &Session{sessionManager: &FSSessionManager{rater: mc, timezone: time.UTC.String()}, eventStart: FSEvent{SETUP_TIME: time.Now().Format(time.RFC3339)}} ts := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), TimeEnd: time.Date(2015, 6, 10, 14, 7, 30, 0, time.UTC), @@ -132,7 +132,7 @@ func TestSessionRefundAll(t *testing.T) { func TestSessionRefundManyAll(t *testing.T) { mc := &MockRpcClient{} - s := &Session{sessionManager: &FSSessionManager{rater: mc}} + s := &Session{sessionManager: &FSSessionManager{rater: mc, timezone: time.UTC.String()}, eventStart: FSEvent{SETUP_TIME: time.Now().Format(time.RFC3339)}} ts1 := &engine.TimeSpan{ TimeStart: time.Date(2015, 6, 10, 14, 7, 0, 0, time.UTC), TimeEnd: time.Date(2015, 6, 10, 14, 7, 30, 0, time.UTC), diff --git a/test.sh b/test.sh index 0c62795fc..e0c162a50 100755 --- a/test.sh +++ b/test.sh @@ -3,6 +3,7 @@ go test -i github.com/cgrates/cgrates/apier/v1 go test -i github.com/cgrates/cgrates/apier/v2 go test -i github.com/cgrates/cgrates/engine +go test -i github.com/cgrates/cgrates/general_tests go test -i github.com/cgrates/cgrates/sessionmanager go test -i github.com/cgrates/cgrates/config go test -i github.com/cgrates/cgrates/cmd/cgr-engine From 8490d8abcc7f43e70768fefb19faa7a028dc04f1 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 30 Nov 2015 20:18:35 +0200 Subject: [PATCH 008/227] updated tests --- engine/aliases_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/engine/aliases_test.go b/engine/aliases_test.go index 340735b59..dbbb7057b 100644 --- a/engine/aliases_test.go +++ b/engine/aliases_test.go @@ -47,7 +47,7 @@ func TestAliasesGetMatchingAlias(t *testing.T) { func TestAliasesSetters(t *testing.T) { var out string - if err := aliasService.SetAlias(Alias{ + if err := aliasService.Call("AliasesV1.SetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -63,7 +63,7 @@ func TestAliasesSetters(t *testing.T) { t.Error("Error setting alias: ", err, out) } r := &Alias{} - if err := aliasService.GetAlias(Alias{ + if err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -74,7 +74,7 @@ func TestAliasesSetters(t *testing.T) { t.Errorf("Error getting alias: %+v", r) } - if err := aliasService.UpdateAlias(Alias{ + if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -89,7 +89,7 @@ func TestAliasesSetters(t *testing.T) { }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } - if err := aliasService.GetAlias(Alias{ + if err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -103,7 +103,7 @@ func TestAliasesSetters(t *testing.T) { r.Values[0].Pairs["Account"]["1234"] != "1235" { t.Errorf("Error getting alias: %+v", r.Values[0]) } - if err := aliasService.UpdateAlias(Alias{ + if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -118,7 +118,7 @@ func TestAliasesSetters(t *testing.T) { }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } - if err := aliasService.GetAlias(Alias{ + if err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -128,7 +128,7 @@ func TestAliasesSetters(t *testing.T) { }, r); err != nil || len(r.Values) != 1 || len(r.Values[0].Pairs) != 2 || r.Values[0].Pairs["Subject"]["1111"] != "2222" { t.Errorf("Error getting alias: %+v", r.Values[0].Pairs["Subject"]) } - if err := aliasService.UpdateAlias(Alias{ + if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", @@ -143,7 +143,7 @@ func TestAliasesSetters(t *testing.T) { }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } - if err := aliasService.GetAlias(Alias{ + if err := aliasService.Call("AliasesV1.GetAlias", &Alias{ Direction: "*out", Tenant: "cgrates.org", Category: "call", From 58732e1a34aeabcf896c2050f95b31c7276724b5 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 30 Nov 2015 21:01:11 +0200 Subject: [PATCH 009/227] using rpc connection pool --- cmd/cgr-engine/cgr-engine.go | 44 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 13c28408d..c05f9f680 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -138,14 +138,15 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") - var raterConn, cdrsConn rpcclient.RpcClientConnection + raterConn := &rpcclient.RpcClientPool{} + cdrsConn := &rpcclient.RpcClientPool{} var client *rpcclient.RpcClient var err error // Connect to rater for _, raterCfg := range cfg.SmGenericConfig.HaRater { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections + raterConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -154,7 +155,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal exitChan <- true return } - raterConn = client + raterConn.AddClient(client) } } // Connect to CDRS @@ -164,7 +165,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal for _, cdrsCfg := range cfg.SmGenericConfig.HaCdrs { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - cdrsConn = resp + cdrsConn.AddClient(client) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -173,7 +174,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal exitChan <- true return } - cdrsConn = client + cdrsConn.AddClient(client) } } } @@ -226,14 +227,15 @@ func startDiameterAgent(internalSMGChan chan rpcclient.RpcClientConnection, exit func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-FreeSWITCH service.") - var raterConn, cdrsConn rpcclient.RpcClientConnection + raterConn := &rpcclient.RpcClientPool{} + cdrsConn := &rpcclient.RpcClientPool{} var client *rpcclient.RpcClient var err error // Connect to rater for _, raterCfg := range cfg.SmFsConfig.HaRater { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections + raterConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -242,7 +244,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd exitChan <- true return } - raterConn = client + raterConn.AddClient(client) } } // Connect to CDRS @@ -252,7 +254,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd for _, cdrsCfg := range cfg.SmFsConfig.HaCdrs { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - cdrsConn = resp + cdrsConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -261,7 +263,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd exitChan <- true return } - cdrsConn = client + cdrsConn.AddClient(client) } } } @@ -275,14 +277,15 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Kamailio service.") - var raterConn, cdrsConn rpcclient.RpcClientConnection + raterConn := &rpcclient.RpcClientPool{} + cdrsConn := &rpcclient.RpcClientPool{} var client *rpcclient.RpcClient var err error // Connect to rater for _, raterCfg := range cfg.SmKamConfig.HaRater { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections + raterConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -291,7 +294,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - raterConn = client + raterConn.AddClient(client) } } // Connect to CDRS @@ -301,7 +304,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS for _, cdrsCfg := range cfg.SmKamConfig.HaCdrs { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - cdrsConn = resp + cdrsConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -310,7 +313,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - cdrsConn = client + cdrsConn.AddClient(client) } } } @@ -324,14 +327,15 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-OpenSIPS service.") - var raterConn, cdrsConn rpcclient.RpcClientConnection + raterConn := &rpcclient.RpcClientPool{} + cdrsConn := &rpcclient.RpcClientPool{} var client *rpcclient.RpcClient var err error // Connect to rater for _, raterCfg := range cfg.SmOsipsConfig.HaRater { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections + raterConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -340,7 +344,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - raterConn = client + raterConn.AddClient(client) } } // Connect to CDRS @@ -350,7 +354,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS for _, cdrsCfg := range cfg.SmOsipsConfig.HaCdrs { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan - cdrsConn = resp + cdrsConn.AddClient(resp) internalRaterChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) @@ -359,7 +363,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true return } - cdrsConn = client + cdrsConn.AddClient(client) } } } From fd40b64f0245ae38b30c26d61198f22b1e364fb5 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 30 Nov 2015 21:14:05 +0200 Subject: [PATCH 010/227] use pointers as first args --- apier/v1/aliases.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index ea22fa0ed..93ebf639e 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -52,7 +52,7 @@ func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, als := engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Subject}, "Subject": map[string]string{alias: attrs.Subject}}, Weight: 10.0}}} - if err := aliases.Call("AliasesV1.SetAlias", als, &ignr); err != nil { + if err := aliases.Call("AliasesV1.SetAlias", &als, &ignr); err != nil { return utils.NewErrServerError(err) } } @@ -104,7 +104,7 @@ func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *strin als := engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Account}, "Subject": map[string]string{alias: attrs.Account}}, Weight: 10.0}}} - if err := aliases.Call("AliasesV1.SetAlias", als, &ignr); err != nil { + if err := aliases.Call("AliasesV1.SetAlias", &als, &ignr); err != nil { return utils.NewErrServerError(err) } } From d8d1ce2efcedf6d7efdd46924ec87ab6bb05a568 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 30 Nov 2015 22:00:05 +0200 Subject: [PATCH 011/227] unified Set and Update Aliases fixes #301 --- engine/aliases.go | 80 +++++++++++++++--------------- engine/aliases_test.go | 108 +++++++++++++++++++++++------------------ 2 files changed, 98 insertions(+), 90 deletions(-) diff --git a/engine/aliases.go b/engine/aliases.go index 55589ee99..d4fdc736b 100644 --- a/engine/aliases.go +++ b/engine/aliases.go @@ -167,62 +167,58 @@ func NewAliasHandler(accountingDb AccountingStorage) *AliasHandler { } } -func (am *AliasHandler) SetAlias(al *Alias, reply *string) error { - am.mu.Lock() - defer am.mu.Unlock() - - if err := am.accountingDb.SetAlias(al); err != nil { - *reply = err.Error() - return err - } //add to cache - - aliasesChanged := []string{utils.ALIASES_PREFIX + al.GetId()} - if err := am.accountingDb.CacheAccountingPrefixValues(map[string][]string{utils.ALIASES_PREFIX: aliasesChanged}); err != nil { - return utils.NewErrServerError(err) - } - *reply = utils.OK - return nil +type AttrAddAlias struct { + Alias *Alias + Overwrite bool } -func (am *AliasHandler) UpdateAlias(al *Alias, reply *string) error { +func (am *AliasHandler) SetAlias(attr *AttrAddAlias, reply *string) error { am.mu.Lock() defer am.mu.Unlock() - // get previous value - oldAlias, err := am.accountingDb.GetAlias(al.GetId(), false) - if err != nil { - return err + + var oldAlias *Alias + if !attr.Overwrite { // get previous value + oldAlias, _ = am.accountingDb.GetAlias(attr.Alias.GetId(), false) } - for _, value := range al.Values { - found := false - if value.DestinationId == "" { - value.DestinationId = utils.ANY + + if attr.Overwrite || oldAlias == nil { + if err := am.accountingDb.SetAlias(attr.Alias); err != nil { + *reply = err.Error() + return err } - for _, oldValue := range oldAlias.Values { - if oldValue.DestinationId == value.DestinationId { - for target, origAliasMap := range value.Pairs { - for orig, alias := range origAliasMap { - if oldValue.Pairs[target] == nil { - oldValue.Pairs[target] = make(map[string]string) + } else { + for _, value := range attr.Alias.Values { + found := false + if value.DestinationId == "" { + value.DestinationId = utils.ANY + } + for _, oldValue := range oldAlias.Values { + if oldValue.DestinationId == value.DestinationId { + for target, origAliasMap := range value.Pairs { + for orig, alias := range origAliasMap { + if oldValue.Pairs[target] == nil { + oldValue.Pairs[target] = make(map[string]string) + } + oldValue.Pairs[target][orig] = alias } - oldValue.Pairs[target][orig] = alias } + oldValue.Weight = value.Weight + found = true + break } - oldValue.Weight = value.Weight - found = true - break + } + if !found { + oldAlias.Values = append(oldAlias.Values, value) } } - if !found { - oldAlias.Values = append(oldAlias.Values, value) + if err := am.accountingDb.SetAlias(oldAlias); err != nil { + *reply = err.Error() + return err } } - if err := am.accountingDb.SetAlias(oldAlias); err != nil { - *reply = err.Error() - return err - } //add to cache - - aliasesChanged := []string{utils.ALIASES_PREFIX + al.GetId()} + //add to cache + aliasesChanged := []string{utils.ALIASES_PREFIX + attr.Alias.GetId()} if err := am.accountingDb.CacheAccountingPrefixValues(map[string][]string{utils.ALIASES_PREFIX: aliasesChanged}); err != nil { return utils.NewErrServerError(err) } diff --git a/engine/aliases_test.go b/engine/aliases_test.go index dbbb7057b..43708d128 100644 --- a/engine/aliases_test.go +++ b/engine/aliases_test.go @@ -47,18 +47,21 @@ func TestAliasesGetMatchingAlias(t *testing.T) { func TestAliasesSetters(t *testing.T) { var out string - if err := aliasService.Call("AliasesV1.SetAlias", &Alias{ - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "set", - Subject: "set", - Context: "*rating", - Values: AliasValues{&AliasValue{ - DestinationId: utils.ANY, - Pairs: AliasPairs{"Account": map[string]string{"1234": "1235"}}, - Weight: 10, - }}, + if err := aliasService.Call("AliasesV1.SetAlias", &AttrAddAlias{ + Alias: &Alias{ + Direction: "*out", + Tenant: "cgrates.org", + Category: "call", + Account: "set", + Subject: "set", + Context: "*rating", + Values: AliasValues{&AliasValue{ + DestinationId: utils.ANY, + Pairs: AliasPairs{"Account": map[string]string{"1234": "1235"}}, + Weight: 10, + }}, + }, + Overwrite: true, }, &out); err != nil || out != utils.OK { t.Error("Error setting alias: ", err, out) } @@ -74,18 +77,21 @@ func TestAliasesSetters(t *testing.T) { t.Errorf("Error getting alias: %+v", r) } - if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "set", - Subject: "set", - Context: "*rating", - Values: AliasValues{&AliasValue{ - DestinationId: utils.ANY, - Pairs: AliasPairs{"Subject": map[string]string{"1234": "1235"}}, - Weight: 10, - }}, + if err := aliasService.Call("AliasesV1.SetAlias", &AttrAddAlias{ + Alias: &Alias{ + Direction: "*out", + Tenant: "cgrates.org", + Category: "call", + Account: "set", + Subject: "set", + Context: "*rating", + Values: AliasValues{&AliasValue{ + DestinationId: utils.ANY, + Pairs: AliasPairs{"Subject": map[string]string{"1234": "1235"}}, + Weight: 10, + }}, + }, + Overwrite: false, }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } @@ -103,18 +109,21 @@ func TestAliasesSetters(t *testing.T) { r.Values[0].Pairs["Account"]["1234"] != "1235" { t.Errorf("Error getting alias: %+v", r.Values[0]) } - if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "set", - Subject: "set", - Context: "*rating", - Values: AliasValues{&AliasValue{ - DestinationId: utils.ANY, - Pairs: AliasPairs{"Subject": map[string]string{"1111": "2222"}}, - Weight: 10, - }}, + if err := aliasService.Call("AliasesV1.SetAlias", &AttrAddAlias{ + Alias: &Alias{ + Direction: "*out", + Tenant: "cgrates.org", + Category: "call", + Account: "set", + Subject: "set", + Context: "*rating", + Values: AliasValues{&AliasValue{ + DestinationId: utils.ANY, + Pairs: AliasPairs{"Subject": map[string]string{"1111": "2222"}}, + Weight: 10, + }}, + }, + Overwrite: false, }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } @@ -128,18 +137,21 @@ func TestAliasesSetters(t *testing.T) { }, r); err != nil || len(r.Values) != 1 || len(r.Values[0].Pairs) != 2 || r.Values[0].Pairs["Subject"]["1111"] != "2222" { t.Errorf("Error getting alias: %+v", r.Values[0].Pairs["Subject"]) } - if err := aliasService.Call("AliasesV1.UpdateAlias", &Alias{ - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "set", - Subject: "set", - Context: "*rating", - Values: AliasValues{&AliasValue{ - DestinationId: "NAT", - Pairs: AliasPairs{"Subject": map[string]string{"3333": "4444"}}, - Weight: 10, - }}, + if err := aliasService.Call("AliasesV1.SetAlias", &AttrAddAlias{ + Alias: &Alias{ + Direction: "*out", + Tenant: "cgrates.org", + Category: "call", + Account: "set", + Subject: "set", + Context: "*rating", + Values: AliasValues{&AliasValue{ + DestinationId: "NAT", + Pairs: AliasPairs{"Subject": map[string]string{"3333": "4444"}}, + Weight: 10, + }}, + }, + Overwrite: false, }, &out); err != nil || out != utils.OK { t.Error("Error updateing alias: ", err, out) } From 4aea0eb2a6e6c9993c28b27b712717c7a321e6d7 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 1 Dec 2015 14:26:00 +0200 Subject: [PATCH 012/227] engine using pool first strategy --- cmd/cgr-engine/cgr-engine.go | 16 ++++++++-------- cmd/cgr-engine/rater.go | 1 + engine/responder.go | 8 ++------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index c05f9f680..d5c221e77 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -138,8 +138,8 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") - raterConn := &rpcclient.RpcClientPool{} - cdrsConn := &rpcclient.RpcClientPool{} + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) var client *rpcclient.RpcClient var err error // Connect to rater @@ -227,8 +227,8 @@ func startDiameterAgent(internalSMGChan chan rpcclient.RpcClientConnection, exit func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-FreeSWITCH service.") - raterConn := &rpcclient.RpcClientPool{} - cdrsConn := &rpcclient.RpcClientPool{} + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) var client *rpcclient.RpcClient var err error // Connect to rater @@ -277,8 +277,8 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Kamailio service.") - raterConn := &rpcclient.RpcClientPool{} - cdrsConn := &rpcclient.RpcClientPool{} + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) var client *rpcclient.RpcClient var err error // Connect to rater @@ -327,8 +327,8 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-OpenSIPS service.") - raterConn := &rpcclient.RpcClientPool{} - cdrsConn := &rpcclient.RpcClientPool{} + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) var client *rpcclient.RpcClient var err error // Connect to rater diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 83aa71506..76239b479 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -238,6 +238,7 @@ func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan c } responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats} + responder.SetTimeToLive(cfg.ResponseCacheTTL) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer} apierRpcV2 := &v2.ApierV2{ diff --git a/engine/responder.go b/engine/responder.go index 8c228bf27..46e26a80a 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -57,12 +57,8 @@ type Responder struct { responseCache *cache2go.ResponseCache } -func NewResponder(exitChan chan bool, cdrSrv *CdrServer, stats rpcclient.RpcClientConnection, timeToLive time.Duration) *Responder { - return &Responder{ - ExitChan: exitChan, - Stats: stats, - responseCache: cache2go.NewResponseCache(timeToLive), - } +func (rs *Responder) SetTimeToLive(timeToLive time.Duration) { + rs.responseCache = cache2go.NewResponseCache(timeToLive) } func (rs *Responder) getCache() *cache2go.ResponseCache { From 7ce8936ad973d2a77b5dae84ba58ee096d82f289 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 2 Dec 2015 13:17:49 +0200 Subject: [PATCH 013/227] updated aliases console commands --- console/aliases_set.go | 6 ++-- console/aliases_update.go | 66 --------------------------------------- 2 files changed, 3 insertions(+), 69 deletions(-) delete mode 100644 console/aliases_update.go diff --git a/console/aliases_set.go b/console/aliases_set.go index 991836167..e091be6d3 100644 --- a/console/aliases_set.go +++ b/console/aliases_set.go @@ -27,7 +27,7 @@ func init() { c := &CmdSetAliases{ name: "aliases_set", rpcMethod: "AliasesV1.SetAlias", - rpcParams: &engine.Alias{Direction: utils.OUT}, + rpcParams: &engine.AttrAddAlias{Alias: &engine.Alias{Direction: utils.OUT}}, } commands[c.Name()] = c c.CommandExecuter = &CommandExecuter{c} @@ -37,7 +37,7 @@ func init() { type CmdSetAliases struct { name string rpcMethod string - rpcParams *engine.Alias + rpcParams *engine.AttrAddAlias *CommandExecuter } @@ -51,7 +51,7 @@ func (self *CmdSetAliases) RpcMethod() string { func (self *CmdSetAliases) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - self.rpcParams = &engine.Alias{Direction: utils.OUT} + self.rpcParams = &engine.AttrAddAlias{Alias: &engine.Alias{Direction: utils.OUT}} } return self.rpcParams } diff --git a/console/aliases_update.go b/console/aliases_update.go deleted file mode 100644 index 7063141be..000000000 --- a/console/aliases_update.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Rating system designed to be used in VoIP Carriers World -Copyright (C) 2012-2015 ITsysCOM - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdUpdateAliases{ - name: "aliases_update", - rpcMethod: "AliasesV1.UpdateAlias", - rpcParams: &engine.Alias{Direction: utils.OUT}, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -// Commander implementation -type CmdUpdateAliases struct { - name string - rpcMethod string - rpcParams *engine.Alias - *CommandExecuter -} - -func (self *CmdUpdateAliases) Name() string { - return self.name -} - -func (self *CmdUpdateAliases) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdUpdateAliases) RpcParams(reset bool) interface{} { - if reset || self.rpcParams == nil { - self.rpcParams = &engine.Alias{Direction: utils.OUT} - } - return self.rpcParams -} - -func (self *CmdUpdateAliases) PostprocessRpcParams() error { - return nil -} - -func (self *CmdUpdateAliases) RpcResult() interface{} { - var s string - return &s -} From faefc9d86e2ef9ceaad11e38c9e2f0d8ec215a4e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 2 Dec 2015 21:25:20 +0200 Subject: [PATCH 014/227] responder cache fixes --- apier/v1/aliases.go | 4 ++-- cache2go/response_cache.go | 8 +++++++- cmd/cgr-engine/rater.go | 2 +- engine/responder.go | 9 +++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index 93ebf639e..7800b6d72 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -52,7 +52,7 @@ func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, als := engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Subject}, "Subject": map[string]string{alias: attrs.Subject}}, Weight: 10.0}}} - if err := aliases.Call("AliasesV1.SetAlias", &als, &ignr); err != nil { + if err := aliases.Call("AliasesV1.SetAlias", &engine.AttrAddAlias{Alias: &als}, &ignr); err != nil { return utils.NewErrServerError(err) } } @@ -104,7 +104,7 @@ func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *strin als := engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING, Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY, Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Account}, "Subject": map[string]string{alias: attrs.Account}}, Weight: 10.0}}} - if err := aliases.Call("AliasesV1.SetAlias", &als, &ignr); err != nil { + if err := aliases.Call("AliasesV1.SetAlias", &engine.AttrAddAlias{Alias: &als}, &ignr); err != nil { return utils.NewErrServerError(err) } } diff --git a/cache2go/response_cache.go b/cache2go/response_cache.go index c27fdecbf..eba9dcf2e 100644 --- a/cache2go/response_cache.go +++ b/cache2go/response_cache.go @@ -54,10 +54,16 @@ func (rc *ResponseCache) Get(key string) (*CacheItem, error) { if rc.ttl == 0 { return nil, utils.ErrNotImplemented } + rc.mu.RLock() + item, ok := rc.cache[key] + rc.mu.RUnlock() + if ok { + return item, nil + } rc.wait(key) // wait for other goroutine processsing this key rc.mu.RLock() defer rc.mu.RUnlock() - item, ok := rc.cache[key] + item, ok = rc.cache[key] if !ok { return nil, ErrNotFound } diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 7ce1cd254..08b4f2de7 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -239,7 +239,7 @@ func startRater(internalRaterChan chan *engine.Responder, cacheDoneChan chan str } responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats} - responder.SetTimeToLive(cfg.ResponseCacheTTL) + responder.SetTimeToLive(cfg.ResponseCacheTTL, nil) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer} apierRpcV2 := &v2.ApierV2{ diff --git a/engine/responder.go b/engine/responder.go index 46e26a80a..ded672c1e 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -57,8 +57,9 @@ type Responder struct { responseCache *cache2go.ResponseCache } -func (rs *Responder) SetTimeToLive(timeToLive time.Duration) { +func (rs *Responder) SetTimeToLive(timeToLive time.Duration, out *int) error { rs.responseCache = cache2go.NewResponseCache(timeToLive) + return nil } func (rs *Responder) getCache() *cache2go.ResponseCache { @@ -263,7 +264,7 @@ func (rs *Responder) GetMaxSessionTime(arg *CallDescriptor, reply *float64) (err func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) error { cacheKey := "GetDerivedMaxSessionTime" + ev.CgrId if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - *reply = *(item.Value.(*float64)) + *reply = item.Value.(float64) return item.Err } if rs.Bal != nil { @@ -367,7 +368,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *StoredCdr, reply *float64) err func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { cacheKey := "GetSessionRuns" + ev.CgrId if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - *sRuns = *(item.Value.(*[]*SessionRun)) + *sRuns = item.Value.([]*SessionRun) return item.Err } if rs.Bal != nil { @@ -446,7 +447,7 @@ func (rs *Responder) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *u func (rs *Responder) ProcessCdr(cdr *StoredCdr, reply *string) error { cacheKey := "ProcessCdr" + cdr.CgrId if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - *reply = *(item.Value.(*string)) + *reply = item.Value.(string) return item.Err } if rs.CdrSrv == nil { From 735ad5b47e24b131eb51465081c85d655210be09 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 2 Dec 2015 21:45:21 +0200 Subject: [PATCH 015/227] fix getMaxDebit local test --- general_tests/tutorial_local_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 5cbf192df..5b0b5904b 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -381,6 +381,7 @@ func TestTutLocalMaxDebit(t *testing.T) { TimeStart: tStart, TimeEnd: tStart.Add(time.Duration(120) * time.Second), } + cd.CgrId = "1" if err := tutLocalRpc.Call("Responder.MaxDebit", cd, &cc); err != nil { t.Error("Got error on Responder.GetCost: ", err.Error()) } else if cc.GetDuration() == 120 { @@ -397,6 +398,7 @@ func TestTutLocalMaxDebit(t *testing.T) { TimeStart: tStart, TimeEnd: tStart.Add(time.Duration(120) * time.Second), } + cd.CgrId = "2" if err := tutLocalRpc.Call("Responder.MaxDebit", cd, &cc); err != nil { t.Error("Got error on Responder.GetCost: ", err.Error()) } else if cc.GetDuration() != time.Duration(62)*time.Second { // We have as strategy *dsconnect From ce222cf4468e81c06811c436a0f103bd0d2333e6 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 3 Dec 2015 11:34:18 +0200 Subject: [PATCH 016/227] updated config for ha --- config/config_defaults.go | 32 +++++++--- config/config_json_test.go | 52 +++++++++++----- config/libconfig_json.go | 21 ++++--- config/smconfig.go | 77 ++++++++++++++++++------ data/conf/cgrates/cgrates.json | 34 ++++++++--- data/conf/samples/dmtagent/cgrates.json | 8 ++- data/conf/samples/smgeneric/cgrates.json | 32 +++++++--- devel.yaml | 1 + 8 files changed, 190 insertions(+), 67 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 0ffd77b09..d8c3e153c 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -201,8 +201,12 @@ const CGRATES_CFG_JSON = ` "sm_generic": { "enabled": false, // starts SessionManager service: "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server <""|internal|x.y.z.y:1234> + "ha_rater": [ + {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + ], + "ha_cdrs": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], "debit_interval": "0s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this "max_call_duration": "3h", // maximum call duration a prepaid call can last @@ -211,8 +215,12 @@ const CGRATES_CFG_JSON = ` "sm_freeswitch": { "enabled": false, // starts SessionManager service: - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> + "ha_rater": [ + {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + ], + "ha_cdrs": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], "create_cdr": false, // create CDR out of events and sends them to CDRS component "extra_fields": [], // extra fields to store in auth/CDRs when creating them "debit_interval": "10s", // interval to perform debits on. @@ -232,8 +240,12 @@ const CGRATES_CFG_JSON = ` "sm_kamailio": { "enabled": false, // starts SessionManager service: - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> + "ha_rater": [ + {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + ], + "ha_cdrs": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], "create_cdr": false, // create CDR out of events and sends them to CDRS component "debit_interval": "10s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this @@ -247,8 +259,12 @@ const CGRATES_CFG_JSON = ` "sm_opensips": { "enabled": false, // starts SessionManager service: "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> + "ha_rater": [ + {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + ], + "ha_cdrs": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], "reconnects": 5, // number of reconnects if connection is lost "create_cdr": false, // create CDR out of events and sends them to CDRS component "debit_interval": "10s", // interval to perform debits on. diff --git a/config/config_json_test.go b/config/config_json_test.go index b618bb1c8..5784001c7 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -326,10 +326,16 @@ func TestDfCdrcJsonCfg(t *testing.T) { func TestSmGenericJsonCfg(t *testing.T) { eCfg := &SmGenericJsonCfg{ - Enabled: utils.BoolPointer(false), - Listen_bijson: utils.StringPointer("127.0.0.1:2014"), - Rater: utils.StringPointer("internal"), - Cdrs: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Listen_bijson: utils.StringPointer("127.0.0.1:2014"), + Ha_rater: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, + Ha_cdrs: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, Debit_interval: utils.StringPointer("0s"), Min_call_duration: utils.StringPointer("0s"), Max_call_duration: utils.StringPointer("3h"), @@ -343,9 +349,15 @@ func TestSmGenericJsonCfg(t *testing.T) { func TestSmFsJsonCfg(t *testing.T) { eCfg := &SmFsJsonCfg{ - Enabled: utils.BoolPointer(false), - Rater: utils.StringPointer("internal"), - Cdrs: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Ha_rater: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, + Ha_cdrs: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, Create_cdr: utils.BoolPointer(false), Extra_fields: utils.StringSlicePointer([]string{}), Debit_interval: utils.StringPointer("10s"), @@ -373,9 +385,15 @@ func TestSmFsJsonCfg(t *testing.T) { func TestSmKamJsonCfg(t *testing.T) { eCfg := &SmKamJsonCfg{ - Enabled: utils.BoolPointer(false), - Rater: utils.StringPointer("internal"), - Cdrs: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Ha_rater: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, + Ha_cdrs: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, Create_cdr: utils.BoolPointer(false), Debit_interval: utils.StringPointer("10s"), Min_call_duration: utils.StringPointer("0s"), @@ -396,10 +414,16 @@ func TestSmKamJsonCfg(t *testing.T) { func TestSmOsipsJsonCfg(t *testing.T) { eCfg := &SmOsipsJsonCfg{ - Enabled: utils.BoolPointer(false), - Listen_udp: utils.StringPointer("127.0.0.1:2020"), - Rater: utils.StringPointer("internal"), - Cdrs: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Listen_udp: utils.StringPointer("127.0.0.1:2020"), + Ha_rater: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, + Ha_cdrs: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, Create_cdr: utils.BoolPointer(false), Debit_interval: utils.StringPointer("10s"), Min_call_duration: utils.StringPointer("0s"), diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 82240478d..69fc4902c 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -164,8 +164,8 @@ type CdrcJsonCfg struct { type SmGenericJsonCfg struct { Enabled *bool Listen_bijson *string - Rater *string - Cdrs *string + Ha_rater *[]*HaPoolJsonCfg + Ha_cdrs *[]*HaPoolJsonCfg Debit_interval *string Min_call_duration *string Max_call_duration *string @@ -174,8 +174,8 @@ type SmGenericJsonCfg struct { // SM-FreeSWITCH config section type SmFsJsonCfg struct { Enabled *bool - Rater *string - Cdrs *string + Ha_rater *[]*HaPoolJsonCfg + Ha_cdrs *[]*HaPoolJsonCfg Create_cdr *bool Extra_fields *[]string Debit_interval *string @@ -190,6 +190,11 @@ type SmFsJsonCfg struct { Connections *[]*FsConnJsonCfg } +// Represents one connection instance towards a rater/cdrs server +type HaPoolJsonCfg struct { + Server *string +} + // Represents one connection instance towards FreeSWITCH type FsConnJsonCfg struct { Server *string @@ -200,8 +205,8 @@ type FsConnJsonCfg struct { // SM-Kamailio config section type SmKamJsonCfg struct { Enabled *bool - Rater *string - Cdrs *string + Ha_rater *[]*HaPoolJsonCfg + Ha_cdrs *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string Min_call_duration *string @@ -219,8 +224,8 @@ type KamConnJsonCfg struct { type SmOsipsJsonCfg struct { Enabled *bool Listen_udp *string - Rater *string - Cdrs *string + Ha_rater *[]*HaPoolJsonCfg + Ha_cdrs *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string Min_call_duration *string diff --git a/config/smconfig.go b/config/smconfig.go index b06ad7c20..82c1d14b6 100644 --- a/config/smconfig.go +++ b/config/smconfig.go @@ -35,8 +35,17 @@ func NewDfltHaPoolConfig() *HaPoolConfig { // One connection to Rater type HaPoolConfig struct { - Server string - Timeout time.Duration + Server string +} + +func (self *HaPoolConfig) loadFromJsonCfg(jsnCfg *HaPoolJsonCfg) error { + if jsnCfg == nil { + return nil + } + if jsnCfg.Server != nil { + self.Server = *jsnCfg.Server + } + return nil } // Returns the first cached default value for a SM-FreeSWITCH connection @@ -92,11 +101,19 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { if jsnCfg.Listen_bijson != nil { self.ListenBijson = *jsnCfg.Listen_bijson } - if jsnCfg.Rater != nil { - self.HaRater = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Rater, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_rater != nil { + self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) + for idx, jsnHaCfg := range *jsnCfg.Ha_rater { + self.HaRater[idx] = NewDfltHaPoolConfig() + self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnCfg.Cdrs != nil { - self.HaCdrs = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Cdrs, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_cdrs != nil { + self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) + for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { + self.HaCdrs[idx] = NewDfltHaPoolConfig() + self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Debit_interval != nil { if self.DebitInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Debit_interval); err != nil { @@ -142,11 +159,19 @@ func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Rater != nil { - self.HaRater = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Rater, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_rater != nil { + self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) + for idx, jsnHaCfg := range *jsnCfg.Ha_rater { + self.HaRater[idx] = NewDfltHaPoolConfig() + self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnCfg.Cdrs != nil { - self.HaCdrs = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Cdrs, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_cdrs != nil { + self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) + for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { + self.HaCdrs[idx] = NewDfltHaPoolConfig() + self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Create_cdr != nil { self.CreateCdr = *jsnCfg.Create_cdr @@ -251,11 +276,19 @@ func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Rater != nil { - self.HaRater = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Rater, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_rater != nil { + self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) + for idx, jsnHaCfg := range *jsnCfg.Ha_rater { + self.HaRater[idx] = NewDfltHaPoolConfig() + self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnCfg.Cdrs != nil { - self.HaCdrs = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Cdrs, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_cdrs != nil { + self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) + for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { + self.HaCdrs[idx] = NewDfltHaPoolConfig() + self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Create_cdr != nil { self.CreateCdr = *jsnCfg.Create_cdr @@ -323,11 +356,19 @@ func (self *SmOsipsConfig) loadFromJsonCfg(jsnCfg *SmOsipsJsonCfg) error { if jsnCfg.Listen_udp != nil { self.ListenUdp = *jsnCfg.Listen_udp } - if jsnCfg.Rater != nil { - self.HaRater = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Rater, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_rater != nil { + self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) + for idx, jsnHaCfg := range *jsnCfg.Ha_rater { + self.HaRater[idx] = NewDfltHaPoolConfig() + self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnCfg.Cdrs != nil { - self.HaCdrs = []*HaPoolConfig{&HaPoolConfig{Server: *jsnCfg.Cdrs, Timeout: time.Duration(1) * time.Second}} + if jsnCfg.Ha_cdrs != nil { + self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) + for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { + self.HaCdrs[idx] = NewDfltHaPoolConfig() + self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Create_cdr != nil { self.CreateCdr = *jsnCfg.Create_cdr diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index e35dd57c7..d31fa7e98 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -180,8 +180,12 @@ //"sm_generic": { // "enabled": false, // starts SessionManager service: // "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "debit_interval": "0s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this // "max_call_duration": "3h", // maximum call duration a prepaid call can last @@ -190,8 +194,12 @@ //"sm_freeswitch": { // "enabled": false, // starts SessionManager service: -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "extra_fields": [], // extra fields to store in auth/CDRs when creating them // "debit_interval": "10s", // interval to perform debits on. @@ -211,8 +219,12 @@ //"sm_kamailio": { // "enabled": false, // starts SessionManager service: -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "debit_interval": "10s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this @@ -226,8 +238,12 @@ //"sm_opensips": { // "enabled": false, // starts SessionManager service: // "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "reconnects": 5, // number of reconnects if connection is lost // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "debit_interval": "10s", // interval to perform debits on. @@ -334,4 +350,4 @@ // "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> //}, -} \ No newline at end of file +} diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index ff1387e85..d955b34fc 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -47,8 +47,12 @@ "sm_generic": { "enabled": true, - "rater": "internal", - "cdrs": "internal", + "ha_rater": [ + {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + ], + "ha_cdrs": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], }, "diameter_agent": { diff --git a/data/conf/samples/smgeneric/cgrates.json b/data/conf/samples/smgeneric/cgrates.json index 7907e7a47..385242808 100644 --- a/data/conf/samples/smgeneric/cgrates.json +++ b/data/conf/samples/smgeneric/cgrates.json @@ -179,8 +179,12 @@ "sm_generic": { "enabled": true, // starts SessionManager service: // "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "debit_interval": "10s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this // "max_call_duration": "3h", // maximum call duration a prepaid call can last @@ -189,8 +193,12 @@ //"sm_freeswitch": { // "enabled": false, // starts SessionManager service: -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "extra_fields": [], // extra fields to store in auth/CDRs when creating them // "debit_interval": "10s", // interval to perform debits on. @@ -210,8 +218,12 @@ //"sm_kamailio": { // "enabled": false, // starts SessionManager service: -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "debit_interval": "10s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this @@ -225,8 +237,12 @@ //"sm_opensips": { // "enabled": false, // starts SessionManager service: // "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> -// "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> +// "ha_rater": [ +// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> +// ], +// "ha_cdrs": [ +// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing +// ], // "reconnects": 5, // number of reconnects if connection is lost // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "debit_interval": "10s", // interval to perform debits on. diff --git a/devel.yaml b/devel.yaml index 3af8b74b5..d2a7d0226 100644 --- a/devel.yaml +++ b/devel.yaml @@ -2,6 +2,7 @@ package: github.com/cgrates/cgrates import: - package: github.com/ugorji/go - package: github.com/jinzhu/gorm + - package: github.com/jinzhu/inflection - package: golang.org/x/net - package: github.com/DisposaBoy/JsonConfigReader - package: github.com/go-sql-driver/mysql From 7ac4f7c38fdad9a31ef9a25e5d0370055e3dd57e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 3 Dec 2015 12:44:34 +0200 Subject: [PATCH 017/227] generated usagerecord id --- engine/storedcdr.go | 7 ++++++- engine/storedcdr_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/engine/storedcdr.go b/engine/storedcdr.go index 8c45f4c71..fba8ac848 100644 --- a/engine/storedcdr.go +++ b/engine/storedcdr.go @@ -762,7 +762,7 @@ type UsageRecord struct { func (self *UsageRecord) AsStoredCdr(timezone string) (*StoredCdr, error) { var err error - storedCdr := &StoredCdr{TOR: self.TOR, ReqType: self.ReqType, Direction: self.Direction, Tenant: self.Tenant, Category: self.Category, + storedCdr := &StoredCdr{CgrId: self.GetId(), TOR: self.TOR, ReqType: self.ReqType, Direction: self.Direction, Tenant: self.Tenant, Category: self.Category, Account: self.Account, Subject: self.Subject, Destination: self.Destination} if storedCdr.SetupTime, err = utils.ParseTimeDetectLayout(self.SetupTime, timezone); err != nil { return nil, err @@ -785,6 +785,7 @@ func (self *UsageRecord) AsStoredCdr(timezone string) (*StoredCdr, error) { func (self *UsageRecord) AsCallDescriptor(timezone string) (*CallDescriptor, error) { var err error cd := &CallDescriptor{ + CgrId: self.GetId(), TOR: self.TOR, Direction: self.Direction, Tenant: self.Tenant, @@ -813,3 +814,7 @@ func (self *UsageRecord) AsCallDescriptor(timezone string) (*CallDescriptor, err } return cd, nil } + +func (self *UsageRecord) GetId() string { + return utils.Sha1(self.TOR, self.ReqType, self.Direction, self.Tenant, self.Category, self.Account, self.Subject, self.Destination, self.SetupTime, self.AnswerTime, self.Usage) +} diff --git a/engine/storedcdr_test.go b/engine/storedcdr_test.go index 4a58aae1e..670c3bdee 100644 --- a/engine/storedcdr_test.go +++ b/engine/storedcdr_test.go @@ -570,7 +570,7 @@ func TestUsageReqAsCD(t *testing.T) { Account: "1001", Subject: "1001", Destination: "1002", SetupTime: "2013-11-07T08:42:20Z", AnswerTime: "2013-11-07T08:42:26Z", Usage: "0.00000001", } - eCD := &CallDescriptor{TOR: req.TOR, Direction: req.Direction, + eCD := &CallDescriptor{CgrId: "9473e7b2e075d168b9da10ae957ee68fe5a217e4", TOR: req.TOR, Direction: req.Direction, Tenant: req.Tenant, Category: req.Category, Account: req.Account, Subject: req.Subject, Destination: req.Destination, TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), TimeEnd: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).Add(time.Duration(10))} if cd, err := req.AsCallDescriptor(""); err != nil { From 44d893c17bfaa997fbc13ba236c9de29ed8f09cc Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 4 Dec 2015 17:04:42 +0200 Subject: [PATCH 018/227] ha for diameter agent renamed ha_rater/ha_cdrs to rater_conns/cdrs_conns --- agents/dmtagent.go | 4 +- cmd/cgr-engine/cgr-engine.go | 61 ++++++++------- config/config.go | 34 ++++----- config/config_defaults.go | 20 ++--- config/config_json_test.go | 35 +++++---- config/daconfig.go | 10 ++- config/libconfig_json.go | 24 +++--- config/smconfig.go | 96 ++++++++++++------------ data/conf/cgrates/cgrates.json | 20 ++--- data/conf/samples/dmtagent/cgrates.json | 6 +- data/conf/samples/smgeneric/cgrates.json | 18 ++--- 11 files changed, 172 insertions(+), 156 deletions(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index f8c6e4a67..5af5f2571 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -29,7 +29,7 @@ import ( "github.com/fiorix/go-diameter/diam/sm" ) -func NewDiameterAgent(cgrCfg *config.CGRConfig, smg *rpcclient.RpcClient) (*DiameterAgent, error) { +func NewDiameterAgent(cgrCfg *config.CGRConfig, smg rpcclient.RpcClientConnection) (*DiameterAgent, error) { da := &DiameterAgent{cgrCfg: cgrCfg, smg: smg} dictsDir := cgrCfg.DiameterAgentCfg().DictionariesDir if len(dictsDir) != 0 { @@ -42,7 +42,7 @@ func NewDiameterAgent(cgrCfg *config.CGRConfig, smg *rpcclient.RpcClient) (*Diam type DiameterAgent struct { cgrCfg *config.CGRConfig - smg *rpcclient.RpcClient // Connection towards CGR-SMG component + smg rpcclient.RpcClientConnection // Connection towards CGR-SMG component } // Creates the message handlers diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 5ac4753a4..04fe96494 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -143,7 +143,7 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal var client *rpcclient.RpcClient var err error // Connect to rater - for _, raterCfg := range cfg.SmGenericConfig.HaRater { + for _, raterCfg := range cfg.SmGenericConfig.RaterConns { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan raterConn.AddClient(resp) @@ -159,10 +159,10 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal } } // Connect to CDRS - if reflect.DeepEqual(cfg.SmGenericConfig.HaCdrs, cfg.SmGenericConfig.HaRater) { + if reflect.DeepEqual(cfg.SmGenericConfig.CdrsConns, cfg.SmGenericConfig.RaterConns) { cdrsConn = raterConn - } else if len(cfg.SmGenericConfig.HaCdrs) != 0 { - for _, cdrsCfg := range cfg.SmGenericConfig.HaCdrs { + } else if len(cfg.SmGenericConfig.CdrsConns) != 0 { + for _, cdrsCfg := range cfg.SmGenericConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan cdrsConn.AddClient(client) @@ -199,19 +199,24 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal func startDiameterAgent(internalSMGChan chan rpcclient.RpcClientConnection, exitChan chan bool) { utils.Logger.Info("Starting CGRateS DiameterAgent service.") - var smgConn *rpcclient.RpcClient + smgConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + var client *rpcclient.RpcClient var err error - if cfg.DiameterAgentCfg().SMGeneric == utils.INTERNAL { - smgRpc := <-internalSMGChan - internalSMGChan <- smgRpc - smgConn, err = rpcclient.NewRpcClient("", "", 0, 0, rpcclient.INTERNAL_RPC, smgRpc) - } else { - smgConn, err = rpcclient.NewRpcClient("tcp", cfg.DiameterAgentCfg().SMGeneric, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - } - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to SMG: %s", err.Error())) - exitChan <- true - return + for _, smgCfg := range cfg.DiameterAgentCfg().SMGenericConns { + if smgCfg.Server == utils.INTERNAL { + smgRpc := <-internalSMGChan + internalSMGChan <- smgRpc + client, _ = rpcclient.NewRpcClient("", "", 0, 0, rpcclient.INTERNAL_RPC, smgRpc) + smgConn.AddClient(client) + } else { + client, err = rpcclient.NewRpcClient("tcp", smgCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to SMG: %s", err.Error())) + exitChan <- true + return + } + smgConn.AddClient(client) + } } da, err := agents.NewDiameterAgent(cfg, smgConn) if err != nil { @@ -232,7 +237,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd var client *rpcclient.RpcClient var err error // Connect to rater - for _, raterCfg := range cfg.SmFsConfig.HaRater { + for _, raterCfg := range cfg.SmFsConfig.RaterConns { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan raterConn.AddClient(resp) @@ -248,10 +253,10 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd } } // Connect to CDRS - if reflect.DeepEqual(cfg.SmFsConfig.HaCdrs, cfg.SmFsConfig.HaRater) { + if reflect.DeepEqual(cfg.SmFsConfig.CdrsConns, cfg.SmFsConfig.RaterConns) { cdrsConn = raterConn - } else if len(cfg.SmFsConfig.HaCdrs) != 0 { - for _, cdrsCfg := range cfg.SmFsConfig.HaCdrs { + } else if len(cfg.SmFsConfig.CdrsConns) != 0 { + for _, cdrsCfg := range cfg.SmFsConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan cdrsConn.AddClient(resp) @@ -282,7 +287,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS var client *rpcclient.RpcClient var err error // Connect to rater - for _, raterCfg := range cfg.SmKamConfig.HaRater { + for _, raterCfg := range cfg.SmKamConfig.RaterConns { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan raterConn.AddClient(resp) @@ -298,10 +303,10 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS } } // Connect to CDRS - if reflect.DeepEqual(cfg.SmKamConfig.HaCdrs, cfg.SmKamConfig.HaRater) { + if reflect.DeepEqual(cfg.SmKamConfig.CdrsConns, cfg.SmKamConfig.RaterConns) { cdrsConn = raterConn - } else if len(cfg.SmKamConfig.HaCdrs) != 0 { - for _, cdrsCfg := range cfg.SmKamConfig.HaCdrs { + } else if len(cfg.SmKamConfig.CdrsConns) != 0 { + for _, cdrsCfg := range cfg.SmKamConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan cdrsConn.AddClient(resp) @@ -332,7 +337,7 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS var client *rpcclient.RpcClient var err error // Connect to rater - for _, raterCfg := range cfg.SmOsipsConfig.HaRater { + for _, raterCfg := range cfg.SmOsipsConfig.RaterConns { if raterCfg.Server == utils.INTERNAL { resp := <-internalRaterChan raterConn.AddClient(resp) @@ -348,10 +353,10 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS } } // Connect to CDRS - if reflect.DeepEqual(cfg.SmOsipsConfig.HaCdrs, cfg.SmOsipsConfig.HaRater) { + if reflect.DeepEqual(cfg.SmOsipsConfig.CdrsConns, cfg.SmOsipsConfig.RaterConns) { cdrsConn = raterConn - } else if len(cfg.SmOsipsConfig.HaCdrs) != 0 { - for _, cdrsCfg := range cfg.SmOsipsConfig.HaCdrs { + } else if len(cfg.SmOsipsConfig.CdrsConns) != 0 { + for _, cdrsCfg := range cfg.SmOsipsConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalRaterChan cdrsConn.AddClient(resp) diff --git a/config/config.go b/config/config.go index 0be3b935d..bafba8128 100644 --- a/config/config.go +++ b/config/config.go @@ -320,67 +320,67 @@ func (self *CGRConfig) checkConfigSanity() error { } // SM-Generic checks if self.SmGenericConfig.Enabled { - if len(self.SmGenericConfig.HaRater) == 0 { + if len(self.SmGenericConfig.RaterConns) == 0 { return errors.New("Rater definition is mandatory!") } - if len(self.SmGenericConfig.HaCdrs) == 0 { + if len(self.SmGenericConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmGenericConfig.HaRater[0].Server == utils.INTERNAL && !self.RaterEnabled { + if self.SmGenericConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { return errors.New("Rater not enabled but requested by SM-Generic component.") } - if self.SmGenericConfig.HaCdrs[0].Server == utils.INTERNAL && !self.CDRSEnabled { + if self.SmGenericConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { return errors.New("CDRS not enabled but referenced by SM-Generic component") } } // SM-FreeSWITCH checks if self.SmFsConfig.Enabled { - if len(self.SmFsConfig.HaRater) == 0 { + if len(self.SmFsConfig.RaterConns) == 0 { return errors.New("Rater definition is mandatory!") } - if len(self.SmFsConfig.HaCdrs) == 0 { + if len(self.SmFsConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmFsConfig.HaRater[0].Server == utils.INTERNAL && !self.RaterEnabled { + if self.SmFsConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { return errors.New("Rater not enabled but requested by SM-FreeSWITCH component.") } - if self.SmFsConfig.HaCdrs[0].Server == utils.INTERNAL && !self.CDRSEnabled { + if self.SmFsConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { return errors.New("CDRS not enabled but referenced by SM-FreeSWITCH component") } } // SM-Kamailio checks if self.SmKamConfig.Enabled { - if len(self.SmKamConfig.HaRater) == 0 { + if len(self.SmKamConfig.RaterConns) == 0 { return errors.New("Rater definition is mandatory!") } - if len(self.SmKamConfig.HaCdrs) == 0 { + if len(self.SmKamConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmKamConfig.HaRater[0].Server == utils.INTERNAL && !self.RaterEnabled { + if self.SmKamConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { return errors.New("Rater not enabled but requested by SM-Kamailio component.") } - if self.SmKamConfig.HaCdrs[0].Server == utils.INTERNAL && !self.CDRSEnabled { + if self.SmKamConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { return errors.New("CDRS not enabled but referenced by SM-Kamailio component") } } // SM-OpenSIPS checks if self.SmOsipsConfig.Enabled { - if len(self.SmOsipsConfig.HaRater) == 0 { + if len(self.SmOsipsConfig.RaterConns) == 0 { return errors.New("Rater definition is mandatory!") } - if len(self.SmOsipsConfig.HaCdrs) == 0 { + if len(self.SmOsipsConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmOsipsConfig.HaRater[0].Server == utils.INTERNAL && !self.RaterEnabled { + if self.SmOsipsConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { return errors.New("Rater not enabled but requested by SM-OpenSIPS component.") } - if self.SmOsipsConfig.HaCdrs[0].Server == utils.INTERNAL && !self.CDRSEnabled { + if self.SmOsipsConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { return errors.New("CDRS not enabled but referenced by SM-OpenSIPS component") } } // DAgent checks if self.diameterAgentCfg.Enabled { - if self.diameterAgentCfg.SMGeneric == utils.INTERNAL && !self.SmGenericConfig.Enabled { + if self.diameterAgentCfg.SMGenericConns[0].Server == utils.INTERNAL && !self.SmGenericConfig.Enabled { return errors.New("SMGeneric not enabled but referenced by DiameterAgent component") } } diff --git a/config/config_defaults.go b/config/config_defaults.go index d8c3e153c..2b9542248 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -201,10 +201,10 @@ const CGRATES_CFG_JSON = ` "sm_generic": { "enabled": false, // starts SessionManager service: "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests - "ha_rater": [ + "rater_conns": [ {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> ], - "ha_cdrs": [ + "cdrs_conns": [ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing ], "debit_interval": "0s", // interval to perform debits on. @@ -215,10 +215,10 @@ const CGRATES_CFG_JSON = ` "sm_freeswitch": { "enabled": false, // starts SessionManager service: - "ha_rater": [ + "rater_conns": [ {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> ], - "ha_cdrs": [ + "cdrs_conns": [ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing ], "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -240,10 +240,10 @@ const CGRATES_CFG_JSON = ` "sm_kamailio": { "enabled": false, // starts SessionManager service: - "ha_rater": [ + "rater_conns": [ {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> ], - "ha_cdrs": [ + "cdrs_conns": [ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing ], "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -259,10 +259,10 @@ const CGRATES_CFG_JSON = ` "sm_opensips": { "enabled": false, // starts SessionManager service: "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS - "ha_rater": [ + "rater_conns": [ {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> ], - "ha_cdrs": [ + "cdrs_conns": [ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing ], "reconnects": 5, // number of reconnects if connection is lost @@ -279,7 +279,9 @@ const CGRATES_CFG_JSON = ` "enabled": false, // enables the diameter agent: "listen": "127.0.0.1:3868", // address where to listen for diameter requests "dictionaries_dir": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load - "sm_generic": "internal", // connection towards SMG component for session management + "sm_generic_conns": [ + {"server": "internal"} // connection towards SMG component for session management + ], "debit_interval": "5m", // interval for CCR updates "timezone": "", // timezone for timestamps where not specified, empty for general defaults <""|UTC|Local|$IANA_TZ_DB> "dialect": "huawei", // the diameter dialect used in the communication, supported: diff --git a/config/config_json_test.go b/config/config_json_test.go index 5784001c7..9bb995366 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -328,11 +328,11 @@ func TestSmGenericJsonCfg(t *testing.T) { eCfg := &SmGenericJsonCfg{ Enabled: utils.BoolPointer(false), Listen_bijson: utils.StringPointer("127.0.0.1:2014"), - Ha_rater: &[]*HaPoolJsonCfg{ + Rater_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, - Ha_cdrs: &[]*HaPoolJsonCfg{ + Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, @@ -350,11 +350,11 @@ func TestSmGenericJsonCfg(t *testing.T) { func TestSmFsJsonCfg(t *testing.T) { eCfg := &SmFsJsonCfg{ Enabled: utils.BoolPointer(false), - Ha_rater: &[]*HaPoolJsonCfg{ + Rater_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, - Ha_cdrs: &[]*HaPoolJsonCfg{ + Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, @@ -386,11 +386,11 @@ func TestSmFsJsonCfg(t *testing.T) { func TestSmKamJsonCfg(t *testing.T) { eCfg := &SmKamJsonCfg{ Enabled: utils.BoolPointer(false), - Ha_rater: &[]*HaPoolJsonCfg{ + Rater_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, - Ha_cdrs: &[]*HaPoolJsonCfg{ + Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, @@ -416,11 +416,11 @@ func TestSmOsipsJsonCfg(t *testing.T) { eCfg := &SmOsipsJsonCfg{ Enabled: utils.BoolPointer(false), Listen_udp: utils.StringPointer("127.0.0.1:2020"), - Ha_rater: &[]*HaPoolJsonCfg{ + Rater_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, - Ha_cdrs: &[]*HaPoolJsonCfg{ + Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ Server: utils.StringPointer("internal"), }}, @@ -443,14 +443,17 @@ func TestDiameterAgentJsonCfg(t *testing.T) { Enabled: utils.BoolPointer(false), Listen: utils.StringPointer("127.0.0.1:3868"), Dictionaries_dir: utils.StringPointer("/usr/share/cgrates/diameter/dict/"), - Sm_generic: utils.StringPointer("internal"), - Debit_interval: utils.StringPointer("5m"), - Timezone: utils.StringPointer(""), - Dialect: utils.StringPointer("huawei"), - Origin_host: utils.StringPointer("CGR-DA"), - Origin_realm: utils.StringPointer("cgrates.org"), - Vendor_id: utils.IntPointer(0), - Product_name: utils.StringPointer("CGRateS"), + Sm_generic_conns: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, + Debit_interval: utils.StringPointer("5m"), + Timezone: utils.StringPointer(""), + Dialect: utils.StringPointer("huawei"), + Origin_host: utils.StringPointer("CGR-DA"), + Origin_realm: utils.StringPointer("cgrates.org"), + Vendor_id: utils.IntPointer(0), + Product_name: utils.StringPointer("CGRateS"), Request_processors: &[]*DARequestProcessorJsnCfg{ &DARequestProcessorJsnCfg{ Id: utils.StringPointer("*default"), diff --git a/config/daconfig.go b/config/daconfig.go index c358dfa8a..5e6f0a379 100644 --- a/config/daconfig.go +++ b/config/daconfig.go @@ -28,7 +28,7 @@ type DiameterAgentCfg struct { Enabled bool // enables the diameter agent: Listen string // address where to listen for diameter requests DictionariesDir string - SMGeneric string // connection towards SMG component + SMGenericConns []*HaPoolConfig // connections towards SMG component DebitInterval time.Duration Timezone string // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> Dialect string // the diameter dialect used in the implementation @@ -52,8 +52,12 @@ func (self *DiameterAgentCfg) loadFromJsonCfg(jsnCfg *DiameterAgentJsonCfg) erro if jsnCfg.Dictionaries_dir != nil { self.DictionariesDir = *jsnCfg.Dictionaries_dir } - if jsnCfg.Sm_generic != nil { - self.SMGeneric = *jsnCfg.Sm_generic + if jsnCfg.Sm_generic_conns != nil { + self.SMGenericConns = make([]*HaPoolConfig, len(*jsnCfg.Sm_generic_conns)) + for idx, jsnHaCfg := range *jsnCfg.Sm_generic_conns { + self.SMGenericConns[idx] = NewDfltHaPoolConfig() + self.SMGenericConns[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Debit_interval != nil { var err error diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 69fc4902c..fcaf6a2aa 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -164,8 +164,8 @@ type CdrcJsonCfg struct { type SmGenericJsonCfg struct { Enabled *bool Listen_bijson *string - Ha_rater *[]*HaPoolJsonCfg - Ha_cdrs *[]*HaPoolJsonCfg + Rater_conns *[]*HaPoolJsonCfg + Cdrs_conns *[]*HaPoolJsonCfg Debit_interval *string Min_call_duration *string Max_call_duration *string @@ -174,8 +174,8 @@ type SmGenericJsonCfg struct { // SM-FreeSWITCH config section type SmFsJsonCfg struct { Enabled *bool - Ha_rater *[]*HaPoolJsonCfg - Ha_cdrs *[]*HaPoolJsonCfg + Rater_conns *[]*HaPoolJsonCfg + Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Extra_fields *[]string Debit_interval *string @@ -205,8 +205,8 @@ type FsConnJsonCfg struct { // SM-Kamailio config section type SmKamJsonCfg struct { Enabled *bool - Ha_rater *[]*HaPoolJsonCfg - Ha_cdrs *[]*HaPoolJsonCfg + Rater_conns *[]*HaPoolJsonCfg + Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string Min_call_duration *string @@ -224,8 +224,8 @@ type KamConnJsonCfg struct { type SmOsipsJsonCfg struct { Enabled *bool Listen_udp *string - Ha_rater *[]*HaPoolJsonCfg - Ha_cdrs *[]*HaPoolJsonCfg + Rater_conns *[]*HaPoolJsonCfg + Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string Min_call_duration *string @@ -242,10 +242,10 @@ type OsipsConnJsonCfg struct { // DiameterAgent configuration type DiameterAgentJsonCfg struct { - Enabled *bool // enables the diameter agent: - Listen *string // address where to listen for diameter requests - Dictionaries_dir *string // path towards additional dictionaries - Sm_generic *string // Connection towards generic SM + Enabled *bool // enables the diameter agent: + Listen *string // address where to listen for diameter requests + Dictionaries_dir *string // path towards additional dictionaries + Sm_generic_conns *[]*HaPoolJsonCfg // Connections towards generic SM Debit_interval *string Timezone *string // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> Dialect *string diff --git a/config/smconfig.go b/config/smconfig.go index 82c1d14b6..1feebd934 100644 --- a/config/smconfig.go +++ b/config/smconfig.go @@ -83,8 +83,8 @@ func (self *FsConnConfig) loadFromJsonCfg(jsnCfg *FsConnJsonCfg) error { type SmGenericConfig struct { Enabled bool ListenBijson string - HaRater []*HaPoolConfig - HaCdrs []*HaPoolConfig + RaterConns []*HaPoolConfig + CdrsConns []*HaPoolConfig DebitInterval time.Duration MinCallDuration time.Duration MaxCallDuration time.Duration @@ -101,18 +101,18 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { if jsnCfg.Listen_bijson != nil { self.ListenBijson = *jsnCfg.Listen_bijson } - if jsnCfg.Ha_rater != nil { - self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) - for idx, jsnHaCfg := range *jsnCfg.Ha_rater { - self.HaRater[idx] = NewDfltHaPoolConfig() - self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rater_conns != nil { + self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rater_conns { + self.RaterConns[idx] = NewDfltHaPoolConfig() + self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) } } - if jsnCfg.Ha_cdrs != nil { - self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) - for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { - self.HaCdrs[idx] = NewDfltHaPoolConfig() - self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Cdrs_conns != nil { + self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { + self.CdrsConns[idx] = NewDfltHaPoolConfig() + self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Debit_interval != nil { @@ -135,8 +135,8 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { type SmFsConfig struct { Enabled bool - HaRater []*HaPoolConfig - HaCdrs []*HaPoolConfig + RaterConns []*HaPoolConfig + CdrsConns []*HaPoolConfig CreateCdr bool ExtraFields []*utils.RSRField DebitInterval time.Duration @@ -159,18 +159,18 @@ func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Ha_rater != nil { - self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) - for idx, jsnHaCfg := range *jsnCfg.Ha_rater { - self.HaRater[idx] = NewDfltHaPoolConfig() - self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rater_conns != nil { + self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rater_conns { + self.RaterConns[idx] = NewDfltHaPoolConfig() + self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) } } - if jsnCfg.Ha_cdrs != nil { - self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) - for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { - self.HaCdrs[idx] = NewDfltHaPoolConfig() - self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Cdrs_conns != nil { + self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { + self.CdrsConns[idx] = NewDfltHaPoolConfig() + self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { @@ -259,8 +259,8 @@ func (self *KamConnConfig) loadFromJsonCfg(jsnCfg *KamConnJsonCfg) error { // SM-Kamailio config section type SmKamConfig struct { Enabled bool - HaRater []*HaPoolConfig - HaCdrs []*HaPoolConfig + RaterConns []*HaPoolConfig + CdrsConns []*HaPoolConfig CreateCdr bool DebitInterval time.Duration MinCallDuration time.Duration @@ -276,18 +276,18 @@ func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Ha_rater != nil { - self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) - for idx, jsnHaCfg := range *jsnCfg.Ha_rater { - self.HaRater[idx] = NewDfltHaPoolConfig() - self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rater_conns != nil { + self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rater_conns { + self.RaterConns[idx] = NewDfltHaPoolConfig() + self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) } } - if jsnCfg.Ha_cdrs != nil { - self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) - for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { - self.HaCdrs[idx] = NewDfltHaPoolConfig() - self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Cdrs_conns != nil { + self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { + self.CdrsConns[idx] = NewDfltHaPoolConfig() + self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { @@ -338,8 +338,8 @@ func (self *OsipsConnConfig) loadFromJsonCfg(jsnCfg *OsipsConnJsonCfg) error { type SmOsipsConfig struct { Enabled bool ListenUdp string - HaRater []*HaPoolConfig - HaCdrs []*HaPoolConfig + RaterConns []*HaPoolConfig + CdrsConns []*HaPoolConfig CreateCdr bool DebitInterval time.Duration MinCallDuration time.Duration @@ -356,18 +356,18 @@ func (self *SmOsipsConfig) loadFromJsonCfg(jsnCfg *SmOsipsJsonCfg) error { if jsnCfg.Listen_udp != nil { self.ListenUdp = *jsnCfg.Listen_udp } - if jsnCfg.Ha_rater != nil { - self.HaRater = make([]*HaPoolConfig, len(*jsnCfg.Ha_rater)) - for idx, jsnHaCfg := range *jsnCfg.Ha_rater { - self.HaRater[idx] = NewDfltHaPoolConfig() - self.HaRater[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rater_conns != nil { + self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rater_conns { + self.RaterConns[idx] = NewDfltHaPoolConfig() + self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) } } - if jsnCfg.Ha_cdrs != nil { - self.HaCdrs = make([]*HaPoolConfig, len(*jsnCfg.Ha_cdrs)) - for idx, jsnHaCfg := range *jsnCfg.Ha_cdrs { - self.HaCdrs[idx] = NewDfltHaPoolConfig() - self.HaCdrs[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Cdrs_conns != nil { + self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { + self.CdrsConns[idx] = NewDfltHaPoolConfig() + self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index d31fa7e98..b1b8a4d0f 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -180,10 +180,10 @@ //"sm_generic": { // "enabled": false, // starts SessionManager service: // "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_conns": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "debit_interval": "0s", // interval to perform debits on. @@ -194,10 +194,10 @@ //"sm_freeswitch": { // "enabled": false, // starts SessionManager service: -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_conns": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -219,10 +219,10 @@ //"sm_kamailio": { // "enabled": false, // starts SessionManager service: -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_conns": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -238,10 +238,10 @@ //"sm_opensips": { // "enabled": false, // starts SessionManager service: // "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_conns": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "reconnects": 5, // number of reconnects if connection is lost @@ -258,7 +258,9 @@ // "enabled": false, // enables the diameter agent: // "listen": "127.0.0.1:3868", // address where to listen for diameter requests // "dictionaries_dir": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load -// "sm_generic": "internal", // connection towards SMG component for session management +// "sm_generic": [ +// {"server":"internal"} // connection towards SMG component for session management +// ] // "timezone": "", // timezone for timestamps where not specified, empty for general defaults <""|UTC|Local|$IANA_TZ_DB> // "origin_host": "CGR-DA", // diameter Origin-Host AVP used in replies // "origin_realm": "cgrates.org", // diameter Origin-Realm AVP used in replies diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index d955b34fc..005c8625c 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -47,12 +47,12 @@ "sm_generic": { "enabled": true, - "ha_rater": [ + "rater_conns": [ {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> ], - "ha_cdrs": [ + "rater_conns": [ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing - ], + ], }, "diameter_agent": { diff --git a/data/conf/samples/smgeneric/cgrates.json b/data/conf/samples/smgeneric/cgrates.json index 385242808..9a370c4dd 100644 --- a/data/conf/samples/smgeneric/cgrates.json +++ b/data/conf/samples/smgeneric/cgrates.json @@ -179,10 +179,10 @@ "sm_generic": { "enabled": true, // starts SessionManager service: // "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_conns": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "debit_interval": "10s", // interval to perform debits on. @@ -193,10 +193,10 @@ //"sm_freeswitch": { // "enabled": false, // starts SessionManager service: -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -218,10 +218,10 @@ //"sm_kamailio": { // "enabled": false, // starts SessionManager service: -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component @@ -237,10 +237,10 @@ //"sm_opensips": { // "enabled": false, // starts SessionManager service: // "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "ha_rater": [ +// "rater_conns": [ // {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> // ], -// "ha_cdrs": [ +// "cdrs_": [ // {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing // ], // "reconnects": 5, // number of reconnects if connection is lost @@ -313,4 +313,4 @@ // "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> //}, -} \ No newline at end of file +} From 09dd9299e3a3b0eb557f25f893562edd6929fd77 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 4 Dec 2015 17:14:32 +0200 Subject: [PATCH 019/227] broadcast for smg --- cmd/cgr-engine/cgr-engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 04fe96494..c21808016 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -138,8 +138,8 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") - raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) - cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) var client *rpcclient.RpcClient var err error // Connect to rater From a1d2f7166734d52ff9a352ee73c0c66395e3b1ef Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 7 Dec 2015 12:48:45 +0200 Subject: [PATCH 020/227] fix for scheduler_queue console command --- console/scheduler_queue.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/console/scheduler_queue.go b/console/scheduler_queue.go index a409c3a68..c70b9e547 100644 --- a/console/scheduler_queue.go +++ b/console/scheduler_queue.go @@ -24,7 +24,7 @@ func init() { c := &CmdGetScheduledActions{ name: "scheduler_queue", rpcMethod: "ApierV1.GetScheduledActions", - rpcParams: &v1.AttrsGetScheduledActions{}, + rpcParams: []*v1.AttrsGetScheduledActions{}, } commands[c.Name()] = c c.CommandExecuter = &CommandExecuter{c} @@ -34,7 +34,7 @@ func init() { type CmdGetScheduledActions struct { name string rpcMethod string - rpcParams *v1.AttrsGetScheduledActions + rpcParams []*v1.AttrsGetScheduledActions *CommandExecuter } @@ -48,7 +48,7 @@ func (self *CmdGetScheduledActions) RpcMethod() string { func (self *CmdGetScheduledActions) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - self.rpcParams = &v1.AttrsGetScheduledActions{} + self.rpcParams = make([]*v1.AttrsGetScheduledActions, 0) } return self.rpcParams } From 7e7ea02d11e3c5bbe8be5db0efcb9ad47350d6f0 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 7 Dec 2015 13:00:01 +0200 Subject: [PATCH 021/227] use pointer on scheduler action slice --- console/scheduler_queue.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/console/scheduler_queue.go b/console/scheduler_queue.go index c70b9e547..81af38f19 100644 --- a/console/scheduler_queue.go +++ b/console/scheduler_queue.go @@ -24,7 +24,7 @@ func init() { c := &CmdGetScheduledActions{ name: "scheduler_queue", rpcMethod: "ApierV1.GetScheduledActions", - rpcParams: []*v1.AttrsGetScheduledActions{}, + rpcParams: &[]*v1.AttrsGetScheduledActions{}, } commands[c.Name()] = c c.CommandExecuter = &CommandExecuter{c} @@ -34,7 +34,7 @@ func init() { type CmdGetScheduledActions struct { name string rpcMethod string - rpcParams []*v1.AttrsGetScheduledActions + rpcParams *[]*v1.AttrsGetScheduledActions *CommandExecuter } @@ -48,7 +48,8 @@ func (self *CmdGetScheduledActions) RpcMethod() string { func (self *CmdGetScheduledActions) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - self.rpcParams = make([]*v1.AttrsGetScheduledActions, 0) + scha := make([]*v1.AttrsGetScheduledActions, 0) + self.rpcParams = &scha } return self.rpcParams } From 2a3dd4a4af621bd1d4c88329cf9e90af13bc0e09 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 7 Dec 2015 15:33:11 +0200 Subject: [PATCH 022/227] rpc results fix only --- console/scheduler_queue.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/console/scheduler_queue.go b/console/scheduler_queue.go index 81af38f19..079575373 100644 --- a/console/scheduler_queue.go +++ b/console/scheduler_queue.go @@ -24,7 +24,7 @@ func init() { c := &CmdGetScheduledActions{ name: "scheduler_queue", rpcMethod: "ApierV1.GetScheduledActions", - rpcParams: &[]*v1.AttrsGetScheduledActions{}, + rpcParams: &v1.AttrsGetScheduledActions{}, } commands[c.Name()] = c c.CommandExecuter = &CommandExecuter{c} @@ -34,7 +34,7 @@ func init() { type CmdGetScheduledActions struct { name string rpcMethod string - rpcParams *[]*v1.AttrsGetScheduledActions + rpcParams *v1.AttrsGetScheduledActions *CommandExecuter } @@ -48,8 +48,7 @@ func (self *CmdGetScheduledActions) RpcMethod() string { func (self *CmdGetScheduledActions) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - scha := make([]*v1.AttrsGetScheduledActions, 0) - self.rpcParams = &scha + self.rpcParams = &v1.AttrsGetScheduledActions{} } return self.rpcParams } @@ -59,6 +58,6 @@ func (self *CmdGetScheduledActions) PostprocessRpcParams() error { } func (self *CmdGetScheduledActions) RpcResult() interface{} { - s := v1.ScheduledActions{} + s := make([]*v1.ScheduledActions, 0) return &s } From 6162e21d91593f287a487ae14bb6a9d45c881b7f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 11:56:02 +0200 Subject: [PATCH 023/227] compilation fix --- engine/cdrs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/cdrs.go b/engine/cdrs.go index 95aa2754c..eb16467f2 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -68,7 +68,7 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater rpcclient.RpcClientConnection, pubsub rpcclient.RpcClientConnection, users rpcclient.RpcClientConnection, aliases rpcclient.RpcClientConnection, stats rpcclient.RpcClientConnection) (*CdrServer, error) { - return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, rater: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{locksMap: make(map[string]chan bool)}}, nil + return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, client: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{locksMap: make(map[string]chan bool)}}, nil } type CdrServer struct { From dccb5d3cae557bb1f1fb3e7a6dc41d12e3790314 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 12:16:17 +0200 Subject: [PATCH 024/227] started hapool tests --- agents/hapool_it_test.go | 306 ++++++++++++++++++ .../samples/hapool/cgrrater1/cgradmin.json | 48 +++ .../samples/hapool/cgrrater2/cgradmin.json | 48 +++ .../conf/samples/hapool/cgrsmg1/cgradmin.json | 30 ++ .../conf/samples/hapool/cgrsmg2/cgradmin.json | 23 ++ 5 files changed, 455 insertions(+) create mode 100644 agents/hapool_it_test.go create mode 100644 data/conf/samples/hapool/cgrrater1/cgradmin.json create mode 100644 data/conf/samples/hapool/cgrrater2/cgradmin.json create mode 100644 data/conf/samples/hapool/cgrsmg1/cgradmin.json create mode 100644 data/conf/samples/hapool/cgrsmg2/cgradmin.json diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go new file mode 100644 index 000000000..0588793e8 --- /dev/null +++ b/agents/hapool_it_test.go @@ -0,0 +1,306 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can Storagetribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITH*out ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package agents + +import ( + "net/rpc/jsonrpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" + "github.com/fiorix/go-diameter/diam/dict" +) + +func TestHaPoolInitCfg(t *testing.T) { + if !*testIntegration { + return + } + daCfgPath = path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater1") + // Init config first + var err error + daCfg, err = config.NewCGRConfigFromFolder(daCfgPath) + if err != nil { + t.Error(err) + } + daCfg.DataFolderPath = *dataDir // Share DataFolderPath through config towards StoreDb for Flush() + config.SetCgrConfig(daCfg) +} + +// Remove data in both rating and accounting db +func TestHaPoolResetDataDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitDataDb(daCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func TestHaPoolResetStorDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitStorDb(daCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func TestHaPoolStartEngine(t *testing.T) { + if !*testIntegration { + return + } + engine.KillEngine(*waitRater) // just to make sure + + cgrRater1 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater1") + if _, err := engine.StartEngine(cgrRater1, *waitRater); err != nil { + t.Fatal("cgrRater1: ", err) + } + cgrRater2 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater2") + if _, err := engine.StartEngine(cgrRater2, *waitRater); err != nil { + t.Fatal("cgrRater2: ", err) + } + cgrSmg1 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrsmg1") + if _, err := engine.StartEngine(cgrSmg1, *waitRater); err != nil { + t.Fatal("cgrSmg1: ", err) + } + cgrSmg2 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrsmg2") + if _, err := engine.StartEngine(cgrSmg2, *waitRater); err != nil { + t.Fatal("cgrSmg2: ", err) + } + +} + +// Connect rpc client to rater +func TestHaPoolApierRpcConn(t *testing.T) { + if !*testIntegration { + return + } + var err error + apierRpc, err = jsonrpc.Dial("tcp", daCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +// Load the tariff plan, creating accounts and their balances +func TestHaPoolTPFromFolder(t *testing.T) { + if !*testIntegration { + return + } + attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")} + var loadInst engine.LoadInstance + if err := apierRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &loadInst); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups +} + +// cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:47:26Z"' +func TestHaPoolSendCCRInit(t *testing.T) { + if !*testIntegration { + return + } + dmtClient, err = NewDiameterClient(daCfg.DiameterAgentCfg().Listen, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, + daCfg.DiameterAgentCfg().VendorId, daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DictionariesDir) + if err != nil { + t.Fatal(err) + } + cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, + AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", + SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(0) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, + } + ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, + daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) + m, err := ccr.AsDiameterMessage() + if err != nil { + t.Error(err) + } + if err := dmtClient.SendMessage(m); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(100) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Granted-Service-Unit not found") + } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { + t.Errorf("Expecting 300, received: %s", strCCTime) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 9.484 + if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + +// cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:52:26Z"' +func TestHaPoolSendCCRUpdate(t *testing.T) { + if !*testIntegration { + return + } + cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, + AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", + SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(300) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, + } + ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, + daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) + m, err := ccr.AsDiameterMessage() + if err != nil { + t.Error(err) + } + if err := dmtClient.SendMessage(m); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(100) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Granted-Service-Unit not found") + } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { + t.Errorf("Expecting 300, received: %s", strCCTime) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 9.214 + if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + +// cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:57:26Z"' +func TestHaPoolSendCCRUpdate2(t *testing.T) { + if !*testIntegration { + return + } + cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, + AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", + SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(600) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, + } + ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, + daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) + m, err := ccr.AsDiameterMessage() + if err != nil { + t.Error(err) + } + if err := dmtClient.SendMessage(m); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(100) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Granted-Service-Unit not found") + } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { + t.Errorf("Expecting 300, received: %s", strCCTime) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 8.944 + if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + +func TestHaPoolSendCCRTerminate(t *testing.T) { + if !*testIntegration { + return + } + cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, + AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", + SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(610) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, + } + ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, + daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, true) + m, err := ccr.AsDiameterMessage() + if err != nil { + t.Error(err) + } + if err := dmtClient.SendMessage(m); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(100) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Granted-Service-Unit not found") + } else if strCCTime := avpValAsString(avps[0]); strCCTime != "0" { + t.Errorf("Expecting 0, received: %s", strCCTime) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 9.205 + if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { // Should also consider derived charges which double the cost of 6m10s - 2x0.7584 + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + +func TestHaPoolCdrs(t *testing.T) { + if !*testIntegration { + return + } + var cdrs []*engine.ExternalCdr + req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}} + if err := apierRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Usage != "610" { + t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) + } + if cdrs[0].Cost != 0.795 { + t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) + } + } +} + +func TestHaPoolStopEngine(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.KillEngine(*waitRater); err != nil { + t.Error(err) + } +} diff --git a/data/conf/samples/hapool/cgrrater1/cgradmin.json b/data/conf/samples/hapool/cgrrater1/cgradmin.json new file mode 100644 index 000000000..4c55524fd --- /dev/null +++ b/data/conf/samples/hapool/cgrrater1/cgradmin.json @@ -0,0 +1,48 @@ +{ +// CGRateS Configuration file +// +// Used for cgradmin +// Starts rater, scheduler + +"listen": { + "rpc_json": ":2014", // RPC JSON listening address + "rpc_gob": ":2015", // RPC GOB listening address + "http": ":2081", // HTTP listening address +}, + +"rater": { + "enabled": true, // enable Rater service: + "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> + "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> + "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> + "aliases": "internal", +}, + +"scheduler": { + "enabled": true, // start Scheduler service: +}, + +"cdrs": { + "enabled": true, // start the CDR Server service: + "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> +}, + +"cdrstats": { + "enabled": true, // starts the cdrstats service: +}, + +"pubsubs": { + "enabled": true, // starts PubSub service: . +}, + +"aliases": { + "enabled": true, // starts Aliases service: . +}, + +"users": { + "enabled": true, + "indexes": ["SubscriberId"], +}, + +} diff --git a/data/conf/samples/hapool/cgrrater2/cgradmin.json b/data/conf/samples/hapool/cgrrater2/cgradmin.json new file mode 100644 index 000000000..846b4affb --- /dev/null +++ b/data/conf/samples/hapool/cgrrater2/cgradmin.json @@ -0,0 +1,48 @@ +{ +// CGRateS Configuration file +// +// Used for cgradmin +// Starts rater, scheduler + +"listen": { + "rpc_json": ":2016", // RPC JSON listening address + "rpc_gob": ":2017", // RPC GOB listening address + "http": ":2082", // HTTP listening address +}, + +"rater": { + "enabled": true, // enable Rater service: + "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> + "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> + "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> + "aliases": "internal", +}, + +"scheduler": { + "enabled": true, // start Scheduler service: +}, + +"cdrs": { + "enabled": true, // start the CDR Server service: + "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> +}, + +"cdrstats": { + "enabled": true, // starts the cdrstats service: +}, + +"pubsubs": { + "enabled": true, // starts PubSub service: . +}, + +"aliases": { + "enabled": true, // starts Aliases service: . +}, + +"users": { + "enabled": true, + "indexes": ["SubscriberId"], +}, + +} diff --git a/data/conf/samples/hapool/cgrsmg1/cgradmin.json b/data/conf/samples/hapool/cgrsmg1/cgradmin.json new file mode 100644 index 000000000..f224c3c44 --- /dev/null +++ b/data/conf/samples/hapool/cgrsmg1/cgradmin.json @@ -0,0 +1,30 @@ +{ +// CGRateS Configuration file +// +// Used for cgradmin +// Starts rater, scheduler + + +"cdrs": { + "enabled": true, // start the CDR Server service: + "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> +}, + +"sm_generic": { + "enabled": true, + "rater_conns": [ + {"server": "127.0.0.1:2014"}, + {"server": "127.0.0.1:2016"} + ], + "cdrs_conns": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], +}, + +"diameter_agent": { + "enabled": true, +}, + + +} diff --git a/data/conf/samples/hapool/cgrsmg2/cgradmin.json b/data/conf/samples/hapool/cgrsmg2/cgradmin.json new file mode 100644 index 000000000..db742c071 --- /dev/null +++ b/data/conf/samples/hapool/cgrsmg2/cgradmin.json @@ -0,0 +1,23 @@ +{ +// CGRateS Configuration file +// +// Used for cgradmin +// Starts rater, scheduler + +"sm_generic": { + "enabled": true, + "rater_conns": [ + {"server": "127.0.0.1:2014"}, + {"server": "127.0.0.1:2016"} + ], + "cdrs_conns": [ + {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + ], +}, + +"diameter_agent": { + "enabled": true, +}, + + +} \ No newline at end of file From a6448cf32094b794e91d5bde00f7f517256c498d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 13:43:34 +0200 Subject: [PATCH 025/227] updated conf fo second generic session manager --- data/conf/samples/hapool/cgrsmg2/cgradmin.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/data/conf/samples/hapool/cgrsmg2/cgradmin.json b/data/conf/samples/hapool/cgrsmg2/cgradmin.json index db742c071..f224c3c44 100644 --- a/data/conf/samples/hapool/cgrsmg2/cgradmin.json +++ b/data/conf/samples/hapool/cgrsmg2/cgradmin.json @@ -4,6 +4,13 @@ // Used for cgradmin // Starts rater, scheduler + +"cdrs": { + "enabled": true, // start the CDR Server service: + "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> +}, + "sm_generic": { "enabled": true, "rater_conns": [ @@ -20,4 +27,4 @@ }, -} \ No newline at end of file +} From d91dc98ebcf68496a67bcc5c7ffe72b1021c2940 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 16:42:46 +0200 Subject: [PATCH 026/227] more hapool tests conf files --- agents/hapool_it_test.go | 4 ++++ .../hapool/cgrrater1/{cgradmin.json => cgr.json} | 5 ----- .../hapool/cgrrater2/{cgradmin.json => cgr.json} | 4 ---- .../{cgrsmg2/cgradmin.json => cgrsmg1/cgr.json} | 13 +------------ .../{cgrsmg1/cgradmin.json => cgrsmg2/cgr.json} | 12 +----------- data/conf/samples/hapool/dagent/cgr.json | 10 ++++++++++ 6 files changed, 16 insertions(+), 32 deletions(-) rename data/conf/samples/hapool/cgrrater1/{cgradmin.json => cgr.json} (94%) rename data/conf/samples/hapool/cgrrater2/{cgradmin.json => cgr.json} (94%) rename data/conf/samples/hapool/{cgrsmg2/cgradmin.json => cgrsmg1/cgr.json} (83%) rename data/conf/samples/hapool/{cgrsmg1/cgradmin.json => cgrsmg2/cgr.json} (83%) create mode 100644 data/conf/samples/hapool/dagent/cgr.json diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go index 0588793e8..08d788528 100644 --- a/agents/hapool_it_test.go +++ b/agents/hapool_it_test.go @@ -88,6 +88,10 @@ func TestHaPoolStartEngine(t *testing.T) { if _, err := engine.StartEngine(cgrSmg2, *waitRater); err != nil { t.Fatal("cgrSmg2: ", err) } + cgrDa := path.Join(*dataDir, "conf", "samples", "hapool", "dagent") + if _, err := engine.StartEngine(cgrDa, *waitRater); err != nil { + t.Fatal("cgrSmg2: ", err) + } } diff --git a/data/conf/samples/hapool/cgrrater1/cgradmin.json b/data/conf/samples/hapool/cgrrater1/cgr.json similarity index 94% rename from data/conf/samples/hapool/cgrrater1/cgradmin.json rename to data/conf/samples/hapool/cgrrater1/cgr.json index 4c55524fd..74af2d95c 100644 --- a/data/conf/samples/hapool/cgrrater1/cgradmin.json +++ b/data/conf/samples/hapool/cgrrater1/cgr.json @@ -1,9 +1,4 @@ { -// CGRateS Configuration file -// -// Used for cgradmin -// Starts rater, scheduler - "listen": { "rpc_json": ":2014", // RPC JSON listening address "rpc_gob": ":2015", // RPC GOB listening address diff --git a/data/conf/samples/hapool/cgrrater2/cgradmin.json b/data/conf/samples/hapool/cgrrater2/cgr.json similarity index 94% rename from data/conf/samples/hapool/cgrrater2/cgradmin.json rename to data/conf/samples/hapool/cgrrater2/cgr.json index 846b4affb..00c31a61b 100644 --- a/data/conf/samples/hapool/cgrrater2/cgradmin.json +++ b/data/conf/samples/hapool/cgrrater2/cgr.json @@ -1,8 +1,4 @@ { -// CGRateS Configuration file -// -// Used for cgradmin -// Starts rater, scheduler "listen": { "rpc_json": ":2016", // RPC JSON listening address diff --git a/data/conf/samples/hapool/cgrsmg2/cgradmin.json b/data/conf/samples/hapool/cgrsmg1/cgr.json similarity index 83% rename from data/conf/samples/hapool/cgrsmg2/cgradmin.json rename to data/conf/samples/hapool/cgrsmg1/cgr.json index f224c3c44..a6d002f38 100644 --- a/data/conf/samples/hapool/cgrsmg2/cgradmin.json +++ b/data/conf/samples/hapool/cgrsmg1/cgr.json @@ -1,10 +1,4 @@ { -// CGRateS Configuration file -// -// Used for cgradmin -// Starts rater, scheduler - - "cdrs": { "enabled": true, // start the CDR Server service: "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> @@ -13,6 +7,7 @@ "sm_generic": { "enabled": true, + "listen_bijson": "127.0.0.1:3014", "rater_conns": [ {"server": "127.0.0.1:2014"}, {"server": "127.0.0.1:2016"} @@ -21,10 +16,4 @@ {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing ], }, - -"diameter_agent": { - "enabled": true, -}, - - } diff --git a/data/conf/samples/hapool/cgrsmg1/cgradmin.json b/data/conf/samples/hapool/cgrsmg2/cgr.json similarity index 83% rename from data/conf/samples/hapool/cgrsmg1/cgradmin.json rename to data/conf/samples/hapool/cgrsmg2/cgr.json index f224c3c44..400a47eda 100644 --- a/data/conf/samples/hapool/cgrsmg1/cgradmin.json +++ b/data/conf/samples/hapool/cgrsmg2/cgr.json @@ -1,10 +1,4 @@ { -// CGRateS Configuration file -// -// Used for cgradmin -// Starts rater, scheduler - - "cdrs": { "enabled": true, // start the CDR Server service: "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> @@ -13,6 +7,7 @@ "sm_generic": { "enabled": true, + "listen_bijson": "127.0.0.1:3015", "rater_conns": [ {"server": "127.0.0.1:2014"}, {"server": "127.0.0.1:2016"} @@ -22,9 +17,4 @@ ], }, -"diameter_agent": { - "enabled": true, -}, - - } diff --git a/data/conf/samples/hapool/dagent/cgr.json b/data/conf/samples/hapool/dagent/cgr.json new file mode 100644 index 000000000..6a3bd6388 --- /dev/null +++ b/data/conf/samples/hapool/dagent/cgr.json @@ -0,0 +1,10 @@ +{ +"diameter_agent": { + "enabled": true, + "listen": "127.0.0.1:3868", + "sm_generic_conns": [ + {"server": "127.0.0.1:3014"}, + {"server": "127.0.0.1:3015"}, + ], +}, +} \ No newline at end of file From 7a72d1d42d2d610887376bc9a9ab73fc865eab20 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 20:23:56 +0200 Subject: [PATCH 027/227] updated listening ports --- agents/hapool_it_test.go | 17 ++++++++++------- data/conf/samples/hapool/cgrsmg1/cgr.json | 7 ++++++- data/conf/samples/hapool/cgrsmg2/cgr.json | 7 ++++++- data/conf/samples/hapool/dagent/cgr.json | 6 +++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go index 08d788528..6ca260be6 100644 --- a/agents/hapool_it_test.go +++ b/agents/hapool_it_test.go @@ -20,6 +20,7 @@ package agents import ( "net/rpc/jsonrpc" + "os/exec" "path" "testing" "time" @@ -30,6 +31,8 @@ import ( "github.com/fiorix/go-diameter/diam/dict" ) +var cgrRater1Cmd, cgrSmg1Cmd *exec.Cmd + func TestHaPoolInitCfg(t *testing.T) { if !*testIntegration { return @@ -71,26 +74,26 @@ func TestHaPoolStartEngine(t *testing.T) { return } engine.KillEngine(*waitRater) // just to make sure - + var err error cgrRater1 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater1") - if _, err := engine.StartEngine(cgrRater1, *waitRater); err != nil { + if cgrRater1Cmd, err = engine.StartEngine(cgrRater1, *waitRater); err != nil { t.Fatal("cgrRater1: ", err) } cgrRater2 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater2") - if _, err := engine.StartEngine(cgrRater2, *waitRater); err != nil { + if _, err = engine.StartEngine(cgrRater2, *waitRater); err != nil { t.Fatal("cgrRater2: ", err) } cgrSmg1 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrsmg1") - if _, err := engine.StartEngine(cgrSmg1, *waitRater); err != nil { + if cgrSmg1Cmd, err = engine.StartEngine(cgrSmg1, *waitRater); err != nil { t.Fatal("cgrSmg1: ", err) } cgrSmg2 := path.Join(*dataDir, "conf", "samples", "hapool", "cgrsmg2") - if _, err := engine.StartEngine(cgrSmg2, *waitRater); err != nil { + if _, err = engine.StartEngine(cgrSmg2, *waitRater); err != nil { t.Fatal("cgrSmg2: ", err) } cgrDa := path.Join(*dataDir, "conf", "samples", "hapool", "dagent") - if _, err := engine.StartEngine(cgrDa, *waitRater); err != nil { - t.Fatal("cgrSmg2: ", err) + if _, err = engine.StartEngine(cgrDa, *waitRater); err != nil { + t.Fatal("cgrDa: ", err) } } diff --git a/data/conf/samples/hapool/cgrsmg1/cgr.json b/data/conf/samples/hapool/cgrsmg1/cgr.json index a6d002f38..71d9e4cb0 100644 --- a/data/conf/samples/hapool/cgrsmg1/cgr.json +++ b/data/conf/samples/hapool/cgrsmg1/cgr.json @@ -1,4 +1,10 @@ { +"listen": { + "rpc_json": "127.0.0.1:2018", // RPC JSON listening address + "rpc_gob": "127.0.0.1:2019", // RPC GOB listening address + "http": "127.0.0.1:2083", // HTTP listening address +}, + "cdrs": { "enabled": true, // start the CDR Server service: "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> @@ -7,7 +13,6 @@ "sm_generic": { "enabled": true, - "listen_bijson": "127.0.0.1:3014", "rater_conns": [ {"server": "127.0.0.1:2014"}, {"server": "127.0.0.1:2016"} diff --git a/data/conf/samples/hapool/cgrsmg2/cgr.json b/data/conf/samples/hapool/cgrsmg2/cgr.json index 400a47eda..e0d98bc7f 100644 --- a/data/conf/samples/hapool/cgrsmg2/cgr.json +++ b/data/conf/samples/hapool/cgrsmg2/cgr.json @@ -1,4 +1,10 @@ { +"listen": { + "rpc_json": "127.0.0.1:2020", // RPC JSON listening address + "rpc_gob": "127.0.0.1:2021", // RPC GOB listening address + "http": "127.0.0.1:2084", // HTTP listening address +}, + "cdrs": { "enabled": true, // start the CDR Server service: "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> @@ -7,7 +13,6 @@ "sm_generic": { "enabled": true, - "listen_bijson": "127.0.0.1:3015", "rater_conns": [ {"server": "127.0.0.1:2014"}, {"server": "127.0.0.1:2016"} diff --git a/data/conf/samples/hapool/dagent/cgr.json b/data/conf/samples/hapool/dagent/cgr.json index 6a3bd6388..164f9bfec 100644 --- a/data/conf/samples/hapool/dagent/cgr.json +++ b/data/conf/samples/hapool/dagent/cgr.json @@ -3,8 +3,8 @@ "enabled": true, "listen": "127.0.0.1:3868", "sm_generic_conns": [ - {"server": "127.0.0.1:3014"}, - {"server": "127.0.0.1:3015"}, + {"server": "127.0.0.1:3018"}, + {"server": "127.0.0.1:3020"}, ], }, -} \ No newline at end of file +} From 2f66d81ff953ed01a23fe23dbff509d09316bac3 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 21 Dec 2015 19:27:40 +0100 Subject: [PATCH 028/227] Channels for CDRS sent to SM --- cmd/cgr-engine/cgr-engine.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 5131850d2..370af7e54 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -136,7 +136,7 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * } } -func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, server *utils.Server, exitChan chan bool) { +func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, internalCDRSChan chan *engine.CdrServer, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) @@ -164,9 +164,9 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal } else if len(cfg.SmGenericConfig.CdrsConns) != 0 { for _, cdrsCfg := range cfg.SmGenericConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { - resp := <-internalRaterChan - cdrsConn.AddClient(client) - internalRaterChan <- resp + resp := <-internalCDRSChan + cdrsConn.AddClient(resp) + internalCDRSChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -230,7 +230,7 @@ func startDiameterAgent(internalSMGChan chan rpcclient.RpcClientConnection, exit exitChan <- true } -func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { +func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, internalCDRSChan chan *engine.CdrServer, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-FreeSWITCH service.") raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) @@ -258,9 +258,9 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd } else if len(cfg.SmFsConfig.CdrsConns) != 0 { for _, cdrsCfg := range cfg.SmFsConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { - resp := <-internalRaterChan + resp := <-internalCDRSChan cdrsConn.AddClient(resp) - internalRaterChan <- resp + internalCDRSChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -280,7 +280,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.Cd exitChan <- true } -func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { +func startSmKamailio(internalRaterChan chan *engine.Responder, internalCDRSChan chan *engine.CdrServer, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Kamailio service.") raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) @@ -308,9 +308,9 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS } else if len(cfg.SmKamConfig.CdrsConns) != 0 { for _, cdrsCfg := range cfg.SmKamConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { - resp := <-internalRaterChan + resp := <-internalCDRSChan cdrsConn.AddClient(resp) - internalRaterChan <- resp + internalCDRSChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -330,7 +330,7 @@ func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS exitChan <- true } -func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) { +func startSmOpenSIPS(internalRaterChan chan *engine.Responder, internalCDRSChan chan *engine.CdrServer, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-OpenSIPS service.") raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) @@ -358,9 +358,9 @@ func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrS } else if len(cfg.SmOsipsConfig.CdrsConns) != 0 { for _, cdrsCfg := range cfg.SmOsipsConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { - resp := <-internalRaterChan + resp := <-internalCDRSChan cdrsConn.AddClient(resp) - internalRaterChan <- resp + internalCDRSChan <- resp } else { client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) if err != nil { @@ -718,23 +718,23 @@ func main() { // Start SM-Generic if cfg.SmGenericConfig.Enabled { - go startSmGeneric(internalSMGChan, internalRaterChan, server, exitChan) + go startSmGeneric(internalSMGChan, internalRaterChan, internalCdrSChan, server, exitChan) } // Start SM-FreeSWITCH if cfg.SmFsConfig.Enabled { - go startSmFreeSWITCH(internalRaterChan, cdrDb, exitChan) + go startSmFreeSWITCH(internalRaterChan, internalCdrSChan, cdrDb, exitChan) // close all sessions on shutdown go shutdownSessionmanagerSingnalHandler(exitChan) } // Start SM-Kamailio if cfg.SmKamConfig.Enabled { - go startSmKamailio(internalRaterChan, cdrDb, exitChan) + go startSmKamailio(internalRaterChan, internalCdrSChan, cdrDb, exitChan) } // Start SM-OpenSIPS if cfg.SmOsipsConfig.Enabled { - go startSmOpenSIPS(internalRaterChan, cdrDb, exitChan) + go startSmOpenSIPS(internalRaterChan, internalCdrSChan, cdrDb, exitChan) } // Register session manager service // FixMe: make sure this is thread safe From bcd9eccf7491855898c0934f90b9f86a04a7b93c Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 20:36:47 +0200 Subject: [PATCH 029/227] make cdrs implement RpcClient --- engine/cdrs.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/engine/cdrs.go b/engine/cdrs.go index eb16467f2..8dd1c7d61 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "net/http" "path" + "reflect" + "strings" "time" "github.com/cgrates/cgrates/config" @@ -454,3 +456,31 @@ func (self *CdrServer) replicateCdr(cdr *StoredCdr) error { } return nil } + +func (cdrsrv *CdrServer) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(cdrsrv).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented + } + + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} + + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err +} From 5103204194b8d488dd01e3db7cec4fc3b70193b0 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 20:42:53 +0200 Subject: [PATCH 030/227] fixed da smg ports --- data/conf/samples/hapool/dagent/cgr.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/conf/samples/hapool/dagent/cgr.json b/data/conf/samples/hapool/dagent/cgr.json index 164f9bfec..829cb1b32 100644 --- a/data/conf/samples/hapool/dagent/cgr.json +++ b/data/conf/samples/hapool/dagent/cgr.json @@ -3,8 +3,8 @@ "enabled": true, "listen": "127.0.0.1:3868", "sm_generic_conns": [ - {"server": "127.0.0.1:3018"}, - {"server": "127.0.0.1:3020"}, + {"server": "127.0.0.1:2018"}, + {"server": "127.0.0.1:2020"}, ], }, } From 9d264930679c4e941308e1444ca2fe1679943111 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 21 Dec 2015 23:40:56 +0200 Subject: [PATCH 031/227] parallel stress addaccount tool --- cmd/cgr-tester/parallel/parallel.go | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 cmd/cgr-tester/parallel/parallel.go diff --git a/cmd/cgr-tester/parallel/parallel.go b/cmd/cgr-tester/parallel/parallel.go new file mode 100644 index 000000000..b249dec36 --- /dev/null +++ b/cmd/cgr-tester/parallel/parallel.go @@ -0,0 +1,44 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "sync" +) + +func main() { + log.Print("Start!") + var wg sync.WaitGroup + for i := 1; i < 1002; i++ { + go func(index int) { + wg.Add(1) + resp, err := http.Post("http://localhost:2080/jsonrpc", "application/json", bytes.NewBuffer([]byte(fmt.Sprintf(`{"method": "ApierV1.SetAccount","params": [{"Tenant":"reglo","Account":"100%d","ActionPlanId":"PACKAGE_NEW_FOR795", "ReloadScheduler":false}], "id":%d}`, index, index)))) + if err != nil { + log.Print("Post error: ", err) + } + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Print("Body error: ", err) + } + log.Printf("SetAccount(%d): %s", index, string(contents)) + wg.Done() + }(i) + } + wg.Wait() + for index := 1; index < 1002; index++ { + resp, err := http.Post("http://localhost:2080/jsonrpc", "application/json", bytes.NewBuffer([]byte(fmt.Sprintf(`{"method": "ApierV1.GetAccountActionPlan","params": [{"Tenant":"reglo","Account":"100%d"}], "id":%d}`, index, index)))) + if err != nil { + log.Print("Post error: ", err) + } + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Print("Body error: ", err) + } + log.Printf("GetAccountActionPlan(%d): %s", index, string(contents)) + } + + log.Print("Done!") +} From d6713343b4c4dc0736498827878a53c5687c52ac Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 22 Dec 2015 19:59:39 +0200 Subject: [PATCH 032/227] use rpc pool for cdrs rater connection --- cmd/cgr-engine/cgr-engine.go | 90 ++++++++----------- config/config.go | 13 ++- config/config_defaults.go | 4 +- config/config_json_test.go | 11 ++- config/libconfig_json.go | 2 +- data/conf/cgrates/cgrates.json | 4 +- data/conf/samples/apier/apier.json | 4 +- data/conf/samples/cdrstats/cdrstats.json | 4 +- data/conf/samples/cdrsv2mongo/cdrsv2psql.json | 6 +- .../conf/samples/cdrsv2mysql/cdrsv2mysql.json | 6 +- data/conf/samples/cdrsv2psql/cdrsv2psql.json | 6 +- data/conf/samples/dmtagent/cgrates.json | 4 +- data/conf/samples/fscsv/cgrates.json | 4 +- data/conf/samples/hapool/cgrrater1/cgr.json | 6 -- data/conf/samples/hapool/cgrrater2/cgr.json | 6 -- data/conf/samples/hapool/cgrsmg1/cgr.json | 5 +- data/conf/samples/hapool/cgrsmg2/cgr.json | 5 +- data/conf/samples/smgeneric/cgrates.json | 4 +- data/conf/samples/tutlocal/cgrates.json | 4 +- 19 files changed, 99 insertions(+), 89 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 370af7e54..6a63579c1 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -388,19 +388,21 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, var err error var client *rpcclient.RpcClient // Rater connection init - var raterConn rpcclient.RpcClientConnection - if cfg.CDRSRater == utils.INTERNAL { - responder := <-internalRaterChan // Wait for rater to come up before start querying - raterConn = responder - internalRaterChan <- responder // Put back the connection since there might be other entities waiting for it - } else if len(cfg.CDRSRater) != 0 { - client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSRater, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to rater: %s", err.Error())) - exitChan <- true - return + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + for _, raterCfg := range cfg.SmOsipsConfig.RaterConns { + if raterCfg.Server == utils.INTERNAL { + responder := <-internalRaterChan // Wait for rater to come up before start querying + raterConn.AddClient(responder) + internalRaterChan <- responder // Put back the connection since there might be other entities waiting for it + } else if len(raterCfg.Server) != 0 { + client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to rater: %s", err.Error())) + exitChan <- true + return + } + raterConn.AddClient(client) } - raterConn = client } // Pubsub connection init var pubSubConn rpcclient.RpcClientConnection @@ -409,17 +411,13 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, pubSubConn = pubSubs internalPubSubSChan <- pubSubs } else if len(cfg.CDRSPubSub) != 0 { - if cfg.CDRSRater == cfg.CDRSPubSub { - pubSubConn = client - } else { - client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSPubSub, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to pubsub server: %s", err.Error())) - exitChan <- true - return - } - pubSubConn = client + client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSPubSub, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to pubsub server: %s", err.Error())) + exitChan <- true + return } + pubSubConn = client } // Users connection init var usersConn rpcclient.RpcClientConnection @@ -428,17 +426,13 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, usersConn = userS internalUserSChan <- userS } else if len(cfg.CDRSUsers) != 0 { - if cfg.CDRSRater == cfg.CDRSUsers { - usersConn = client - } else { - client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSUsers, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to users server: %s", err.Error())) - exitChan <- true - return - } - usersConn = client + client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSUsers, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to users server: %s", err.Error())) + exitChan <- true + return } + usersConn = client } // Aliases connection init var aliasesConn rpcclient.RpcClientConnection @@ -447,17 +441,13 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, aliasesConn = aliaseS internalAliaseSChan <- aliaseS } else if len(cfg.CDRSAliases) != 0 { - if cfg.CDRSRater == cfg.CDRSAliases { - aliasesConn = client - } else { - client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSAliases, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to aliases server: %s", err.Error())) - exitChan <- true - return - } - aliasesConn = client + client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSAliases, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to aliases server: %s", err.Error())) + exitChan <- true + return } + aliasesConn = client } // Stats connection init var statsConn rpcclient.RpcClientConnection @@ -466,17 +456,13 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, statsConn = statS internalCdrStatSChan <- statS } else if len(cfg.CDRSStats) != 0 { - if cfg.CDRSRater == cfg.CDRSStats { - statsConn = client - } else { - client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to stats server: %s", err.Error())) - exitChan <- true - return - } - statsConn = client + client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to stats server: %s", err.Error())) + exitChan <- true + return } + statsConn = client } cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, raterConn, pubSubConn, usersConn, aliasesConn, statsConn) diff --git a/config/config.go b/config/config.go index bafba8128..943f8dc8f 100644 --- a/config/config.go +++ b/config/config.go @@ -218,7 +218,7 @@ type CGRConfig struct { CDRSEnabled bool // Enable CDR Server service CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs CDRSStoreCdrs bool // store cdrs in storDb - CDRSRater string // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> + CDRSRaterConns []*HaPoolConfig // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> CDRSPubSub string // address where to reach the pubsub service: <""|internal|x.y.z.y:1234> CDRSUsers string // address where to reach the users service: <""|internal|x.y.z.y:1234> CDRSAliases string // address where to reach the aliases service: <""|internal|x.y.z.y:1234> @@ -277,7 +277,7 @@ func (self *CGRConfig) checkConfigSanity() error { } // CDRServer checks if self.CDRSEnabled { - if self.CDRSRater == utils.INTERNAL && !self.RaterEnabled { + if self.CDRSRaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { return errors.New("Rater not enabled but requested by CDRS component.") } if self.CDRSPubSub == utils.INTERNAL && !self.PubSubServerEnabled { @@ -684,8 +684,13 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnCdrsCfg.Store_cdrs != nil { self.CDRSStoreCdrs = *jsnCdrsCfg.Store_cdrs } - if jsnCdrsCfg.Rater != nil { - self.CDRSRater = *jsnCdrsCfg.Rater + + if jsnCdrsCfg.Rater_conns != nil { + self.CDRSRaterConns = make([]*HaPoolConfig, len(*jsnCdrsCfg.Rater_conns)) + for idx, jsnHaCfg := range *jsnCdrsCfg.Rater_conns { + self.CDRSRaterConns[idx] = NewDfltHaPoolConfig() + self.CDRSRaterConns[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCdrsCfg.Pubsubs != nil { self.CDRSPubSub = *jsnCdrsCfg.Pubsubs diff --git a/config/config_defaults.go b/config/config_defaults.go index 403bc530f..c30bb0d43 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -110,7 +110,9 @@ const CGRATES_CFG_JSON = ` "enabled": false, // start the CDR Server service: "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs "store_cdrs": true, // store cdrs in storDb - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> diff --git a/config/config_json_test.go b/config/config_json_test.go index e629680f8..ffccdba82 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -146,10 +146,13 @@ func TestDfSchedulerJsonCfg(t *testing.T) { func TestDfCdrsJsonCfg(t *testing.T) { eCfg := &CdrsJsonCfg{ - Enabled: utils.BoolPointer(false), - Extra_fields: utils.StringSlicePointer([]string{}), - Store_cdrs: utils.BoolPointer(true), - Rater: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Extra_fields: utils.StringSlicePointer([]string{}), + Store_cdrs: utils.BoolPointer(true), + Rater_conns: &[]*HaPoolJsonCfg{ + &HaPoolJsonCfg{ + Server: utils.StringPointer("internal"), + }}, Pubsubs: utils.StringPointer(""), Users: utils.StringPointer(""), Aliases: utils.StringPointer(""), diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 26d09123b..556d3be69 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -82,7 +82,7 @@ type CdrsJsonCfg struct { Enabled *bool Extra_fields *[]string Store_cdrs *bool - Rater *string + Rater_conns *[]*HaPoolJsonCfg Pubsubs *string Users *string Aliases *string diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index b1b8a4d0f..d33530d7c 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -89,7 +89,9 @@ // "enabled": false, // start the CDR Server service: // "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs // "store_cdrs": true, // store cdrs in storDb -// "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + // "rater_conns": [ + // {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + //], // "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> // "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> // "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> diff --git a/data/conf/samples/apier/apier.json b/data/conf/samples/apier/apier.json index 6b80a486b..e7385e90e 100644 --- a/data/conf/samples/apier/apier.json +++ b/data/conf/samples/apier/apier.json @@ -21,7 +21,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], }, "aliases": { diff --git a/data/conf/samples/cdrstats/cdrstats.json b/data/conf/samples/cdrstats/cdrstats.json index 0c32dd32b..8c91dbc9d 100644 --- a/data/conf/samples/cdrstats/cdrstats.json +++ b/data/conf/samples/cdrstats/cdrstats.json @@ -17,7 +17,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: "store_cdrs": false, // store cdrs in storDb - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], "cdrstats": "internal", // address where to reach the cdrstats service. Empty to disable stats gathering out of mediated CDRs <""|internal|x.y.z.y:1234> }, diff --git a/data/conf/samples/cdrsv2mongo/cdrsv2psql.json b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json index e62a46f95..d3590440f 100644 --- a/data/conf/samples/cdrsv2mongo/cdrsv2psql.json +++ b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json @@ -16,7 +16,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], }, -} \ No newline at end of file +} diff --git a/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json b/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json index e05ac0535..e1f724ad2 100644 --- a/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json +++ b/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json @@ -10,7 +10,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], }, -} \ No newline at end of file +} diff --git a/data/conf/samples/cdrsv2psql/cdrsv2psql.json b/data/conf/samples/cdrsv2psql/cdrsv2psql.json index 9f78c6df8..fe166b1d3 100644 --- a/data/conf/samples/cdrsv2psql/cdrsv2psql.json +++ b/data/conf/samples/cdrsv2psql/cdrsv2psql.json @@ -16,7 +16,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], }, -} \ No newline at end of file +} diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index 005c8625c..b33df0fed 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -24,7 +24,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, diff --git a/data/conf/samples/fscsv/cgrates.json b/data/conf/samples/fscsv/cgrates.json index c7c8abba9..8ec9bb202 100644 --- a/data/conf/samples/fscsv/cgrates.json +++ b/data/conf/samples/fscsv/cgrates.json @@ -17,7 +17,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, diff --git a/data/conf/samples/hapool/cgrrater1/cgr.json b/data/conf/samples/hapool/cgrrater1/cgr.json index 74af2d95c..cfde7c390 100644 --- a/data/conf/samples/hapool/cgrrater1/cgr.json +++ b/data/conf/samples/hapool/cgrrater1/cgr.json @@ -17,12 +17,6 @@ "enabled": true, // start Scheduler service: }, -"cdrs": { - "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> -}, - "cdrstats": { "enabled": true, // starts the cdrstats service: }, diff --git a/data/conf/samples/hapool/cgrrater2/cgr.json b/data/conf/samples/hapool/cgrrater2/cgr.json index 00c31a61b..dd64d4442 100644 --- a/data/conf/samples/hapool/cgrrater2/cgr.json +++ b/data/conf/samples/hapool/cgrrater2/cgr.json @@ -18,12 +18,6 @@ "enabled": true, // start Scheduler service: }, -"cdrs": { - "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> -}, - "cdrstats": { "enabled": true, // starts the cdrstats service: }, diff --git a/data/conf/samples/hapool/cgrsmg1/cgr.json b/data/conf/samples/hapool/cgrsmg1/cgr.json index 71d9e4cb0..9b8ec46b1 100644 --- a/data/conf/samples/hapool/cgrsmg1/cgr.json +++ b/data/conf/samples/hapool/cgrsmg1/cgr.json @@ -7,7 +7,10 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server": "127.0.0.1:2014"}, + {"server": "127.0.0.1:2016"} + ], "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, diff --git a/data/conf/samples/hapool/cgrsmg2/cgr.json b/data/conf/samples/hapool/cgrsmg2/cgr.json index e0d98bc7f..c61812057 100644 --- a/data/conf/samples/hapool/cgrsmg2/cgr.json +++ b/data/conf/samples/hapool/cgrsmg2/cgr.json @@ -7,7 +7,10 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "127.0.0.1:2014", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server": "127.0.0.1:2014"}, + {"server": "127.0.0.1:2016"} + ], "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, diff --git a/data/conf/samples/smgeneric/cgrates.json b/data/conf/samples/smgeneric/cgrates.json index 9a370c4dd..e54c93b48 100644 --- a/data/conf/samples/smgeneric/cgrates.json +++ b/data/conf/samples/smgeneric/cgrates.json @@ -89,7 +89,9 @@ "enabled": true, // start the CDR Server service: // "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs // "store_cdrs": true, // store cdrs in storDb -// "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + // "rater_conns": [ + //{"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + //], // "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> // "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> // "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> diff --git a/data/conf/samples/tutlocal/cgrates.json b/data/conf/samples/tutlocal/cgrates.json index 14e933058..df77fa492 100644 --- a/data/conf/samples/tutlocal/cgrates.json +++ b/data/conf/samples/tutlocal/cgrates.json @@ -23,7 +23,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rater_conns": [ + {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + ], "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, From a43a695a3ba0f5a62293a42cf4866509cb9783a8 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Dec 2015 00:18:16 +0200 Subject: [PATCH 033/227] cdrs expose rpc methods --- apier/v1/cdrsv1.go | 4 +- cmd/cgr-engine/cgr-engine.go | 1 + engine/cdrs.go | 63 +++++++++++++++++++++++++----- engine/responder.go | 41 ------------------- sessionmanager/fssessionmanager.go | 2 +- sessionmanager/kamailiosm.go | 2 +- sessionmanager/osipssm.go | 2 +- sessionmanager/session.go | 2 +- sessionmanager/smg_session.go | 2 +- sessionmanager/smgeneric.go | 2 +- 10 files changed, 62 insertions(+), 59 deletions(-) diff --git a/apier/v1/cdrsv1.go b/apier/v1/cdrsv1.go index cdf43a646..c0913766a 100644 --- a/apier/v1/cdrsv1.go +++ b/apier/v1/cdrsv1.go @@ -32,7 +32,7 @@ type CdrsV1 struct { // Designed for CGR internal usage func (self *CdrsV1) ProcessCdr(cdr *engine.StoredCdr, reply *string) error { - if err := self.CdrSrv.ProcessCdr(cdr); err != nil { + if err := self.CdrSrv.LocalProcessCdr(cdr); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK @@ -74,7 +74,7 @@ func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { } func (self *CdrsV1) LogCallCost(ccl *engine.CallCostLog, reply *string) error { - if err := self.CdrSrv.LogCallCost(ccl); err != nil { + if err := self.CdrSrv.LocalLogCallCost(ccl); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 6a63579c1..2ec7f35ac 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -473,6 +473,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, server.RpcRegister(&cdrSrv) server.RpcRegister(&v2.CdrsV2{CdrsV1: cdrSrv}) // Make the cdr server available for internal communication + server.RpcRegister(cdrServer) // register CdrServer for internal usage (TODO: refactor this) responder := <-internalRaterChan // Retrieve again the responder responder.CdrSrv = cdrServer // Attach connection to cdrServer in responder, so it can be used later internalRaterChan <- responder // Put back the connection for the rest of the system diff --git a/engine/cdrs.go b/engine/cdrs.go index 8dd1c7d61..6eeeb0c14 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/cgrates/rpcclient" @@ -74,19 +75,31 @@ func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater rpcclient.Rp } type CdrServer struct { - cgrCfg *config.CGRConfig - cdrDb CdrStorage - client rpcclient.RpcClientConnection - pubsub rpcclient.RpcClientConnection - users rpcclient.RpcClientConnection - aliases rpcclient.RpcClientConnection - stats rpcclient.RpcClientConnection - guard *GuardianLock + cgrCfg *config.CGRConfig + cdrDb CdrStorage + client rpcclient.RpcClientConnection + pubsub rpcclient.RpcClientConnection + users rpcclient.RpcClientConnection + aliases rpcclient.RpcClientConnection + stats rpcclient.RpcClientConnection + guard *GuardianLock + responseCache *cache2go.ResponseCache } func (self *CdrServer) Timezone() string { return self.cgrCfg.DefaultTimezone } +func (self *CdrServer) SetTimeToLive(timeToLive time.Duration, out *int) error { + self.responseCache = cache2go.NewResponseCache(timeToLive) + return nil +} + +func (self *CdrServer) getCache() *cache2go.ResponseCache { + if self.responseCache == nil { + self.responseCache = cache2go.NewResponseCache(0) + } + return self.responseCache +} func (self *CdrServer) RegisterHandlersToServer(server *utils.Server) { cdrServer = self // Share the server object for handlers @@ -94,8 +107,23 @@ func (self *CdrServer) RegisterHandlersToServer(server *utils.Server) { server.RegisterHttpFunc("/freeswitch_json", fsCdrHandler) } +func (self *CdrServer) ProcessCdr(cdr *StoredCdr, reply *string) error { + cacheKey := "ProcessCdr" + cdr.CgrId + if item, err := self.getCache().Get(cacheKey); err == nil && item != nil { + *reply = item.Value.(string) + return item.Err + } + if err := self.LocalProcessCdr(cdr); err != nil { + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return utils.NewErrServerError(err) + } + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) + *reply = utils.OK + return nil +} + // RPC method, used to internally process CDR -func (self *CdrServer) ProcessCdr(cdr *StoredCdr) error { +func (self *CdrServer) LocalProcessCdr(cdr *StoredCdr) error { return self.processCdr(cdr) } @@ -108,8 +136,23 @@ func (self *CdrServer) ProcessExternalCdr(cdr *ExternalCdr) error { return self.processCdr(storedCdr) } +func (self *CdrServer) LogCallCost(ccl *CallCostLog, reply *string) error { + cacheKey := "LogCallCost" + ccl.CgrId + if item, err := self.getCache().Get(cacheKey); err == nil && item != nil { + *reply = item.Value.(string) + return item.Err + } + if err := self.LocalLogCallCost(ccl); err != nil { + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return utils.NewErrServerError(err) + } + *reply = utils.OK + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) + return nil +} + // RPC method, used to log callcosts to db -func (self *CdrServer) LogCallCost(ccl *CallCostLog) error { +func (self *CdrServer) LocalLogCallCost(ccl *CallCostLog) error { if ccl.CheckDuplicate { _, err := self.guard.Guard(func() (interface{}, error) { cc, err := self.cdrDb.GetCallCostLog(ccl.CgrId, ccl.Source, ccl.RunId) diff --git a/engine/responder.go b/engine/responder.go index ded672c1e..c16b1423b 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -50,7 +50,6 @@ type AttrGetLcr struct { type Responder struct { Bal *balancer2go.Balancer ExitChan chan bool - CdrSrv *CdrServer Stats rpcclient.RpcClientConnection Timezone string cnt int64 @@ -444,46 +443,6 @@ func (rs *Responder) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *u return nil } -func (rs *Responder) ProcessCdr(cdr *StoredCdr, reply *string) error { - cacheKey := "ProcessCdr" + cdr.CgrId - if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - *reply = item.Value.(string) - return item.Err - } - if rs.CdrSrv == nil { - err := errors.New("CDR_SERVER_NOT_RUNNING") - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) - return err - } - if err := rs.CdrSrv.ProcessCdr(cdr); err != nil { - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) - return err - } - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) - *reply = utils.OK - return nil -} - -func (rs *Responder) LogCallCost(ccl *CallCostLog, reply *string) error { - cacheKey := "LogCallCost" + ccl.CgrId - if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - *reply = item.Value.(string) - return item.Err - } - if rs.CdrSrv == nil { - err := errors.New("CDR_SERVER_NOT_RUNNING") - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) - return err - } - if err := rs.CdrSrv.LogCallCost(ccl); err != nil { - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) - return err - } - *reply = utils.OK - rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) - return nil -} - func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { cacheKey := "GetLCR" + attrs.CgrId if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 102e5e4dd..bc7d7982c 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -298,7 +298,7 @@ func (sm *FSSessionManager) DisconnectSession(ev engine.Event, connId, notify st func (sm *FSSessionManager) ProcessCdr(storedCdr *engine.StoredCdr) error { var reply string - if err := sm.cdrsrv.Call("Responder.ProcessCdr", storedCdr, &reply); err != nil { + if err := sm.cdrsrv.Call("CdrServer.ProcessCdr", storedCdr, &reply); err != nil { utils.Logger.Err(fmt.Sprintf(" Failed processing CDR, cgrid: %s, accid: %s, error: <%s>", storedCdr.CgrId, storedCdr.AccId, err.Error())) } return nil diff --git a/sessionmanager/kamailiosm.go b/sessionmanager/kamailiosm.go index 26754a200..50034d7df 100644 --- a/sessionmanager/kamailiosm.go +++ b/sessionmanager/kamailiosm.go @@ -210,7 +210,7 @@ func (self *KamailioSessionManager) ProcessCdr(cdr *engine.StoredCdr) error { return nil } var reply string - if err := self.cdrsrv.Call("Responder.ProcessCdr", cdr, &reply); err != nil { + if err := self.cdrsrv.Call("CdrServer.ProcessCdr", cdr, &reply); err != nil { utils.Logger.Err(fmt.Sprintf(" Failed processing CDR, cgrid: %s, accid: %s, error: <%s>", cdr.CgrId, cdr.AccId, err.Error())) } return nil diff --git a/sessionmanager/osipssm.go b/sessionmanager/osipssm.go index f3a91e4a5..86ffe40f7 100644 --- a/sessionmanager/osipssm.go +++ b/sessionmanager/osipssm.go @@ -153,7 +153,7 @@ func (osm *OsipsSessionManager) Shutdown() error { // Process the CDR with CDRS component func (osm *OsipsSessionManager) ProcessCdr(storedCdr *engine.StoredCdr) error { var reply string - return osm.cdrsrv.Call("Responder.ProcessCdr", storedCdr, &reply) + return osm.cdrsrv.Call("CdrServer.ProcessCdr", storedCdr, &reply) } // Disconnects the session diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 31ba88eeb..9320bd66e 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -234,7 +234,7 @@ func (s *Session) SaveOperations() { } var reply string - err := s.sessionManager.CdrSrv().Call("Responder.LogCallCost", &engine.CallCostLog{ + err := s.sessionManager.CdrSrv().Call("CdrServer.LogCallCost", &engine.CallCostLog{ CgrId: s.eventStart.GetCgrId(s.sessionManager.Timezone()), Source: utils.SESSION_MANAGER_SOURCE, RunId: sr.DerivedCharger.RunId, diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 97f3226e9..c12e1e5c8 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -205,7 +205,7 @@ func (self *SMGSession) saveOperations() error { firstCC.Merge(cc) } var reply string - err := self.cdrsrv.Call("Responder.LogCallCost", &engine.CallCostLog{ + err := self.cdrsrv.Call("CdrServer.LogCallCost", &engine.CallCostLog{ CgrId: self.eventStart.GetCgrId(self.timezone), Source: utils.SESSION_MANAGER_SOURCE, RunId: self.runId, diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 1afe6d893..b12cb7e9f 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -201,7 +201,7 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { var reply string - if err := self.cdrsrv.Call("Responder.ProcessCdr", gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { + if err := self.cdrsrv.Call("CdrServer.ProcessCdr", gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err } return nil From 970c47ccea0d6fc7fc2a55e93b4b52ea0a0687fc Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Dec 2015 00:25:01 +0200 Subject: [PATCH 034/227] set cache time for cdrs --- cmd/cgr-engine/cgr-engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 2ec7f35ac..94da809c6 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -466,6 +466,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, } cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, raterConn, pubSubConn, usersConn, aliasesConn, statsConn) + cdrServer.SetTimeToLive(cfg.ResponseCacheTTL, nil) utils.Logger.Info("Registering CDRS HTTP Handlers.") cdrServer.RegisterHandlersToServer(server) utils.Logger.Info("Registering CDRS RPC service.") @@ -475,7 +476,6 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, // Make the cdr server available for internal communication server.RpcRegister(cdrServer) // register CdrServer for internal usage (TODO: refactor this) responder := <-internalRaterChan // Retrieve again the responder - responder.CdrSrv = cdrServer // Attach connection to cdrServer in responder, so it can be used later internalRaterChan <- responder // Put back the connection for the rest of the system internalCdrSChan <- cdrServer // Signal that cdrS is operational } From a92fae45b2d3985f7556da8bc6aa5b1a5ec40d0e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Dec 2015 12:17:53 +0200 Subject: [PATCH 035/227] remove responder cdrs dependency --- cmd/cgr-engine/cgr-engine.go | 6 ++---- glide.lock | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 94da809c6..70f3e7745 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -474,10 +474,8 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, server.RpcRegister(&cdrSrv) server.RpcRegister(&v2.CdrsV2{CdrsV1: cdrSrv}) // Make the cdr server available for internal communication - server.RpcRegister(cdrServer) // register CdrServer for internal usage (TODO: refactor this) - responder := <-internalRaterChan // Retrieve again the responder - internalRaterChan <- responder // Put back the connection for the rest of the system - internalCdrSChan <- cdrServer // Signal that cdrS is operational + server.RpcRegister(cdrServer) // register CdrServer for internal usage (TODO: refactor this) + internalCdrSChan <- cdrServer // Signal that cdrS is operational } func startScheduler(internalSchedulerChan chan *scheduler.Scheduler, cacheDoneChan chan struct{}, ratingDb engine.RatingStorage, exitChan chan bool) { diff --git a/glide.lock b/glide.lock index 97ec30d38..5c0143f2f 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -updated: 2015-12-18T18:02:12.669823411+02:00 +updated: 2015-12-23T11:12:22.098552467+02:00 imports: - name: github.com/cenkalti/hub version: 57d753b5f4856e77b3cf8ecce78c97215a7d324d @@ -36,20 +36,20 @@ imports: - name: github.com/lib/pq version: 11fc39a580a008f1f39bb3d11d984fb34ed778d9 - name: github.com/mediocregopher/radix.v2 - version: b49c6f79790ba8a13a70b42e6a53d72f2239cf63 + version: 91435107718b55ff544323a2b0f25fdd8475d283 subpackages: - /pool - redis - name: github.com/peterh/liner version: 3f1c20449d1836aa4cbe38731b96f95cdf89634d - name: github.com/ugorji/go - version: cd43bdd6be4b5675a0d1e75c4af55ee1dc0d9c5e + version: 646ae4a518c1c3be0739df898118d9bccf993858 subpackages: - /codec - name: golang.org/x/crypto version: f18420efc3b4f8e9f3d51f6bd2476e92c46260e9 - name: golang.org/x/net - version: 28273ec927bee3bea305f112fc28ceee575ea893 + version: 6c89489cafabcbc76df9dbf84ebf07204673fecf subpackages: - /websocket - name: golang.org/x/text From 58d033484fc6db0bd8cd89644103b2b083eeca7c Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 14:21:09 +0100 Subject: [PATCH 036/227] Fix on starting CDRS in cmd/cgr-engine, config parsing optimization for multiple connections in hapool --- cmd/cgr-engine/cgr-engine.go | 2 +- config/config.go | 62 ++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 70f3e7745..69fdc5349 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -389,7 +389,7 @@ func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, var client *rpcclient.RpcClient // Rater connection init raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) - for _, raterCfg := range cfg.SmOsipsConfig.RaterConns { + for _, raterCfg := range cfg.CDRSRaterConns { if raterCfg.Server == utils.INTERNAL { responder := <-internalRaterChan // Wait for rater to come up before start querying raterConn.AddClient(responder) diff --git a/config/config.go b/config/config.go index 943f8dc8f..dac9dd6a6 100644 --- a/config/config.go +++ b/config/config.go @@ -277,8 +277,10 @@ func (self *CGRConfig) checkConfigSanity() error { } // CDRServer checks if self.CDRSEnabled { - if self.CDRSRaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by CDRS component.") + for _, cdrsRaterConn := range self.CDRSRaterConns { + if cdrsRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + return errors.New("Rater not enabled but requested by CDRS component.") + } } if self.CDRSPubSub == utils.INTERNAL && !self.PubSubServerEnabled { return errors.New("PubSub service not enabled but requested by CDRS component.") @@ -326,11 +328,15 @@ func (self *CGRConfig) checkConfigSanity() error { if len(self.SmGenericConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmGenericConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-Generic component.") + for _, smgRaterConn := range self.SmGenericConfig.RaterConns { + if smgRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + return errors.New("Rater not enabled but requested by SM-Generic component.") + } } - if self.SmGenericConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-Generic component") + for _, smgCDRSConn := range self.SmGenericConfig.CdrsConns { + if smgCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced by SM-Generic component") + } } } // SM-FreeSWITCH checks @@ -339,13 +345,17 @@ func (self *CGRConfig) checkConfigSanity() error { return errors.New("Rater definition is mandatory!") } if len(self.SmFsConfig.CdrsConns) == 0 { - return errors.New("Cdrs definition is mandatory!") + return errors.New("CDRS definition is mandatory!") } - if self.SmFsConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-FreeSWITCH component.") + for _, smFSRaterConn := range self.SmFsConfig.RaterConns { + if smFSRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + return errors.New("Rater not enabled but requested by SM-FreeSWITCH component.") + } } - if self.SmFsConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-FreeSWITCH component") + for _, smFSCDRSConn := range self.SmFsConfig.CdrsConns { + if smFSCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced by SM-FreeSWITCH component") + } } } // SM-Kamailio checks @@ -356,11 +366,15 @@ func (self *CGRConfig) checkConfigSanity() error { if len(self.SmKamConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmKamConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-Kamailio component.") + for _, smKamRaterConn := range self.SmKamConfig.RaterConns { + if smKamRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + return errors.New("Rater not enabled but requested by SM-Kamailio component.") + } } - if self.SmKamConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-Kamailio component") + for _, smKamCDRSConn := range self.SmKamConfig.CdrsConns { + if smKamCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced by SM-Kamailio component") + } } } // SM-OpenSIPS checks @@ -371,17 +385,23 @@ func (self *CGRConfig) checkConfigSanity() error { if len(self.SmOsipsConfig.CdrsConns) == 0 { return errors.New("Cdrs definition is mandatory!") } - if self.SmOsipsConfig.RaterConns[0].Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-OpenSIPS component.") + for _, smOsipsRaterConn := range self.SmOsipsConfig.RaterConns { + if smOsipsRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + return errors.New("Rater not enabled but requested by SM-OpenSIPS component.") + } } - if self.SmOsipsConfig.CdrsConns[0].Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-OpenSIPS component") + for _, smOsipsCDRSConn := range self.SmOsipsConfig.CdrsConns { + if smOsipsCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced by SM-OpenSIPS component") + } } } // DAgent checks if self.diameterAgentCfg.Enabled { - if self.diameterAgentCfg.SMGenericConns[0].Server == utils.INTERNAL && !self.SmGenericConfig.Enabled { - return errors.New("SMGeneric not enabled but referenced by DiameterAgent component") + for _, daSMGConn := range self.diameterAgentCfg.SMGenericConns { + if daSMGConn.Server == utils.INTERNAL && !self.SmGenericConfig.Enabled { + return errors.New("SMGeneric not enabled but referenced by DiameterAgent component") + } } } return nil From fb60f37f784cd8cf0328c4dcf4b5c1e3005e55cc Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 10:20:21 +0100 Subject: [PATCH 037/227] Diameter serializeAVPValueFromString with tests --- agents/libdmt.go | 41 +++++++++++++++++++++++++++++++++++++++++ agents/libdmt_test.go | 26 ++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/agents/libdmt.go b/agents/libdmt.go index fe3e7581e..13e2e441d 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -27,6 +27,7 @@ import ( "fmt" "math" "math/rand" + "net" "os" "path/filepath" "strconv" @@ -269,6 +270,46 @@ func composedFieldvalue(m *diam.Message, outTpl utils.RSRFields, avpIdx int) str return outVal } +// Used to return the encoded value based on what AVP understands for it's type +func serializeAVPValueFromString(dictAVP *dict.AVP, valStr, timezone string) ([]byte, error) { + switch dictAVP.Data.Type { + case datatype.OctetStringType, datatype.DiameterIdentityType, datatype.DiameterURIType, datatype.IPFilterRuleType, datatype.QoSFilterRuleType, datatype.UTF8StringType: + return []byte(valStr), nil + case datatype.AddressType: + return []byte(net.ParseIP(valStr)), nil + case datatype.EnumeratedType, datatype.Integer32Type, datatype.Integer64Type, datatype.Unsigned32Type, datatype.Unsigned64Type: + i, err := strconv.Atoi(valStr) + if err != nil { + return nil, err + } + return datatype.Enumerated(i).Serialize(), nil + case datatype.Float32Type: + f, err := strconv.ParseFloat(valStr, 32) + if err != nil { + return nil, err + } + return datatype.Float32(f).Serialize(), nil + case datatype.Float64Type: + f, err := strconv.ParseFloat(valStr, 64) + if err != nil { + return nil, err + } + return datatype.Float64(f).Serialize(), nil + case datatype.GroupedType: + return nil, errors.New("GroupedType not supported for serialization") + case datatype.IPv4Type: + return datatype.IPv4(net.ParseIP(valStr)).Serialize(), nil + case datatype.TimeType: + t, err := utils.ParseTimeDetectLayout(valStr, timezone) + if err != nil { + return nil, err + } + return datatype.Time(t).Serialize(), nil + default: + return nil, fmt.Errorf("Unsupported type for serialization: %v", dictAVP.Data.Type) + } +} + func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interface{}) (fmtValOut string, err error) { var outVal string switch cfgFld.Type { diff --git a/agents/libdmt_test.go b/agents/libdmt_test.go index cfba48efa..3bbbe1dfc 100644 --- a/agents/libdmt_test.go +++ b/agents/libdmt_test.go @@ -20,6 +20,7 @@ package agents import ( "bytes" + "encoding/binary" "reflect" "testing" "time" @@ -29,6 +30,7 @@ import ( "github.com/fiorix/go-diameter/diam" "github.com/fiorix/go-diameter/diam/avp" "github.com/fiorix/go-diameter/diam/datatype" + "github.com/fiorix/go-diameter/diam/dict" ) func TestDisectUsageForCCR(t *testing.T) { @@ -138,6 +140,24 @@ func TestFieldOutVal(t *testing.T) { } } +func TestSerializeAVPValueFromString(t *testing.T) { + dictAVP, _ := dict.Default.FindAVP(4, "Session-Id") + eValByte := []byte("simuhuawei;1449573472;00002") + if valByte, err := serializeAVPValueFromString(dictAVP, "simuhuawei;1449573472;00002", "UTC"); err != nil { + t.Error(err) + } else if !bytes.Equal(eValByte, valByte) { + t.Errorf("Expecting: %+v, received: %+v", eValByte, valByte) + } + dictAVP, _ = dict.Default.FindAVP(4, "Result-Code") + eValByte = make([]byte, 4) + binary.BigEndian.PutUint32(eValByte, uint32(5031)) + if valByte, err := serializeAVPValueFromString(dictAVP, "5031", "UTC"); err != nil { + t.Error(err) + } else if !bytes.Equal(eValByte, valByte) { + t.Errorf("Expecting: %+v, received: %+v", eValByte, valByte) + } +} + func TestMessageSetAVPsWithPath(t *testing.T) { eMessage := diam.NewRequest(diam.CreditControl, 4, nil) eMessage.NewAVP("Session-Id", avp.Mbit, 0, datatype.UTF8String("simuhuawei;1449573472;00002")) @@ -244,9 +264,11 @@ func TestCCASetProcessorAVPs(t *testing.T) { reqProcessor := &config.DARequestProcessor{Id: "UNIT_TEST", // Set template for tests CCAFields: []*config.CfgCdrField{ &config.CfgCdrField{Tag: "Subscription-Id/Subscription-Id-Type", Type: utils.META_COMPOSED, - Value: utils.ParseRSRFieldsMustCompile("Subscription-Id>Subscription-Id-Type", utils.INFIELD_SEP), Mandatory: true}, + FieldId: "Subscription-Id>Subscription-Id-Type", + Value: utils.ParseRSRFieldsMustCompile("Subscription-Id>Subscription-Id-Type", utils.INFIELD_SEP), Mandatory: true}, &config.CfgCdrField{Tag: "Subscription-Id/Subscription-Id-Data", Type: utils.META_COMPOSED, - Value: utils.ParseRSRFieldsMustCompile("Subscription-Id>Subscription-Id-Data", utils.INFIELD_SEP), Mandatory: true}, + FieldId: "Subscription-Id>Subscription-Id-Data", + Value: utils.ParseRSRFieldsMustCompile("Subscription-Id>Subscription-Id-Data", utils.INFIELD_SEP), Mandatory: true}, }, } eMessage := cca.AsDiameterMessage() From a01f953eead1c9d5ea54c2699441c516ec4c7f11 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 12:52:55 +0100 Subject: [PATCH 038/227] Diameter proper serialization in CCA, fixes #334 --- agents/libdmt.go | 10 ++++++--- agents/libdmt_test.go | 43 ++++++++++++++++++--------------------- config/config_defaults.go | 2 +- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/agents/libdmt.go b/agents/libdmt.go index 13e2e441d..361fe3bd7 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -356,7 +356,7 @@ func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interfa // messageAddAVPsWithPath will dynamically add AVPs into the message // append: append to the message, on false overwrite if AVP is single or add to group if AVP is Grouped -func messageSetAVPsWithPath(m *diam.Message, path []interface{}, avpValByte []byte, appnd bool) error { +func messageSetAVPsWithPath(m *diam.Message, path []interface{}, avpValStr string, appnd bool, timezone string) error { if len(path) == 0 { return errors.New("Empty path as AVP filter") } @@ -377,8 +377,11 @@ func messageSetAVPsWithPath(m *diam.Message, path []interface{}, avpValByte []by lastAVPIdx := len(path) - 1 for i := lastAVPIdx; i >= 0; i-- { var typeVal datatype.Type - var err error if i == lastAVPIdx { + avpValByte, err := serializeAVPValueFromString(dictAVPs[i], avpValStr, timezone) + if err != nil { + return err + } typeVal, err = datatype.Decode(dictAVPs[i].Data.Type, avpValByte) if err != nil { return err @@ -585,6 +588,7 @@ type CCA struct { ccrMessage *diam.Message diamMessage *diam.Message debitInterval time.Duration + timezone string } // AsBareDiameterMessage converts CCA into a bare DiameterMessage @@ -613,7 +617,7 @@ func (self *CCA) SetProcessorAVPs(reqProcessor *config.DARequestProcessor, maxUs if err != nil { return err } - if err := messageSetAVPsWithPath(self.diamMessage, splitIntoInterface(cfgFld.FieldId, utils.HIERARCHY_SEP), []byte(fmtOut), cfgFld.Append); err != nil { + if err := messageSetAVPsWithPath(self.diamMessage, splitIntoInterface(cfgFld.FieldId, utils.HIERARCHY_SEP), fmtOut, cfgFld.Append, self.timezone); err != nil { return err } } diff --git a/agents/libdmt_test.go b/agents/libdmt_test.go index 3bbbe1dfc..6a60b5896 100644 --- a/agents/libdmt_test.go +++ b/agents/libdmt_test.go @@ -162,17 +162,17 @@ func TestMessageSetAVPsWithPath(t *testing.T) { eMessage := diam.NewRequest(diam.CreditControl, 4, nil) eMessage.NewAVP("Session-Id", avp.Mbit, 0, datatype.UTF8String("simuhuawei;1449573472;00002")) m := diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) - if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id", "Unknown"}, []byte("simuhuawei;1449573472;00002"), false); err == nil || err.Error() != "Could not find AVP Unknown" { + if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id", "Unknown"}, "simuhuawei;1449573472;00002", false, "UTC"); err == nil || err.Error() != "Could not find AVP Unknown" { t.Error(err) } - if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, []byte("simuhuawei;1449573472;00002"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, "simuhuawei;1449573472;00002", false, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMessage, m) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) } // test append eMessage.NewAVP("Session-Id", avp.Mbit, 0, datatype.UTF8String("simuhuawei;1449573472;00003")) - if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, []byte("simuhuawei;1449573472;00003"), true); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, "simuhuawei;1449573472;00003", true, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMessage, m) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) @@ -181,10 +181,10 @@ func TestMessageSetAVPsWithPath(t *testing.T) { eMessage = diam.NewRequest(diam.CreditControl, 4, nil) eMessage.NewAVP("Session-Id", avp.Mbit, 0, datatype.UTF8String("simuhuawei;1449573472;00002")) m = diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) - if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, []byte("simuhuawei;1449573472;00001"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, "simuhuawei;1449573472;00001", false, "UTC"); err != nil { t.Error(err) } - if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, []byte("simuhuawei;1449573472;00002"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Session-Id"}, "simuhuawei;1449573472;00002", false, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMessage, m) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) @@ -195,18 +195,17 @@ func TestMessageSetAVPsWithPath(t *testing.T) { diam.NewAVP(444, avp.Mbit, 0, datatype.UTF8String("33708000003")), // Subscription-Id-Data }}) m = diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) - if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Data"}, []byte("33708000003"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Data"}, "33708000003", false, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMessage, m) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) } - // diam.NewAVP(450, avp.Mbit, 0, datatype.Enumerated(subscriptionId.SubscriptionIdType)), // Subscription-Id-Type // test append eMessage.NewAVP("Subscription-Id", avp.Mbit, 0, &diam.GroupedAVP{ AVP: []*diam.AVP{ diam.NewAVP(450, avp.Mbit, 0, datatype.Enumerated(0)), // Subscription-Id-Data }}) - if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Type"}, []byte("0"), true); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Type"}, "0", true, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMessage, m) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) @@ -220,10 +219,10 @@ func TestMessageSetAVPsWithPath(t *testing.T) { }}) eMsgSrl, _ := eMessage.Serialize() m = diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) - if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Type"}, []byte("0"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Type"}, "0", false, "UTC"); err != nil { t.Error(err) } - if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Data"}, []byte("33708000003"), false); err != nil { + if err := messageSetAVPsWithPath(m, []interface{}{"Subscription-Id", "Subscription-Id-Data"}, "33708000003", false, "UTC"); err != nil { t.Error(err) } else { mSrl, _ := m.Serialize() @@ -231,19 +230,17 @@ func TestMessageSetAVPsWithPath(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eMessage, m) } } - /* - eMessage = diam.NewRequest(diam.CreditControl, 4, nil) - eMessage.NewAVP("Granted-Service-Unit", avp.Mbit, 0, &diam.GroupedAVP{ - AVP: []*diam.AVP{ - diam.NewAVP(420, avp.Mbit, 0, datatype.UTF8String("300")), // Subscription-Id-Data - }}) - m = diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) - if err := messageSetAVPsWithPath(m, []interface{}{"Granted-Service-Unit", "CC-Time"}, []byte("300"), false); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eMessage, m) { - t.Errorf("Expecting: %+v, received: %+v", eMessage, m) - } - */ + eMessage = diam.NewRequest(diam.CreditControl, 4, nil) + eMessage.NewAVP("Granted-Service-Unit", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(300)), // Subscription-Id-Data + }}) + m = diam.NewMessage(diam.CreditControl, diam.RequestFlag, 4, eMessage.Header.HopByHopID, eMessage.Header.EndToEndID, nil) + if err := messageSetAVPsWithPath(m, []interface{}{"Granted-Service-Unit", "CC-Time"}, "300", false, "UTC"); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eMessage, m) { + t.Errorf("Expecting: %+v, received: %+v", eMessage, m) + } } func TestCCASetProcessorAVPs(t *testing.T) { diff --git a/config/config_defaults.go b/config/config_defaults.go index 81f0a949b..fa1a60cb8 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -313,7 +313,7 @@ const CGRATES_CFG_JSON = ` {"tag": "SubscriberID", "field_id": "SubscriberId", "type": "*composed", "value": "Subscription-Id>Subscription-Id-Data", "mandatory": true}, ], "cca_fields":[ // fields returned in CCA - {"tag": "GrantedUnits", "type": "*handler", "handler_id": "*cca_usage", "value": "Granted-Service-Unit>CC-Time", "mandatory": true}, + {"tag": "GrantedUnits", "field_id": "Granted-Service-Unit>CC-Time", "type": "*handler", "handler_id": "*cca_usage", "mandatory": true}, ], }, ], From c8f2c101788b786a6d09c07f9de93c78243b4452 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 16:19:51 +0100 Subject: [PATCH 039/227] Diameter enable field filters for all formated field types --- agents/libdmt.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/agents/libdmt.go b/agents/libdmt.go index 361fe3bd7..9bccd0cce 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -312,6 +312,21 @@ func serializeAVPValueFromString(dictAVP *dict.AVP, valStr, timezone string) ([] func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interface{}) (fmtValOut string, err error) { var outVal string + passAtIndex := -1 + passedAllFilters := true + for _, fldFilter := range cfgFld.FieldFilter { + var pass bool + if pass, passAtIndex = passesFieldFilter(m, fldFilter); !pass { + passedAllFilters = false + break + } + } + if !passedAllFilters { + return "", nil // Not matching field filters, will have it empty + } + if passAtIndex == -1 { + passAtIndex = 0 // No filter + } switch cfgFld.Type { case utils.META_FILLER: outVal = cfgFld.Value.Id() @@ -330,21 +345,6 @@ func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interfa case utils.META_COMPOSED: outVal = composedFieldvalue(m, cfgFld.Value, 0) case utils.MetaGrouped: // GroupedAVP - passAtIndex := -1 - matchedAllFilters := true - for _, fldFilter := range cfgFld.FieldFilter { - var pass bool - if pass, passAtIndex = passesFieldFilter(m, fldFilter); !pass { - matchedAllFilters = false - break - } - } - if !matchedAllFilters { - return "", nil // Not matching field filters, will have it empty - } - if passAtIndex == -1 { - passAtIndex = 0 // No filter - } outVal = composedFieldvalue(m, cfgFld.Value, passAtIndex) } if fmtValOut, err = utils.FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { From c46a2e2ec4b45d9d4579ac8c0dfac08e41d8d73e Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 17:25:27 +0100 Subject: [PATCH 040/227] Diameter DryRun for request processors --- agents/dmtagent.go | 11 ++++++++--- agents/libdmt.go | 3 ++- agents/libdmt_test.go | 2 +- config/config_json_test.go | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index 3bf0cc677..2a6877592 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -79,6 +79,14 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro if err != nil { return nil, err } + cca := NewBareCCAFromCCR(ccr, self.cgrCfg.DiameterAgentCfg().OriginHost, self.cgrCfg.DiameterAgentCfg().OriginRealm) + if reqProcessor.DryRun { // DryRun does not send over network + utils.Logger.Info(fmt.Sprintf(" RequestProcessor: %s", reqProcessor.Id)) + utils.Logger.Info(fmt.Sprintf(" CCR message: %+v", ccr)) + utils.Logger.Info(fmt.Sprintf(" SMGenericEvent: %+v", smgEv)) + cca.ResultCode = diam.LimitedSuccess + return cca, nil + } var maxUsage float64 switch ccr.CCRequestType { case 1: @@ -92,9 +100,6 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro err = errCdr } } - cca := NewBareCCAFromCCR(ccr) - cca.OriginHost = self.cgrCfg.DiameterAgentCfg().OriginHost - cca.OriginRealm = self.cgrCfg.DiameterAgentCfg().OriginRealm if err != nil { cca.ResultCode = DiameterRatingFailed utils.Logger.Err(fmt.Sprintf(" Processing message: %+v, error: %s", ccr.diamMessage, err)) diff --git a/agents/libdmt.go b/agents/libdmt.go index 9bccd0cce..11f8962ee 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -564,8 +564,9 @@ func (self *CCR) AsSMGenericEvent(cfgFlds []*config.CfgCdrField) (sessionmanager return sessionmanager.SMGenericEvent(utils.ConvertMapValStrIf(outMap)), nil } -func NewBareCCAFromCCR(ccr *CCR) *CCA { +func NewBareCCAFromCCR(ccr *CCR, originHost, originRealm string) *CCA { cca := &CCA{SessionId: ccr.SessionId, AuthApplicationId: ccr.AuthApplicationId, CCRequestType: ccr.CCRequestType, CCRequestNumber: ccr.CCRequestNumber, + OriginHost: originHost, OriginRealm: originRealm, diamMessage: diam.NewMessage(ccr.diamMessage.Header.CommandCode, ccr.diamMessage.Header.CommandFlags&^diam.RequestFlag, ccr.diamMessage.Header.ApplicationID, ccr.diamMessage.Header.HopByHopID, ccr.diamMessage.Header.EndToEndID, ccr.diamMessage.Dictionary()), ccrMessage: ccr.diamMessage, debitInterval: ccr.debitInterval, } diff --git a/agents/libdmt_test.go b/agents/libdmt_test.go index 6a60b5896..73297edd0 100644 --- a/agents/libdmt_test.go +++ b/agents/libdmt_test.go @@ -257,7 +257,7 @@ func TestCCASetProcessorAVPs(t *testing.T) { diam.NewAVP(444, avp.Mbit, 0, datatype.UTF8String("33708000003")), // Subscription-Id-Data }}) ccr.debitInterval = time.Duration(300) * time.Second - cca := NewBareCCAFromCCR(ccr) + cca := NewBareCCAFromCCR(ccr, "CGR-DA", "cgrates.org") reqProcessor := &config.DARequestProcessor{Id: "UNIT_TEST", // Set template for tests CCAFields: []*config.CfgCdrField{ &config.CfgCdrField{Tag: "Subscription-Id/Subscription-Id-Type", Type: utils.META_COMPOSED, diff --git a/config/config_json_test.go b/config/config_json_test.go index c87c67aee..dba9f6f9d 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -493,7 +493,7 @@ func TestDiameterAgentJsonCfg(t *testing.T) { }, CCA_fields: &[]*CdrFieldJsonCfg{ &CdrFieldJsonCfg{Tag: utils.StringPointer("GrantedUnits"), Type: utils.StringPointer(utils.META_HANDLER), Handler_id: utils.StringPointer("*cca_usage"), - Value: utils.StringPointer("Granted-Service-Unit>CC-Time"), Mandatory: utils.BoolPointer(true)}, + Field_id: utils.StringPointer("Granted-Service-Unit>CC-Time"), Mandatory: utils.BoolPointer(true)}, }, }, }, From f46cb6915830f11123be83ee4530137d7da8a5c0 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 17:56:28 +0100 Subject: [PATCH 041/227] Diameter DryRun improvement to show the real diameter message received --- agents/dmtagent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index 2a6877592..ee89e987b 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -82,7 +82,7 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro cca := NewBareCCAFromCCR(ccr, self.cgrCfg.DiameterAgentCfg().OriginHost, self.cgrCfg.DiameterAgentCfg().OriginRealm) if reqProcessor.DryRun { // DryRun does not send over network utils.Logger.Info(fmt.Sprintf(" RequestProcessor: %s", reqProcessor.Id)) - utils.Logger.Info(fmt.Sprintf(" CCR message: %+v", ccr)) + utils.Logger.Info(fmt.Sprintf(" CCR message: %s", ccr.diamMessage)) utils.Logger.Info(fmt.Sprintf(" SMGenericEvent: %+v", smgEv)) cca.ResultCode = diam.LimitedSuccess return cca, nil From 20d9848cdf574b4520d892edd076815c44a0ba30 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 18:07:20 +0100 Subject: [PATCH 042/227] Diameter filterNotPasing error handling in field formatting --- agents/libdmt.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/agents/libdmt.go b/agents/libdmt.go index 11f8962ee..9129872dc 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -310,6 +310,8 @@ func serializeAVPValueFromString(dictAVP *dict.AVP, valStr, timezone string) ([] } } +var ErrFilterNotPassing = errors.New("Filter not passing") + func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interface{}) (fmtValOut string, err error) { var outVal string passAtIndex := -1 @@ -322,7 +324,7 @@ func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interfa } } if !passedAllFilters { - return "", nil // Not matching field filters, will have it empty + return "", ErrFilterNotPassing // Not matching field filters, will have it empty } if passAtIndex == -1 { passAtIndex = 0 // No filter @@ -552,6 +554,9 @@ func (self *CCR) AsSMGenericEvent(cfgFlds []*config.CfgCdrField) (sessionmanager for _, cfgFld := range cfgFlds { fmtOut, err := fieldOutVal(self.diamMessage, cfgFld, self.debitInterval) if err != nil { + if err == ErrFilterNotPassing { + continue // Do nothing in case of Filter not passing + } return nil, err } if _, hasKey := outMap[cfgFld.FieldId]; hasKey && cfgFld.Append { @@ -616,6 +621,9 @@ func (self *CCA) SetProcessorAVPs(reqProcessor *config.DARequestProcessor, maxUs for _, cfgFld := range reqProcessor.CCAFields { fmtOut, err := fieldOutVal(self.ccrMessage, cfgFld, maxUsage) if err != nil { + if err == ErrFilterNotPassing { + continue + } return err } if err := messageSetAVPsWithPath(self.diamMessage, splitIntoInterface(cfgFld.FieldId, utils.HIERARCHY_SEP), fmtOut, cfgFld.Append, self.timezone); err != nil { From 7395aaa43c1ecc3d8a1d69ff74da4a3e69319d40 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 23 Dec 2015 18:42:36 +0100 Subject: [PATCH 043/227] Responder.GetSessionRuns ads ExtraFields in SesssionRuns.CD, Diameter DryRun able to simulate full reply --- agents/dmtagent.go | 5 +++++ engine/responder.go | 3 ++- engine/responder_test.go | 9 ++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index ee89e987b..bc247aeeb 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -85,6 +85,11 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro utils.Logger.Info(fmt.Sprintf(" CCR message: %s", ccr.diamMessage)) utils.Logger.Info(fmt.Sprintf(" SMGenericEvent: %+v", smgEv)) cca.ResultCode = diam.LimitedSuccess + if err := cca.SetProcessorAVPs(reqProcessor, 0); err != nil { + cca.ResultCode = DiameterRatingFailed + utils.Logger.Err(fmt.Sprintf(" Processing message: %+v, error: %s", ccr.diamMessage, err)) + return cca, nil + } return cca, nil } var maxUsage float64 diff --git a/engine/responder.go b/engine/responder.go index c16b1423b..79000e44f 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -423,7 +423,8 @@ func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { Subject: ev.GetSubject(dc.SubjectField), Account: ev.GetAccount(dc.AccountField), Destination: ev.GetDestination(dc.DestinationField), - TimeStart: startTime} + TimeStart: startTime, + ExtraFields: ev.GetExtraFields()} sesRuns = append(sesRuns, &SessionRun{DerivedCharger: dc, CallDescriptor: cd}) } *sRuns = sesRuns diff --git a/engine/responder_test.go b/engine/responder_test.go index 74e87ccdd..7b6941b5a 100644 --- a/engine/responder_test.go +++ b/engine/responder_test.go @@ -150,11 +150,14 @@ func TestResponderGetSessionRuns(t *testing.T) { sesRuns := make([]*SessionRun, 0) eSRuns := []*SessionRun{ &SessionRun{DerivedCharger: extra1DC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "0", Tenant: "vdf", Subject: "rif", Account: "minitsboy", Destination: "0256", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC)}}, + CallDescriptor: &CallDescriptor{Direction: "*out", Category: "0", Tenant: "vdf", Subject: "rif", Account: "minitsboy", Destination: "0256", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: extra2DC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "ivo", Account: "ivo", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC)}}, + CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "ivo", Account: "ivo", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: dfDC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "dan2", Account: "dan2", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC)}}} + CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "dan2", Account: "dan2", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}} if err := rsponder.GetSessionRuns(cdr, &sesRuns); err != nil { t.Error(err) } else if !reflect.DeepEqual(eSRuns, sesRuns) { From 939f6257df1877127220c9aa7cc6b9783b69c838 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Dec 2015 11:22:41 +0100 Subject: [PATCH 044/227] MaxDebit debug log for users --- engine/responder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/responder.go b/engine/responder.go index 79000e44f..2a2ddf4cf 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -177,6 +177,7 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) r, e := rs.getCallCost(arg, "Responder.MaxDebit") *reply, err = *r, e } else { + utils.Logger.Debug(fmt.Sprintf("### MaxDebit, cd: %+v", arg)) r, e := arg.MaxDebit() if e != nil { rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: e}) From be1d3a67c008ba2ad4d0520eeb68ec5f8aaa4c47 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Dec 2015 11:51:52 +0100 Subject: [PATCH 045/227] More logging on MaxDebit --- engine/responder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/responder.go b/engine/responder.go index 2a2ddf4cf..64710363a 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -148,6 +148,7 @@ func (rs *Responder) Debit(arg *CallDescriptor, reply *CallCost) (err error) { func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) { cacheKey := "MaxDebit" + arg.CgrId + strconv.FormatFloat(arg.LoopIndex, 'f', -1, 64) if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + utils.Logger.Debug(fmt.Sprintf("### MaxDebit out of cache, cd: %+v, error: %+v", arg, item.Err)) *reply = *(item.Value.(*CallCost)) return item.Err } From 1c3e1ad17eabc3b816135dc89248ae04f7dc32f3 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Dec 2015 11:59:06 +0100 Subject: [PATCH 046/227] Even more logging --- engine/responder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/responder.go b/engine/responder.go index 64710363a..9eb3426d6 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -147,6 +147,7 @@ func (rs *Responder) Debit(arg *CallDescriptor, reply *CallCost) (err error) { func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) { cacheKey := "MaxDebit" + arg.CgrId + strconv.FormatFloat(arg.LoopIndex, 'f', -1, 64) + utils.Logger.Debug(fmt.Sprintf("### MaxDebit 1 cd: %+v", arg)) if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { utils.Logger.Debug(fmt.Sprintf("### MaxDebit out of cache, cd: %+v, error: %+v", arg, item.Err)) *reply = *(item.Value.(*CallCost)) @@ -169,11 +170,13 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } + utils.Logger.Debug(fmt.Sprintf("### MaxDebit after aliasing, cd: %+v", arg)) // replace user profile fields if err := LoadUserProfile(arg, utils.EXTRA_FIELDS); err != nil { rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } + utils.Logger.Debug(fmt.Sprintf("### MaxDebit after users, cd: %+v", arg)) if rs.Bal != nil { r, e := rs.getCallCost(arg, "Responder.MaxDebit") *reply, err = *r, e From 61f0e469ad5eafaf75e4db7c3cc02d155446f758 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Dec 2015 12:09:49 +0100 Subject: [PATCH 047/227] Debug out --- engine/responder.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/engine/responder.go b/engine/responder.go index 9eb3426d6..79000e44f 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -147,9 +147,7 @@ func (rs *Responder) Debit(arg *CallDescriptor, reply *CallCost) (err error) { func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) { cacheKey := "MaxDebit" + arg.CgrId + strconv.FormatFloat(arg.LoopIndex, 'f', -1, 64) - utils.Logger.Debug(fmt.Sprintf("### MaxDebit 1 cd: %+v", arg)) if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { - utils.Logger.Debug(fmt.Sprintf("### MaxDebit out of cache, cd: %+v, error: %+v", arg, item.Err)) *reply = *(item.Value.(*CallCost)) return item.Err } @@ -170,18 +168,15 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } - utils.Logger.Debug(fmt.Sprintf("### MaxDebit after aliasing, cd: %+v", arg)) // replace user profile fields if err := LoadUserProfile(arg, utils.EXTRA_FIELDS); err != nil { rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } - utils.Logger.Debug(fmt.Sprintf("### MaxDebit after users, cd: %+v", arg)) if rs.Bal != nil { r, e := rs.getCallCost(arg, "Responder.MaxDebit") *reply, err = *r, e } else { - utils.Logger.Debug(fmt.Sprintf("### MaxDebit, cd: %+v", arg)) r, e := arg.MaxDebit() if e != nil { rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: e}) From 808089455d56ca6fbf289b49e64693ca4bf6d5ed Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Dec 2015 17:38:31 +0100 Subject: [PATCH 048/227] Populating CgrId within CallDescriptor --- engine/responder.go | 1 + engine/responder_test.go | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/engine/responder.go b/engine/responder.go index 79000e44f..7843c609a 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -417,6 +417,7 @@ func (rs *Responder) GetSessionRuns(ev *StoredCdr, sRuns *[]*SessionRun) error { return err } cd := &CallDescriptor{ + CgrId: ev.GetCgrId(rs.Timezone), Direction: ev.GetDirection(dc.DirectionField), Tenant: ev.GetTenant(dc.TenantField), Category: ev.GetCategory(dc.CategoryField), diff --git a/engine/responder_test.go b/engine/responder_test.go index 7b6941b5a..1e7a23889 100644 --- a/engine/responder_test.go +++ b/engine/responder_test.go @@ -150,13 +150,16 @@ func TestResponderGetSessionRuns(t *testing.T) { sesRuns := make([]*SessionRun, 0) eSRuns := []*SessionRun{ &SessionRun{DerivedCharger: extra1DC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "0", Tenant: "vdf", Subject: "rif", Account: "minitsboy", Destination: "0256", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + CallDescriptor: &CallDescriptor{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Direction: "*out", Category: "0", + Tenant: "vdf", Subject: "rif", Account: "minitsboy", Destination: "0256", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: extra2DC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "ivo", Account: "ivo", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + CallDescriptor: &CallDescriptor{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Direction: "*out", Category: "call", + Tenant: "vdf", Subject: "ivo", Account: "ivo", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: dfDC, - CallDescriptor: &CallDescriptor{Direction: "*out", Category: "call", Tenant: "vdf", Subject: "dan2", Account: "dan2", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + CallDescriptor: &CallDescriptor{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Direction: "*out", Category: "call", + Tenant: "vdf", Subject: "dan2", Account: "dan2", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}} if err := rsponder.GetSessionRuns(cdr, &sesRuns); err != nil { t.Error(err) From 0519500b60e0fc60939bdbd0cf25ce76ff3a4d2f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 24 Dec 2015 18:44:17 +0200 Subject: [PATCH 049/227] glide update --- glide.lock | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index 5c0143f2f..94e964ee3 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -updated: 2015-12-23T11:12:22.098552467+02:00 +hash: 0ca45753122a2e205a1b401e7f38b17e58ea22d3b105894604e504aeace503cb +updated: 2015-12-24T18:29:52.686317738+02:00 imports: - name: github.com/cenkalti/hub version: 57d753b5f4856e77b3cf8ecce78c97215a7d324d @@ -28,9 +28,10 @@ imports: - name: github.com/gorhill/cronexpr version: a557574d6c024ed6e36acc8b610f5f211c91568a - name: github.com/jinzhu/gorm - version: d209be3138acbe304daffee637bc495499c1e70e + version: 4a821a5beff551a4f445283fc232e892e2cff324 - name: github.com/jinzhu/inflection version: 3272df6c21d04180007eb3349844c89a3856bc25 + repo: https://github.com/jinzhu/inflection - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq @@ -49,11 +50,12 @@ imports: - name: golang.org/x/crypto version: f18420efc3b4f8e9f3d51f6bd2476e92c46260e9 - name: golang.org/x/net - version: 6c89489cafabcbc76df9dbf84ebf07204673fecf + version: ea6dba8c93880aa07d6ebed83c3c680cd9faa63a subpackages: - /websocket - name: golang.org/x/text version: cf4986612c83df6c55578ba198316d1684a9a287 + repo: https://golang.org/x/text - name: gopkg.in/fsnotify.v1 version: 508915b7500b6e42a87132e4afeb4729cebc7cbb - name: gopkg.in/mgo.v2 @@ -62,4 +64,5 @@ imports: - bson - name: gopkg.in/tomb.v2 version: 14b3d72120e8d10ea6e6b7f87f7175734b1faab8 + repo: https://gopkg.in/tomb.v2 devImports: [] From 8be03ff940564682fa285b08ef9165231531f667 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Dec 2015 19:24:17 +0200 Subject: [PATCH 050/227] integration tests improvements --- agents/hapool_it_test.go | 213 ++--------------------- data/conf/samples/dmtagent/cgrates.json | 2 +- data/conf/samples/hapool/dagent/cgr.json | 6 +- 3 files changed, 15 insertions(+), 206 deletions(-) diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go index 6ca260be6..4b6311c55 100644 --- a/agents/hapool_it_test.go +++ b/agents/hapool_it_test.go @@ -19,16 +19,12 @@ along with this program. If not, see package agents import ( - "net/rpc/jsonrpc" "os/exec" "path" "testing" - "time" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "github.com/fiorix/go-diameter/diam/dict" ) var cgrRater1Cmd, cgrSmg1Cmd *exec.Cmd @@ -37,7 +33,7 @@ func TestHaPoolInitCfg(t *testing.T) { if !*testIntegration { return } - daCfgPath = path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater1") + daCfgPath = path.Join(*dataDir, "conf", "samples", "hapool", "dagent") // Init config first var err error daCfg, err = config.NewCGRConfigFromFolder(daCfgPath) @@ -50,22 +46,12 @@ func TestHaPoolInitCfg(t *testing.T) { // Remove data in both rating and accounting db func TestHaPoolResetDataDb(t *testing.T) { - if !*testIntegration { - return - } - if err := engine.InitDataDb(daCfg); err != nil { - t.Fatal(err) - } + TestDmtAgentResetDataDb(t) } // Wipe out the cdr database func TestHaPoolResetStorDb(t *testing.T) { - if !*testIntegration { - return - } - if err := engine.InitStorDb(daCfg); err != nil { - t.Fatal(err) - } + TestDmtAgentResetStorDb(t) } // Start CGR Engine @@ -100,214 +86,37 @@ func TestHaPoolStartEngine(t *testing.T) { // Connect rpc client to rater func TestHaPoolApierRpcConn(t *testing.T) { - if !*testIntegration { - return - } - var err error - apierRpc, err = jsonrpc.Dial("tcp", daCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal(err) - } + TestDmtAgentApierRpcConn(t) } // Load the tariff plan, creating accounts and their balances func TestHaPoolTPFromFolder(t *testing.T) { - if !*testIntegration { - return - } - attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")} - var loadInst engine.LoadInstance - if err := apierRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &loadInst); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups + TestDmtAgentTPFromFolder(t) } // cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:47:26Z"' func TestHaPoolSendCCRInit(t *testing.T) { - if !*testIntegration { - return - } - dmtClient, err = NewDiameterClient(daCfg.DiameterAgentCfg().Listen, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, - daCfg.DiameterAgentCfg().VendorId, daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DictionariesDir) - if err != nil { - t.Fatal(err) - } - cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, - AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", - SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(0) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, - } - ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, - daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) - m, err := ccr.AsDiameterMessage() - if err != nil { - t.Error(err) - } - if err := dmtClient.SendMessage(m); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(100) * time.Millisecond) - msg := dmtClient.ReceivedMessage() - if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { - t.Error(err) - } else if len(avps) == 0 { - t.Error("Granted-Service-Unit not found") - } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { - t.Errorf("Expecting 300, received: %s", strCCTime) - } - var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} - eAcntVal := 9.484 - if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) - } + TestDmtAgentSendCCRInit(t) } // cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:52:26Z"' func TestHaPoolSendCCRUpdate(t *testing.T) { - if !*testIntegration { - return - } - cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, - AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", - SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(300) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, - } - ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, - daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) - m, err := ccr.AsDiameterMessage() - if err != nil { - t.Error(err) - } - if err := dmtClient.SendMessage(m); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(100) * time.Millisecond) - msg := dmtClient.ReceivedMessage() - if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { - t.Error(err) - } else if len(avps) == 0 { - t.Error("Granted-Service-Unit not found") - } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { - t.Errorf("Expecting 300, received: %s", strCCTime) - } - var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} - eAcntVal := 9.214 - if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) - } + TestDmtAgentSendCCRUpdate(t) } // cgr-console 'cost Category="call" Tenant="cgrates.org" Subject="1001" Destination="1004" TimeStart="2015-11-07T08:42:26Z" TimeEnd="2015-11-07T08:57:26Z"' func TestHaPoolSendCCRUpdate2(t *testing.T) { - if !*testIntegration { - return - } - cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, - AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", - SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(600) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, - } - ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, - daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, false) - m, err := ccr.AsDiameterMessage() - if err != nil { - t.Error(err) - } - if err := dmtClient.SendMessage(m); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(100) * time.Millisecond) - msg := dmtClient.ReceivedMessage() - if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { - t.Error(err) - } else if len(avps) == 0 { - t.Error("Granted-Service-Unit not found") - } else if strCCTime := avpValAsString(avps[0]); strCCTime != "300" { - t.Errorf("Expecting 300, received: %s", strCCTime) - } - var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} - eAcntVal := 8.944 - if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) - } + TestDmtAgentSendCCRUpdate2(t) } func TestHaPoolSendCCRTerminate(t *testing.T) { - if !*testIntegration { - return - } - cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, - AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1004", Supplier: "SUPPL1", - SetupTime: time.Date(2015, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2015, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(610) * time.Second, Pdd: time.Duration(7) * time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, - } - ccr := storedCdrToCCR(cdr, "UNIT_TEST", daCfg.DiameterAgentCfg().OriginRealm, daCfg.DiameterAgentCfg().VendorId, - daCfg.DiameterAgentCfg().ProductName, utils.DIAMETER_FIRMWARE_REVISION, daCfg.DiameterAgentCfg().DebitInterval, true) - m, err := ccr.AsDiameterMessage() - if err != nil { - t.Error(err) - } - if err := dmtClient.SendMessage(m); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(100) * time.Millisecond) - msg := dmtClient.ReceivedMessage() - if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { - t.Error(err) - } else if len(avps) == 0 { - t.Error("Granted-Service-Unit not found") - } else if strCCTime := avpValAsString(avps[0]); strCCTime != "0" { - t.Errorf("Expecting 0, received: %s", strCCTime) - } - var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} - eAcntVal := 9.205 - if err := apierRpc.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { // Should also consider derived charges which double the cost of 6m10s - 2x0.7584 - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) - } + TestDmtAgentSendCCRTerminate(t) } func TestHaPoolCdrs(t *testing.T) { - if !*testIntegration { - return - } - var cdrs []*engine.ExternalCdr - req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}} - if err := apierRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Usage != "610" { - t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) - } - if cdrs[0].Cost != 0.795 { - t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) - } - } + TestDmtAgentCdrs(t) } func TestHaPoolStopEngine(t *testing.T) { - if !*testIntegration { - return - } - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } + TestDmtAgentStopEngine(t) } diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index b33df0fed..676d330bc 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -24,7 +24,7 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": [ + "rater_conns": [ {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> ], "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> diff --git a/data/conf/samples/hapool/dagent/cgr.json b/data/conf/samples/hapool/dagent/cgr.json index 829cb1b32..2f02d5669 100644 --- a/data/conf/samples/hapool/dagent/cgr.json +++ b/data/conf/samples/hapool/dagent/cgr.json @@ -3,8 +3,8 @@ "enabled": true, "listen": "127.0.0.1:3868", "sm_generic_conns": [ - {"server": "127.0.0.1:2018"}, - {"server": "127.0.0.1:2020"}, - ], + {"server": "127.0.0.1:2018"}, + {"server": "127.0.0.1:2020"} + ], }, } From 4dc8311843d178e3f9038746780823d758f06db5 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 30 Dec 2015 15:30:57 +0200 Subject: [PATCH 051/227] use raters conf file --- agents/hapool_it_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go index 4b6311c55..81908d728 100644 --- a/agents/hapool_it_test.go +++ b/agents/hapool_it_test.go @@ -33,7 +33,7 @@ func TestHaPoolInitCfg(t *testing.T) { if !*testIntegration { return } - daCfgPath = path.Join(*dataDir, "conf", "samples", "hapool", "dagent") + daCfgPath = path.Join(*dataDir, "conf", "samples", "hapool", "cgrrater1") // Init config first var err error daCfg, err = config.NewCGRConfigFromFolder(daCfgPath) From c122d4e74a34ae0bcb4b4601f1c43ef28f58956d Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 6 Jan 2016 19:48:55 +0100 Subject: [PATCH 052/227] Fix cdrsConn for SMG in engine --- cmd/cgr-engine/cgr-engine.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 469267e4d..49ec1aed5 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -138,8 +138,8 @@ func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan * func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan chan *engine.Responder, internalCDRSChan chan *engine.CdrServer, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") - raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) - cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_BROADCAST) + raterConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + cdrsConn := rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) var client *rpcclient.RpcClient var err error // Connect to rater @@ -253,9 +253,7 @@ func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, internalCDRSCha } } // Connect to CDRS - if reflect.DeepEqual(cfg.SmFsConfig.CdrsConns, cfg.SmFsConfig.RaterConns) { - cdrsConn = raterConn - } else if len(cfg.SmFsConfig.CdrsConns) != 0 { + if len(cfg.SmFsConfig.CdrsConns) != 0 { for _, cdrsCfg := range cfg.SmFsConfig.CdrsConns { if cdrsCfg.Server == utils.INTERNAL { resp := <-internalCDRSChan From 16c7a3ce8dcbe55804f168523120a76f097e8ecd Mon Sep 17 00:00:00 2001 From: rbarrabe Date: Tue, 22 Mar 2016 13:52:57 +0100 Subject: [PATCH 053/227] Update CONTRIBUTORS.md --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b2a3821bc..6c3e9a5e2 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -42,6 +42,7 @@ information, please see the [`CONTRIBUTING.md`](CONTRIBUTING.md) file. | @bhepp | Brice Heppner | | @noahmehl | Noah Mehl | | @elfranne | Tom Braarup Cuykens | +| @rbarrabe | Régis Barrabé | From 2058e885566a100d741f79970f29cf539b011cdb Mon Sep 17 00:00:00 2001 From: rbarrabe Date: Tue, 22 Mar 2016 13:59:27 +0100 Subject: [PATCH 054/227] Update huawei.xml --- data/diameter/dict/huawei/huawei.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/diameter/dict/huawei/huawei.xml b/data/diameter/dict/huawei/huawei.xml index b180087a7..8553ad05b 100644 --- a/data/diameter/dict/huawei/huawei.xml +++ b/data/diameter/dict/huawei/huawei.xml @@ -133,7 +133,7 @@ - + @@ -143,4 +143,4 @@ - \ No newline at end of file + From 31e33aa8d092f0a27627fbd166947257f498c9ca Mon Sep 17 00:00:00 2001 From: rbarrabe Date: Tue, 22 Mar 2016 14:01:38 +0100 Subject: [PATCH 055/227] Update vodafone.xml --- data/diameter/dict/huawei/vodafone.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/diameter/dict/huawei/vodafone.xml b/data/diameter/dict/huawei/vodafone.xml index ff56332c0..be03b5746 100644 --- a/data/diameter/dict/huawei/vodafone.xml +++ b/data/diameter/dict/huawei/vodafone.xml @@ -43,8 +43,8 @@ - + - \ No newline at end of file + From fbbc067a5d2f6d5ec83e56dbf55b7cd26f0b15cf Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 22 Mar 2016 17:48:04 +0200 Subject: [PATCH 056/227] more lastusage tests --- sessionmanager/smg_it_test.go | 184 ++++++++++++++++++++++++++++++++++ sessionmanager/smg_session.go | 15 +-- 2 files changed, 193 insertions(+), 6 deletions(-) diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index cd246596e..30e7af12f 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -422,3 +422,187 @@ func TestSMGLastUsed(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } } + +func TestSMGLastUsedEnd(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 7.59000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "2m", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 6.190020 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "2m", + utils.LastUsed: "30s", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 6.090030 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0s", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 6.590000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + +func TestSMGLastUsedNotFixed(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 6.59000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "2m", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 5.190020 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "2m", + utils.LastUsed: "13s", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 5.123360 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0s", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 5.590000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} + + diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 6f94b1a0e..936f4ac56 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -85,12 +85,14 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D } else { // We have debitted less than we have consumed, add the difference to duration debitted lastUsedCorrection = lastUsed - self.lastUsage } + + // apply the lastUsed correction + dur += lastUsedCorrection + self.totalUsage += dur // Should reflect the total usage so far + } else { + // apply correction from previous run + dur -= self.extraDuration } - // apply the lastUsed correction - dur += lastUsedCorrection - self.totalUsage += dur // Should reflect the total usage so far - // apply correction from previous run - dur -= self.extraDuration self.extraDuration = 0 if self.cd.LoopIndex > 0 { self.cd.TimeStart = self.cd.TimeEnd @@ -189,7 +191,7 @@ func (self *SMGSession) close(endTime time.Time) error { for _, cc := range self.callCosts[1:] { firstCC.Merge(cc) } - //utils.Logger.Debug(fmt.Sprintf("MergedCC: %+v", firstCC)) + //utils.Logger.Debug("MergedCC: " + utils.ToJSON(firstCC)) end := firstCC.GetEndTime() refundDuration := end.Sub(endTime) self.refund(refundDuration) @@ -223,6 +225,7 @@ func (self *SMGSession) saveOperations() error { } firstCC := self.callCosts[0] // was merged in close method firstCC.Round() + //utils.Logger.Debug("Saved CC: " + utils.ToJSON(firstCC)) roundIncrements := firstCC.GetRoundIncrements() if len(roundIncrements) != 0 { cd := firstCC.CreateCallDescriptor() From e07acf23f74a70a975531150d33891c0bbcbfc20 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 22 Mar 2016 22:04:26 +0200 Subject: [PATCH 057/227] started smg data tests --- data/tariffplans/testtp/AccountActions.csv | 3 +- data/tariffplans/testtp/ActionPlans.csv | 3 +- data/tariffplans/testtp/Actions.csv | 3 +- data/tariffplans/testtp/DestinationRates.csv | 1 + data/tariffplans/testtp/Destinations.csv | 1 + data/tariffplans/testtp/Rates.csv | 2 + data/tariffplans/testtp/RatingPlans.csv | 1 + data/tariffplans/testtp/RatingProfiles.csv | 1 + sessionmanager/data_it_test.go | 184 +++++++++++++++++++ sessionmanager/smg_it_test.go | 13 -- 10 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 sessionmanager/data_it_test.go diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index ca940949b..4b6dddf29 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -4,4 +4,5 @@ cgrates.org,1002,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1003,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1004,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1005,PREPAID_10,STANDARD_TRIGGERS,, -cgrates.org,1009,TEST_EXE,,, \ No newline at end of file +cgrates.org,1009,TEST_EXE,,, +cgrates.org,1010,TEST_DATA_r,,, diff --git a/data/tariffplans/testtp/ActionPlans.csv b/data/tariffplans/testtp/ActionPlans.csv index 670cf8c93..f41668f9b 100644 --- a/data/tariffplans/testtp/ActionPlans.csv +++ b/data/tariffplans/testtp/ActionPlans.csv @@ -1,4 +1,5 @@ #Tag,ActionsTag,TimingTag,Weight PREPAID_10,PREPAID_10,ASAP,10 PREPAID_10,BONUS_1,ASAP,10 -TEST_EXE,TOPUP_EXE,ALWAYS,10 \ No newline at end of file +TEST_EXE,TOPUP_EXE,ALWAYS,10 +TEST_DATA_r,TOPUP_DATA_r,ASAP,10 \ No newline at end of file diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 045fc0666..0c39073a9 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -4,4 +4,5 @@ BONUS_1,*topup,,,,*monetary,*out,,*any,,,*unlimited,,1,10,false,false,10 LOG_BALANCE,*log,,,,,,,,,,,,,,false,false,10 CDRST_WARN_HTTP,*call_url,http://localhost:8080,,,,,,,,,,,,,false,false,10 CDRST_LOG,*log,,,,,,,,,,,,,,false,false,10 -TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 \ No newline at end of file +TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 +TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,,,*unlimited,,50000000000,10,false,false,10 \ No newline at end of file diff --git a/data/tariffplans/testtp/DestinationRates.csv b/data/tariffplans/testtp/DestinationRates.csv index b5d2474b3..9be903aef 100644 --- a/data/tariffplans/testtp/DestinationRates.csv +++ b/data/tariffplans/testtp/DestinationRates.csv @@ -3,3 +3,4 @@ DR_RETAIL,GERMANY,RT_1CENT,*up,4,0, DR_RETAIL,GERMANY_MOBILE,RT_1CENT,*up,4,0, DR_DATA_1,*any,RT_DATA_2c,*up,4,0, DR_SMS_1,*any,RT_SMS_5c,*up,4,0, +DR_DATA_r,DATA_DEST,RT_DATA_r,*up,5,0, \ No newline at end of file diff --git a/data/tariffplans/testtp/Destinations.csv b/data/tariffplans/testtp/Destinations.csv index 192146546..37385ea00 100644 --- a/data/tariffplans/testtp/Destinations.csv +++ b/data/tariffplans/testtp/Destinations.csv @@ -3,3 +3,4 @@ GERMANY,+49 GERMANY_MOBILE,+4915 GERMANY_MOBILE,+4916 GERMANY_MOBILE,+4917 +DATA_DEST,222 diff --git a/data/tariffplans/testtp/Rates.csv b/data/tariffplans/testtp/Rates.csv index 991ce0720..dd4dd35a7 100644 --- a/data/tariffplans/testtp/Rates.csv +++ b/data/tariffplans/testtp/Rates.csv @@ -2,3 +2,5 @@ RT_1CENT,0,1,1s,1s,0s RT_DATA_2c,0,0.002,10,10,0 RT_SMS_5c,0,0.005,1,1,0 +RT_DATA_r,0,0,1,10240000,0 +RT_DATA_r,0,0,1,10240000,60 \ No newline at end of file diff --git a/data/tariffplans/testtp/RatingPlans.csv b/data/tariffplans/testtp/RatingPlans.csv index 7d15f1a92..f7003dcf4 100644 --- a/data/tariffplans/testtp/RatingPlans.csv +++ b/data/tariffplans/testtp/RatingPlans.csv @@ -2,3 +2,4 @@ RP_RETAIL,DR_RETAIL,ALWAYS,10 RP_DATA1,DR_DATA_1,ALWAYS,10 RP_SMS1,DR_SMS_1,ALWAYS,10 +RP_DATAr,DR_DATA_r,ALWAYS,10 diff --git a/data/tariffplans/testtp/RatingProfiles.csv b/data/tariffplans/testtp/RatingProfiles.csv index db758126b..fed2e73c6 100644 --- a/data/tariffplans/testtp/RatingProfiles.csv +++ b/data/tariffplans/testtp/RatingProfiles.csv @@ -2,3 +2,4 @@ *out,cgrates.org,call,*any,2012-01-01T00:00:00Z,RP_RETAIL,, *out,cgrates.org,data,*any,2012-01-01T00:00:00Z,RP_DATA1,, *out,cgrates.org,sms,*any,2012-01-01T00:00:00Z,RP_SMS1,, +*out,cgrates.org,data,*any,2016-01-01T00:00:00Z,RP_DATAr,, diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go new file mode 100644 index 000000000..2cef5654e --- /dev/null +++ b/sessionmanager/data_it_test.go @@ -0,0 +1,184 @@ +package sessionmanager + +import ( + "flag" + "net/rpc" + "net/rpc/jsonrpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var testIntegration = flag.Bool("integration", false, "Perform the tests in integration mode, not by default.") // This flag will be passed here via "go test -local" args +var waitRater = flag.Int("wait_rater", 150, "Number of miliseconds to wait for rater to start and cache") +var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") + +var daCfgPath string +var daCfg *config.CGRConfig +var smgRPC *rpc.Client +var err error + +func TestSMGDataInitCfg(t *testing.T) { + if !*testIntegration { + return + } + daCfgPath = path.Join(*dataDir, "conf", "samples", "smg") + // Init config first + var err error + daCfg, err = config.NewCGRConfigFromFolder(daCfgPath) + if err != nil { + t.Error(err) + } + daCfg.DataFolderPath = *dataDir // Share DataFolderPath through config towards StoreDb for Flush() + config.SetCgrConfig(daCfg) +} + +// Remove data in both rating and accounting db +func TestSMGDataResetDataDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitDataDb(daCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func TestSMGDataResetStorDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitStorDb(daCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func TestSMGDataStartEngine(t *testing.T) { + if !*testIntegration { + return + } + if _, err := engine.StopStartEngine(daCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func TestSMGDataApierRpcConn(t *testing.T) { + if !*testIntegration { + return + } + var err error + smgRPC, err = jsonrpc.Dial("tcp", daCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +// Load the tariff plan, creating accounts and their balances +func TestSMGDataTPFromFolder(t *testing.T) { + if !*testIntegration { + return + } + attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "testtp")} + var loadInst engine.LoadInstance + if err := smgRPC.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &loadInst); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups +} + +func TestSMGDataLastUsedNotFixed(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 6.59000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "2m", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 5.190020 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "2m", + utils.LastUsed: "13s", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 5.123360 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0s", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 5.590000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 30e7af12f..129505986 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -19,8 +19,6 @@ along with this program. If not, see package sessionmanager import ( - "flag" - "net/rpc" "net/rpc/jsonrpc" "path" "testing" @@ -31,15 +29,6 @@ import ( "github.com/cgrates/cgrates/utils" ) -var testIntegration = flag.Bool("integration", false, "Perform the tests in integration mode, not by default.") // This flag will be passed here via "go test -local" args -var waitRater = flag.Int("wait_rater", 150, "Number of miliseconds to wait for rater to start and cache") -var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") - -var daCfgPath string -var daCfg *config.CGRConfig -var smgRPC *rpc.Client -var err error - func TestSMGInitCfg(t *testing.T) { if !*testIntegration { return @@ -604,5 +593,3 @@ func TestSMGLastUsedNotFixed(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } } - - From 5fc1efc055458bccd31a3c3fe1d7783f6063f4bf Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 11:08:21 +0200 Subject: [PATCH 058/227] updated data tests --- data/tariffplans/testtp/Rates.csv | 3 +- sessionmanager/data_it_test.go | 58 +++++++++++++++---------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/data/tariffplans/testtp/Rates.csv b/data/tariffplans/testtp/Rates.csv index dd4dd35a7..bff768bf1 100644 --- a/data/tariffplans/testtp/Rates.csv +++ b/data/tariffplans/testtp/Rates.csv @@ -2,5 +2,4 @@ RT_1CENT,0,1,1s,1s,0s RT_DATA_2c,0,0.002,10,10,0 RT_SMS_5c,0,0.005,1,1,0 -RT_DATA_r,0,0,1,10240000,0 -RT_DATA_r,0,0,1,10240000,60 \ No newline at end of file +RT_DATA_r,0,0.19,1048576,10240,0 diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 2cef5654e..d99a287e2 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -92,32 +92,32 @@ func TestSMGDataTPFromFolder(t *testing.T) { time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups } -func TestSMGDataLastUsedNotFixed(t *testing.T) { +func TestSMGDataLastUsedData(t *testing.T) { if !*testIntegration { return } var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} eAcntVal := 6.59000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.VOICE, + utils.TOR: utils.DATA, utils.ACCID: "12349", utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1001", - utils.SUBJECT: "1001", - utils.DESTINATION: "1006", - utils.CATEGORY: "call", + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, utils.SETUP_TIME: "2016-01-05 18:30:49", utils.ANSWER_TIME: "2016-01-05 18:31:05", - utils.USAGE: "2m", + utils.USAGE: "1048576", } var maxUsage float64 if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { @@ -129,22 +129,22 @@ func TestSMGDataLastUsedNotFixed(t *testing.T) { eAcntVal = 5.190020 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.VOICE, + utils.TOR: utils.DATA, utils.ACCID: "12349", utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1001", - utils.SUBJECT: "1001", - utils.DESTINATION: "1006", - utils.CATEGORY: "call", + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.USAGE: "2m", - utils.LastUsed: "13s", + utils.USAGE: "1048576", + utils.LastUsed: "20000", } if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { t.Error(err) @@ -155,21 +155,21 @@ func TestSMGDataLastUsedNotFixed(t *testing.T) { eAcntVal = 5.123360 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.VOICE, + utils.TOR: utils.DATA, utils.ACCID: "12349", utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1001", - utils.SUBJECT: "1001", - utils.DESTINATION: "1006", - utils.CATEGORY: "call", + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.LastUsed: "0s", + utils.LastUsed: "0", } var rpl string if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { @@ -178,7 +178,7 @@ func TestSMGDataLastUsedNotFixed(t *testing.T) { eAcntVal = 5.590000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } } From 1170a7ba21c9e43e934cac6f3c4ab137c69e43dd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 13:24:49 +0200 Subject: [PATCH 059/227] updated test data --- data/tariffplans/testtp/Actions.csv | 2 +- data/tariffplans/testtp/RatingProfiles.csv | 2 +- sessionmanager/data_it_test.go | 23 ++++++---------------- sessionmanager/smg_it_test.go | 11 +++++++++++ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 0c39073a9..0f478dd4f 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -5,4 +5,4 @@ LOG_BALANCE,*log,,,,,,,,,,,,,,false,false,10 CDRST_WARN_HTTP,*call_url,http://localhost:8080,,,,,,,,,,,,,false,false,10 CDRST_LOG,*log,,,,,,,,,,,,,,false,false,10 TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 -TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,,,*unlimited,,50000000000,10,false,false,10 \ No newline at end of file +TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 \ No newline at end of file diff --git a/data/tariffplans/testtp/RatingProfiles.csv b/data/tariffplans/testtp/RatingProfiles.csv index fed2e73c6..ffa6bf2f6 100644 --- a/data/tariffplans/testtp/RatingProfiles.csv +++ b/data/tariffplans/testtp/RatingProfiles.csv @@ -2,4 +2,4 @@ *out,cgrates.org,call,*any,2012-01-01T00:00:00Z,RP_RETAIL,, *out,cgrates.org,data,*any,2012-01-01T00:00:00Z,RP_DATA1,, *out,cgrates.org,sms,*any,2012-01-01T00:00:00Z,RP_SMS1,, -*out,cgrates.org,data,*any,2016-01-01T00:00:00Z,RP_DATAr,, +*out,cgrates.org,data,datar,2016-01-01T00:00:00Z,RP_DATAr,, diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index d99a287e2..d3804dfcf 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -1,8 +1,6 @@ package sessionmanager import ( - "flag" - "net/rpc" "net/rpc/jsonrpc" "path" "testing" @@ -13,15 +11,6 @@ import ( "github.com/cgrates/cgrates/utils" ) -var testIntegration = flag.Bool("integration", false, "Perform the tests in integration mode, not by default.") // This flag will be passed here via "go test -local" args -var waitRater = flag.Int("wait_rater", 150, "Number of miliseconds to wait for rater to start and cache") -var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") - -var daCfgPath string -var daCfg *config.CGRConfig -var smgRPC *rpc.Client -var err error - func TestSMGDataInitCfg(t *testing.T) { if !*testIntegration { return @@ -98,7 +87,7 @@ func TestSMGDataLastUsedData(t *testing.T) { } var acnt *engine.Account attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} - eAcntVal := 6.59000 + eAcntVal := 50000000000.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -123,10 +112,10 @@ func TestSMGDataLastUsedData(t *testing.T) { if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { t.Error(err) } - if maxUsage != 120 { + if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 5.190020 + eAcntVal = 49998951424.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -149,10 +138,10 @@ func TestSMGDataLastUsedData(t *testing.T) { if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { t.Error(err) } - if maxUsage != 120 { + if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 5.123360 + eAcntVal = 49998931424.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -175,7 +164,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { t.Error(err) } - eAcntVal = 5.590000 + eAcntVal = 49998931424.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 129505986..435778513 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -19,6 +19,8 @@ along with this program. If not, see package sessionmanager import ( + "flag" + "net/rpc" "net/rpc/jsonrpc" "path" "testing" @@ -29,6 +31,15 @@ import ( "github.com/cgrates/cgrates/utils" ) +var testIntegration = flag.Bool("integration", false, "Perform the tests in integration mode, not by default.") // This flag will be passed here via "go test -local" args +var waitRater = flag.Int("wait_rater", 150, "Number of miliseconds to wait for rater to start and cache") +var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") + +var daCfgPath string +var daCfg *config.CGRConfig +var smgRPC *rpc.Client +var err error + func TestSMGInitCfg(t *testing.T) { if !*testIntegration { return From c40efaad382fbb7c308505e5e076aa8f61d09465 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 13:31:51 +0200 Subject: [PATCH 060/227] allow negative in test account --- data/tariffplans/testtp/AccountActions.csv | 2 +- data/tariffplans/testtp/Rates.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index 4b6dddf29..4cf06f896 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -5,4 +5,4 @@ cgrates.org,1003,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1004,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1005,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1009,TEST_EXE,,, -cgrates.org,1010,TEST_DATA_r,,, +cgrates.org,1010,TEST_DATA_r,,true, diff --git a/data/tariffplans/testtp/Rates.csv b/data/tariffplans/testtp/Rates.csv index bff768bf1..21c9abc4a 100644 --- a/data/tariffplans/testtp/Rates.csv +++ b/data/tariffplans/testtp/Rates.csv @@ -2,4 +2,4 @@ RT_1CENT,0,1,1s,1s,0s RT_DATA_2c,0,0.002,10,10,0 RT_SMS_5c,0,0.005,1,1,0 -RT_DATA_r,0,0.19,1048576,10240,0 +RT_DATA_r,0,0.1,1048576,10240,0 From ca2375aba4b26becc2e426f03bfc564dfba9c3af Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 15:49:08 +0200 Subject: [PATCH 061/227] data monetary balance --- data/tariffplans/testtp/Actions.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 0f478dd4f..666038a38 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -5,4 +5,5 @@ LOG_BALANCE,*log,,,,,,,,,,,,,,false,false,10 CDRST_WARN_HTTP,*call_url,http://localhost:8080,,,,,,,,,,,,,false,false,10 CDRST_LOG,*log,,,,,,,,,,,,,,false,false,10 TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 +TOPUP_DATA_r,*topup,,,,*monetary,*out,,DATA_DEST,,,*unlimited,,5000000,10,false,false,10 TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 \ No newline at end of file From a1118d24236fea8e57245494a8ceedaa96ae98a6 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 21:52:43 +0200 Subject: [PATCH 062/227] usage corrections --- sessionmanager/data_it_test.go | 177 ++++++++++++++++++++++++++++++++- sessionmanager/smg_session.go | 2 +- 2 files changed, 175 insertions(+), 4 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index d3804dfcf..826f8653c 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -115,7 +115,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998951424.000000 + eAcntVal = 49998945280.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -141,7 +141,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998931424.000000 + eAcntVal = 49998924800.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -164,7 +164,178 @@ func TestSMGDataLastUsedData(t *testing.T) { if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { t.Error(err) } - eAcntVal = 49998931424.000000 + eAcntVal = 49999979520.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } +} + +func TestSMGDataLastUsedMultipleData(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} + eAcntVal := 49999979520.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "1048576", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998924800.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "20000", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998904320.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "20000", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998883840.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "20000", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998904320.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "20000", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998883840.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 49999948800.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 936f4ac56..6eace1506 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -88,7 +88,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D // apply the lastUsed correction dur += lastUsedCorrection - self.totalUsage += dur // Should reflect the total usage so far + self.totalUsage += lastUsed // Should reflect the total usage so far } else { // apply correction from previous run dur -= self.extraDuration From 4f87aae12af39af1c1cdf66d6e0e85b993d729bf Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 23 Mar 2016 21:56:59 +0200 Subject: [PATCH 063/227] test fixes --- sessionmanager/data_it_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 826f8653c..69b6d95a5 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -285,7 +285,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998904320.000000 + eAcntVal = 49998863360.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -311,7 +311,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998883840.000000 + eAcntVal = 49998842880.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -335,7 +335,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { t.Error(err) } - eAcntVal = 49999948800.000000 + eAcntVal = 49999897600.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { From bf745d1ea5bded866da735e3975b2821c7c00a56 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 24 Mar 2016 13:17:38 +0200 Subject: [PATCH 064/227] add cgdr usage in lastusage scenario --- sessionmanager/smg_session.go | 1 + sessionmanager/smgeneric.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 6eace1506..53a655db6 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -225,6 +225,7 @@ func (self *SMGSession) saveOperations() error { } firstCC := self.callCosts[0] // was merged in close method firstCC.Round() + self.totalUsage = time.Duration(firstCC.RatedUsage) // save final usage //utils.Logger.Debug("Saved CC: " + utils.ToJSON(firstCC)) roundIncrements := firstCC.GetRoundIncrements() if len(roundIncrements) != 0 { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index e80417211..ff5931434 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -333,8 +333,18 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { + cdr := gev.AsStoredCdr(self.cgrCfg, self.timezone) + if cdr.Usage == 0 { + var s *SMGSession + for _, s = range self.getSession(gev.GetUUID()) { + break + } + if s != nil { + cdr.Usage = s.TotalUsage() + } + } var reply string - if err := self.cdrsrv.ProcessCdr(gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { + if err := self.cdrsrv.ProcessCdr(cdr, &reply); err != nil { return err } return nil From 73286e3c7184c2cd2d83ef591e654c397219c73d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 24 Mar 2016 13:24:31 +0200 Subject: [PATCH 065/227] update integration test --- apier/v1/apier_local_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 47da716b5..88ad8c819 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1294,10 +1294,10 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) } else { - if rcvStats.Destinations != 4 || - rcvStats.RatingPlans != 3 || - rcvStats.RatingProfiles != 3 || - rcvStats.Actions != 6 || + if rcvStats.Destinations != 5 || + rcvStats.RatingPlans != 4 || + rcvStats.RatingProfiles != 4 || + rcvStats.Actions != 7 || rcvStats.DerivedChargers != 2 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) } From 4ff66e088378f762aefb199db7ff932e6e2bf7d9 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 24 Mar 2016 17:39:47 +0200 Subject: [PATCH 066/227] populate cdr Usage --- apier/v1/cdrs.go | 8 +++--- data/storage/mysql/create_cdrs_tables.sql | 1 + data/storage/postgres/create_cdrs_tables.sql | 1 + engine/cdrs.go | 16 +++++++++--- engine/guardian_test.go | 20 ++++++++------- engine/models.go | 1 + engine/pubsub.go | 4 +-- engine/storage_cdrs_it_test.go | 8 +++--- engine/storage_interface.go | 4 +-- engine/storage_mongo_stordb.go | 8 +++--- engine/storage_sql.go | 27 ++++++++++++-------- engine/storage_utils.go | 1 + sessionmanager/smg_session.go | 2 +- sessionmanager/smgeneric.go | 13 ++-------- 14 files changed, 63 insertions(+), 51 deletions(-) diff --git a/apier/v1/cdrs.go b/apier/v1/cdrs.go index fe52b49ab..39dcd5378 100644 --- a/apier/v1/cdrs.go +++ b/apier/v1/cdrs.go @@ -26,19 +26,19 @@ import ( ) // Retrieves the callCost out of CGR logDb -func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine.CallCost) error { +func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine.SMCost) error { if attrs.CgrId == "" { return utils.NewErrMandatoryIeMissing("CgrId") } if attrs.RunId == "" { attrs.RunId = utils.META_DEFAULT } - if cc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, attrs.RunId); err != nil { + if smc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, attrs.RunId); err != nil { return utils.NewErrServerError(err) - } else if cc == nil { + } else if smc == nil { return utils.ErrNotFound } else { - *reply = *cc + *reply = *smc } return nil } diff --git a/data/storage/mysql/create_cdrs_tables.sql b/data/storage/mysql/create_cdrs_tables.sql index 3f5e0269d..463ea922e 100644 --- a/data/storage/mysql/create_cdrs_tables.sql +++ b/data/storage/mysql/create_cdrs_tables.sql @@ -42,6 +42,7 @@ CREATE TABLE sm_costs ( cgrid char(40) NOT NULL, run_id varchar(64) NOT NULL, cost_source varchar(64) NOT NULL, + `usage` DECIMAL(30,9) NOT NULL, cost_details text, created_at TIMESTAMP, deleted_at TIMESTAMP, diff --git a/data/storage/postgres/create_cdrs_tables.sql b/data/storage/postgres/create_cdrs_tables.sql index d4425fb89..2c710dc4e 100644 --- a/data/storage/postgres/create_cdrs_tables.sql +++ b/data/storage/postgres/create_cdrs_tables.sql @@ -45,6 +45,7 @@ CREATE TABLE sm_costs ( cgrid CHAR(40) NOT NULL, run_id VARCHAR(64) NOT NULL, cost_source VARCHAR(64) NOT NULL, + usage NUMERIC(30,9) NOT NULL, cost_details jsonb, created_at TIMESTAMP, deleted_at TIMESTAMP, diff --git a/engine/cdrs.go b/engine/cdrs.go index 27cd49dc6..e8b7de17d 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -38,6 +38,7 @@ type CallCostLog struct { CgrId string Source string RunId string + Usage float64 // real usage (not increment rounded) CallCost *CallCost CheckDuplicate bool } @@ -119,11 +120,11 @@ func (self *CdrServer) LogCallCost(ccl *CallCostLog) error { if cc != nil { return nil, utils.ErrExists } - return nil, self.cdrDb.LogCallCost(ccl.CgrId, ccl.RunId, ccl.Source, ccl.CallCost) + return nil, self.cdrDb.LogCallCost(&SMCost{CGRID: ccl.CgrId, RunID: ccl.RunId, CostSource: ccl.Source, Usage: ccl.Usage, CostDetails: ccl.CallCost}) }, 0, ccl.CgrId) return err } - return self.cdrDb.LogCallCost(ccl.CgrId, ccl.RunId, ccl.Source, ccl.CallCost) + return self.cdrDb.LogCallCost(&SMCost{CGRID: ccl.CgrId, RunID: ccl.RunId, CostSource: ccl.Source, Usage: ccl.Usage, CostDetails: ccl.CallCost}) } // Called by rate/re-rate API @@ -332,12 +333,16 @@ func (self *CdrServer) rateCDR(cdr *CDR) error { if cdr.RequestType == utils.META_NONE { return nil } - if utils.IsSliceMember([]string{utils.META_PREPAID, utils.PREPAID}, cdr.RequestType) && cdr.Usage != 0 { // ToDo: Get rid of PREPAID as soon as we don't want to support it backwards + _, hasLastUsed := cdr.ExtraFields["LastUsed"] + if utils.IsSliceMember([]string{utils.META_PREPAID, utils.PREPAID}, cdr.RequestType) && (cdr.Usage != 0 || hasLastUsed) { // ToDo: Get rid of PREPAID as soon as we don't want to support it backwards // Should be previously calculated and stored in DB delay := utils.Fib() + var usage float64 for i := 0; i < 4; i++ { - qryCC, err = self.cdrDb.GetCallCostLog(cdr.CGRID, cdr.RunID) + qrySMC, err := self.cdrDb.GetCallCostLog(cdr.CGRID, cdr.RunID) if err == nil { + qryCC = qrySMC.CostDetails + usage = qrySMC.Usage break } time.Sleep(delay()) @@ -346,6 +351,9 @@ func (self *CdrServer) rateCDR(cdr *CDR) error { utils.Logger.Warning(fmt.Sprintf(" WARNING: Could not find CallCostLog for cgrid: %s, source: %s, runid: %s, will recalculate", cdr.CGRID, utils.SESSION_MANAGER_SOURCE, cdr.RunID)) qryCC, err = self.getCostFromRater(cdr) } + if cdr.Usage == 0 { + cdr.Usage = time.Duration(usage) + } } else { qryCC, err = self.getCostFromRater(cdr) diff --git a/engine/guardian_test.go b/engine/guardian_test.go index a8e3cc4f8..6e436e9d5 100644 --- a/engine/guardian_test.go +++ b/engine/guardian_test.go @@ -19,31 +19,33 @@ along with this program. If not, see package engine import ( - "log" "testing" "time" ) -func ATestAccountLock(t *testing.T) { +func BenchmarkGuard(b *testing.B) { for i := 0; i < 100; i++ { go Guardian.Guard(func() (interface{}, error) { - log.Print("first 1") time.Sleep(1 * time.Millisecond) - log.Print("end first 1") return 0, nil }, 0, "1") go Guardian.Guard(func() (interface{}, error) { - log.Print("first 2") time.Sleep(1 * time.Millisecond) - log.Print("end first 2") return 0, nil }, 0, "2") go Guardian.Guard(func() (interface{}, error) { - log.Print("second 1") time.Sleep(1 * time.Millisecond) - log.Print("end second 1") return 0, nil }, 0, "1") } - time.Sleep(10 * time.Second) + +} + +func BenchmarkGuardian(b *testing.B) { + for i := 0; i < 100; i++ { + go Guardian.Guard(func() (interface{}, error) { + time.Sleep(1 * time.Millisecond) + return 0, nil + }, 0, "1") + } } diff --git a/engine/models.go b/engine/models.go index 85ce94362..1da2db987 100644 --- a/engine/models.go +++ b/engine/models.go @@ -452,6 +452,7 @@ type TBLSMCosts struct { Cgrid string RunID string CostSource string + Usage float64 CostDetails string CreatedAt time.Time DeletedAt time.Time diff --git a/engine/pubsub.go b/engine/pubsub.go index 2ae544df0..0d4dd17c1 100644 --- a/engine/pubsub.go +++ b/engine/pubsub.go @@ -139,13 +139,13 @@ func (ps *PubSub) Publish(evt CgrEvent, reply *string) error { } transport := split[0] address := split[1] - + ttlVerify := ps.ttlVerify switch transport { case utils.META_HTTP_POST: go func() { delay := utils.Fib() for i := 0; i < 5; i++ { // Loop so we can increase the success rate on best effort - if _, err := ps.pubFunc(address, ps.ttlVerify, evt); err == nil { + if _, err := ps.pubFunc(address, ttlVerify, evt); err == nil { break // Success, no need to reinterate } else if i == 4 { // Last iteration, syslog the warning utils.Logger.Warning(fmt.Sprintf(" Failed calling url: [%s], error: [%s], event type: %s", address, err.Error(), evt["EventName"])) diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index f264ec237..4e934d4e4 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -222,13 +222,13 @@ func testSMCosts(cfg *config.CGRConfig) error { }, TOR: utils.VOICE, } - if err := cdrStorage.LogCallCost("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT, utils.UNIT_TEST, cc); err != nil { + if err := cdrStorage.LogCallCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e41", RunID: utils.META_DEFAULT, CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { return err } - if rcvCC, err := cdrStorage.GetCallCostLog("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT); err != nil { + if rcvSMC, err := cdrStorage.GetCallCostLog("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT); err != nil { return err - } else if len(cc.Timespans) != len(rcvCC.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) - return fmt.Errorf("Expecting: %+v, received: %+v", cc, rcvCC) + } else if len(cc.Timespans) != len(rcvSMC.CostDetails.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) + return fmt.Errorf("Expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC)) } return nil } diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 8d082ae14..95d2f8d0e 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -97,8 +97,8 @@ type AccountingStorage interface { type CdrStorage interface { Storage SetCDR(*CDR, bool) error - LogCallCost(cgrid, runid, source string, cc *CallCost) error - GetCallCostLog(cgrid, runid string) (*CallCost, error) + LogCallCost(smc *SMCost) error + GetCallCostLog(cgrid, runid string) (*SMCost, error) GetCDRs(*utils.CDRsFilter, bool) ([]*CDR, int64, error) } diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index 0e0090def..a0aaffe09 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -698,16 +698,16 @@ func (ms *MongoStorage) LogActionTiming(source string, at *ActionTiming, as Acti }{at, as, time.Now(), source}) } -func (ms *MongoStorage) LogCallCost(cgrid, runid, source string, cc *CallCost) error { - return ms.db.C(utils.TBLSMCosts).Insert(&SMCost{CGRID: cgrid, RunID: runid, CostSource: source, CostDetails: cc}) +func (ms *MongoStorage) LogCallCost(smc *SMCost) error { + return ms.db.C(utils.TBLSMCosts).Insert(smc) } -func (ms *MongoStorage) GetCallCostLog(cgrid, runid string) (cc *CallCost, err error) { +func (ms *MongoStorage) GetCallCostLog(cgrid, runid string) (smc *SMCost, err error) { var result SMCost if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{CGRIDLow: cgrid, RunIDLow: runid}).One(&result); err != nil { return nil, err } - return result.CostDetails, nil + return &result, nil } func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) (err error) { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index e7796e34f..cac8c86fc 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -569,21 +569,22 @@ func (self *SQLStorage) SetTpAccountActions(aas []TpAccountAction) error { return nil } -func (self *SQLStorage) LogCallCost(cgrid, runid, source string, cc *CallCost) error { - if cc == nil { +func (self *SQLStorage) LogCallCost(smc *SMCost) error { + if smc.CostDetails == nil { return nil } - tss, err := json.Marshal(cc) + tss, err := json.Marshal(smc.CostDetails) if err != nil { utils.Logger.Err(fmt.Sprintf("Error marshalling timespans to json: %v", err)) return err } tx := self.db.Begin() cd := &TBLSMCosts{ - Cgrid: cgrid, - RunID: runid, - CostSource: source, + Cgrid: smc.CGRID, + RunID: smc.RunID, + CostSource: smc.CostSource, CostDetails: string(tss), + Usage: smc.Usage, CreatedAt: time.Now(), } if tx.Save(cd).Error != nil { // Check further since error does not properly reflect duplicates here (sql: no rows in result set) @@ -594,7 +595,7 @@ func (self *SQLStorage) LogCallCost(cgrid, runid, source string, cc *CallCost) e return nil } -func (self *SQLStorage) GetCallCostLog(cgrid, runid string) (*CallCost, error) { +func (self *SQLStorage) GetCallCostLog(cgrid, runid string) (*SMCost, error) { var tpCostDetail TBLSMCosts if err := self.db.Where(&TBLSMCosts{Cgrid: cgrid, RunID: runid}).First(&tpCostDetail).Error; err != nil { return nil, err @@ -602,11 +603,17 @@ func (self *SQLStorage) GetCallCostLog(cgrid, runid string) (*CallCost, error) { if len(tpCostDetail.CostDetails) == 0 { return nil, nil // No costs returned } - var cc CallCost - if err := json.Unmarshal([]byte(tpCostDetail.CostDetails), &cc); err != nil { + smc := &SMCost{ + CGRID: tpCostDetail.Cgrid, + RunID: tpCostDetail.RunID, + CostSource: tpCostDetail.CostSource, + Usage: tpCostDetail.Usage, + CostDetails: &CallCost{}, + } + if err := json.Unmarshal([]byte(tpCostDetail.CostDetails), smc.CostDetails); err != nil { return nil, err } - return &cc, nil + return smc, nil } func (self *SQLStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) (err error) { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 8994020fd..16012f2e1 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -157,5 +157,6 @@ type SMCost struct { CGRID string RunID string CostSource string + Usage float64 CostDetails *CallCost } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 53a655db6..46ade6019 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -225,7 +225,6 @@ func (self *SMGSession) saveOperations() error { } firstCC := self.callCosts[0] // was merged in close method firstCC.Round() - self.totalUsage = time.Duration(firstCC.RatedUsage) // save final usage //utils.Logger.Debug("Saved CC: " + utils.ToJSON(firstCC)) roundIncrements := firstCC.GetRoundIncrements() if len(roundIncrements) != 0 { @@ -242,6 +241,7 @@ func (self *SMGSession) saveOperations() error { CgrId: self.eventStart.GetCgrId(self.timezone), Source: utils.SESSION_MANAGER_SOURCE, RunId: self.runId, + Usage: float64(self.totalUsage), CallCost: firstCC, CheckDuplicate: true, }, &reply) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index ff5931434..2f16b2f71 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -117,6 +117,7 @@ func (self *SMGeneric) sessionEnd(sessionId string, usage time.Duration) error { return nil, nil // Did not find the session so no need to close it anymore } for idx, s := range ss { + s.totalUsage = usage // save final usage as totalUsage //utils.Logger.Info(fmt.Sprintf(" Ending session: %s, runId: %s", sessionId, s.runId)) if idx == 0 && s.stopDebit != nil { close(s.stopDebit) // Stop automatic debits @@ -333,18 +334,8 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { - cdr := gev.AsStoredCdr(self.cgrCfg, self.timezone) - if cdr.Usage == 0 { - var s *SMGSession - for _, s = range self.getSession(gev.GetUUID()) { - break - } - if s != nil { - cdr.Usage = s.TotalUsage() - } - } var reply string - if err := self.cdrsrv.ProcessCdr(cdr, &reply); err != nil { + if err := self.cdrsrv.ProcessCdr(gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err } return nil From 7a4b3f24af8b67f6f67664cab1d90bb2eed200e3 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 24 Mar 2016 18:27:16 +0200 Subject: [PATCH 067/227] don't skip LastUsed in extra fields' --- sessionmanager/smg_event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sessionmanager/smg_event.go b/sessionmanager/smg_event.go index 643c8693f..ced2b5e0c 100644 --- a/sessionmanager/smg_event.go +++ b/sessionmanager/smg_event.go @@ -228,7 +228,7 @@ func (self SMGenericEvent) GetCdrSource() string { func (self SMGenericEvent) GetExtraFields() map[string]string { extraFields := make(map[string]string) for key, val := range self { - primaryFields := append(utils.PrimaryCdrFields, utils.EVENT_NAME, utils.LastUsed) + primaryFields := append(utils.PrimaryCdrFields, utils.EVENT_NAME) if utils.IsSliceMember(primaryFields, key) { continue } From 29bd0c5dbb18488647db63ae92b2f39471fe4e10 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Mar 2016 20:13:04 +0100 Subject: [PATCH 068/227] Diameter - fix metaSum to consider filter indexes --- agents/libdmt.go | 6 ++-- agents/libdmt_test.go | 81 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/agents/libdmt.go b/agents/libdmt.go index bb60b4f75..f3ee75dee 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -244,8 +244,8 @@ func metaValueExponent(m *diam.Message, argsTpl utils.RSRFields, roundingDecimal return strconv.FormatFloat(utils.Round(res, roundingDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil } -func metaSum(m *diam.Message, argsTpl utils.RSRFields, roundingDecimals int) (string, error) { - valStr := composedFieldvalue(m, argsTpl, 0, nil) +func metaSum(m *diam.Message, argsTpl utils.RSRFields, passAtIndex, roundingDecimals int) (string, error) { + valStr := composedFieldvalue(m, argsTpl, passAtIndex, nil) handlerArgs := strings.Split(valStr, utils.HandlerArgSep) var summed float64 for _, arg := range handlerArgs { @@ -398,7 +398,7 @@ func fieldOutVal(m *diam.Message, cfgFld *config.CfgCdrField, extraParam interfa case META_VALUE_EXPONENT: outVal, err = metaValueExponent(m, cfgFld.Value, 10) // FixMe: add here configured number of decimals case META_SUM: - outVal, err = metaSum(m, cfgFld.Value, 10) + outVal, err = metaSum(m, cfgFld.Value, passAtIndex, 10) default: outVal, err = metaHandler(m, cfgFld.HandlerId, cfgFld.Layout, extraParam.(time.Duration)) if err != nil { diff --git a/agents/libdmt_test.go b/agents/libdmt_test.go index 7d6924c42..5bb1d7c01 100644 --- a/agents/libdmt_test.go +++ b/agents/libdmt_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/sessionmanager" "github.com/cgrates/cgrates/utils" "github.com/fiorix/go-diameter/diam" "github.com/fiorix/go-diameter/diam/avp" @@ -133,12 +134,12 @@ func TestMetaSum(t *testing.T) { }), }, }) - if val, err := metaSum(m, utils.ParseRSRFieldsMustCompile("Requested-Service-Unit>CC-Money>Unit-Value>Value-Digits;^|;Requested-Service-Unit>CC-Money>Unit-Value>Exponent", utils.INFIELD_SEP), 10); err != nil { + if val, err := metaSum(m, utils.ParseRSRFieldsMustCompile("Requested-Service-Unit>CC-Money>Unit-Value>Value-Digits;^|;Requested-Service-Unit>CC-Money>Unit-Value>Exponent", utils.INFIELD_SEP), 0, 10); err != nil { t.Error(err) } else if val != "9995" { t.Error("Received: ", val) } - if _, err = metaSum(m, utils.ParseRSRFieldsMustCompile("Requested-Service-Unit>CC-Money>Unit-Value>Value-Digits;Requested-Service-Unit>CC-Money>Unit-Value>Exponent", utils.INFIELD_SEP), 10); err == nil { + if _, err = metaSum(m, utils.ParseRSRFieldsMustCompile("Requested-Service-Unit>CC-Money>Unit-Value>Value-Digits;Requested-Service-Unit>CC-Money>Unit-Value>Exponent", utils.INFIELD_SEP), 0, 10); err == nil { t.Error("Should have received error") // Insufficient number arguments } } @@ -396,3 +397,79 @@ func TestCCASetProcessorAVPs(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eMessage, ccaMsg) } } + +func TestCCRAsSMGenericEvent(t *testing.T) { + ccr := &CCR{ // Bare information, just the one needed for answer + SessionId: "ccrasgen1", + AuthApplicationId: 4, + CCRequestType: 3, + } + ccr.diamMessage = ccr.AsBareDiameterMessage() + ccr.diamMessage.NewAVP("Multiple-Services-Credit-Control", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit + AVP: []*diam.AVP{ + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(17)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1341)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(3079)), // CC-Output-Octets + }, + }), + diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(99)), + }, + }) + ccr.diamMessage.NewAVP("Multiple-Services-Credit-Control", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit + AVP: []*diam.AVP{ + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(8046)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(46193)), // CC-Output-Octets + }, + }), + diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(1)), + }, + }) + ccr.diamMessage.NewAVP("FramedIPAddress", avp.Mbit, 0, datatype.OctetString("0AE40041")) + cfgFlds := make([]*config.CfgCdrField, 0) + eSMGEv := sessionmanager.SMGenericEvent{"EventName": "DIAMETER_CCR"} + if rSMGEv, err := ccr.AsSMGenericEvent(cfgFlds); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eSMGEv, rSMGEv) { + t.Errorf("Expecting: %+v, received: %+v", eSMGEv, rSMGEv) + } + cfgFlds = []*config.CfgCdrField{ + &config.CfgCdrField{ + Tag: "LastUsed", + FieldFilter: utils.ParseRSRFieldsMustCompile("~Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets:s/^(.*)$/test/(test);Multiple-Services-Credit-Control>Rating-Group(1)", utils.INFIELD_SEP), + FieldId: "LastUsed", + Type: "*handler", + HandlerId: "*sum", + Value: utils.ParseRSRFieldsMustCompile("Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets", utils.INFIELD_SEP), + Mandatory: true, + }, + } + eSMGEv = sessionmanager.SMGenericEvent{"EventName": "DIAMETER_CCR", "LastUsed": "54239"} + if rSMGEv, err := ccr.AsSMGenericEvent(cfgFlds); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eSMGEv, rSMGEv) { + t.Errorf("Expecting: %+v, received: %+v", eSMGEv, rSMGEv) + } + cfgFlds = []*config.CfgCdrField{ + &config.CfgCdrField{ + Tag: "LastUsed", + FieldFilter: utils.ParseRSRFieldsMustCompile("~Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets:s/^(.*)$/test/(test);Multiple-Services-Credit-Control>Rating-Group(99)", utils.INFIELD_SEP), + FieldId: "LastUsed", + Type: "*handler", + HandlerId: "*sum", + Value: utils.ParseRSRFieldsMustCompile("Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets", utils.INFIELD_SEP), + Mandatory: true, + }, + } + eSMGEv = sessionmanager.SMGenericEvent{"EventName": "DIAMETER_CCR", "LastUsed": "4420"} + if rSMGEv, err := ccr.AsSMGenericEvent(cfgFlds); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eSMGEv, rSMGEv) { + t.Errorf("Expecting: %+v, received: %+v", eSMGEv, rSMGEv) + } +} From 069af112a6f5a4b543cc72be8c7801582d9077cf Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 24 Mar 2016 20:19:09 +0100 Subject: [PATCH 069/227] Fix SMGEventTest --- sessionmanager/smg_event_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sessionmanager/smg_event_test.go b/sessionmanager/smg_event_test.go index 4cc80781c..5cc65583b 100644 --- a/sessionmanager/smg_event_test.go +++ b/sessionmanager/smg_event_test.go @@ -127,7 +127,7 @@ func TestSMGenericEventParseFields(t *testing.T) { if smGev.GetOriginatorIP(utils.META_DEFAULT) != "127.0.0.1" { t.Error("Unexpected: ", smGev.GetOriginatorIP(utils.META_DEFAULT)) } - if extrFlds := smGev.GetExtraFields(); !reflect.DeepEqual(extrFlds, map[string]string{"Extra1": "Value1", "Extra2": "5"}) { + if extrFlds := smGev.GetExtraFields(); !reflect.DeepEqual(extrFlds, map[string]string{"Extra1": "Value1", "Extra2": "5", "LastUsed": "21s"}) { t.Error("Unexpected: ", extrFlds) } } From 393d893ef7505368d0fec5bdae3f21d68756008f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 25 Mar 2016 15:11:02 +0200 Subject: [PATCH 070/227] make tests use glide novendor --- .travis.yml | 2 +- local_test.sh | 31 ++----------------------------- test.sh | 50 ++------------------------------------------------ 3 files changed, 5 insertions(+), 78 deletions(-) diff --git a/.travis.yml b/.travis.yml index 899b7c359..9b4f37383 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - go get github.com/Masterminds/glide - glide install -script: $TRAVIS_BUILD_DIR/test.sh +script: go test -v $(glide novendor) branches: only: master diff --git a/local_test.sh b/local_test.sh index 05577d629..4218bd3bd 100755 --- a/local_test.sh +++ b/local_test.sh @@ -1,32 +1,5 @@ #! /usr/bin/env sh ./test.sh gen=$? -echo 'go test github.com/cgrates/cgrates/apier/v1 -local' -go test github.com/cgrates/cgrates/apier/v1 -local -ap1=$? -echo 'go test github.com/cgrates/cgrates/apier/v2 -local' -go test github.com/cgrates/cgrates/apier/v2 -local -ap2=$? -echo 'go test github.com/cgrates/cgrates/engine -local -integration' -go test github.com/cgrates/cgrates/engine -local -integration -en=$? -echo 'go test github.com/cgrates/cgrates/cdrc -local' -go test github.com/cgrates/cgrates/cdrc -local -cdrc=$? -echo 'go test github.com/cgrates/cgrates/config -local' -go test github.com/cgrates/cgrates/config -local -cfg=$? -echo 'go test github.com/cgrates/cgrates/utils -local' -go test github.com/cgrates/cgrates/utils -local -utl=$? -echo 'go test github.com/cgrates/cgrates/general_tests -local -integration' -go test github.com/cgrates/cgrates/general_tests -local -integration -gnr=$? -echo 'go test github.com/cgrates/cgrates/agents -integration' -go test github.com/cgrates/cgrates/agents -integration -agts=$? -echo 'go test github.com/cgrates/cgrates/sessionmanager -integration' -go test github.com/cgrates/cgrates/sessionmanager -integration -smg=$? - -exit $gen && $ap1 && $ap2 && $en && $cdrc && $cfg && $utl && $gnr && $agts && $smg +go test -local -integration $(glide novendor) +exit $gen && $? diff --git a/test.sh b/test.sh index 3a4a57e95..8c6ab0558 100755 --- a/test.sh +++ b/test.sh @@ -1,50 +1,4 @@ #! /usr/bin/env sh ./build.sh - -go test -i github.com/cgrates/cgrates/apier/v1 -go test -i github.com/cgrates/cgrates/apier/v2 -go test -i github.com/cgrates/cgrates/engine -go test -i github.com/cgrates/cgrates/sessionmanager -go test -i github.com/cgrates/cgrates/config -go test -i github.com/cgrates/cgrates/cmd/cgr-engine -go test -i github.com/cgrates/cgrates/cache2go -go test -i github.com/cgrates/cgrates/cdrc -go test -i github.com/cgrates/cgrates/utils -go test -i github.com/cgrates/cgrates/history -go test -i github.com/cgrates/cgrates/cdre -go test -i github.com/cgrates/cgrates/agents -go test -i github.com/cgrates/cgrates/structmatcher - -go test github.com/cgrates/cgrates/apier/v1 -v1=$? -go test github.com/cgrates/cgrates/apier/v2 -v2=$? -go test github.com/cgrates/cgrates/engine -en=$? -go test github.com/cgrates/cgrates/general_tests -gt=$? -go test github.com/cgrates/cgrates/sessionmanager -sm=$? -go test github.com/cgrates/cgrates/config -cfg=$? -go test github.com/cgrates/cgrates/cmd/cgr-engine -cr=$? -go test github.com/cgrates/cgrates/console -con=$? -go test github.com/cgrates/cgrates/cdrc -cdrcs=$? -go test github.com/cgrates/cgrates/utils -ut=$? -go test github.com/cgrates/cgrates/history -hs=$? -go test github.com/cgrates/cgrates/cache2go -c2g=$? -go test github.com/cgrates/cgrates/cdre -cdre=$? -go test github.com/cgrates/cgrates/agents -ag=$? -go test github.com/cgrates/cgrates/structmatcher -sc=$? - - -exit $v1 && $v2 && $en && $gt && $sm && $cfg && $bl && $cr && $con && $cdrc && $ut && $hs && $c2g && $cdre && $ag && $sc +go test $(glide novendor) +exit $? From 7721a99edd5efa501ca1f186e7e3ebce7292ef4a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 25 Mar 2016 15:32:21 +0200 Subject: [PATCH 071/227] fix GetAccounts with mongo --- apier/v1/accounts.go | 2 +- apier/v2/accounts.go | 2 +- data/conf/samples/cgradmin/cgradmin.json | 26 +++++------ engine/storage_interface.go | 2 +- engine/storage_map.go | 15 +++--- engine/storage_mongo_datadb.go | 58 ++++++++++++++++++++++-- engine/storage_redis.go | 13 ++++-- engine/storage_sql.go | 2 +- engine/tp_reader.go | 2 +- 9 files changed, 90 insertions(+), 32 deletions(-) diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index 6d006da25..c6c161331 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -318,7 +318,7 @@ func (self *ApierV1) GetAccounts(attr utils.AttrGetAccounts, reply *[]interface{ var accountKeys []string var err error if len(attr.AccountIds) == 0 { - if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + attr.Tenant); err != nil { + if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+attr.Tenant, true); err != nil { return err } } else { diff --git a/apier/v2/accounts.go b/apier/v2/accounts.go index c3c495edd..08f6832dd 100644 --- a/apier/v2/accounts.go +++ b/apier/v2/accounts.go @@ -33,7 +33,7 @@ func (self *ApierV2) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Ac var accountKeys []string var err error if len(attr.AccountIds) == 0 { - if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + utils.ConcatenatedKey(attr.Tenant)); err != nil { + if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Tenant), true); err != nil { return err } } else { diff --git a/data/conf/samples/cgradmin/cgradmin.json b/data/conf/samples/cgradmin/cgradmin.json index 2f6f9e4a1..9dd30abcf 100644 --- a/data/conf/samples/cgradmin/cgradmin.json +++ b/data/conf/samples/cgradmin/cgradmin.json @@ -10,19 +10,19 @@ "http": ":2080", // HTTP listening address }, -//"tariffplan_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mongo", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 27017, // the port to reach the stordb -// "db_name": "tpdb", -//}, -// -//"data_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mongo", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 27017, // the port to reach the stordb -// "db_name": "datadb", -//}, +"tariffplan_db": { // database used to store offline tariff plans and CDRs + "db_type": "mongo", // stor database type to use: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 27017, // the port to reach the stordb + "db_name": "tpdb", +}, + +"data_db": { // database used to store offline tariff plans and CDRs + "db_type": "mongo", // stor database type to use: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 27017, // the port to reach the stordb + "db_name": "datadb", +}, "stor_db": { // database used to store offline tariff plans and CDRs "db_type": "mongo", // stor database type to use: diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 95d2f8d0e..dbf8a8632 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -33,7 +33,7 @@ import ( type Storage interface { Close() Flush(string) error - GetKeysForPrefix(string) ([]string, error) + GetKeysForPrefix(string, bool) ([]string, error) } // Interface for storage providers. diff --git a/engine/storage_map.go b/engine/storage_map.go index 0e77e3ae1..5a898fa32 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -53,14 +53,17 @@ func (ms *MapStorage) Flush(ignore string) error { return nil } -func (ms *MapStorage) GetKeysForPrefix(prefix string) ([]string, error) { - keysForPrefix := make([]string, 0) - for key := range ms.dict { - if strings.HasPrefix(key, prefix) { - keysForPrefix = append(keysForPrefix, key) +func (ms *MapStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + if skipCache { + keysForPrefix := make([]string, 0) + for key := range ms.dict { + if strings.HasPrefix(key, prefix) { + keysForPrefix = append(keysForPrefix, key) + } } + return keysForPrefix, nil } - return keysForPrefix, nil + return cache2go.GetEntriesKeys(prefix), nil } func (ms *MapStorage) CacheRatingAll() error { diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 4fb29e84b..a208ce6cb 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -244,8 +244,60 @@ func (ms *MongoStorage) Close() { ms.session.Close() } -func (ms *MongoStorage) GetKeysForPrefix(prefix string) ([]string, error) { - return nil, nil +func (ms *MongoStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + var category, subject string + length := len(utils.DESTINATION_PREFIX) + if len(prefix) >= length { + category = prefix[:length] // prefix lenght + subject = fmt.Sprintf("^%s", prefix[length:]) + } else { + return nil, fmt.Errorf("unsupported prefix in GetKeysForPrefix: %s", prefix) + } + var result []string + if skipCache { + keyResult := struct{ Key string }{} + idResult := struct{ Id string }{} + switch category { + case utils.DESTINATION_PREFIX: + iter := ms.db.C(colDst).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.DESTINATION_PREFIX+keyResult.Key) + } + return result, nil + case utils.RATING_PLAN_PREFIX: + iter := ms.db.C(colRpl).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.RATING_PLAN_PREFIX+keyResult.Key) + } + return result, nil + case utils.RATING_PROFILE_PREFIX: + iter := ms.db.C(colRpf).Find(bson.M{"id": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"id": 1}).Iter() + for iter.Next(&idResult) { + result = append(result, utils.RATING_PROFILE_PREFIX+idResult.Id) + } + return result, nil + case utils.ACTION_PREFIX: + iter := ms.db.C(colAct).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.ACTION_PREFIX+keyResult.Key) + } + return result, nil + case utils.ACTION_PLAN_PREFIX: + iter := ms.db.C(colApl).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.ACTION_PLAN_PREFIX+keyResult.Key) + } + return result, nil + case utils.ACCOUNT_PREFIX: + iter := ms.db.C(colAcc).Find(bson.M{"id": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"id": 1}).Iter() + for iter.Next(&idResult) { + result = append(result, utils.ACCOUNT_PREFIX+idResult.Id) + } + return result, nil + } + return result, fmt.Errorf("unsupported prefix in GetKeysForPrefix: %s", prefix) + } + return cache2go.GetEntriesKeys(prefix), nil } func (ms *MongoStorage) Flush(ignore string) (err error) { @@ -627,7 +679,7 @@ func (ms *MongoStorage) HasData(category, subject string) (bool, error) { count, err := ms.db.C(colAcc).Find(bson.M{"id": subject}).Count() return count > 0, err } - return false, errors.New("Unsupported category in HasData") + return false, errors.New("unsupported category in HasData") } func (ms *MongoStorage) GetRatingPlan(key string, skipCache bool) (rp *RatingPlan, err error) { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 52d4bb3a7..8d8b1660a 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -84,12 +84,15 @@ func (rs *RedisStorage) Flush(ignore string) error { return rs.db.Cmd("FLUSHDB").Err } -func (rs *RedisStorage) GetKeysForPrefix(prefix string) ([]string, error) { - r := rs.db.Cmd("KEYS", prefix+"*") - if r.Err != nil { - return nil, r.Err +func (rs *RedisStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + if skipCache { + r := rs.db.Cmd("KEYS", prefix+"*") + if r.Err != nil { + return nil, r.Err + } + return r.List() } - return r.List() + return cache2go.GetEntriesKeys(prefix), nil } func (rs *RedisStorage) CacheRatingAll() error { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index cac8c86fc..1e0f5368a 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -55,7 +55,7 @@ func (self *SQLStorage) Flush(scriptsPath string) (err error) { return nil } -func (self *SQLStorage) GetKeysForPrefix(prefix string) ([]string, error) { +func (self *SQLStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { return nil, utils.ErrNotImplemented } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index bf662a335..e458e130e 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -433,7 +433,7 @@ func (tpr *TpReader) LoadLCRs() (err error) { } } if !found && tpr.ratingStorage != nil { - if keys, err := tpr.ratingStorage.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX + ratingProfileSearchKey); err != nil { + if keys, err := tpr.ratingStorage.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX+ratingProfileSearchKey, true); err != nil { return fmt.Errorf("[LCR] error querying ratingDb %s", err.Error()) } else if len(keys) != 0 { found = true From 4c6514bf4f55a15606068b5cf95fa5bf14279298 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 25 Mar 2016 15:54:57 +0200 Subject: [PATCH 072/227] revert local tests till we found a better solution --- local_test.sh | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/local_test.sh b/local_test.sh index 4218bd3bd..05577d629 100755 --- a/local_test.sh +++ b/local_test.sh @@ -1,5 +1,32 @@ #! /usr/bin/env sh ./test.sh gen=$? -go test -local -integration $(glide novendor) -exit $gen && $? +echo 'go test github.com/cgrates/cgrates/apier/v1 -local' +go test github.com/cgrates/cgrates/apier/v1 -local +ap1=$? +echo 'go test github.com/cgrates/cgrates/apier/v2 -local' +go test github.com/cgrates/cgrates/apier/v2 -local +ap2=$? +echo 'go test github.com/cgrates/cgrates/engine -local -integration' +go test github.com/cgrates/cgrates/engine -local -integration +en=$? +echo 'go test github.com/cgrates/cgrates/cdrc -local' +go test github.com/cgrates/cgrates/cdrc -local +cdrc=$? +echo 'go test github.com/cgrates/cgrates/config -local' +go test github.com/cgrates/cgrates/config -local +cfg=$? +echo 'go test github.com/cgrates/cgrates/utils -local' +go test github.com/cgrates/cgrates/utils -local +utl=$? +echo 'go test github.com/cgrates/cgrates/general_tests -local -integration' +go test github.com/cgrates/cgrates/general_tests -local -integration +gnr=$? +echo 'go test github.com/cgrates/cgrates/agents -integration' +go test github.com/cgrates/cgrates/agents -integration +agts=$? +echo 'go test github.com/cgrates/cgrates/sessionmanager -integration' +go test github.com/cgrates/cgrates/sessionmanager -integration +smg=$? + +exit $gen && $ap1 && $ap2 && $en && $cdrc && $cfg && $utl && $gnr && $agts && $smg From 8b94e0c8e3adfc1aceee7d508772ea344229fb85 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 25 Mar 2016 16:44:05 +0100 Subject: [PATCH 073/227] SMG - SessionRelocate through the use of InitialOriginID --- sessionmanager/smg_event.go | 12 ++++++++++++ sessionmanager/smg_event_test.go | 16 ++++++++++++++++ sessionmanager/smgeneric.go | 30 ++++++++++++++++++++++++++++++ utils/consts.go | 2 ++ 4 files changed, 60 insertions(+) diff --git a/sessionmanager/smg_event.go b/sessionmanager/smg_event.go index ced2b5e0c..890e9a1c9 100644 --- a/sessionmanager/smg_event.go +++ b/sessionmanager/smg_event.go @@ -238,6 +238,18 @@ func (self SMGenericEvent) GetExtraFields() map[string]string { return extraFields } +func (self SMGenericEvent) GetFieldAsString(fieldName string) (string, error) { + valIf, hasVal := self[fieldName] + if !hasVal { + return "", utils.ErrNotFound + } + result, converted := utils.ConvertIfaceToString(valIf) + if !converted { + return "", utils.ErrNotConvertible + } + return result, nil +} + func (self SMGenericEvent) MissingParameter(timezone string) bool { switch self.GetName() { case utils.CGR_AUTHORIZATION: diff --git a/sessionmanager/smg_event_test.go b/sessionmanager/smg_event_test.go index 5cc65583b..336996943 100644 --- a/sessionmanager/smg_event_test.go +++ b/sessionmanager/smg_event_test.go @@ -191,3 +191,19 @@ func TestSMGenericEventAsLcrRequest(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eLcrReq, lcrReq) } } + +func TestSMGenericEventGetFieldAsString(t *testing.T) { + smGev := SMGenericEvent{} + smGev[utils.EVENT_NAME] = "TEST_EVENT" + smGev[utils.TOR] = utils.VOICE + smGev[utils.ACCID] = "12345" + smGev[utils.DIRECTION] = utils.OUT + smGev[utils.ACCOUNT] = "account1" + smGev[utils.SUBJECT] = "subject1" + eFldVal := utils.VOICE + if strVal, err := smGev.GetFieldAsString(utils.TOR); err != nil { + t.Error(err) + } else if strVal != eFldVal { + t.Errorf("Expecting: %s, received: %s", eFldVal, strVal) + } +} diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 2f16b2f71..a5b102a77 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -139,6 +139,27 @@ func (self *SMGeneric) sessionEnd(sessionId string, usage time.Duration) error { return err } +// Used when an update will relocate an initial session (eg multiple data streams) +func (self *SMGeneric) sessionRelocate(sessionID, initialID string) error { + _, err := self.guard.Guard(func() (interface{}, error) { // Lock it on initialID level + if utils.IsSliceMember([]string{sessionID, initialID}, "") { + return nil, utils.ErrMandatoryIeMissing + } + ss := self.getSession(initialID) + if len(ss) == 0 { // No need of relocation + return nil, utils.ErrNotFound + } + for i, s := range ss { + self.indexSession(sessionID, s) + if i == 0 { + self.unindexSession(initialID) + } + } + return nil, nil + }, time.Duration(2)*time.Second, initialID) + return err +} + // Methods to apply on sessions, mostly exported through RPC/Bi-RPC //Calculates maximum usage allowed for gevent func (self *SMGeneric) GetMaxUsage(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { @@ -170,6 +191,15 @@ func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([ // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { + if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { + err := self.sessionRelocate(gev.GetUUID(), initialID) + if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update + err = self.sessionStart(gev, getClientConnId(clnt)) + } + if err != nil { + return nilDuration, err + } + } evLastUsed, err := gev.GetLastUsed(utils.META_DEFAULT) if err != nil && err != utils.ErrNotFound { return nilDuration, err diff --git a/utils/consts.go b/utils/consts.go index 72c15b145..36001e3c9 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -31,6 +31,7 @@ var ( ErrAccountDisabled = errors.New("ACCOUNT_DISABLED") ErrUserNotFound = errors.New("USER_NOT_FOUND") ErrInsufficientCredit = errors.New("INSUFFICENT_CREDIT") + ErrNotConvertible = errors.New("NOT_CONVERTIBLE") ) const ( @@ -113,6 +114,7 @@ const ( TOR = "ToR" ORDERID = "OrderID" ACCID = "OriginID" + InitialOriginID = "InitialOriginID" CDRSOURCE = "Source" CDRHOST = "OriginHost" REQTYPE = "RequestType" From fb16ce988228fe5784d0616c62b5450d6f8fc482 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 25 Mar 2016 17:25:44 +0100 Subject: [PATCH 074/227] SMG - SessionRelocate also for terminate message --- sessionmanager/smgeneric.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index a5b102a77..4d6d10464 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -142,9 +142,13 @@ func (self *SMGeneric) sessionEnd(sessionId string, usage time.Duration) error { // Used when an update will relocate an initial session (eg multiple data streams) func (self *SMGeneric) sessionRelocate(sessionID, initialID string) error { _, err := self.guard.Guard(func() (interface{}, error) { // Lock it on initialID level - if utils.IsSliceMember([]string{sessionID, initialID}, "") { + if utils.IsSliceMember([]string{sessionID, initialID}, "") { // Not allowed empty params here return nil, utils.ErrMandatoryIeMissing } + ssNew := self.getSession(sessionID) // Already relocated + if len(ss) != 0 { + return nil, nil + } ss := self.getSession(initialID) if len(ss) == 0 { // No need of relocation return nil, utils.ErrNotFound @@ -232,6 +236,15 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time // Called on session end, should stop debit loop func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { + if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { + err := self.sessionRelocate(gev.GetUUID(), initialID) + if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update + err = self.sessionStart(gev, getClientConnId(clnt)) + } + if err != nil { + return nilDuration, err + } + } usage, err := gev.GetUsage(utils.META_DEFAULT) if err != nil { if err != utils.ErrNotFound { From 4ac4ddf9ba911564885380f2f30904b714b0cc9d Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 25 Mar 2016 17:26:45 +0100 Subject: [PATCH 075/227] Build fix --- sessionmanager/smgeneric.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 4d6d10464..f7fc47c58 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -146,7 +146,7 @@ func (self *SMGeneric) sessionRelocate(sessionID, initialID string) error { return nil, utils.ErrMandatoryIeMissing } ssNew := self.getSession(sessionID) // Already relocated - if len(ss) != 0 { + if len(ssNew) != 0 { return nil, nil } ss := self.getSession(initialID) @@ -242,7 +242,7 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { err = self.sessionStart(gev, getClientConnId(clnt)) } if err != nil { - return nilDuration, err + return err } } usage, err := gev.GetUsage(utils.META_DEFAULT) From 38719cbab2f4ef7bdf51a6859029bdbfe8b0ee02 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 25 Mar 2016 18:28:20 +0200 Subject: [PATCH 076/227] updated vagrant and started smg simplification --- data/vagrant/Vagrantfile | 2 +- data/vagrant/cgrates_devel.yml | 38 +++++++++++++--------------------- sessionmanager/smg_session.go | 33 +++++++++++++---------------- 3 files changed, 29 insertions(+), 44 deletions(-) diff --git a/data/vagrant/Vagrantfile b/data/vagrant/Vagrantfile index 680947427..85d56090a 100644 --- a/data/vagrant/Vagrantfile +++ b/data/vagrant/Vagrantfile @@ -38,7 +38,7 @@ Vagrant.configure(2) do |config| # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder "../../", "/home/vagrant/code/src/github.com/cgrates/cgrates", owner: "vagrant", group: "vagrant" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. diff --git a/data/vagrant/cgrates_devel.yml b/data/vagrant/cgrates_devel.yml index 4002907c0..ed3e065e1 100644 --- a/data/vagrant/cgrates_devel.yml +++ b/data/vagrant/cgrates_devel.yml @@ -24,8 +24,7 @@ - name: install dependency apt: pkg={{ item }} update_cache=yes state=latest with_items: - - git - - bzr + - git - mercurial - redis-server - mysql-server @@ -33,10 +32,11 @@ - mongodb-org - freeswitch-meta-vanilla - freeswitch-mod-json-cdr - - libyuv-dev + - libyuv-dev + - python-mysqldb - name: update mysql root password for root account - mysql_user: name=cgrates host=localhost password={{ root_db_password }} + mysql_user: name=root host=localhost password={{ root_db_password }} state=present - name: copy .my.cnf template: src=my.cnf dest=/root/.my.cnf mode=0600 @@ -47,35 +47,25 @@ go_version: 1.6 tasks: + - name: create cgrates path + file: path=/home/vagrant/code/src/github.com/cgrates state=directory + - name: get golang - unarchive: src=https://storage.googleapis.com/golang/go{{ go_version }}.linux-amd64.tar.gz dest=~/go creates=~/go copy=no + unarchive: src=https://storage.googleapis.com/golang/go{{ go_version }}.linux-amd64.tar.gz dest=~/ creates=~/go copy=no - name: add variables to variables /etc/profile copy: src=golang.sh dest=/etc/profile.d/golang.sh become: yes - - - name: get cgrates - git: repo=https://github.com/cgrates/cgrates.git dest=/home/vagrant/code/src/github.com/cgrates/cgrates - - - name: get glide - shell: GOROOT=/home/vagrant/go GOPATH=/home/vagrant/code ~/go/bin/go get -u -v github.com/Masterminds/glide - - - name: install cgrates - shell: cd /home/vagrant/code/src/github.com/cgrates/cgrates; ~/code/bin/glide install - - - name: create cgr-engine link - file: src=/home/vagrant/code/bin/cgr-engine dest=/usr/bin/cgr-engine state=link - become: yes - - name: create a link to data dir - become: yes + - name: create a link to data dir file: src=/home/vagrant/code/src/github.com/cgrates/cgrates/data dest=/usr/share/cgrates state=link + become: yes - - name: expand freeswitch json conf - command: tar -xzvf /usr/share/cgrates/tutorials/fs_json/freeswitch/etc/freeswitch_conf.tar.gz + #- name: expand freeswitch json conf + # unarchive: src=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/freeswitch_conf.tar.gz dest=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/ copy=no - - name: expand freeswitch csv conf - command: tar -xzvf /usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/freeswitch_conf.tar.gz + #- name: expand freeswitch csv conf + # unarchive: src=/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/freeswitch_conf.tar.gz dest=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/ copy=no - name: setup database tables shell: chdir=/usr/share/cgrates/storage/mysql ./setup_cgr_db.sh root {{ root_db_password }} localhost diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 46ade6019..fc06f8057 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -41,7 +41,7 @@ type SMGSession struct { sessionCds []*engine.CallDescriptor callCosts []*engine.CallCost extraDuration time.Duration // keeps the current duration debited on top of what heas been asked - lastUsage time.Duration // Keep record of the last debit for LastUsed functionality + lastUsage time.Duration totalUsage time.Duration } @@ -78,21 +78,20 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { // Attempts to debit a duration, returns maximum duration which can be debitted or error func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.Duration, error) { - lastUsedCorrection := time.Duration(0) // Used if lastUsed influences the debit - if self.cd.DurationIndex != 0 && lastUsed != 0 { - if self.lastUsage > lastUsed { // We have debitted more than we have used, refund in the duration debitted - lastUsedCorrection = -(self.lastUsage - lastUsed) - } else { // We have debitted less than we have consumed, add the difference to duration debitted - lastUsedCorrection = lastUsed - self.lastUsage - } + self.totalUsage += lastUsed // Should reflect the total usage so far - // apply the lastUsed correction - dur += lastUsedCorrection - self.totalUsage += lastUsed // Should reflect the total usage so far - } else { - // apply correction from previous run - dur -= self.extraDuration + if lastUsed > 0 { + self.extraDuration = self.lastUsage - lastUsed } + // apply correction from previous run + if self.extraDuration < dur { + dur -= self.extraDuration + } else { + ccDuration := self.extraDuration // fake ccDuration + self.extraDuration -= dur + return ccDuration, nil + } + self.extraDuration = 0 if self.cd.LoopIndex > 0 { self.cd.TimeStart = self.cd.TimeEnd @@ -117,11 +116,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.LoopIndex += 1 self.sessionCds = append(self.sessionCds, self.cd.Clone()) self.callCosts = append(self.callCosts, cc) - ccDuration -= lastUsedCorrection - if ccDuration < 0 { // if correction has pushed ccDuration bellow 0 - ccDuration = 0 - } - self.lastUsage = ccDuration // Reset the lastUsage for later reference + self.lastUsage = ccDuration return ccDuration, nil } From d357663aac1a65e9d05b491e49d3dd74a7318ff0 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 25 Mar 2016 19:13:06 +0100 Subject: [PATCH 077/227] SMG - OriginIDPrefix support to terminate sessions starting with same prefix --- engine/cdrs.go | 2 +- sessionmanager/smgeneric.go | 26 +++++++++++++++++++++++--- utils/consts.go | 1 + 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/engine/cdrs.go b/engine/cdrs.go index e8b7de17d..5fc633805 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -333,7 +333,7 @@ func (self *CdrServer) rateCDR(cdr *CDR) error { if cdr.RequestType == utils.META_NONE { return nil } - _, hasLastUsed := cdr.ExtraFields["LastUsed"] + _, hasLastUsed := cdr.ExtraFields[utils.LastUsed] if utils.IsSliceMember([]string{utils.META_PREPAID, utils.PREPAID}, cdr.RequestType) && (cdr.Usage != 0 || hasLastUsed) { // ToDo: Get rid of PREPAID as soon as we don't want to support it backwards // Should be previously calculated and stored in DB delay := utils.Fib() diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index f7fc47c58..74ead925a 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -21,6 +21,7 @@ package sessionmanager import ( "errors" "fmt" + "strings" "sync" "time" @@ -73,6 +74,18 @@ func (self *SMGeneric) getSessions() map[string][]*SMGSession { return self.sessions } +func (self *SMGeneric) getSessionIDsForPrefix(prefix string) []string { + self.sessionsMux.Lock() + defer self.sessionsMux.Unlock() + sessionIDs := make([]string, 0) + for sessionID := range self.sessions { + if strings.HasPrefix(sessionID, prefix) { + sessionIDs = append(sessionIDs, sessionID) + } + } + return sessionIDs +} + // Returns sessions/derived for a specific uuid func (self *SMGeneric) getSession(uuid string) []*SMGSession { self.sessionsMux.Lock() @@ -267,10 +280,17 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { } usage = s.TotalUsage() + lastUsed } - if err := self.sessionEnd(gev.GetUUID(), usage); err != nil { - return err + sessionIDs := []string{gev.GetUUID()} + if sessionIDPrefix, err := gev.GetFieldAsString(utils.OriginIDPrefix); err == nil { // OriginIDPrefix is present, OriginID will not be anymore considered + sessionIDs = self.getSessionIDsForPrefix(sessionIDPrefix) } - return nil + var interimError error + for _, sessionID := range sessionIDs { + if err := self.sessionEnd(sessionID, usage); err != nil { + interimError = err // Last error will be the one returned as API result + } + } + return interimError } // Processes one time events (eg: SMS) diff --git a/utils/consts.go b/utils/consts.go index 36001e3c9..0bacf323e 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -115,6 +115,7 @@ const ( ORDERID = "OrderID" ACCID = "OriginID" InitialOriginID = "InitialOriginID" + OriginIDPrefix = "OriginIDPrefix" CDRSOURCE = "Source" CDRHOST = "OriginHost" REQTYPE = "RequestType" From fb6d04c103375b49801b6277c694eb2666a8dfdf Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 25 Mar 2016 19:31:00 +0100 Subject: [PATCH 078/227] Diameter integration test fixes --- agents/dmtagent_it_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index eff5e81ee..f62c93f25 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -626,7 +626,7 @@ func TestDmtAgentSendCCRSimpaEvent(t *testing.T) { }), diam.NewAVP(29000, avp.Mbit, 2011, &diam.GroupedAVP{ // MC-Information AVP: []*diam.AVP{ - diam.NewAVP(29001, avp.Mbit, 2011, datatype.OctetString("0x38924012914528")), // HighLayerCharacteristics + diam.NewAVP(29938, avp.Mbit, 2011, datatype.OctetString("0x38924012914528")), // HighLayerCharacteristics diam.NewAVP(29002, avp.Mbit, 2011, datatype.UTF8String("12928471313847173")), // MC-Service-Id diam.NewAVP(29003, avp.Mbit, 2011, datatype.UTF8String("SPV123456012123")), // TransparentData diam.NewAVP(1201, avp.Mbit, 10415, &diam.GroupedAVP{ // MC-Information From 608f430db72a70015d468a14a53551ad41873b9d Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 26 Mar 2016 22:17:10 +0100 Subject: [PATCH 079/227] Storage - SetSMCost and GetSMCost deprecating LogCallCost and GetLogCallCost, CDRS.StoreSMCost method, adding sm_costs indexes for cgrid+runid and originhost+originid in all dbs supported --- agents/dmtagent_it_test.go | 8 +++---- apier/v1/cdrs.go | 2 +- apier/v1/cdrsv1.go | 4 ++-- cdre/cdrexporter.go | 2 +- data/storage/mysql/create_cdrs_tables.sql | 3 +++ data/storage/postgres/create_cdrs_tables.sql | 7 ++++++ engine/cdrs.go | 22 ++++++++++------- engine/models.go | 2 ++ engine/responder.go | 23 +++++++++--------- engine/storage_cdrs_it_test.go | 5 ++-- engine/storage_interface.go | 4 ++-- engine/storage_map.go | 10 ++++---- engine/storage_mongo_datadb.go | 22 +++++++++++++++++ engine/storage_mongo_stordb.go | 10 +++++--- engine/storage_sql.go | 12 +++++++--- engine/storage_utils.go | 7 ++++++ sessionmanager/session.go | 16 +++++++------ sessionmanager/session_test.go | 2 +- sessionmanager/smg_session.go | 25 ++++++++++---------- sessionmanager/smgeneric.go | 16 +++++++------ 20 files changed, 130 insertions(+), 72 deletions(-) diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index f62c93f25..3abf823eb 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -242,7 +242,7 @@ func TestDmtAgentSendCCRInit(t *testing.T) { if err := dmtClient.SendMessage(m); err != nil { t.Error(err) } - time.Sleep(time.Duration(100) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { t.Error(err) @@ -286,7 +286,7 @@ func TestDmtAgentSendCCRUpdate(t *testing.T) { if err := dmtClient.SendMessage(m); err != nil { t.Error(err) } - time.Sleep(time.Duration(100) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { t.Error(err) @@ -325,7 +325,7 @@ func TestDmtAgentSendCCRUpdate2(t *testing.T) { if err := dmtClient.SendMessage(m); err != nil { t.Error(err) } - time.Sleep(time.Duration(100) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() if avps, err := msg.FindAVPsWithPath([]interface{}{"Granted-Service-Unit", "CC-Time"}, dict.UndefinedVendorID); err != nil { t.Error(err) @@ -363,7 +363,7 @@ func TestDmtAgentSendCCRTerminate(t *testing.T) { if err := dmtClient.SendMessage(m); err != nil { t.Error(err) } - time.Sleep(time.Duration(150) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() if msg == nil { t.Fatal("No answer to CCR terminate received") diff --git a/apier/v1/cdrs.go b/apier/v1/cdrs.go index 39dcd5378..b8fc0deec 100644 --- a/apier/v1/cdrs.go +++ b/apier/v1/cdrs.go @@ -33,7 +33,7 @@ func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine. if attrs.RunId == "" { attrs.RunId = utils.META_DEFAULT } - if smc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, attrs.RunId); err != nil { + if smc, err := apier.CdrDb.GetSMCost(attrs.CgrId, attrs.RunId, "", ""); err != nil { return utils.NewErrServerError(err) } else if smc == nil { return utils.ErrNotFound diff --git a/apier/v1/cdrsv1.go b/apier/v1/cdrsv1.go index db5681a03..e96292369 100644 --- a/apier/v1/cdrsv1.go +++ b/apier/v1/cdrsv1.go @@ -59,8 +59,8 @@ func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { return nil } -func (self *CdrsV1) LogCallCost(ccl *engine.CallCostLog, reply *string) error { - if err := self.CdrSrv.LogCallCost(ccl); err != nil { +func (self *CdrsV1) StoreSMCost(smCost *engine.SMCost, reply *string) error { + if err := self.CdrSrv.StoreSMCost(smCost, false); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index a98a26c95..112f135fd 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -116,7 +116,7 @@ type CdrExporter struct { // Return Json marshaled callCost attached to // Keep it separately so we test only this part in local tests func (cdre *CdrExporter) getCdrCostDetails(CGRID, runId string) (string, error) { - cc, err := cdre.cdrDb.GetCallCostLog(CGRID, runId) + cc, err := cdre.cdrDb.GetSMCost(CGRID, runId, "", "") if err != nil { return "", err } else if cc == nil { diff --git a/data/storage/mysql/create_cdrs_tables.sql b/data/storage/mysql/create_cdrs_tables.sql index 463ea922e..5fe75add6 100644 --- a/data/storage/mysql/create_cdrs_tables.sql +++ b/data/storage/mysql/create_cdrs_tables.sql @@ -41,6 +41,8 @@ CREATE TABLE sm_costs ( id int(11) NOT NULL AUTO_INCREMENT, cgrid char(40) NOT NULL, run_id varchar(64) NOT NULL, + origin_host varchar(64) NOT NULL, + origin_id varchar(64) NOT NULL, cost_source varchar(64) NOT NULL, `usage` DECIMAL(30,9) NOT NULL, cost_details text, @@ -48,5 +50,6 @@ CREATE TABLE sm_costs ( deleted_at TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY costid (cgrid,run_id), + KEY origin_idx (origin_host, origin_id), KEY deleted_at_idx (deleted_at) ); diff --git a/data/storage/postgres/create_cdrs_tables.sql b/data/storage/postgres/create_cdrs_tables.sql index 2c710dc4e..157aa34d4 100644 --- a/data/storage/postgres/create_cdrs_tables.sql +++ b/data/storage/postgres/create_cdrs_tables.sql @@ -44,6 +44,8 @@ CREATE TABLE sm_costs ( id SERIAL PRIMARY KEY, cgrid CHAR(40) NOT NULL, run_id VARCHAR(64) NOT NULL, + origin_host VARCHAR(64) NOT NULL, + origin_id VARCHAR(64) NOT NULL, cost_source VARCHAR(64) NOT NULL, usage NUMERIC(30,9) NOT NULL, cost_details jsonb, @@ -51,5 +53,10 @@ CREATE TABLE sm_costs ( deleted_at TIMESTAMP, UNIQUE (cgrid, run_id) ); +DROP INDEX IF EXISTS cgrid_smcost_idx; +CREATE INDEX cgrid_smcost_idx ON sm_costs (cgrid, run_id); +DROP INDEX IF EXISTS origin_smcost_idx; +CREATE INDEX origin_smcost_idx ON sm_costs (origin_host, origin_id); DROP INDEX IF EXISTS deleted_at_smcost_idx; CREATE INDEX deleted_at_smcost_idx ON sm_costs (deleted_at); + diff --git a/engine/cdrs.go b/engine/cdrs.go index 5fc633805..babf82f2c 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -108,23 +108,27 @@ func (self *CdrServer) ProcessExternalCdr(eCDR *ExternalCDR) error { } // RPC method, used to log callcosts to db -func (self *CdrServer) LogCallCost(ccl *CallCostLog) error { - ccl.CallCost.UpdateCost() // make sure the total cost reflect the increments - ccl.CallCost.UpdateRatedUsage() // make sure rated usage is updated - if ccl.CheckDuplicate { +func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { + smCost.CostDetails.UpdateCost() // make sure the total cost reflect the increments + smCost.CostDetails.UpdateRatedUsage() // make sure rated usage is updated + lockKey := smCost.CGRID + smCost.RunID // Will lock on this ID + if smCost.CGRID == "" && smCost.OriginID != "" { + lockKey = smCost.OriginHost + smCost.OriginID + } + if checkDuplicate { _, err := self.guard.Guard(func() (interface{}, error) { - cc, err := self.cdrDb.GetCallCostLog(ccl.CgrId, ccl.RunId) + cc, err := self.cdrDb.GetSMCost(smCost.CGRID, smCost.RunID, "", "") if err != nil && err != gorm.RecordNotFound && err != mgov2.ErrNotFound { return nil, err } if cc != nil { return nil, utils.ErrExists } - return nil, self.cdrDb.LogCallCost(&SMCost{CGRID: ccl.CgrId, RunID: ccl.RunId, CostSource: ccl.Source, Usage: ccl.Usage, CostDetails: ccl.CallCost}) - }, 0, ccl.CgrId) + return nil, self.cdrDb.SetSMCost(smCost) + }, 0, lockKey) // FixMe: Possible deadlock with Guard from SMG session close() return err } - return self.cdrDb.LogCallCost(&SMCost{CGRID: ccl.CgrId, RunID: ccl.RunId, CostSource: ccl.Source, Usage: ccl.Usage, CostDetails: ccl.CallCost}) + return self.cdrDb.SetSMCost(smCost) } // Called by rate/re-rate API @@ -339,7 +343,7 @@ func (self *CdrServer) rateCDR(cdr *CDR) error { delay := utils.Fib() var usage float64 for i := 0; i < 4; i++ { - qrySMC, err := self.cdrDb.GetCallCostLog(cdr.CGRID, cdr.RunID) + qrySMC, err := self.cdrDb.GetSMCost(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) if err == nil { qryCC = qrySMC.CostDetails usage = qrySMC.Usage diff --git a/engine/models.go b/engine/models.go index 1da2db987..e5c686892 100644 --- a/engine/models.go +++ b/engine/models.go @@ -451,6 +451,8 @@ type TBLSMCosts struct { ID int64 Cgrid string RunID string + OriginHost string + OriginID string CostSource string Usage float64 CostDetails string diff --git a/engine/responder.go b/engine/responder.go index b6d12c545..1b211b3f5 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -512,26 +512,26 @@ func (rs *Responder) ProcessCdr(cdr *CDR, reply *string) error { return nil } -func (rs *Responder) LogCallCost(ccl *CallCostLog, reply *string) error { - if item, err := rs.getCache().Get(utils.LOG_CALL_COST_CACHE_PREFIX + ccl.CgrId); err == nil && item != nil { +func (rs *Responder) StoreSMCost(attrs AttrCDRSStoreSMCost, reply *string) error { + if item, err := rs.getCache().Get(utils.LOG_CALL_COST_CACHE_PREFIX + attrs.SMCost.CGRID); err == nil && item != nil { *reply = item.Value.(string) return item.Err } if rs.CdrSrv == nil { err := errors.New("CDR_SERVER_NOT_RUNNING") - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ Err: err, }) return err } - if err := rs.CdrSrv.LogCallCost(ccl); err != nil { - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ + if err := rs.CdrSrv.StoreSMCost(attrs.SMCost, attrs.CheckDuplicate); err != nil { + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ Err: err, }) return err } *reply = utils.OK - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+ccl.CgrId, &cache2go.CacheItem{ + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ Value: utils.OK, }) return nil @@ -740,7 +740,7 @@ type Connector interface { GetDerivedMaxSessionTime(*CDR, *float64) error GetSessionRuns(*CDR, *[]*SessionRun) error ProcessCdr(*CDR, *string) error - LogCallCost(*CallCostLog, *string) error + StoreSMCost(AttrCDRSStoreSMCost, *string) error GetLCR(*AttrGetLcr, *LCRCost) error GetTimeout(int, *time.Duration) error } @@ -790,8 +790,8 @@ func (rcc *RPCClientConnector) ProcessCdr(cdr *CDR, reply *string) error { return rcc.Client.Call("CdrsV1.ProcessCdr", cdr, reply) } -func (rcc *RPCClientConnector) LogCallCost(ccl *CallCostLog, reply *string) error { - return rcc.Client.Call("CdrsV1.LogCallCost", ccl, reply) +func (rcc *RPCClientConnector) StoreSMCost(attrs AttrCDRSStoreSMCost, reply *string) error { + return rcc.Client.Call("CdrsV1.StoreSMCost", attrs, reply) } func (rcc *RPCClientConnector) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { @@ -1005,15 +1005,14 @@ func (cp ConnectorPool) ProcessCdr(cdr *CDR, reply *string) error { return utils.ErrTimedOut } -func (cp ConnectorPool) LogCallCost(ccl *CallCostLog, reply *string) error { +func (cp ConnectorPool) StoreSMCost(attrs AttrCDRSStoreSMCost, reply *string) error { for _, con := range cp { c := make(chan error, 1) var r string var timeout time.Duration con.GetTimeout(0, &timeout) - - go func() { c <- con.LogCallCost(ccl, &r) }() + go func() { c <- con.StoreSMCost(attrs, &r) }() select { case err := <-c: *reply = r diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index 4e934d4e4..5e23e7976 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -222,10 +222,11 @@ func testSMCosts(cfg *config.CGRConfig) error { }, TOR: utils.VOICE, } - if err := cdrStorage.LogCallCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e41", RunID: utils.META_DEFAULT, CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { + if err := cdrStorage.SetSMCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e41", RunID: utils.META_DEFAULT, OriginHost: "localhost", OriginID: "12345", + CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { return err } - if rcvSMC, err := cdrStorage.GetCallCostLog("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT); err != nil { + if rcvSMC, err := cdrStorage.GetSMCost("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT, "", ""); err != nil { return err } else if len(cc.Timespans) != len(rcvSMC.CostDetails.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) return fmt.Errorf("Expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC)) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index dbf8a8632..53f990880 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -97,8 +97,8 @@ type AccountingStorage interface { type CdrStorage interface { Storage SetCDR(*CDR, bool) error - LogCallCost(smc *SMCost) error - GetCallCostLog(cgrid, runid string) (*SMCost, error) + SetSMCost(smc *SMCost) error + GetSMCost(cgrid, runid, originHost, originIDPrfx string) (*SMCost, error) GetCDRs(*utils.CDRsFilter, bool) ([]*CDR, int64, error) } diff --git a/engine/storage_map.go b/engine/storage_map.go index 5a898fa32..d87fdcddd 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -782,15 +782,15 @@ func (ms *MapStorage) GetAllCdrStats() (css []*CdrStats, err error) { return } -func (ms *MapStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) error { - result, err := ms.ms.Marshal(cc) - ms.dict[utils.LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid] = result +func (ms *MapStorage) SetSMCost(smCost *SMCost) error { + result, err := ms.ms.Marshal(smCost) + ms.dict[utils.LOG_CALL_COST_PREFIX+smCost.CostSource+smCost.RunID+"_"+smCost.CGRID] = result return err } -func (ms *MapStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { +func (ms *MapStorage) GetSMCost(cgrid, source, runid, originHost, originID string) (smCost *SMCost, err error) { if values, ok := ms.dict[utils.LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid]; ok { - err = ms.ms.Unmarshal(values, &cc) + err = ms.ms.Unmarshal(values, &smCost) } else { return nil, utils.ErrNotFound } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index a208ce6cb..f6a032188 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -59,6 +59,8 @@ var ( CGRIDLow = strings.ToLower(utils.CGRID) RunIDLow = strings.ToLower(utils.MEDI_RUNID) OrderIDLow = strings.ToLower(utils.ORDERID) + OriginHostLow = strings.ToLower(utils.CDRHOST) + OriginIDLow = strings.ToLower(utils.ACCID) ToRLow = strings.ToLower(utils.TOR) CDRHostLow = strings.ToLower(utils.CDRHOST) CDRSourceLow = strings.ToLower(utils.CDRSOURCE) @@ -237,6 +239,26 @@ func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string) (* return nil, err } } + index = mgo.Index{ + Key: []string{CGRIDLow, RunIDLow}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + if err = ndb.C(utils.TBLSMCosts).EnsureIndex(index); err != nil { + return nil, err + } + index = mgo.Index{ + Key: []string{OriginHostLow, OriginIDLow}, + Unique: false, + DropDups: false, + Background: false, + Sparse: false, + } + if err = ndb.C(utils.TBLSMCosts).EnsureIndex(index); err != nil { + return nil, err + } return &MongoStorage{db: ndb, session: session, ms: NewCodecMsgpackMarshaler()}, err } diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index a0aaffe09..c6c557862 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -698,13 +698,17 @@ func (ms *MongoStorage) LogActionTiming(source string, at *ActionTiming, as Acti }{at, as, time.Now(), source}) } -func (ms *MongoStorage) LogCallCost(smc *SMCost) error { +func (ms *MongoStorage) SetSMCost(smc *SMCost) error { return ms.db.C(utils.TBLSMCosts).Insert(smc) } -func (ms *MongoStorage) GetCallCostLog(cgrid, runid string) (smc *SMCost, err error) { +func (ms *MongoStorage) GetSMCost(cgrid, runid, originHost, originIDPrefix string) (smc *SMCost, err error) { var result SMCost - if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{CGRIDLow: cgrid, RunIDLow: runid}).One(&result); err != nil { + if originIDPrefix != "" { + if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{OriginHostLow: originHost, OriginIDLow: originIDPrefix, RunIDLow: runid}).One(&result); err != nil { // FixMe for prefix + return nil, err + } + } else if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{CGRIDLow: cgrid, RunIDLow: runid}).One(&result); err != nil { return nil, err } return &result, nil diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 1e0f5368a..40b316cf4 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -569,7 +569,7 @@ func (self *SQLStorage) SetTpAccountActions(aas []TpAccountAction) error { return nil } -func (self *SQLStorage) LogCallCost(smc *SMCost) error { +func (self *SQLStorage) SetSMCost(smc *SMCost) error { if smc.CostDetails == nil { return nil } @@ -582,6 +582,8 @@ func (self *SQLStorage) LogCallCost(smc *SMCost) error { cd := &TBLSMCosts{ Cgrid: smc.CGRID, RunID: smc.RunID, + OriginHost: smc.OriginHost, + OriginID: smc.OriginID, CostSource: smc.CostSource, CostDetails: string(tss), Usage: smc.Usage, @@ -595,9 +597,13 @@ func (self *SQLStorage) LogCallCost(smc *SMCost) error { return nil } -func (self *SQLStorage) GetCallCostLog(cgrid, runid string) (*SMCost, error) { +func (self *SQLStorage) GetSMCost(cgrid, runid, originHost, originIDPrefix string) (*SMCost, error) { var tpCostDetail TBLSMCosts - if err := self.db.Where(&TBLSMCosts{Cgrid: cgrid, RunID: runid}).First(&tpCostDetail).Error; err != nil { + if originIDPrefix != "" { + if err := self.db.Where(&TBLSMCosts{OriginHost: originHost, OriginID: originIDPrefix, RunID: runid}).First(&tpCostDetail).Error; err != nil { // FixMe with originPrefix + return nil, err + } + } else if err := self.db.Where(&TBLSMCosts{Cgrid: cgrid, RunID: runid}).First(&tpCostDetail).Error; err != nil { return nil, err } if len(tpCostDetail.CostDetails) == 0 { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 16012f2e1..5ec8f6f4e 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -156,7 +156,14 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, type SMCost struct { CGRID string RunID string + OriginHost string + OriginID string CostSource string Usage float64 CostDetails *CallCost } + +type AttrCDRSStoreSMCost struct { + SMCost *SMCost + CheckDuplicate bool +} diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 31942cfdf..a6a5d1709 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -240,13 +240,15 @@ func (s *Session) SaveOperations() { } } var reply string - err := s.sessionManager.CdrSrv().LogCallCost(&engine.CallCostLog{ - CgrId: s.eventStart.GetCgrId(s.sessionManager.Timezone()), - Source: utils.SESSION_MANAGER_SOURCE, - RunId: sr.DerivedCharger.RunID, - CallCost: firstCC, - CheckDuplicate: true, - }, &reply) + smCost := &engine.SMCost{ + CGRID: s.eventStart.GetCgrId(s.sessionManager.Timezone()), + CostSource: utils.SESSION_MANAGER_SOURCE, + RunID: sr.DerivedCharger.RunID, + OriginHost: s.eventStart.GetOriginatorIP(utils.META_DEFAULT), + OriginID: s.eventStart.GetUUID(), + CostDetails: firstCC, + } + err := s.sessionManager.CdrSrv().StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply) // this is a protection against the case when the close event is missed for some reason // when the cdr arrives to cdrserver because our callcost is not there it will be rated // as postpaid. When the close event finally arives we have to refund everything diff --git a/sessionmanager/session_test.go b/sessionmanager/session_test.go index 209577806..0ab6c9c0a 100644 --- a/sessionmanager/session_test.go +++ b/sessionmanager/session_test.go @@ -101,7 +101,7 @@ func (mc *MockConnector) GetDerivedChargers(*utils.AttrDerivedChargers, *utils.D func (mc *MockConnector) GetDerivedMaxSessionTime(*engine.CDR, *float64) error { return nil } func (mc *MockConnector) GetSessionRuns(*engine.CDR, *[]*engine.SessionRun) error { return nil } func (mc *MockConnector) ProcessCdr(*engine.CDR, *string) error { return nil } -func (mc *MockConnector) LogCallCost(*engine.CallCostLog, *string) error { return nil } +func (mc *MockConnector) StoreSMCost(engine.AttrCDRSStoreSMCost, *string) error { return nil } func (mc *MockConnector) GetLCR(*engine.AttrGetLcr, *engine.LCRCost) error { return nil } func (mc *MockConnector) GetTimeout(int, *time.Duration) error { return nil } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 46ade6019..542776bf9 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -235,20 +235,19 @@ func (self *SMGSession) saveOperations() error { return err } } - var reply string - err := self.cdrsrv.LogCallCost(&engine.CallCostLog{ - CgrId: self.eventStart.GetCgrId(self.timezone), - Source: utils.SESSION_MANAGER_SOURCE, - RunId: self.runId, - Usage: float64(self.totalUsage), - CallCost: firstCC, - CheckDuplicate: true, - }, &reply) - // this is a protection against the case when the close event is missed for some reason - // when the cdr arrives to cdrserver because our callcost is not there it will be rated - // as postpaid. When the close event finally arives we have to refund everything - if err != nil { + smCost := &engine.SMCost{ + CGRID: self.eventStart.GetCgrId(self.timezone), + CostSource: utils.SESSION_MANAGER_SOURCE, + RunID: self.runId, + OriginHost: self.eventStart.GetOriginatorIP(utils.META_DEFAULT), + OriginID: self.eventStart.GetUUID(), + CostDetails: firstCC, + } + if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply); err != nil { + // this is a protection against the case when the close event is missed for some reason + // when the cdr arrives to cdrserver because our callcost is not there it will be rated + // as postpaid. When the close event finally arives we have to refund everything if err == utils.ErrExists { self.refund(self.cd.GetDuration()) // Refund entire duration } else { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 74ead925a..a0df22aab 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -379,13 +379,15 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } var reply string - if err := self.cdrsrv.LogCallCost(&engine.CallCostLog{ - CgrId: gev.GetCgrId(self.timezone), - Source: utils.SESSION_MANAGER_SOURCE, - RunId: sR.DerivedCharger.RunID, - CallCost: cc, - CheckDuplicate: true, - }, &reply); err != nil && err != utils.ErrExists { + smCost := &engine.SMCost{ + CGRID: gev.GetCgrId(self.timezone), + CostSource: utils.SESSION_MANAGER_SOURCE, + RunID: sR.DerivedCharger.RunID, + OriginHost: gev.GetOriginatorIP(utils.META_DEFAULT), + OriginID: gev.GetUUID(), + CostDetails: cc, + } + if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply); err != nil && err != utils.ErrExists { withErrors = true utils.Logger.Err(fmt.Sprintf(" Could not save CC: %+v, RunID: %s error: %s", cc, sR.DerivedCharger.RunID, err.Error())) } From ee53a1b3de73417d11201dce5be595fcedb51e4a Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 27 Mar 2016 20:45:23 +0200 Subject: [PATCH 080/227] Storage.GetSMCost -> Storage.GetSMCosts, prefix match for OriginID when querying --- apier/v1/apier_local_test.go | 2 +- apier/v1/cdrs.go | 6 ++--- cdre/cdrexporter.go | 6 ++--- engine/cdrs.go | 14 +++++------ engine/storage_cdrs_it_test.go | 23 ++++++++++++++---- engine/storage_interface.go | 2 +- engine/storage_mongo_stordb.go | 20 ++++++++++------ engine/storage_sql.go | 44 +++++++++++++++++++--------------- 8 files changed, 72 insertions(+), 45 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 88ad8c819..3f8a03b7a 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1359,7 +1359,7 @@ func TestApierGetCallCostLog(t *testing.T) { } attrs.CgrId = "dummyid" attrs.RunId = "default" - if err := rater.Call("ApierV1.GetCallCostLog", attrs, &cc); err == nil || err.Error() != "SERVER_ERROR: record not found" { + if err := rater.Call("ApierV1.GetCallCostLog", attrs, &cc); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Error("ApierV1.GetCallCostLog: should return NOT_FOUND, got:", err) } } diff --git a/apier/v1/cdrs.go b/apier/v1/cdrs.go index b8fc0deec..4e87a2120 100644 --- a/apier/v1/cdrs.go +++ b/apier/v1/cdrs.go @@ -33,12 +33,12 @@ func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine. if attrs.RunId == "" { attrs.RunId = utils.META_DEFAULT } - if smc, err := apier.CdrDb.GetSMCost(attrs.CgrId, attrs.RunId, "", ""); err != nil { + if smcs, err := apier.CdrDb.GetSMCosts(attrs.CgrId, attrs.RunId, "", ""); err != nil { return utils.NewErrServerError(err) - } else if smc == nil { + } else if len(smcs) == 0 { return utils.ErrNotFound } else { - *reply = *smc + *reply = *smcs[0] } return nil } diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index 112f135fd..e90b88e03 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -116,13 +116,13 @@ type CdrExporter struct { // Return Json marshaled callCost attached to // Keep it separately so we test only this part in local tests func (cdre *CdrExporter) getCdrCostDetails(CGRID, runId string) (string, error) { - cc, err := cdre.cdrDb.GetSMCost(CGRID, runId, "", "") + smcs, err := cdre.cdrDb.GetSMCosts(CGRID, runId, "", "") if err != nil { return "", err - } else if cc == nil { + } else if len(smcs) == 0 { return "", nil } - ccJson, _ := json.Marshal(cc) + ccJson, _ := json.Marshal(smcs[0].CostDetails) return string(ccJson), nil } diff --git a/engine/cdrs.go b/engine/cdrs.go index babf82f2c..1b2d2e563 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -117,11 +117,11 @@ func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { } if checkDuplicate { _, err := self.guard.Guard(func() (interface{}, error) { - cc, err := self.cdrDb.GetSMCost(smCost.CGRID, smCost.RunID, "", "") - if err != nil && err != gorm.RecordNotFound && err != mgov2.ErrNotFound { + smCosts, err := self.cdrDb.GetSMCosts(smCost.CGRID, smCost.RunID, "", "") + if err != nil { return nil, err } - if cc != nil { + if len(smCosts) != 0 { return nil, utils.ErrExists } return nil, self.cdrDb.SetSMCost(smCost) @@ -343,10 +343,10 @@ func (self *CdrServer) rateCDR(cdr *CDR) error { delay := utils.Fib() var usage float64 for i := 0; i < 4; i++ { - qrySMC, err := self.cdrDb.GetSMCost(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) - if err == nil { - qryCC = qrySMC.CostDetails - usage = qrySMC.Usage + smCosts, err := self.cdrDb.GetSMCosts(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) + if err == nil && len(smCosts) != 0 { + qryCC = smCosts[0].CostDetails + usage = smCosts[0].Usage break } time.Sleep(delay()) diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index 5e23e7976..656098885 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -19,10 +19,11 @@ along with this program. If not, see package engine import ( + "errors" "flag" "fmt" "path" - //"reflect" + "strconv" "testing" "time" @@ -226,10 +227,24 @@ func testSMCosts(cfg *config.CGRConfig) error { CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { return err } - if rcvSMC, err := cdrStorage.GetSMCost("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT, "", ""); err != nil { + if rcvSMC, err := cdrStorage.GetSMCosts("164b0422fdc6a5117031b427439482c6a4f90e41", utils.META_DEFAULT, "", ""); err != nil { return err - } else if len(cc.Timespans) != len(rcvSMC.CostDetails.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) - return fmt.Errorf("Expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC)) + } else if len(rcvSMC) == 0 { + return errors.New("No SMCosts received") + } else if len(cc.Timespans) != len(rcvSMC[0].CostDetails.Timespans) { // cc.Timespans[0].RateInterval.Rating.Rates[0], rcvCC.Timespans[0].RateInterval.Rating.Rates[0]) + return fmt.Errorf("Expecting: %+v, received: %+s", cc, utils.ToIJSON(rcvSMC[0])) + } + // Test query per prefix + for i := 0; i < 3; i++ { + if err := cdrStorage.SetSMCost(&SMCost{CGRID: "164b0422fdc6a5117031b427439482c6a4f90e5" + strconv.Itoa(i), RunID: utils.META_DEFAULT, OriginHost: "localhost", OriginID: "abc" + strconv.Itoa(i), + CostSource: utils.UNIT_TEST, CostDetails: cc}); err != nil { + return err + } + } + if rcvSMC, err := cdrStorage.GetSMCosts("", utils.META_DEFAULT, "localhost", "abc"); err != nil { + return err + } else if len(rcvSMC) != 3 { + return fmt.Errorf("Expecting 3, received: %d", len(rcvSMC)) } return nil } diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 53f990880..c2123970e 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -98,7 +98,7 @@ type CdrStorage interface { Storage SetCDR(*CDR, bool) error SetSMCost(smc *SMCost) error - GetSMCost(cgrid, runid, originHost, originIDPrfx string) (*SMCost, error) + GetSMCosts(cgrid, runid, originHost, originIDPrfx string) ([]*SMCost, error) GetCDRs(*utils.CDRsFilter, bool) ([]*CDR, int64, error) } diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index c6c557862..4edd9ae8e 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -1,6 +1,7 @@ package engine import ( + "fmt" "regexp" "strings" "time" @@ -702,16 +703,21 @@ func (ms *MongoStorage) SetSMCost(smc *SMCost) error { return ms.db.C(utils.TBLSMCosts).Insert(smc) } -func (ms *MongoStorage) GetSMCost(cgrid, runid, originHost, originIDPrefix string) (smc *SMCost, err error) { - var result SMCost +func (ms *MongoStorage) GetSMCosts(cgrid, runid, originHost, originIDPrefix string) (smcs []*SMCost, err error) { + filter := bson.M{CGRIDLow: cgrid, RunIDLow: runid} if originIDPrefix != "" { - if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{OriginHostLow: originHost, OriginIDLow: originIDPrefix, RunIDLow: runid}).One(&result); err != nil { // FixMe for prefix - return nil, err - } - } else if err = ms.db.C(utils.TBLSMCosts).Find(bson.M{CGRIDLow: cgrid, RunIDLow: runid}).One(&result); err != nil { + filter = bson.M{OriginIDLow: bson.M{"$regex": bson.RegEx{Pattern: fmt.Sprintf("^%s", originIDPrefix)}}, OriginHostLow: originHost, RunIDLow: runid} + } + // Execute query + iter := ms.db.C(utils.TBLSMCosts).Find(filter).Iter() + var smCost SMCost + for iter.Next(&smCost) { + smcs = append(smcs, &smCost) + } + if err := iter.Err(); err != nil { return nil, err } - return &result, nil + return smcs, nil } func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) (err error) { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 40b316cf4..365211043 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -597,29 +597,35 @@ func (self *SQLStorage) SetSMCost(smc *SMCost) error { return nil } -func (self *SQLStorage) GetSMCost(cgrid, runid, originHost, originIDPrefix string) (*SMCost, error) { - var tpCostDetail TBLSMCosts +// GetSMCosts is used to retrieve one or multiple SMCosts based on filter +func (self *SQLStorage) GetSMCosts(cgrid, runid, originHost, originIDPrefix string) ([]*SMCost, error) { + var smCosts []*SMCost + q := self.db.Where(&TBLSMCosts{Cgrid: cgrid, RunID: runid}) if originIDPrefix != "" { - if err := self.db.Where(&TBLSMCosts{OriginHost: originHost, OriginID: originIDPrefix, RunID: runid}).First(&tpCostDetail).Error; err != nil { // FixMe with originPrefix + q = self.db.Where(&TBLSMCosts{OriginHost: originHost, RunID: runid}).Where(fmt.Sprintf("origin_id LIKE '%s%%'", originIDPrefix)) + } + results := make([]*TBLSMCosts, 0) + if err := q.Find(&results).Error; err != nil { + return nil, err + } + for _, result := range results { + if len(result.CostDetails) == 0 { + continue + } + smc := &SMCost{ + CGRID: result.Cgrid, + RunID: result.RunID, + CostSource: result.CostSource, + Usage: result.Usage, + CostDetails: &CallCost{}, + } + if err := json.Unmarshal([]byte(result.CostDetails), smc.CostDetails); err != nil { return nil, err } - } else if err := self.db.Where(&TBLSMCosts{Cgrid: cgrid, RunID: runid}).First(&tpCostDetail).Error; err != nil { - return nil, err + smCosts = append(smCosts, smc) } - if len(tpCostDetail.CostDetails) == 0 { - return nil, nil // No costs returned - } - smc := &SMCost{ - CGRID: tpCostDetail.Cgrid, - RunID: tpCostDetail.RunID, - CostSource: tpCostDetail.CostSource, - Usage: tpCostDetail.Usage, - CostDetails: &CallCost{}, - } - if err := json.Unmarshal([]byte(tpCostDetail.CostDetails), smc.CostDetails); err != nil { - return nil, err - } - return smc, nil + + return smCosts, nil } func (self *SQLStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) (err error) { From ae003983defa6c246d548112d9b44cd72f8f5bd2 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 28 Mar 2016 15:59:59 +0300 Subject: [PATCH 081/227] local test on docker working --- data/docker/devel/start.sh | 6 ++++-- data/vagrant/cgrates_devel.yml | 35 +++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/data/docker/devel/start.sh b/data/docker/devel/start.sh index fbacacd13..ac7992663 100755 --- a/data/docker/devel/start.sh +++ b/data/docker/devel/start.sh @@ -30,8 +30,10 @@ cd /root/cgr #glide -y devel.yaml install ./build.sh -# create cgr-engine link -ln -s /root/code/bin/cgr-engine /usr/bin/cgr-engine +# create cgr-engine and cgr-loader link +ln -s /root/code/bin/cgr-engine /usr/bin/ +ln -s /root/code/bin/cgr-loader /usr/bin/ + # expand freeswitch conf cd /usr/share/cgrates/tutorials/fs_evsock/freeswitch/etc/ && tar xzf freeswitch_conf.tar.gz diff --git a/data/vagrant/cgrates_devel.yml b/data/vagrant/cgrates_devel.yml index ed3e065e1..9b7d90478 100644 --- a/data/vagrant/cgrates_devel.yml +++ b/data/vagrant/cgrates_devel.yml @@ -24,52 +24,61 @@ - name: install dependency apt: pkg={{ item }} update_cache=yes state=latest with_items: - - git + - git - mercurial - redis-server - - mysql-server + - mysql-server - postgresql-9.4 - mongodb-org - freeswitch-meta-vanilla - freeswitch-mod-json-cdr - - libyuv-dev + - libyuv-dev - python-mysqldb + - python-pymongo - name: update mysql root password for root account mysql_user: name=root host=localhost password={{ root_db_password }} state=present - name: copy .my.cnf template: src=my.cnf dest=/root/.my.cnf mode=0600 - -- hosts: all + +- hosts: all vars: root_db_password: CGRateS.org go_version: 1.6 - + tasks: - name: create cgrates path file: path=/home/vagrant/code/src/github.com/cgrates state=directory - name: get golang - unarchive: src=https://storage.googleapis.com/golang/go{{ go_version }}.linux-amd64.tar.gz dest=~/ creates=~/go copy=no + unarchive: src=https://storage.googleapis.com/golang/go{{ go_version }}.linux-amd64.tar.gz dest=~/ creates=~/go copy=no - name: add variables to variables /etc/profile copy: src=golang.sh dest=/etc/profile.d/golang.sh become: yes - - - name: create a link to data dir + + - name: create a link to data dir file: src=/home/vagrant/code/src/github.com/cgrates/cgrates/data dest=/usr/share/cgrates state=link become: yes #- name: expand freeswitch json conf # unarchive: src=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/freeswitch_conf.tar.gz dest=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/ copy=no - + #- name: expand freeswitch csv conf # unarchive: src=/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/freeswitch_conf.tar.gz dest=/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/ copy=no - - - name: setup database tables + + - name: setup mysql tables shell: chdir=/usr/share/cgrates/storage/mysql ./setup_cgr_db.sh root {{ root_db_password }} localhost - + - name: setup postgress table + shell: chdir=/usr/share/cgrates/storage/postgres ./setup_cgr_db.sh + + - name: create cgrates user for mongo + mongodb_user: database=admin name=cgrates password={{root_db_password}} roles='userAdminAnyDatabase' state=present + - name: create link to cgrates dir file: src=~/code/src/github.com/cgrates/cgrates dest=~/cgr state=link + - name: create var folder + file: path=/var/log/cgrates state=directory owner=vagrant + become: yes From 1a390c9e7d24e174c1726f73bfaf6445835c4cbd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 28 Mar 2016 21:30:26 +0300 Subject: [PATCH 082/227] last usage tests --- sessionmanager/data_it_test.go | 10 +++++----- sessionmanager/smg_it_test.go | 2 +- sessionmanager/smg_session.go | 7 +++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 69b6d95a5..513941df0 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -206,7 +206,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998924800.000000 + eAcntVal = 49998924800.000000 // 1054720 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -232,7 +232,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998904320.000000 + eAcntVal = 49998904320.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -258,7 +258,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998883840.000000 + eAcntVal = 49997849600.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -285,7 +285,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998863360.000000 + eAcntVal = 49997829120.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -311,7 +311,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998842880.000000 + eAcntVal = 49996774400.000000 // 1054720 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 435778513..e553ba148 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -392,7 +392,7 @@ func TestSMGLastUsed(t *testing.T) { if maxUsage != 120 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 6.5901 + eAcntVal = 6.490110 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 631e8fcce..84a7ec7dd 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -78,10 +78,13 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { // Attempts to debit a duration, returns maximum duration which can be debitted or error func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.Duration, error) { + requestedDuration := dur self.totalUsage += lastUsed // Should reflect the total usage so far + //utils.Logger.Debug(fmt.Sprintf("ExtraDuration: %d", self.extraDuration)) if lastUsed > 0 { self.extraDuration = self.lastUsage - lastUsed + //utils.Logger.Debug(fmt.Sprintf("ExtraDuration LastUsed: %d", self.extraDuration)) } // apply correction from previous run if self.extraDuration < dur { @@ -117,6 +120,10 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.sessionCds = append(self.sessionCds, self.cd.Clone()) self.callCosts = append(self.callCosts, cc) self.lastUsage = ccDuration + + if ccDuration >= dur { // we got what we asked to be debited + return requestedDuration, nil + } return ccDuration, nil } From d77ff608507934228f437cf6b661142d273d3d51 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 28 Mar 2016 21:29:49 +0200 Subject: [PATCH 083/227] Diameter tests to emulate multiple data sessions in one CCR --- agents/dmtagent_it_test.go | 243 +++++++++++++++++++++++++++ data/conf/samples/dmtagent/data.json | 108 ++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 data/conf/samples/dmtagent/data.json diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index 3abf823eb..f6d39fd82 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -655,6 +655,249 @@ func TestDmtAgentSendCCRSimpaEvent(t *testing.T) { } } +func TestDmtAgentSendDataGrpInit(t *testing.T) { + if !*testIntegration { + return + } + ccr := diam.NewRequest(diam.CreditControl, 4, nil) + ccr.NewAVP(avp.SessionID, avp.Mbit, 0, datatype.UTF8String("testdatagrp")) + ccr.NewAVP(avp.OriginHost, avp.Mbit, 0, datatype.DiameterIdentity("CGR-DA")) + ccr.NewAVP(avp.OriginRealm, avp.Mbit, 0, datatype.DiameterIdentity("cgrates.org")) + ccr.NewAVP(avp.AuthApplicationID, avp.Mbit, 0, datatype.Unsigned32(4)) + ccr.NewAVP(avp.ServiceContextID, avp.Mbit, 0, datatype.UTF8String("gprs@huawei.com")) + ccr.NewAVP(avp.CCRequestType, avp.Mbit, 0, datatype.Enumerated(1)) + ccr.NewAVP(avp.CCRequestNumber, avp.Mbit, 0, datatype.Unsigned32(1)) + ccr.NewAVP(avp.EventTimestamp, avp.Mbit, 0, datatype.Time(time.Date(2016, 1, 5, 11, 30, 10, 0, time.UTC))) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(0)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("1001")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(1)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("104502200011")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.ServiceIdentifier, avp.Mbit, 0, datatype.Unsigned32(0)) + ccr.NewAVP(avp.RequestedAction, avp.Mbit, 0, datatype.Enumerated(0)) + ccr.NewAVP(avp.RequestedServiceUnit, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.CCTime, avp.Mbit, 0, datatype.Unsigned32(1))}}) + ccr.NewAVP(873, avp.Mbit, 10415, &diam.GroupedAVP{ // + AVP: []*diam.AVP{ + diam.NewAVP(20300, avp.Mbit, 2011, &diam.GroupedAVP{ // IN-Information + AVP: []*diam.AVP{ + diam.NewAVP(20302, avp.Mbit, 2011, datatype.UTF8String("22509")), // Calling-Vlr-Number + diam.NewAVP(20385, avp.Mbit, 2011, datatype.UTF8String("4002")), // Called-Party-NP + }, + }), + diam.NewAVP(2000, avp.Mbit, 10415, &diam.GroupedAVP{ // SMS-Information + AVP: []*diam.AVP{ + diam.NewAVP(886, avp.Mbit, 10415, &diam.GroupedAVP{ // Originator-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1003")), // Address-Data + }}), + diam.NewAVP(1201, avp.Mbit, 10415, &diam.GroupedAVP{ // Recipient-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1002")), // Address-Data + }}), + }, + }), + }}) + if err := dmtClient.SendMessage(ccr); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if msg == nil { + t.Fatal("No message returned") + } + if avps, err := msg.FindAVPsWithPath([]interface{}{"Result-Code"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Result-Code not found") + } else if resCode := avpValAsString(avps[0]); resCode != "2001" { + t.Errorf("Expecting 2001, received: %s", resCode) + } +} + +func TestDmtAgentSendDataGrpUpdate(t *testing.T) { + if !*testIntegration { + return + } + ccr := diam.NewRequest(diam.CreditControl, 4, nil) + ccr.NewAVP(avp.SessionID, avp.Mbit, 0, datatype.UTF8String("testdatagrp")) + ccr.NewAVP(avp.OriginHost, avp.Mbit, 0, datatype.DiameterIdentity("CGR-DA")) + ccr.NewAVP(avp.OriginRealm, avp.Mbit, 0, datatype.DiameterIdentity("cgrates.org")) + ccr.NewAVP(avp.AuthApplicationID, avp.Mbit, 0, datatype.Unsigned32(4)) + ccr.NewAVP(avp.ServiceContextID, avp.Mbit, 0, datatype.UTF8String("gprs@huawei.com")) + ccr.NewAVP(avp.CCRequestType, avp.Mbit, 0, datatype.Enumerated(2)) + ccr.NewAVP(avp.CCRequestNumber, avp.Mbit, 0, datatype.Unsigned32(1)) + ccr.NewAVP(avp.EventTimestamp, avp.Mbit, 0, datatype.Time(time.Date(2016, 1, 5, 11, 30, 10, 0, time.UTC))) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(0)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("1001")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(1)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("104502200011")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.ServiceIdentifier, avp.Mbit, 0, datatype.Unsigned32(0)) + ccr.NewAVP(avp.RequestedAction, avp.Mbit, 0, datatype.Enumerated(0)) + ccr.NewAVP(avp.RequestedServiceUnit, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.CCTime, avp.Mbit, 0, datatype.Unsigned32(1))}}) + ccr.NewAVP(873, avp.Mbit, 10415, &diam.GroupedAVP{ // + AVP: []*diam.AVP{ + diam.NewAVP(20300, avp.Mbit, 2011, &diam.GroupedAVP{ // IN-Information + AVP: []*diam.AVP{ + diam.NewAVP(20302, avp.Mbit, 2011, datatype.UTF8String("22509")), // Calling-Vlr-Number + diam.NewAVP(20385, avp.Mbit, 2011, datatype.UTF8String("4002")), // Called-Party-NP + }, + }), + diam.NewAVP(2000, avp.Mbit, 10415, &diam.GroupedAVP{ // SMS-Information + AVP: []*diam.AVP{ + diam.NewAVP(886, avp.Mbit, 10415, &diam.GroupedAVP{ // Originator-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1003")), // Address-Data + }}), + diam.NewAVP(1201, avp.Mbit, 10415, &diam.GroupedAVP{ // Recipient-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1002")), // Address-Data + }}), + }, + }), + }}) + ccr.NewAVP("Multiple-Services-Credit-Control", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit + AVP: []*diam.AVP{ + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(2)), // CC-Output-Octets + }, + }), + diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(1)), // Data session for group 1 + }, + }) + ccr.NewAVP("Multiple-Services-Credit-Control", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit + AVP: []*diam.AVP{ + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Output-Octets + }, + }), + diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(2)), // Data session for group 2 + }, + }) + if err := dmtClient.SendMessage(ccr); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(500) * time.Millisecond) + /*msg := dmtClient.ReceivedMessage() + if msg == nil { + t.Fatal("No message returned") + } + if avps, err := msg.FindAVPsWithPath([]interface{}{"Result-Code"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Result-Code not found") + } else if resCode := avpValAsString(avps[0]); resCode != "2001" { + t.Errorf("Expecting 2001, received: %s", resCode) + } + */ +} + +func TestDmtAgentSendDataGrpTerminate(t *testing.T) { + if !*testIntegration { + return + } + ccr := diam.NewRequest(diam.CreditControl, 4, nil) + ccr.NewAVP(avp.SessionID, avp.Mbit, 0, datatype.UTF8String("testdatagrp")) + ccr.NewAVP(avp.OriginHost, avp.Mbit, 0, datatype.DiameterIdentity("CGR-DA")) + ccr.NewAVP(avp.OriginRealm, avp.Mbit, 0, datatype.DiameterIdentity("cgrates.org")) + ccr.NewAVP(avp.AuthApplicationID, avp.Mbit, 0, datatype.Unsigned32(4)) + ccr.NewAVP(avp.ServiceContextID, avp.Mbit, 0, datatype.UTF8String("gprs@huawei.com")) + ccr.NewAVP(avp.CCRequestType, avp.Mbit, 0, datatype.Enumerated(3)) + ccr.NewAVP(avp.CCRequestNumber, avp.Mbit, 0, datatype.Unsigned32(1)) + ccr.NewAVP(avp.EventTimestamp, avp.Mbit, 0, datatype.Time(time.Date(2016, 1, 5, 11, 30, 10, 0, time.UTC))) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(0)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("1001")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.SubscriptionID, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.SubscriptionIDType, avp.Mbit, 0, datatype.Enumerated(1)), + diam.NewAVP(avp.SubscriptionIDData, avp.Mbit, 0, datatype.UTF8String("104502200011")), // Subscription-Id-Data + }}) + ccr.NewAVP(avp.ServiceIdentifier, avp.Mbit, 0, datatype.Unsigned32(0)) + ccr.NewAVP(avp.RequestedAction, avp.Mbit, 0, datatype.Enumerated(0)) + ccr.NewAVP(avp.RequestedServiceUnit, avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(avp.CCTime, avp.Mbit, 0, datatype.Unsigned32(1))}}) + ccr.NewAVP(873, avp.Mbit, 10415, &diam.GroupedAVP{ // + AVP: []*diam.AVP{ + diam.NewAVP(20300, avp.Mbit, 2011, &diam.GroupedAVP{ // IN-Information + AVP: []*diam.AVP{ + diam.NewAVP(20302, avp.Mbit, 2011, datatype.UTF8String("22509")), // Calling-Vlr-Number + diam.NewAVP(20385, avp.Mbit, 2011, datatype.UTF8String("4002")), // Called-Party-NP + }, + }), + diam.NewAVP(2000, avp.Mbit, 10415, &diam.GroupedAVP{ // SMS-Information + AVP: []*diam.AVP{ + diam.NewAVP(886, avp.Mbit, 10415, &diam.GroupedAVP{ // Originator-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1003")), // Address-Data + }}), + diam.NewAVP(1201, avp.Mbit, 10415, &diam.GroupedAVP{ // Recipient-Address + AVP: []*diam.AVP{ + diam.NewAVP(899, avp.Mbit, 10415, datatype.Enumerated(1)), // Address-Type + diam.NewAVP(897, avp.Mbit, 10415, datatype.UTF8String("1002")), // Address-Data + }}), + }, + }), + }}) + ccr.NewAVP("Multiple-Services-Credit-Control", avp.Mbit, 0, &diam.GroupedAVP{ + AVP: []*diam.AVP{ + diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit + AVP: []*diam.AVP{ + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(2)), // CC-Output-Octets + }, + }), + }, + }) + if err := dmtClient.SendMessage(ccr); err != nil { + t.Error(err) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) + msg := dmtClient.ReceivedMessage() + if msg == nil { + t.Fatal("No message returned") + } + if avps, err := msg.FindAVPsWithPath([]interface{}{"Result-Code"}, dict.UndefinedVendorID); err != nil { + t.Error(err) + } else if len(avps) == 0 { + t.Error("Result-Code not found") + } else if resCode := avpValAsString(avps[0]); resCode != "2001" { + t.Errorf("Expecting 2001, received: %s", resCode) + } +} + func TestDmtAgentCdrs(t *testing.T) { if !*testIntegration { return diff --git a/data/conf/samples/dmtagent/data.json b/data/conf/samples/dmtagent/data.json new file mode 100644 index 000000000..1c57ae61b --- /dev/null +++ b/data/conf/samples/dmtagent/data.json @@ -0,0 +1,108 @@ + +{ + +"diameter_agent": { + "request_processors": [ + { + "id": "data_init", // formal identifier of this processor + "dry_run": false, // do not send the events to SMG, just log them + "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(1)", // filter requests processed by this processor + "continue_on_success": false, // continue to the next template if executed + "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*data", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*prepaid", "mandatory": true}, + {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "^cgrates.org", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^generic", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*grouped", "value": "Subscription-Id>Subscription-Id-Data", "field_filter":"Subscription-Id>Subscription-Id-Type(0)", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + ], + "cca_fields": [ + {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, + {"tag": "ResultCode", "field_filter": "CGRMaxUsage(0)", "field_id": "Result-Code", "type": "*constant", "value": "4010"}, + ], + }, + { + "id": "data_update_grp1", // formal identifier of this processor + "dry_run": true, // do not send the events to SMG, just log them + "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(2);Multiple-Services-Credit-Control>Rating-Group(1)", // filter requests processed by this processor + "continue_on_success": true, // continue to the next template if executed + "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*data", "mandatory": true}, + {"tag": "InitialOriginID", "field_id": "InitialOriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "^_grp1", "append": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*prepaid", "mandatory": true}, + {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "^cgrates.org", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^generic", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*grouped", "value": "Subscription-Id>Subscription-Id-Data", "field_filter":"Subscription-Id>Subscription-Id-Type(0)", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "LastUsed", "field_id": "Usage", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(1)", "type": "*handler", "handler_id": "*sum", + "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, + ], + "cca_fields": [ + {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, + {"tag": "ResultCode", "field_filter": "CGRMaxUsage(0)", "field_id": "Result-Code", "type": "*constant", "value": "4010"}, + ], + }, + { + "id": "data_update_grp2", // formal identifier of this processor + "dry_run": true, // do not send the events to SMG, just log them + "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(2);Multiple-Services-Credit-Control>Rating-Group(2)", // filter requests processed by this processor + "continue_on_success": true, // continue to the next template if executed + "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*data", "mandatory": true}, + {"tag": "InitialOriginID", "field_id": "InitialOriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "^_grp2", "append": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*prepaid", "mandatory": true}, + {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "^cgrates.org", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^generic", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*grouped", "value": "Subscription-Id>Subscription-Id-Data", "field_filter":"Subscription-Id>Subscription-Id-Type(0)", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "LastUsed", "field_id": "Usage", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(2)", "type": "*handler", "handler_id": "*sum", + "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, + ], + "cca_fields": [ + {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, + {"tag": "ResultCode", "field_filter": "CGRMaxUsage(0)", "field_id": "Result-Code", "type": "*constant", "value": "4010"}, + ], + }, + { + "id": "data_terminate", // formal identifier of this processor + "dry_run": true, // do not send the events to SMG, just log them + "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(3)", // filter requests processed by this processor + "continue_on_success": false, // continue to the next template if executed + "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*data", "mandatory": true}, + {"tag": "OriginIDPrefix", "field_id": "OriginIDPrefix", "type": "*composed", "value": "Session-Id", "mandatory": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*prepaid", "mandatory": true}, + {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "^cgrates.org", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^generic", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*grouped", "value": "Subscription-Id>Subscription-Id-Data", "field_filter":"Subscription-Id>Subscription-Id-Type(0)", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, + {"tag": "LastUsed", "field_id": "Usage", "type": "*handler", "handler_id": "*sum", + "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, + ], + "cca_fields": [ + {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, + {"tag": "ResultCode", "field_filter": "CGRMaxUsage(0)", "field_id": "Result-Code", "type": "*constant", "value": "4010"}, + ], + }, + ] +} + +} \ No newline at end of file From a8309eb0e253a156856ff670247512e9d007cfbc Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 29 Mar 2016 09:57:54 +0200 Subject: [PATCH 084/227] Diameter CCR processing fix for multiple processors activation --- agents/dmtagent.go | 9 ++++++--- agents/dmtagent_it_test.go | 3 +-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index 2b5466bac..bce2fa6db 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -190,10 +190,13 @@ func (self *DiameterAgent) handleCCR(c diam.Conn, m *diam.Message) { return } cca := NewBareCCAFromCCR(ccr, self.cgrCfg.DiameterAgentCfg().OriginHost, self.cgrCfg.DiameterAgentCfg().OriginRealm) - var processed bool + var processed, lclProcessed bool for _, reqProcessor := range self.cgrCfg.DiameterAgentCfg().RequestProcessors { - processed, err = self.processCCR(ccr, reqProcessor, cca) - if err != nil || (processed && !reqProcessor.ContinueOnSuccess) { + lclProcessed, err = self.processCCR(ccr, reqProcessor, cca) + if lclProcessed { // Process local so we don't overwrite globally + processed = lclProcessed + } + if err != nil || (lclProcessed && !reqProcessor.ContinueOnSuccess) { break } } diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index f6d39fd82..723a0ff70 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -804,7 +804,7 @@ func TestDmtAgentSendDataGrpUpdate(t *testing.T) { t.Error(err) } time.Sleep(time.Duration(500) * time.Millisecond) - /*msg := dmtClient.ReceivedMessage() + msg := dmtClient.ReceivedMessage() if msg == nil { t.Fatal("No message returned") } @@ -815,7 +815,6 @@ func TestDmtAgentSendDataGrpUpdate(t *testing.T) { } else if resCode := avpValAsString(avps[0]); resCode != "2001" { t.Errorf("Expecting 2001, received: %s", resCode) } - */ } func TestDmtAgentSendDataGrpTerminate(t *testing.T) { From 33401a78f7f2b1b32960ffbb561ec9ead92f16f2 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 19:33:42 +0300 Subject: [PATCH 085/227] extra duraction improvements --- sessionmanager/smg_session.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 84a7ec7dd..ba02dbb60 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -81,10 +81,11 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D requestedDuration := dur self.totalUsage += lastUsed // Should reflect the total usage so far - //utils.Logger.Debug(fmt.Sprintf("ExtraDuration: %d", self.extraDuration)) + //utils.Logger.Debug(fmt.Sprintf("lastUsage: %f lastUsed: %f", self.lastUsage.Seconds(), lastUsed.Seconds())) + //utils.Logger.Debug(fmt.Sprintf("ExtraDuration: %f", self.extraDuration.Seconds())) if lastUsed > 0 { self.extraDuration = self.lastUsage - lastUsed - //utils.Logger.Debug(fmt.Sprintf("ExtraDuration LastUsed: %d", self.extraDuration)) + //utils.Logger.Debug(fmt.Sprintf("ExtraDuration LastUsed: %f", self.extraDuration.Seconds())) } // apply correction from previous run if self.extraDuration < dur { @@ -94,7 +95,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.extraDuration -= dur return ccDuration, nil } - + //utils.Logger.Debug(fmt.Sprintf("dur: %f", dur.Seconds())) self.extraDuration = 0 if self.cd.LoopIndex > 0 { self.cd.TimeStart = self.cd.TimeEnd @@ -110,6 +111,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.TimeEnd = cc.GetEndTime() // set debited timeEnd // update call duration with real debited duration ccDuration := cc.GetDuration() + initialExtraDuration := self.extraDuration if ccDuration != dur { self.extraDuration = ccDuration - dur } @@ -119,12 +121,12 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.LoopIndex += 1 self.sessionCds = append(self.sessionCds, self.cd.Clone()) self.callCosts = append(self.callCosts, cc) - self.lastUsage = ccDuration + self.lastUsage = initialExtraDuration + ccDuration if ccDuration >= dur { // we got what we asked to be debited return requestedDuration, nil } - return ccDuration, nil + return initialExtraDuration + ccDuration, nil } // Attempts to refund a duration, error on failure From c3feacac7d28bb79e2b7d6b40b956d0ea203d09c Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 19:43:45 +0300 Subject: [PATCH 086/227] last usage test fixes --- sessionmanager/data_it_test.go | 6 +++--- sessionmanager/smg_session.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 513941df0..b281c2985 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -258,7 +258,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49997849600.000000 // 20480 + eAcntVal = 49998883840.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -285,7 +285,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49997829120.000000 // 20480 + eAcntVal = 49998863360.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -311,7 +311,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49996774400.000000 // 1054720 + eAcntVal = 49998842880.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index ba02dbb60..4f4c7ed28 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -96,6 +96,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D return ccDuration, nil } //utils.Logger.Debug(fmt.Sprintf("dur: %f", dur.Seconds())) + initialExtraDuration := self.extraDuration self.extraDuration = 0 if self.cd.LoopIndex > 0 { self.cd.TimeStart = self.cd.TimeEnd @@ -111,7 +112,6 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.TimeEnd = cc.GetEndTime() // set debited timeEnd // update call duration with real debited duration ccDuration := cc.GetDuration() - initialExtraDuration := self.extraDuration if ccDuration != dur { self.extraDuration = ccDuration - dur } From c2c79742331751676b68a0f93c58ef6eacb7c861 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 19:50:57 +0300 Subject: [PATCH 087/227] another test fix --- sessionmanager/smg_it_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index e553ba148..ba131c5e0 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -392,7 +392,7 @@ func TestSMGLastUsed(t *testing.T) { if maxUsage != 120 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 6.490110 + eAcntVal = 6.590100 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { From 587a23e2ccf77e09fd5735d01ad36fab4c569e37 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 20:16:07 +0300 Subject: [PATCH 088/227] fix potential concurent map panic --- engine/action.go | 12 ++++++++++-- engine/cdrs.go | 7 ++++++- utils/httpclient.go | 2 +- utils/httpclient_local_test.go | 8 ++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/engine/action.go b/engine/action.go index fff13356c..6c57e8aaa 100644 --- a/engine/action.go +++ b/engine/action.go @@ -426,9 +426,13 @@ func callUrl(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error if sq != nil { o = sq } + jsn, err := json.Marshal(o) + if err != nil { + return err + } cfg := config.CgrConfig() fallbackPath := path.Join(cfg.HttpFailedDir, fmt.Sprintf("act_%s_%s_%s.json", a.ActionType, a.ExtraParameters, utils.GenUUID())) - _, err := utils.HttpPoster(a.ExtraParameters, cfg.HttpSkipTlsVerify, o, utils.CONTENT_JSON, 1, fallbackPath) + _, err = utils.HttpPoster(a.ExtraParameters, cfg.HttpSkipTlsVerify, jsn, utils.CONTENT_JSON, 1, fallbackPath) return err } @@ -441,9 +445,13 @@ func callUrlAsync(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) if sq != nil { o = sq } + jsn, err := json.Marshal(o) + if err != nil { + return err + } cfg := config.CgrConfig() fallbackPath := path.Join(cfg.HttpFailedDir, fmt.Sprintf("act_%s_%s_%s.json", a.ActionType, a.ExtraParameters, utils.GenUUID())) - go utils.HttpPoster(a.ExtraParameters, cfg.HttpSkipTlsVerify, o, utils.CONTENT_JSON, 3, fallbackPath) + go utils.HttpPoster(a.ExtraParameters, cfg.HttpSkipTlsVerify, jsn, utils.CONTENT_JSON, 3, fallbackPath) return nil } diff --git a/engine/cdrs.go b/engine/cdrs.go index 1b2d2e563..3cca1b6b3 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -19,6 +19,7 @@ along with this program. If not, see package engine import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -424,7 +425,11 @@ func (self *CdrServer) replicateCdr(cdr *CDR) error { body = cdr.AsHttpForm() case utils.META_HTTP_JSON: content = utils.CONTENT_JSON - body = cdr + jsn, err := json.Marshal(cdr) + if err != nil { + return err + } + body = jsn } errChan := make(chan error) go func(body interface{}, rplCfg *config.CdrReplicationCfg, content string, errChan chan error) { diff --git a/utils/httpclient.go b/utils/httpclient.go index 5e048ebd0..0fad55afb 100644 --- a/utils/httpclient.go +++ b/utils/httpclient.go @@ -80,7 +80,7 @@ func HttpPoster(addr string, skipTlsVerify bool, content interface{}, contentTyp var err error switch contentType { case CONTENT_JSON: - body, err = json.Marshal(content) + body = content.([]byte) case CONTENT_FORM: urlData = content.(url.Values) case CONTENT_TEXT: diff --git a/utils/httpclient_local_test.go b/utils/httpclient_local_test.go index 7d73b00bd..ecdf49481 100644 --- a/utils/httpclient_local_test.go +++ b/utils/httpclient_local_test.go @@ -39,15 +39,15 @@ func TestHttpJsonPoster(t *testing.T) { return } content := &TestContent{Var1: "Val1", Var2: "Val2"} + jsn, _ := json.Marshal(content) filePath := "/tmp/cgr_test_http_poster.json" - if _, err := HttpPoster("http://localhost:8080/invalid", true, content, CONTENT_JSON, 3, filePath); err != nil { + if _, err := HttpPoster("http://localhost:8080/invalid", true, jsn, CONTENT_JSON, 3, filePath); err != nil { t.Error(err) } - jsnContent, _ := json.Marshal(content) if readBytes, err := ioutil.ReadFile(filePath); err != nil { t.Error(err) - } else if !reflect.DeepEqual(jsnContent, readBytes) { - t.Errorf("Expecting: %q, received: %q", string(jsnContent), string(readBytes)) + } else if !reflect.DeepEqual(jsn, readBytes) { + t.Errorf("Expecting: %q, received: %q", string(jsn), string(readBytes)) } if err := os.Remove(filePath); err != nil { t.Error("Failed removing file: ", filePath) From 3ac526a05e4e090bf2d6cace08304de46c1aba05 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 29 Mar 2016 19:55:29 +0200 Subject: [PATCH 089/227] CDRS forking rated CDRs depending on smg_costs, smg_costs indexing update to include origin_id also --- agents/dmtagent_it_test.go | 37 +++-- data/conf/samples/dmtagent/data.json | 18 +- data/storage/mysql/create_cdrs_tables.sql | 2 +- data/storage/postgres/create_cdrs_tables.sql | 2 +- engine/cdrs.go | 163 ++++++++++--------- engine/storage_mongo_datadb.go | 2 +- engine/storage_sql.go | 7 +- sessionmanager/smg_session.go | 7 +- sessionmanager/smgeneric.go | 26 +-- 9 files changed, 150 insertions(+), 114 deletions(-) diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index 723a0ff70..4b0179e88 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -655,6 +655,26 @@ func TestDmtAgentSendCCRSimpaEvent(t *testing.T) { } } +func TestDmtAgentCdrs(t *testing.T) { + if !*testIntegration { + return + } + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, ToRs: []string{utils.VOICE}} + if err := apierRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Usage != "610" { + t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) + } + if cdrs[0].Cost != 0.7565 { + t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) + } + } +} + func TestDmtAgentSendDataGrpInit(t *testing.T) { if !*testIntegration { return @@ -803,7 +823,7 @@ func TestDmtAgentSendDataGrpUpdate(t *testing.T) { if err := dmtClient.SendMessage(ccr); err != nil { t.Error(err) } - time.Sleep(time.Duration(500) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() if msg == nil { t.Fatal("No message returned") @@ -883,7 +903,7 @@ func TestDmtAgentSendDataGrpTerminate(t *testing.T) { if err := dmtClient.SendMessage(ccr); err != nil { t.Error(err) } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) + time.Sleep(time.Duration(3000) * time.Millisecond) msg := dmtClient.ReceivedMessage() if msg == nil { t.Fatal("No message returned") @@ -897,23 +917,16 @@ func TestDmtAgentSendDataGrpTerminate(t *testing.T) { } } -func TestDmtAgentCdrs(t *testing.T) { +func TestDmtAgentSendDataGrpCDRs(t *testing.T) { if !*testIntegration { return } var cdrs []*engine.ExternalCDR - req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, ToRs: []string{utils.VOICE}} + req := utils.RPCCDRsFilter{CGRIDs: []string{utils.Sha1("testdatagrp")}} if err := apierRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { + } else if len(cdrs) != 3 { t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Usage != "610" { - t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) - } - if cdrs[0].Cost != 0.7565 { - t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) - } } } diff --git a/data/conf/samples/dmtagent/data.json b/data/conf/samples/dmtagent/data.json index 1c57ae61b..9080766f3 100644 --- a/data/conf/samples/dmtagent/data.json +++ b/data/conf/samples/dmtagent/data.json @@ -28,7 +28,7 @@ }, { "id": "data_update_grp1", // formal identifier of this processor - "dry_run": true, // do not send the events to SMG, just log them + "dry_run": false, // do not send the events to SMG, just log them "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(2);Multiple-Services-Credit-Control>Rating-Group(1)", // filter requests processed by this processor "continue_on_success": true, // continue to the next template if executed "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value @@ -44,7 +44,8 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "LastUsed", "field_id": "Usage", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(1)", "type": "*handler", "handler_id": "*sum", + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + {"tag": "LastUsed", "field_id": "LastUsed", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(1)", "type": "*handler", "handler_id": "*sum", "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, ], "cca_fields": [ @@ -54,7 +55,7 @@ }, { "id": "data_update_grp2", // formal identifier of this processor - "dry_run": true, // do not send the events to SMG, just log them + "dry_run": false, // do not send the events to SMG, just log them "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(2);Multiple-Services-Credit-Control>Rating-Group(2)", // filter requests processed by this processor "continue_on_success": true, // continue to the next template if executed "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value @@ -70,7 +71,8 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "LastUsed", "field_id": "Usage", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(2)", "type": "*handler", "handler_id": "*sum", + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + {"tag": "LastUsed", "field_id": "LastUsed", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(2)", "type": "*handler", "handler_id": "*sum", "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, ], "cca_fields": [ @@ -80,11 +82,12 @@ }, { "id": "data_terminate", // formal identifier of this processor - "dry_run": true, // do not send the events to SMG, just log them + "dry_run": false, // do not send the events to SMG, just log them "request_filter": "Service-Context-Id(^gprs);CC-Request-Type(3)", // filter requests processed by this processor "continue_on_success": false, // continue to the next template if executed "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*data", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, {"tag": "OriginIDPrefix", "field_id": "OriginIDPrefix", "type": "*composed", "value": "Session-Id", "mandatory": true}, {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*prepaid", "mandatory": true}, {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, @@ -94,12 +97,11 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "LastUsed", "field_id": "Usage", "type": "*handler", "handler_id": "*sum", + {"tag": "LastUsed", "field_id": "LastUsed", "type": "*handler", "handler_id": "*sum", "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, ], "cca_fields": [ - {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, - {"tag": "ResultCode", "field_filter": "CGRMaxUsage(0)", "field_id": "Result-Code", "type": "*constant", "value": "4010"}, + {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"} ], }, ] diff --git a/data/storage/mysql/create_cdrs_tables.sql b/data/storage/mysql/create_cdrs_tables.sql index 5fe75add6..2bd6d9a59 100644 --- a/data/storage/mysql/create_cdrs_tables.sql +++ b/data/storage/mysql/create_cdrs_tables.sql @@ -33,7 +33,7 @@ CREATE TABLE cdrs ( updated_at TIMESTAMP, deleted_at TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY cdrrun (cgrid, run_id) + UNIQUE KEY cdrrun (cgrid, run_id, origin_id) ); DROP TABLE IF EXISTS sm_costs; diff --git a/data/storage/postgres/create_cdrs_tables.sql b/data/storage/postgres/create_cdrs_tables.sql index 157aa34d4..a44c24520 100644 --- a/data/storage/postgres/create_cdrs_tables.sql +++ b/data/storage/postgres/create_cdrs_tables.sql @@ -32,7 +32,7 @@ CREATE TABLE cdrs ( created_at TIMESTAMP, updated_at TIMESTAMP, deleted_at TIMESTAMP, - UNIQUE (cgrid, run_id) + UNIQUE (cgrid, run_id, origin_id) ); ; DROP INDEX IF EXISTS deleted_at_cp_idx; diff --git a/engine/cdrs.go b/engine/cdrs.go index 1b2d2e563..b89e70637 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -109,12 +109,9 @@ func (self *CdrServer) ProcessExternalCdr(eCDR *ExternalCDR) error { // RPC method, used to log callcosts to db func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { - smCost.CostDetails.UpdateCost() // make sure the total cost reflect the increments - smCost.CostDetails.UpdateRatedUsage() // make sure rated usage is updated - lockKey := smCost.CGRID + smCost.RunID // Will lock on this ID - if smCost.CGRID == "" && smCost.OriginID != "" { - lockKey = smCost.OriginHost + smCost.OriginID - } + smCost.CostDetails.UpdateCost() // make sure the total cost reflect the increments + smCost.CostDetails.UpdateRatedUsage() // make sure rated usage is updated + lockKey := utils.CDRS_SOURCE + smCost.CGRID + smCost.RunID + smCost.OriginID // Will lock on this ID if checkDuplicate { _, err := self.guard.Guard(func() (interface{}, error) { smCosts, err := self.cdrDb.GetSMCosts(smCost.CGRID, smCost.RunID, "", "") @@ -125,7 +122,7 @@ func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { return nil, utils.ErrExists } return nil, self.cdrDb.SetSMCost(smCost) - }, 0, lockKey) // FixMe: Possible deadlock with Guard from SMG session close() + }, time.Duration(2*time.Second), lockKey) // FixMe: Possible deadlock with Guard from SMG session close() return err } return self.cdrDb.SetSMCost(smCost) @@ -209,60 +206,6 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR) error { return nil } -func (self *CdrServer) rateStoreStatsReplicate(cdr *CDR, sendToStats bool) error { - if cdr.RunID == utils.MetaRaw { // Overwrite *raw with *default for rating - cdr.RunID = utils.META_DEFAULT - } - if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { - return err - } - if err := LoadAlias(&AttrMatchingAlias{ - Destination: cdr.Destination, - Direction: cdr.Direction, - Tenant: cdr.Tenant, - Category: cdr.Category, - Account: cdr.Account, - Subject: cdr.Subject, - Context: utils.ALIAS_CONTEXT_RATING, - }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { - return err - } - - // Rate CDR - if self.rater != nil && !cdr.Rated { - if err := self.rateCDR(cdr); err != nil { - cdr.Cost = -1.0 // If there was an error, mark the CDR - cdr.ExtraInfo = err.Error() - } - } - if cdr.RunID == utils.META_SURETAX { // Request should be processed by SureTax - if err := SureTaxProcessCdr(cdr); err != nil { - cdr.Cost = -1.0 - cdr.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo - } - } - if self.cgrCfg.CDRSStoreCdrs { // Store CDRs - // Store RatedCDR - if cdr.CostDetails != nil { - cdr.CostDetails.UpdateCost() - cdr.CostDetails.UpdateRatedUsage() - } - if err := self.cdrDb.SetCDR(cdr, true); err != nil { - utils.Logger.Err(fmt.Sprintf(" Storing rated CDR %+v, got error: %s", cdr, err.Error())) - } - } - // Attach CDR to stats - if self.stats != nil && sendToStats { // Send CDR to stats - if err := self.stats.AppendCDR(cdr, nil); err != nil { - utils.Logger.Err(fmt.Sprintf(" Could not append CDR to stats: %s", err.Error())) - } - } - if len(self.cgrCfg.CDRSCdrReplication) != 0 { - self.replicateCdr(cdr) - } - return nil -} - func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { cdrRuns := []*CDR{cdr} if cdr.RunID != utils.MetaRaw { // Only derive *raw CDRs @@ -331,44 +274,118 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { return cdrRuns, nil } -func (self *CdrServer) rateCDR(cdr *CDR) error { +func (self *CdrServer) rateStoreStatsReplicate(cdr *CDR, sendToStats bool) error { + if cdr.RunID == utils.MetaRaw { // Overwrite *raw with *default for rating + cdr.RunID = utils.META_DEFAULT + } + if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { + return err + } + if err := LoadAlias(&AttrMatchingAlias{ + Destination: cdr.Destination, + Direction: cdr.Direction, + Tenant: cdr.Tenant, + Category: cdr.Category, + Account: cdr.Account, + Subject: cdr.Subject, + Context: utils.ALIAS_CONTEXT_RATING, + }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + return err + } + // Rate CDR, can receive multiple due to SMCosts for OriginIDPrefix + var ratedCDRs []*CDR + var err error + if cdr.Rated { + ratedCDRs = []*CDR{cdr} + } else if self.rater != nil { + if ratedCDRs, err = self.rateCDR(cdr); err != nil { + cdr.Cost = -1.0 // If there was an error, mark the CDR + cdr.ExtraInfo = err.Error() + ratedCDRs = []*CDR{cdr} + } + } + for _, ratedCDR := range ratedCDRs { + if ratedCDR.RunID == utils.META_SURETAX { // Request should be processed by SureTax + if err := SureTaxProcessCdr(ratedCDR); err != nil { + ratedCDR.Cost = -1.0 + ratedCDR.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo + } + } + } + if self.cgrCfg.CDRSStoreCdrs { // Store CDRs + for _, ratedCDR := range ratedCDRs { + // Store RatedCDR + if ratedCDR.CostDetails != nil { + ratedCDR.CostDetails.UpdateCost() + ratedCDR.CostDetails.UpdateRatedUsage() + } + if err := self.cdrDb.SetCDR(ratedCDR, true); err != nil { + utils.Logger.Err(fmt.Sprintf(" Storing rated CDR %+v, got error: %s", ratedCDR, err.Error())) + } + } + } + // Attach CDR to stats + if self.stats != nil && sendToStats { // Send CDR to stats + for _, ratedCDR := range ratedCDRs { + if err := self.stats.AppendCDR(ratedCDR, nil); err != nil { + utils.Logger.Err(fmt.Sprintf(" Could not append CDR to stats: %s", err.Error())) + } + } + } + if len(self.cgrCfg.CDRSCdrReplication) != 0 { + for _, ratedCDR := range ratedCDRs { + self.replicateCdr(ratedCDR) + } + } + return nil +} + +// rateCDR will populate cost field +// Returns more than one rated CDR in case of SMCost retrieved based on prefix +func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { var qryCC *CallCost var err error if cdr.RequestType == utils.META_NONE { - return nil + return nil, nil } + var cdrsRated []*CDR _, hasLastUsed := cdr.ExtraFields[utils.LastUsed] if utils.IsSliceMember([]string{utils.META_PREPAID, utils.PREPAID}, cdr.RequestType) && (cdr.Usage != 0 || hasLastUsed) { // ToDo: Get rid of PREPAID as soon as we don't want to support it backwards // Should be previously calculated and stored in DB delay := utils.Fib() - var usage float64 + var smCosts []*SMCost for i := 0; i < 4; i++ { - smCosts, err := self.cdrDb.GetSMCosts(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) + smCosts, err = self.cdrDb.GetSMCosts(cdr.CGRID, cdr.RunID, cdr.OriginHost, cdr.ExtraFields[utils.OriginIDPrefix]) if err == nil && len(smCosts) != 0 { - qryCC = smCosts[0].CostDetails - usage = smCosts[0].Usage break } time.Sleep(delay()) } + if len(smCosts) != 0 { // Cost retrieved from SMCost table + for _, smCost := range smCosts { + cdrClone := cdr.Clone() + cdrClone.OriginID = smCost.OriginID + cdrClone.Usage = time.Duration(smCost.Usage * utils.NANO_MULTIPLIER) // Usage is float as seconds, convert back to duration + cdrClone.Cost = smCost.CostDetails.Cost + cdrClone.CostDetails = smCost.CostDetails + cdrsRated = append(cdrsRated, cdrClone) + } + return cdrsRated, nil + } if err != nil && (err == gorm.RecordNotFound || err == mgov2.ErrNotFound) { //calculate CDR as for pseudoprepaid utils.Logger.Warning(fmt.Sprintf(" WARNING: Could not find CallCostLog for cgrid: %s, source: %s, runid: %s, will recalculate", cdr.CGRID, utils.SESSION_MANAGER_SOURCE, cdr.RunID)) qryCC, err = self.getCostFromRater(cdr) } - if cdr.Usage == 0 { - cdr.Usage = time.Duration(usage) - } - } else { qryCC, err = self.getCostFromRater(cdr) } if err != nil { - return err + return nil, err } else if qryCC != nil { cdr.Cost = qryCC.Cost cdr.CostDetails = qryCC } - return nil + return []*CDR{cdr}, nil } // Retrive the cost from engine diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index f6a032188..84503d7a1 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -218,7 +218,7 @@ func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string) (* } } index = mgo.Index{ - Key: []string{CGRIDLow, RunIDLow}, + Key: []string{CGRIDLow, RunIDLow, OriginIDLow}, Unique: true, DropDups: false, Background: false, diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 365211043..07b878a37 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -25,7 +25,6 @@ import ( "fmt" "io/ioutil" "path" - "strconv" "strings" "time" @@ -615,6 +614,8 @@ func (self *SQLStorage) GetSMCosts(cgrid, runid, originHost, originIDPrefix stri smc := &SMCost{ CGRID: result.Cgrid, RunID: result.RunID, + OriginHost: result.OriginHost, + OriginID: result.OriginID, CostSource: result.CostSource, Usage: result.Usage, CostDetails: &CallCost{}, @@ -966,8 +967,8 @@ func (self *SQLStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, return nil, 0, fmt.Errorf("JSON unmarshal callcost error for cgrid: %s, runid: %v, error: %s", result.Cgrid, result.RunID, err.Error()) } } - usageDur, _ := time.ParseDuration(strconv.FormatFloat(result.Usage, 'f', -1, 64) + "s") - pddDur, _ := time.ParseDuration(strconv.FormatFloat(result.Pdd, 'f', -1, 64) + "s") + usageDur := time.Duration(result.Usage * utils.NANO_MULTIPLIER) + pddDur := time.Duration(result.Pdd * utils.NANO_MULTIPLIER) storCdr := &CDR{ CGRID: result.Cgrid, RunID: result.RunID, diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 84a7ec7dd..fa623536b 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -80,7 +80,6 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.Duration, error) { requestedDuration := dur self.totalUsage += lastUsed // Should reflect the total usage so far - //utils.Logger.Debug(fmt.Sprintf("ExtraDuration: %d", self.extraDuration)) if lastUsed > 0 { self.extraDuration = self.lastUsage - lastUsed @@ -221,7 +220,8 @@ func (self *SMGSession) disconnectSession(reason string) error { } // Merge the sum of costs and sends it to CDRS for storage -func (self *SMGSession) saveOperations() error { +// originID could have been changed from original event, hence passing as argument here +func (self *SMGSession) saveOperations(originID string) error { if len(self.callCosts) == 0 { return nil // There are no costs to save, ignore the operation } @@ -243,7 +243,8 @@ func (self *SMGSession) saveOperations() error { CostSource: utils.SESSION_MANAGER_SOURCE, RunID: self.runId, OriginHost: self.eventStart.GetOriginatorIP(utils.META_DEFAULT), - OriginID: self.eventStart.GetUUID(), + OriginID: originID, + Usage: self.TotalUsage().Seconds(), CostDetails: firstCC, } if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply); err != nil { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index a0df22aab..5f7359b1a 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -143,7 +143,7 @@ func (self *SMGeneric) sessionEnd(sessionId string, usage time.Duration) error { if err := s.close(aTime.Add(usage)); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not close session: %s, runId: %s, error: %s", sessionId, s.runId, err.Error())) } - if err := s.saveOperations(); err != nil { + if err := s.saveOperations(sessionId); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not save session: %s, runId: %s, error: %s", sessionId, s.runId, err.Error())) } } @@ -228,8 +228,7 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim } return nilDuration, err } - evUuid := gev.GetUUID() - for _, s := range self.getSession(evUuid) { + for _, s := range self.getSession(gev.GetUUID()) { if maxDur, err := s.debit(evMaxUsage, evLastUsed); err != nil { return nilDuration, err } else if maxDur < evMaxUsage { @@ -254,15 +253,18 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update err = self.sessionStart(gev, getClientConnId(clnt)) } - if err != nil { + if err != nil && err != utils.ErrMandatoryIeMissing { return err } } + sessionIDs := []string{gev.GetUUID()} + if sessionIDPrefix, err := gev.GetFieldAsString(utils.OriginIDPrefix); err == nil { // OriginIDPrefix is present, OriginID will not be anymore considered + sessionIDs = self.getSessionIDsForPrefix(sessionIDPrefix) + } usage, err := gev.GetUsage(utils.META_DEFAULT) if err != nil { if err != utils.ErrNotFound { return err - } lastUsed, err := gev.GetLastUsed(utils.META_DEFAULT) if err != nil { @@ -272,18 +274,19 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { return err } var s *SMGSession - for _, s = range self.getSession(gev.GetUUID()) { - break + for _, sID := range sessionIDs { + for _, s = range self.getSession(sID) { + break + } + if s != nil { + break + } } if s == nil { return nil } usage = s.TotalUsage() + lastUsed } - sessionIDs := []string{gev.GetUUID()} - if sessionIDPrefix, err := gev.GetFieldAsString(utils.OriginIDPrefix); err == nil { // OriginIDPrefix is present, OriginID will not be anymore considered - sessionIDs = self.getSessionIDsForPrefix(sessionIDPrefix) - } var interimError error for _, sessionID := range sessionIDs { if err := self.sessionEnd(sessionID, usage); err != nil { @@ -377,7 +380,6 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu utils.Logger.Err(fmt.Sprintf(" ERROR failed to refund rounding: %v", err)) } } - var reply string smCost := &engine.SMCost{ CGRID: gev.GetCgrId(self.timezone), From 2eb177799a2a9e68206665a6823bc2584baa84db Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 29 Mar 2016 20:07:03 +0200 Subject: [PATCH 090/227] CDRS - correct Usage from sm_costs only when CDR usage is 0 --- engine/cdrs.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/cdrs.go b/engine/cdrs.go index b89e70637..51f1abb24 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -365,7 +365,9 @@ func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { for _, smCost := range smCosts { cdrClone := cdr.Clone() cdrClone.OriginID = smCost.OriginID - cdrClone.Usage = time.Duration(smCost.Usage * utils.NANO_MULTIPLIER) // Usage is float as seconds, convert back to duration + if cdr.Usage == 0 { + cdrClone.Usage = time.Duration(smCost.Usage * utils.NANO_MULTIPLIER) // Usage is float as seconds, convert back to duration + } cdrClone.Cost = smCost.CostDetails.Cost cdrClone.CostDetails = smCost.CostDetails cdrsRated = append(cdrsRated, cdrClone) From 10054d353a2f6a66216aa04560c946ec0f8c4e26 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 22:38:03 +0300 Subject: [PATCH 091/227] attempt to fix concurrent map panic --- cdrc/cdrc_test.go | 4 +- cdrc/csv.go | 10 +++- cdrc/fwv.go | 15 ++++-- cdre/cdrexporter.go | 6 ++- cmd/cgr-tester/cgr-tester.go | 1 - engine/cdr_local_test.go | 7 ++- engine/cdrs.go | 2 +- engine/guardian.go | 4 -- engine/pubsub.go | 9 +++- engine/pubsub_test.go | 40 +------------- engine/storage_map.go | 102 +++++++++++++++++++++++++++++++++++ sessionmanager/sessions.go | 2 +- sessionmanager/smgeneric.go | 2 +- utils/httpclient.go | 9 +--- 14 files changed, 147 insertions(+), 66 deletions(-) diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index ac515b468..e79af9382 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -156,7 +156,7 @@ BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|14 }} cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls", cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), - guard: engine.NewGuardianLock()} + guard: engine.Guardian} cdrsContent := bytes.NewReader([]byte(flatstoreCdrs)) csvReader := csv.NewReader(cdrsContent) csvReader.Comma = '|' @@ -283,7 +283,7 @@ INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Bu }} cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls", cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), - guard: engine.NewGuardianLock()} + guard: engine.Guardian} cdrsContent := bytes.NewReader([]byte(flatstoreCdrs)) csvReader := csv.NewReader(cdrsContent) csvReader.Comma = '|' diff --git a/cdrc/csv.go b/cdrc/csv.go index c6daaaee0..740ff14ac 100644 --- a/cdrc/csv.go +++ b/cdrc/csv.go @@ -20,6 +20,7 @@ package cdrc import ( "encoding/csv" + "encoding/json" "errors" "fmt" "os" @@ -93,7 +94,7 @@ func pairToRecord(part1, part2 *PartialFlatstoreRecord) ([]string, error) { func NewPartialRecordsCache(ttl time.Duration, cdrOutDir string, csvSep rune) (*PartialRecordsCache, error) { return &PartialRecordsCache{ttl: ttl, cdrOutDir: cdrOutDir, csvSep: csvSep, - partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), guard: engine.NewGuardianLock()}, nil + partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), guard: engine.Guardian}, nil } type PartialRecordsCache struct { @@ -323,7 +324,12 @@ func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcId strin for _, rsrFld := range httpFieldCfg.Value { httpAddr += rsrFld.ParseValue("") } - if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory { + var jsn []byte + jsn, err = json.Marshal(storedCdr) + if err != nil { + return nil, err + } + if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { return nil, err } else { fieldVal = string(outValByte) diff --git a/cdrc/fwv.go b/cdrc/fwv.go index b1fffe60d..7940d268d 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -20,16 +20,18 @@ package cdrc import ( "bufio" + "encoding/json" "fmt" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" "io" "net/http" "os" "strconv" "strings" "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) func fwvValue(cdrLine string, indexStart, width int, padding string) string { @@ -214,7 +216,12 @@ func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) for _, rsrFld := range httpFieldCfg.Value { httpAddr += rsrFld.ParseValue("") } - if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory { + var jsn []byte + jsn, err = json.Marshal(storedCdr) + if err != nil { + return nil, err + } + if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { return nil, err } else { fieldVal = string(outValByte) diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index e90b88e03..c269385e1 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -352,9 +352,13 @@ func (cdre *CdrExporter) processCdr(cdr *engine.CDR) error { case utils.META_HTTP_POST: var outValByte []byte httpAddr := cfgFld.Value.Id() + jsn, err := json.Marshal(cdr) + if err != nil { + return err + } if len(httpAddr) == 0 { err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type) - } else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil { + } else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, jsn); err == nil { outVal = string(outValByte) if len(outVal) == 0 && cfgFld.Mandatory { err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag) diff --git a/cmd/cgr-tester/cgr-tester.go b/cmd/cgr-tester/cgr-tester.go index 1fbdc9c66..e4597655d 100644 --- a/cmd/cgr-tester/cgr-tester.go +++ b/cmd/cgr-tester/cgr-tester.go @@ -153,7 +153,6 @@ func durRemoteRater(cd *engine.CallDescriptor) (time.Duration, error) { func main() { flag.Parse() - runtime.GOMAXPROCS(runtime.NumCPU() - 1) if *cpuprofile != "" { f, err := os.Create(*cpuprofile) diff --git a/engine/cdr_local_test.go b/engine/cdr_local_test.go index 823e8f6d5..edf7c8880 100644 --- a/engine/cdr_local_test.go +++ b/engine/cdr_local_test.go @@ -19,10 +19,12 @@ along with this program. If not, see package engine import ( + "encoding/json" "flag" - "github.com/cgrates/cgrates/utils" "testing" "time" + + "github.com/cgrates/cgrates/utils" ) // Arguments received via test command @@ -42,7 +44,8 @@ func TestHttpJsonPost(t *testing.T) { RunID: utils.DEFAULT_RUNID, Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - if _, err := utils.HttpJsonPost("http://localhost:8000", false, cdrOut); err == nil { + jsn, _ := json.Marshal(cdrOut) + if _, err := utils.HttpJsonPost("http://localhost:8000", false, jsn); err == nil { t.Error(err) } } diff --git a/engine/cdrs.go b/engine/cdrs.go index 122bd5fbe..dd164fe03 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -70,7 +70,7 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater Connector, pubsub rpcclient.RpcClientConnection, users UserService, aliases AliasService, stats StatsInterface) (*CdrServer, error) { - return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, rater: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: &GuardianLock{locksMap: make(map[string]chan bool)}}, nil + return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, rater: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: Guardian}, nil } type CdrServer struct { diff --git a/engine/guardian.go b/engine/guardian.go index dc2975aff..6d6664e39 100644 --- a/engine/guardian.go +++ b/engine/guardian.go @@ -26,10 +26,6 @@ import ( // global package variable var Guardian = &GuardianLock{locksMap: make(map[string]chan bool)} -func NewGuardianLock() *GuardianLock { - return &GuardianLock{locksMap: make(map[string]chan bool)} -} - type GuardianLock struct { locksMap map[string]chan bool mu sync.RWMutex diff --git a/engine/pubsub.go b/engine/pubsub.go index 0d4dd17c1..6d9bcd1ea 100644 --- a/engine/pubsub.go +++ b/engine/pubsub.go @@ -1,6 +1,7 @@ package engine import ( + "encoding/json" "errors" "fmt" "sync" @@ -43,7 +44,7 @@ type SubscriberData struct { type PubSub struct { subscribers map[string]*SubscriberData ttlVerify bool - pubFunc func(string, bool, interface{}) ([]byte, error) + pubFunc func(string, bool, []byte) ([]byte, error) mux *sync.Mutex accountDb AccountingStorage } @@ -140,12 +141,16 @@ func (ps *PubSub) Publish(evt CgrEvent, reply *string) error { transport := split[0] address := split[1] ttlVerify := ps.ttlVerify + jsn, err := json.Marshal(evt) + if err != nil { + return err + } switch transport { case utils.META_HTTP_POST: go func() { delay := utils.Fib() for i := 0; i < 5; i++ { // Loop so we can increase the success rate on best effort - if _, err := ps.pubFunc(address, ttlVerify, evt); err == nil { + if _, err := ps.pubFunc(address, ttlVerify, jsn); err == nil { break // Success, no need to reinterate } else if i == 4 { // Last iteration, syslog the warning utils.Logger.Warning(fmt.Sprintf(" Failed calling url: [%s], error: [%s], event type: %s", address, err.Error(), evt["EventName"])) diff --git a/engine/pubsub_test.go b/engine/pubsub_test.go index 5345a06c0..ac28837a8 100644 --- a/engine/pubsub_test.go +++ b/engine/pubsub_test.go @@ -116,43 +116,9 @@ func TestUnsubscribeSave(t *testing.T) { } } -func TestPublish(t *testing.T) { - ps := NewPubSub(accountingStorage, true) - ps.pubFunc = func(url string, ttl bool, obj interface{}) ([]byte, error) { - obj.(CgrEvent)["called"] = url - return nil, nil - } - var r string - if err := ps.Subscribe(SubscribeInfo{ - EventFilter: "EventName/test", - Transport: utils.META_HTTP_POST, - Address: "url", - LifeSpan: time.Second, - }, &r); err != nil { - t.Error("Error subscribing: ", err) - } - m := make(map[string]string) - m["EventFilter"] = "test" - if err := ps.Publish(m, &r); err != nil { - t.Error("Error publishing: ", err) - } - for i := 0; i < 1000; i++ { // wait for the theread to populate map - if len(m) == 2 { - time.Sleep(time.Microsecond) - } else { - break - } - } - if r, exists := m["called"]; !exists || r != "url" { - t.Error("Error calling publish function: ", m) - } -} - func TestPublishExpired(t *testing.T) { ps := NewPubSub(accountingStorage, true) - ps.pubFunc = func(url string, ttl bool, obj interface{}) ([]byte, error) { - m := obj.(map[string]string) - m["called"] = "yes" + ps.pubFunc = func(url string, ttl bool, obj []byte) ([]byte, error) { return nil, nil } var r string @@ -174,9 +140,7 @@ func TestPublishExpired(t *testing.T) { func TestPublishExpiredSave(t *testing.T) { ps := NewPubSub(accountingStorage, true) - ps.pubFunc = func(url string, ttl bool, obj interface{}) ([]byte, error) { - m := obj.(map[string]string) - m["called"] = "yes" + ps.pubFunc = func(url string, ttl bool, obj []byte) ([]byte, error) { return nil, nil } var r string diff --git a/engine/storage_map.go b/engine/storage_map.go index d87fdcddd..f7e21306a 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io/ioutil" + "sync" "strings" "time" @@ -36,6 +37,7 @@ type MapStorage struct { dict map[string][]byte tasks [][]byte ms Marshaler + mu sync.RWMutex } func NewMapStorage() (*MapStorage, error) { @@ -49,11 +51,15 @@ func NewMapStorageJson() (*MapStorage, error) { func (ms *MapStorage) Close() {} func (ms *MapStorage) Flush(ignore string) error { + ms.mu.Lock() + defer ms.mu.Unlock() ms.dict = make(map[string][]byte) return nil } func (ms *MapStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]string, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if skipCache { keysForPrefix := make([]string, 0) for key := range ms.dict { @@ -256,6 +262,8 @@ func (ms *MapStorage) cacheAccounting(alsKeys []string) error { // Used to check if specific subject is stored using prefix key attached to entity func (ms *MapStorage) HasData(categ, subject string) (bool, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() switch categ { case utils.DESTINATION_PREFIX, utils.RATING_PLAN_PREFIX, utils.RATING_PROFILE_PREFIX, utils.ACTION_PREFIX, utils.ACTION_PLAN_PREFIX, utils.ACCOUNT_PREFIX, utils.DERIVEDCHARGERS_PREFIX: _, exists := ms.dict[categ+subject] @@ -265,6 +273,8 @@ func (ms *MapStorage) HasData(categ, subject string) (bool, error) { } func (ms *MapStorage) GetRatingPlan(key string, skipCache bool) (rp *RatingPlan, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.RATING_PLAN_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -294,6 +304,8 @@ func (ms *MapStorage) GetRatingPlan(key string, skipCache bool) (rp *RatingPlan, } func (ms *MapStorage) SetRatingPlan(rp *RatingPlan) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(rp) var b bytes.Buffer w := zlib.NewWriter(&b) @@ -308,6 +320,8 @@ func (ms *MapStorage) SetRatingPlan(rp *RatingPlan) (err error) { } func (ms *MapStorage) GetRatingProfile(key string, skipCache bool) (rpf *RatingProfile, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.RATING_PROFILE_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -328,6 +342,8 @@ func (ms *MapStorage) GetRatingProfile(key string, skipCache bool) (rpf *RatingP } func (ms *MapStorage) SetRatingProfile(rpf *RatingProfile) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(rpf) ms.dict[utils.RATING_PROFILE_PREFIX+rpf.Id] = result response := 0 @@ -338,6 +354,8 @@ func (ms *MapStorage) SetRatingProfile(rpf *RatingProfile) (err error) { } func (ms *MapStorage) RemoveRatingProfile(key string) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() for k := range ms.dict { if strings.HasPrefix(k, key) { delete(ms.dict, key) @@ -353,6 +371,8 @@ func (ms *MapStorage) RemoveRatingProfile(key string) (err error) { } func (ms *MapStorage) GetLCR(key string, skipCache bool) (lcr *LCR, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.LCR_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -371,12 +391,16 @@ func (ms *MapStorage) GetLCR(key string, skipCache bool) (lcr *LCR, err error) { } func (ms *MapStorage) SetLCR(lcr *LCR) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(lcr) ms.dict[utils.LCR_PREFIX+lcr.GetId()] = result return } func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.DESTINATION_PREFIX + key if values, ok := ms.dict[key]; ok { b := bytes.NewBuffer(values) @@ -402,6 +426,8 @@ func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error) } func (ms *MapStorage) SetDestination(dest *Destination) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(dest) var b bytes.Buffer w := zlib.NewWriter(&b) @@ -416,6 +442,8 @@ func (ms *MapStorage) SetDestination(dest *Destination) (err error) { } func (ms *MapStorage) GetActions(key string, skipCache bool) (as Actions, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.ACTION_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -434,12 +462,16 @@ func (ms *MapStorage) GetActions(key string, skipCache bool) (as Actions, err er } func (ms *MapStorage) SetActions(key string, as Actions) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(&as) ms.dict[utils.ACTION_PREFIX+key] = result return } func (ms *MapStorage) GetSharedGroup(key string, skipCache bool) (sg *SharedGroup, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.SHARED_GROUP_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -460,12 +492,16 @@ func (ms *MapStorage) GetSharedGroup(key string, skipCache bool) (sg *SharedGrou } func (ms *MapStorage) SetSharedGroup(sg *SharedGroup) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(sg) ms.dict[utils.SHARED_GROUP_PREFIX+sg.Id] = result return } func (ms *MapStorage) GetAccount(key string) (ub *Account, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if values, ok := ms.dict[utils.ACCOUNT_PREFIX+key]; ok { ub = &Account{ID: key} err = ms.ms.Unmarshal(values, ub) @@ -488,17 +524,23 @@ func (ms *MapStorage) SetAccount(ub *Account) (err error) { ub = ac } } + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(ub) ms.dict[utils.ACCOUNT_PREFIX+ub.ID] = result return } func (ms *MapStorage) RemoveAccount(key string) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() delete(ms.dict, utils.ACCOUNT_PREFIX+key) return } func (ms *MapStorage) GetCdrStatsQueue(key string) (sq *StatsQueue, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if values, ok := ms.dict[utils.CDR_STATS_QUEUE_PREFIX+key]; ok { sq = &StatsQueue{} err = ms.ms.Unmarshal(values, sq) @@ -509,12 +551,16 @@ func (ms *MapStorage) GetCdrStatsQueue(key string) (sq *StatsQueue, err error) { } func (ms *MapStorage) SetCdrStatsQueue(sq *StatsQueue) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(sq) ms.dict[utils.CDR_STATS_QUEUE_PREFIX+sq.GetId()] = result return } func (ms *MapStorage) GetSubscribers() (result map[string]*SubscriberData, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() result = make(map[string]*SubscriberData) for key, value := range ms.dict { if strings.HasPrefix(key, utils.PUBSUB_SUBSCRIBERS_PREFIX) { @@ -527,17 +573,23 @@ func (ms *MapStorage) GetSubscribers() (result map[string]*SubscriberData, err e return } func (ms *MapStorage) SetSubscriber(key string, sub *SubscriberData) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(sub) ms.dict[utils.PUBSUB_SUBSCRIBERS_PREFIX+key] = result return } func (ms *MapStorage) RemoveSubscriber(key string) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() delete(ms.dict, utils.PUBSUB_SUBSCRIBERS_PREFIX+key) return } func (ms *MapStorage) SetUser(up *UserProfile) error { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(up) if err != nil { return err @@ -546,6 +598,8 @@ func (ms *MapStorage) SetUser(up *UserProfile) error { return nil } func (ms *MapStorage) GetUser(key string) (up *UserProfile, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() up = &UserProfile{} if values, ok := ms.dict[utils.USERS_PREFIX+key]; ok { err = ms.ms.Unmarshal(values, &up) @@ -556,6 +610,8 @@ func (ms *MapStorage) GetUser(key string) (up *UserProfile, err error) { } func (ms *MapStorage) GetUsers() (result []*UserProfile, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() for key, value := range ms.dict { if strings.HasPrefix(key, utils.USERS_PREFIX) { up := &UserProfile{} @@ -568,11 +624,15 @@ func (ms *MapStorage) GetUsers() (result []*UserProfile, err error) { } func (ms *MapStorage) RemoveUser(key string) error { + ms.mu.Lock() + defer ms.mu.Unlock() delete(ms.dict, utils.USERS_PREFIX+key) return nil } func (ms *MapStorage) SetAlias(al *Alias) error { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(al.Values) if err != nil { return err @@ -582,6 +642,8 @@ func (ms *MapStorage) SetAlias(al *Alias) error { } func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.ALIASES_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -607,6 +669,8 @@ func (ms *MapStorage) GetAlias(key string, skipCache bool) (al *Alias, err error } func (ms *MapStorage) RemoveAlias(key string) error { + ms.mu.Lock() + defer ms.mu.Unlock() al := &Alias{} al.SetId(key) key = utils.ALIASES_PREFIX + key @@ -622,14 +686,20 @@ func (ms *MapStorage) RemoveAlias(key string) error { } func (ms *MapStorage) GetLoadHistory(limitItems int, skipCache bool) ([]*LoadInstance, error) { + ms.mu.RLock() + defer ms.mu.RUnlock() return nil, nil } func (ms *MapStorage) AddLoadHistory(*LoadInstance, int) error { + ms.mu.Lock() + defer ms.mu.Unlock() return nil } func (ms *MapStorage) GetActionTriggers(key string) (atrs ActionTriggers, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if values, ok := ms.dict[utils.ACTION_TRIGGER_PREFIX+key]; ok { err = ms.ms.Unmarshal(values, &atrs) } else { @@ -639,6 +709,8 @@ func (ms *MapStorage) GetActionTriggers(key string) (atrs ActionTriggers, err er } func (ms *MapStorage) SetActionTriggers(key string, atrs ActionTriggers) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() if len(atrs) == 0 { // delete the key delete(ms.dict, utils.ACTION_TRIGGER_PREFIX+key) @@ -650,6 +722,8 @@ func (ms *MapStorage) SetActionTriggers(key string, atrs ActionTriggers) (err er } func (ms *MapStorage) GetActionPlan(key string, skipCache bool) (ats *ActionPlan, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.ACTION_PLAN_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -669,6 +743,8 @@ func (ms *MapStorage) GetActionPlan(key string, skipCache bool) (ats *ActionPlan func (ms *MapStorage) SetActionPlan(key string, ats *ActionPlan, overwrite bool) (err error) { if len(ats.ActionTimings) == 0 { + ms.mu.Lock() + defer ms.mu.Unlock() // delete the key delete(ms.dict, utils.ACTION_PLAN_PREFIX+key) cache2go.RemKey(utils.ACTION_PLAN_PREFIX + key) @@ -685,12 +761,16 @@ func (ms *MapStorage) SetActionPlan(key string, ats *ActionPlan, overwrite bool) } } } + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(&ats) ms.dict[utils.ACTION_PLAN_PREFIX+key] = result return } func (ms *MapStorage) GetAllActionPlans() (ats map[string]*ActionPlan, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() apls, err := cache2go.GetAllEntries(utils.ACTION_PLAN_PREFIX) if err != nil { return nil, err @@ -706,6 +786,8 @@ func (ms *MapStorage) GetAllActionPlans() (ats map[string]*ActionPlan, err error } func (ms *MapStorage) PushTask(t *Task) error { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(t) if err != nil { return err @@ -715,6 +797,8 @@ func (ms *MapStorage) PushTask(t *Task) error { } func (ms *MapStorage) PopTask() (t *Task, err error) { + ms.mu.Lock() + defer ms.mu.Unlock() if len(ms.tasks) > 0 { var values []byte values, ms.tasks = ms.tasks[0], ms.tasks[1:] @@ -727,6 +811,8 @@ func (ms *MapStorage) PopTask() (t *Task, err error) { } func (ms *MapStorage) GetDerivedChargers(key string, skipCache bool) (dcs *utils.DerivedChargers, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() key = utils.DERIVEDCHARGERS_PREFIX + key if !skipCache { if x, err := cache2go.Get(key); err == nil { @@ -745,6 +831,8 @@ func (ms *MapStorage) GetDerivedChargers(key string, skipCache bool) (dcs *utils } func (ms *MapStorage) SetDerivedChargers(key string, dcs *utils.DerivedChargers) error { + ms.mu.Lock() + defer ms.mu.Unlock() if dcs == nil || len(dcs.Chargers) == 0 { delete(ms.dict, utils.DERIVEDCHARGERS_PREFIX+key) cache2go.RemKey(utils.DERIVEDCHARGERS_PREFIX + key) @@ -756,12 +844,16 @@ func (ms *MapStorage) SetDerivedChargers(key string, dcs *utils.DerivedChargers) } func (ms *MapStorage) SetCdrStats(cs *CdrStats) error { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(cs) ms.dict[utils.CDR_STATS_PREFIX+cs.Id] = result return err } func (ms *MapStorage) GetCdrStats(key string) (cs *CdrStats, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if values, ok := ms.dict[utils.CDR_STATS_PREFIX+key]; ok { err = ms.ms.Unmarshal(values, &cs) } else { @@ -771,6 +863,8 @@ func (ms *MapStorage) GetCdrStats(key string) (cs *CdrStats, err error) { } func (ms *MapStorage) GetAllCdrStats() (css []*CdrStats, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() for key, value := range ms.dict { if !strings.HasPrefix(key, utils.CDR_STATS_PREFIX) { continue @@ -783,12 +877,16 @@ func (ms *MapStorage) GetAllCdrStats() (css []*CdrStats, err error) { } func (ms *MapStorage) SetSMCost(smCost *SMCost) error { + ms.mu.Lock() + defer ms.mu.Unlock() result, err := ms.ms.Marshal(smCost) ms.dict[utils.LOG_CALL_COST_PREFIX+smCost.CostSource+smCost.RunID+"_"+smCost.CGRID] = result return err } func (ms *MapStorage) GetSMCost(cgrid, source, runid, originHost, originID string) (smCost *SMCost, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() if values, ok := ms.dict[utils.LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid]; ok { err = ms.ms.Unmarshal(values, &smCost) } else { @@ -798,6 +896,8 @@ func (ms *MapStorage) GetSMCost(cgrid, source, runid, originHost, originID strin } func (ms *MapStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() mat, err := ms.ms.Marshal(at) if err != nil { return @@ -811,6 +911,8 @@ func (ms *MapStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, a } func (ms *MapStorage) LogActionTiming(source string, at *ActionTiming, as Actions) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() mat, err := ms.ms.Marshal(at) if err != nil { return diff --git a/sessionmanager/sessions.go b/sessionmanager/sessions.go index 65e4a25c1..690a9127a 100644 --- a/sessionmanager/sessions.go +++ b/sessionmanager/sessions.go @@ -28,7 +28,7 @@ import ( func NewSessions() *Sessions { return &Sessions{ sessionsMux: new(sync.Mutex), - guard: engine.NewGuardianLock(), + guard: engine.Guardian, } } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 5f7359b1a..d8f1bd25f 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -35,7 +35,7 @@ var ErrPartiallyExecuted = errors.New("Partially executed") func NewSMGeneric(cgrCfg *config.CGRConfig, rater engine.Connector, cdrsrv engine.Connector, timezone string, extconns *SMGExternalConnections) *SMGeneric { gsm := &SMGeneric{cgrCfg: cgrCfg, rater: rater, cdrsrv: cdrsrv, extconns: extconns, timezone: timezone, - sessions: make(map[string][]*SMGSession), sessionsMux: new(sync.Mutex), guard: engine.NewGuardianLock()} + sessions: make(map[string][]*SMGSession), sessionsMux: new(sync.Mutex), guard: engine.Guardian} return gsm } diff --git a/utils/httpclient.go b/utils/httpclient.go index 0fad55afb..169067914 100644 --- a/utils/httpclient.go +++ b/utils/httpclient.go @@ -22,7 +22,6 @@ import ( "bytes" "crypto/tls" "encoding/gob" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -49,16 +48,12 @@ func GetBytes(content interface{}) ([]byte, error) { } // Post without automatic failover -func HttpJsonPost(url string, skipTlsVerify bool, content interface{}) ([]byte, error) { - body, err := json.Marshal(content) - if err != nil { - return nil, err - } +func HttpJsonPost(url string, skipTlsVerify bool, content []byte) ([]byte, error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTlsVerify}, } client := &http.Client{Transport: tr} - resp, err := client.Post(url, "application/json", bytes.NewBuffer(body)) + resp, err := client.Post(url, "application/json", bytes.NewBuffer(content)) if err != nil { return nil, err } From 97c96dd46d23fa60d72c3fe0a87f87f01bf33dc1 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 29 Mar 2016 23:34:48 +0300 Subject: [PATCH 092/227] better account disabled error reporting --- engine/calldesc.go | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/engine/calldesc.go b/engine/calldesc.go index d5f3c155f..9ab8a9373 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -188,7 +188,11 @@ func (cd *CallDescriptor) getAccount() (ub *Account, err error) { cd.account, err = accountingStorage.GetAccount(cd.GetAccountKey()) } if cd.account != nil && cd.account.Disabled { - return nil, fmt.Errorf("User %s is disabled", cd.account.ID) + return nil, utils.ErrAccountDisabled + } + if err != nil || cd.account == nil { + utils.Logger.Warning(fmt.Sprintf("Account: %s, not found (%v)", cd.GetAccountKey(), err)) + return nil, utils.ErrAccountNotFound } return cd.account, err } @@ -635,13 +639,9 @@ func (origCD *CallDescriptor) getMaxSessionDuration(origAcc *Account) (time.Dura func (cd *CallDescriptor) GetMaxSessionDuration() (duration time.Duration, err error) { cd.account = nil // make sure it's not cached - if account, err := cd.getAccount(); err != nil || account == nil { - utils.Logger.Err(fmt.Sprintf("Account: %s, not found", cd.GetAccountKey())) - return 0, utils.ErrAccountNotFound + if account, err := cd.getAccount(); err != nil { + return 0, err } else { - if account.Disabled { - return 0, utils.ErrAccountDisabled - } if memberIds, err := account.GetUniqueSharedGroupMembers(cd); err == nil { if _, err := Guardian.Guard(func() (interface{}, error) { duration, err = cd.getMaxSessionDuration(account) @@ -705,13 +705,9 @@ func (cd *CallDescriptor) debit(account *Account, dryRun bool, goNegative bool) func (cd *CallDescriptor) Debit() (cc *CallCost, err error) { cd.account = nil // make sure it's not cached // lock all group members - if account, err := cd.getAccount(); err != nil || account == nil { - utils.Logger.Err(fmt.Sprintf("Account: %s, not found", cd.GetAccountKey())) - return nil, utils.ErrAccountNotFound + if account, err := cd.getAccount(); err != nil { + return nil, err } else { - if account.Disabled { - return nil, utils.ErrAccountDisabled - } if memberIds, sgerr := account.GetUniqueSharedGroupMembers(cd); sgerr == nil { _, err = Guardian.Guard(func() (interface{}, error) { cc, err = cd.debit(account, cd.DryRun, true) @@ -730,13 +726,9 @@ func (cd *CallDescriptor) Debit() (cc *CallCost, err error) { // by the GetMaxSessionDuration method. The amount filed has to be filled in call descriptor. func (cd *CallDescriptor) MaxDebit() (cc *CallCost, err error) { cd.account = nil // make sure it's not cached - if account, err := cd.getAccount(); err != nil || account == nil { - utils.Logger.Err(fmt.Sprintf("Account: %s, not found", cd.GetAccountKey())) - return nil, utils.ErrAccountNotFound + if account, err := cd.getAccount(); err != nil { + return nil, err } else { - if account.Disabled { - return nil, utils.ErrAccountDisabled - } //log.Printf("ACC: %+v", account) if memberIDs, err := account.GetUniqueSharedGroupMembers(cd); err == nil { _, err = Guardian.Guard(func() (interface{}, error) { From 2a7087ba5432cb7e5626b49d618da90517ea372e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 30 Mar 2016 00:05:43 +0300 Subject: [PATCH 093/227] empty blocker balance considered --- engine/account.go | 2 +- engine/calldesc_test.go | 53 ++++++++++++++++++++++++++++++++++++++- engine/loader_csv_test.go | 12 ++++++--- engine/storage_test.go | 4 +-- 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/engine/account.go b/engine/account.go index e231875a2..45497e32f 100644 --- a/engine/account.go +++ b/engine/account.go @@ -287,7 +287,7 @@ func (ub *Account) getBalancesForPrefix(prefix, category, direction, tor string, if b.Disabled { continue } - if b.IsExpired() || (len(b.SharedGroups) == 0 && b.GetValue() <= 0) { + if b.IsExpired() || (len(b.SharedGroups) == 0 && b.GetValue() <= 0 && !b.Blocker) { continue } if sharedGroup != "" && b.SharedGroups[sharedGroup] == false { diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index 540bc8dad..ddde0471f 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -566,7 +566,7 @@ func TestGetMaxSessiontWithBlocker(t *testing.T) { MaxCostSoFar: 0, } result, err := cd.GetMaxSessionDuration() - expected := 30 * time.Minute + expected := 17 * time.Minute if result != expected || err != nil { t.Errorf("Expected %v was %v (%v)", expected, result, err) } @@ -588,6 +588,57 @@ func TestGetMaxSessiontWithBlocker(t *testing.T) { } } +func TestGetMaxSessiontWithBlockerEmpty(t *testing.T) { + ap, _ := ratingStorage.GetActionPlan("BLOCK_EMPTY_AT", false) + for _, at := range ap.ActionTimings { + at.accountIDs = ap.AccountIDs + at.Execute() + } + acc, err := accountingStorage.GetAccount("cgrates.org:block_empty") + if err != nil { + t.Error("error getting account: ", err) + } + if len(acc.BalanceMap[utils.MONETARY]) != 2 || + acc.BalanceMap[utils.MONETARY][0].Blocker != true { + for _, b := range acc.BalanceMap[utils.MONETARY] { + t.Logf("B: %+v", b) + } + t.Error("Error executing action plan on account: ", acc.BalanceMap[utils.MONETARY]) + } + cd := &CallDescriptor{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "block", + Account: "block_empty", + Destination: "0723", + TimeStart: time.Date(2016, 1, 13, 14, 0, 0, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 13, 14, 30, 0, 0, time.UTC), + MaxCostSoFar: 0, + } + result, err := cd.GetMaxSessionDuration() + expected := 0 * time.Minute + if result != expected || err != nil { + t.Errorf("Expected %v was %v (%v)", expected, result, err) + } + cd = &CallDescriptor{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "block", + Account: "block_empty", + Destination: "444", + TimeStart: time.Date(2016, 1, 13, 14, 0, 0, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 13, 14, 30, 0, 0, time.UTC), + MaxCostSoFar: 0, + } + result, err = cd.GetMaxSessionDuration() + expected = 30 * time.Minute + if result != expected || err != nil { + t.Errorf("Expected %v was %v (%v)", expected, result, err) + } +} + func TestGetCostWithMaxCost(t *testing.T) { ap, _ := ratingStorage.GetActionPlan("TOPUP10_AT", false) for _, at := range ap.ActionTimings { diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 7c17f02fe..357f56fb8 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -172,8 +172,10 @@ EE0,*topup_reset,,,,*monetary,*out,,,,SG3,*unlimited,,0,10,false,false,10 EE0,*allow_negative,,,,*monetary,*out,,,,,*unlimited,,0,10,false,false,10 DEFEE,*cdrlog,"{""Category"":""^ddi"",""MediationRunId"":""^did_run""}",,,,,,,,,,,,,false,false,10 NEG,*allow_negative,,,,*monetary,*out,,,,,*unlimited,,0,10,false,false,10 -BLOCK,*topup,,,bblocker,*monetary,*out,,NAT,,,*unlimited,,10,20,true,false,20 +BLOCK,*topup,,,bblocker,*monetary,*out,,NAT,,,*unlimited,,1,20,true,false,20 BLOCK,*topup,,,bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 +BLOCK_EMPTY,*topup,,,bblocker,*monetary,*out,,NAT,,,*unlimited,,0,20,true,false,20 +BLOCK_EMPTY,*topup,,,bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 FILTER,*topup,,"{""*and"":[{""Value"":{""*lt"":0}},{""Id"":{""*eq"":""*default""}}]}",bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 EXP,*topup,,,,*voice,*out,,,,,*monthly,*any,300,10,false,false,10 NOEXP,*topup,,,,*voice,*out,,,,,*unlimited,*any,50,10,false,false,10 @@ -188,6 +190,7 @@ TOPUP_SHARED10_AT,SE10,*asap,10 TOPUP_EMPTY_AT,EE0,*asap,10 POST_AT,NEG,*asap,10 BLOCK_AT,BLOCK,*asap,10 +BLOCK_EMPTY_AT,BLOCK_EMPTY,*asap,10 EXP_AT,EXP,*asap,10 ` @@ -216,6 +219,7 @@ vdf,emptyY,TOPUP_EMPTY_AT,,, vdf,post,POST_AT,,, cgrates.org,alodis,TOPUP_EMPTY_AT,,true,true cgrates.org,block,BLOCK_AT,,false,false +cgrates.org,block_empty,BLOCK_EMPTY_AT,,false,false cgrates.org,expo,EXP_AT,,false,false cgrates.org,expnoexp,,,false,false ` @@ -820,7 +824,7 @@ func TestLoadRatingProfiles(t *testing.T) { } func TestLoadActions(t *testing.T) { - if len(csvr.actions) != 13 { + if len(csvr.actions) != 14 { t.Error("Failed to load actions: ", len(csvr.actions)) } as1 := csvr.actions["MINI"] @@ -1006,7 +1010,7 @@ func TestLoadLCRs(t *testing.T) { } func TestLoadActionTimings(t *testing.T) { - if len(csvr.actionPlans) != 8 { + if len(csvr.actionPlans) != 9 { t.Error("Failed to load action timings: ", len(csvr.actionPlans)) } atm := csvr.actionPlans["MORE_MINUTES"] @@ -1101,7 +1105,7 @@ func TestLoadActionTriggers(t *testing.T) { } func TestLoadAccountActions(t *testing.T) { - if len(csvr.accountActions) != 14 { + if len(csvr.accountActions) != 15 { t.Error("Failed to load account actions: ", len(csvr.accountActions)) } aa := csvr.accountActions["vdf:minitsboy"] diff --git a/engine/storage_test.go b/engine/storage_test.go index cdb9d6802..eb007243b 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -274,7 +274,7 @@ func TestDifferentUuid(t *testing.T) { func TestStorageTask(t *testing.T) { // clean previous unused tasks - for i := 0; i < 18; i++ { + for i := 0; i < 19; i++ { ratingStorage.PopTask() } @@ -303,7 +303,7 @@ func TestStorageTask(t *testing.T) { t.Error("Error poping task: ", task, err) } if task, err := ratingStorage.PopTask(); err == nil && task != nil { - t.Errorf("Error poping task %+v, %v: ", task, err) + t.Errorf("Error poping task %+v, %v ", task, err) } } From 666de5469f48064ef8727a138a655da02b5811e1 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 30 Mar 2016 00:16:46 +0300 Subject: [PATCH 094/227] better generic make negative in actions --- engine/action.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/action.go b/engine/action.go index 6c57e8aaa..07152a58d 100644 --- a/engine/action.go +++ b/engine/action.go @@ -370,7 +370,7 @@ func resetCountersAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Ac } func genericMakeNegative(a *Action) { - if a.Balance != nil && a.Balance.GetValue() >= 0 { // only apply if not allready negative + if a.Balance != nil && a.Balance.GetValue() > 0 { // only apply if not allready negative a.Balance.SetValue(-a.Balance.GetValue()) } } From 39d40732ede0b3c6237e83f83430c0a5c24c2b79 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 30 Mar 2016 14:24:22 +0300 Subject: [PATCH 095/227] call sessionEnd explicitly on init error --- sessionmanager/smgeneric.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index d8f1bd25f..1089304a8 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -241,6 +241,7 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim // Called on session start func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { + self.sessionEnd(gev.GetUUID(), 0) return nilDuration, err } return self.SessionUpdate(gev, clnt) From 8f6d56eb13b855273e9f90d7db797e8e7e3b3ad5 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 30 Mar 2016 18:18:03 +0300 Subject: [PATCH 096/227] second try at closing session after init error --- sessionmanager/smgeneric.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 1089304a8..7f3adbc15 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -244,7 +244,11 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time self.sessionEnd(gev.GetUUID(), 0) return nilDuration, err } - return self.SessionUpdate(gev, clnt) + d, err := self.SessionUpdate(gev, clnt) + if err != nil { + self.sessionEnd(gev.GetUUID(), 0) + } + return d, err } // Called on session end, should stop debit loop From a8220e9d9ea9f9e2eabd7d45d2d161a7d2ac112d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 00:44:04 +0300 Subject: [PATCH 097/227] test for #399 --- data/tariffplans/testtp/ActionTriggers.csv | 1 + general_tests/tp_it_test.go | 119 +++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 general_tests/tp_it_test.go diff --git a/data/tariffplans/testtp/ActionTriggers.csv b/data/tariffplans/testtp/ActionTriggers.csv index 795ef6ab8..8a1c88cac 100644 --- a/data/tariffplans/testtp/ActionTriggers.csv +++ b/data/tariffplans/testtp/ActionTriggers.csv @@ -2,6 +2,7 @@ STANDARD_TRIGGERS,,*min_balance,2,false,0,,,,*monetary,*out,,,,,,,,,,,LOG_BALANCE,10 STANDARD_TRIGGERS,,*max_balance,20,false,0,,,,*monetary,*out,,,,,,,,,,,LOG_BALANCE,10 STANDARD_TRIGGERS,,*max_event_counter,15,false,0,,,,*monetary,*out,,FS_USERS,,,,,,,,,LOG_BALANCE,10 +STANDARD_TRIGGERS,,*max_balance_counter,1,false,0,,,,*monetary,*out,,,,,,,,,,,LOG_BALANCE,10 CDRST1_WARN_ASR,,*min_asr,45,true,1h,,,,,,,,,,,,,,,3,CDRST_WARN_HTTP,10 CDRST1_WARN_ACD,,*min_acd,10,true,1h,,,,,,,,,,,,,,,5,CDRST_WARN_HTTP,10 CDRST1_WARN_ACC,,*max_acc,10,true,10m,,,,,,,,,,,,,,,5,CDRST_WARN_HTTP,10 diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go new file mode 100644 index 000000000..3eef405e1 --- /dev/null +++ b/general_tests/tp_it_test.go @@ -0,0 +1,119 @@ +package general_tests + +import ( + "net/rpc" + "net/rpc/jsonrpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var tpCfgPath string +var tpCfg *config.CGRConfig +var tpRPC *rpc.Client +var tpLoadInst engine.LoadInstance // Share load information between tests + +func TestTpInitCfg(t *testing.T) { + if !*testIntegration { + return + } + tpCfgPath = path.Join(*dataDir, "conf", "samples", "tutlocal") + // Init config first + var err error + tpCfg, err = config.NewCGRConfigFromFolder(tpCfgPath) + if err != nil { + t.Error(err) + } + tpCfg.DataFolderPath = *dataDir // Share DataFolderPath through config towards StoreDb for Flush() + config.SetCgrConfig(tpCfg) +} + +// Remove data in both rating and accounting db +func TestTpResetDataDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitDataDb(tpCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func TestTpResetStorDb(t *testing.T) { + if !*testIntegration { + return + } + if err := engine.InitStorDb(tpCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func TestTpStartEngine(t *testing.T) { + if !*testIntegration { + return + } + if _, err := engine.StopStartEngine(tpCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func TestTpRpcConn(t *testing.T) { + if !*testIntegration { + return + } + var err error + tpRPC, err = jsonrpc.Dial("tcp", tpCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +// Load the tariff plan, creating accounts and their balances +func TestTpLoadTariffPlanFromFolder(t *testing.T) { + if !*testIntegration { + return + } + attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "testtp")} + if err := tpRPC.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &tpLoadInst); err != nil { + t.Error(err) + } else if tpLoadInst.LoadId == "" { + t.Error("Empty loadId received, loadInstance: ", tpLoadInst) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups +} + +func TestTpBalanceCounter(t *testing.T) { + if !*testIntegration { + return + } + tStart := time.Date(2016, 3, 31, 0, 0, 0, 0, time.UTC) + cd := engine.CallDescriptor{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "1001", + Destination: "+49", + DurationIndex: 0, + TimeStart: tStart, + TimeEnd: tStart.Add(time.Duration(20) * time.Second), + } + var cc engine.CallCost + if err := tpRPC.Call("Responder.Debit", cd, &cc); err != nil { + t.Error("Got error on Responder.GetCost: ", err.Error()) + } else if cc.GetDuration() == 20 { + t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc.GetDuration()) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } else if acnt.UnitCounters[utils.MONETARY][1].Counters[0].Value != 20.0 { + t.Errorf("Calling ApierV2.GetBalance received: %s", utils.ToIJSON(acnt)) + } +} From ac1aeb9db47270908274e656dfbc771050dbdf76 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 14:29:38 +0300 Subject: [PATCH 098/227] test for derived charges no credit --- data/tariffplans/testtp/AccountActions.csv | 1 + data/tariffplans/testtp/ActionPlans.csv | 3 +- data/tariffplans/testtp/Actions.csv | 3 +- data/tariffplans/testtp/DerivedChargers.csv | 1 + sessionmanager/data_it_test.go | 42 +++++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index 4cf06f896..d04a5bde4 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -6,3 +6,4 @@ cgrates.org,1004,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1005,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1009,TEST_EXE,,, cgrates.org,1010,TEST_DATA_r,,true, +cgrates.org,1011,TEST_VOICE,,, diff --git a/data/tariffplans/testtp/ActionPlans.csv b/data/tariffplans/testtp/ActionPlans.csv index f41668f9b..45e11a89d 100644 --- a/data/tariffplans/testtp/ActionPlans.csv +++ b/data/tariffplans/testtp/ActionPlans.csv @@ -2,4 +2,5 @@ PREPAID_10,PREPAID_10,ASAP,10 PREPAID_10,BONUS_1,ASAP,10 TEST_EXE,TOPUP_EXE,ALWAYS,10 -TEST_DATA_r,TOPUP_DATA_r,ASAP,10 \ No newline at end of file +TEST_DATA_r,TOPUP_DATA_r,ASAP,10 +TEST_VOICE,TOPUP_VOICE,ASAP,10 diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 666038a38..e85a4781e 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -6,4 +6,5 @@ CDRST_WARN_HTTP,*call_url,http://localhost:8080,,,,,,,,,,,,,false,false,10 CDRST_LOG,*log,,,,,,,,,,,,,,false,false,10 TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 TOPUP_DATA_r,*topup,,,,*monetary,*out,,DATA_DEST,,,*unlimited,,5000000,10,false,false,10 -TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 \ No newline at end of file +TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 +TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 \ No newline at end of file diff --git a/data/tariffplans/testtp/DerivedChargers.csv b/data/tariffplans/testtp/DerivedChargers.csv index b50586df9..ff24c376d 100644 --- a/data/tariffplans/testtp/DerivedChargers.csv +++ b/data/tariffplans/testtp/DerivedChargers.csv @@ -3,3 +3,4 @@ *out,cgrates.org,call,dan,dan,,extra2,,,,,,^ivo,^ivo,,,,,,*default,*default,*default,*default *out,cgrates.org,call,dan,dan,,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,,,,*default,*default,*default,*default *out,cgrates.org,call,dan,*any,,extra1,,,,,,^rif2,^rif2,,,,,,*default,*default,*default,*default +*out,cgrates.org,call,1011,1011,GERMANY,extra1,,,+4915,,,,,,,,,,*default,*default,*default,*default \ No newline at end of file diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index b281c2985..a31f1048d 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -342,3 +342,45 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } } + +func TestSMGDataDerivedChargingNoCredit(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1011"} + eAcntVal := 50000.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.VOICE].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1011", + utils.SUBJECT: "1011", + utils.DESTINATION: "+49", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "100", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 100 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 50000.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.VOICE].GetTotalValue()) + } +} From c086b4e853614cf986ff243cd997aba6914bf909 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 31 Mar 2016 13:54:26 +0200 Subject: [PATCH 099/227] SMG handling of subsessions, fix for usage, added *data balance to tutorial files --- agents/dmtagent_it_test.go | 24 +++++----- apier/v1/smgenericv1_it_test.go | 2 +- data/conf/samples/dmtagent/data.json | 6 +-- data/tariffplans/tutorial/ActionPlans.csv | 1 + data/tariffplans/tutorial/Actions.csv | 1 + general_tests/tut_smgeneric_it_test.go | 2 +- general_tests/tutorial_local_test.go | 2 +- sessionmanager/smgeneric.go | 53 +++++++++++------------ 8 files changed, 46 insertions(+), 45 deletions(-) diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index 4b0179e88..ccaacbf1b 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -798,10 +798,10 @@ func TestDmtAgentSendDataGrpUpdate(t *testing.T) { AVP: []*diam.AVP{ diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit AVP: []*diam.AVP{ - diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage - diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time - diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets - diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(2)), // CC-Output-Octets + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1000)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(24)), // CC-Output-Octets }, }), diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(1)), // Data session for group 1 @@ -811,10 +811,10 @@ func TestDmtAgentSendDataGrpUpdate(t *testing.T) { AVP: []*diam.AVP{ diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit AVP: []*diam.AVP{ - diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage - diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time - diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets - diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Output-Octets + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1024)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(512)), // CC-Output-Octets }, }), diam.NewAVP(432, avp.Mbit, 0, datatype.Unsigned32(2)), // Data session for group 2 @@ -892,10 +892,10 @@ func TestDmtAgentSendDataGrpTerminate(t *testing.T) { AVP: []*diam.AVP{ diam.NewAVP(446, avp.Mbit, 0, &diam.GroupedAVP{ // Used-Service-Unit AVP: []*diam.AVP{ - diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage - diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time - diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(1)), // CC-Input-Octets - diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(2)), // CC-Output-Octets + diam.NewAVP(452, avp.Mbit, 0, datatype.Enumerated(0)), // Tariff-Change-Usage + diam.NewAVP(420, avp.Mbit, 0, datatype.Unsigned32(20)), // CC-Time + diam.NewAVP(412, avp.Mbit, 0, datatype.Unsigned64(512)), // CC-Input-Octets + diam.NewAVP(414, avp.Mbit, 0, datatype.Unsigned64(0)), // CC-Output-Octets }, }), }, diff --git a/apier/v1/smgenericv1_it_test.go b/apier/v1/smgenericv1_it_test.go index 6afd87eb4..17da136f8 100644 --- a/apier/v1/smgenericv1_it_test.go +++ b/apier/v1/smgenericv1_it_test.go @@ -116,7 +116,7 @@ func TestSMGV1CacheStats(t *testing.T) { } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 7, ActionPlans: 4, SharedGroups: 1, Aliases: 1, + expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, SharedGroups: 1, Aliases: 1, DerivedChargers: 1, LcrProfiles: 5, CdrStats: 6, Users: 3, LastLoadId: smgV1LoadInst.LoadId, LastLoadTime: smgV1LoadInst.LoadTime.Format(time.RFC3339)} var args utils.AttrCacheStats if err := smgV1Rpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { diff --git a/data/conf/samples/dmtagent/data.json b/data/conf/samples/dmtagent/data.json index 9080766f3..c10765409 100644 --- a/data/conf/samples/dmtagent/data.json +++ b/data/conf/samples/dmtagent/data.json @@ -19,7 +19,7 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "2048"}, ], "cca_fields": [ {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", "value": "^2001"}, @@ -44,7 +44,7 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "2048"}, {"tag": "LastUsed", "field_id": "LastUsed", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(1)", "type": "*handler", "handler_id": "*sum", "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, ], @@ -71,7 +71,7 @@ {"tag": "Destination", "field_id": "Destination", "type": "*constant", "value": "data"}, {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "3"}, + {"tag": "Usage", "field_id": "Usage", "type": "*constant", "value": "2048"}, {"tag": "LastUsed", "field_id": "LastUsed", "field_filter":"Multiple-Services-Credit-Control>Rating-Group(2)", "type": "*handler", "handler_id": "*sum", "value": "Multiple-Services-Credit-Control>Used-Service-Unit>CC-Input-Octets;^|;Multiple-Services-Credit-Control>Used-Service-Unit>CC-Output-Octets"}, ], diff --git a/data/tariffplans/tutorial/ActionPlans.csv b/data/tariffplans/tutorial/ActionPlans.csv index 3d59fe3e9..69d8ee8fb 100644 --- a/data/tariffplans/tutorial/ActionPlans.csv +++ b/data/tariffplans/tutorial/ActionPlans.csv @@ -6,3 +6,4 @@ USE_SHARED_A,SHARED_A_0,*asap,10 PACKAGE_1001,TOPUP_RST_5,*asap,10 PACKAGE_1001,TOPUP_RST_SHARED_5,*asap,10 PACKAGE_1001,TOPUP_120_DST1003,*asap,10 +PACKAGE_1001,TOPUP_RST_DATA_100,*asap,10 diff --git a/data/tariffplans/tutorial/Actions.csv b/data/tariffplans/tutorial/Actions.csv index 93a3faa1f..83c3a9f66 100644 --- a/data/tariffplans/tutorial/Actions.csv +++ b/data/tariffplans/tutorial/Actions.csv @@ -5,6 +5,7 @@ TOPUP_RST_5,*topup_reset,,,,*voice,*out,,DST_1002,SPECIAL_1002,,*unlimited,,90,2 TOPUP_120_DST1003,*topup_reset,,,,*voice,*out,,DST_1003,,,*unlimited,,120,20,false,false,10 TOPUP_RST_SHARED_5,*topup,,,,*monetary,*out,,*any,,SHARED_A,*unlimited,,5,10,false,false,10 SHARED_A_0,*topup_reset,,,,*monetary,*out,,*any,,SHARED_A,*unlimited,,0,10,false,false,10 +TOPUP_RST_DATA_100,*topup_reset,,,,*data,*out,,*any,,,*unlimited,,102400,10,false,false,10 LOG_WARNING,*log,,,,,,,,,,,,,,false,false,10 DISABLE_AND_LOG,*log,,,,,,,,,,,,,,false,false,10 DISABLE_AND_LOG,*disable_account,,,,,,,,,,,,,,false,false,10 diff --git a/general_tests/tut_smgeneric_it_test.go b/general_tests/tut_smgeneric_it_test.go index 889686b1f..0c72cada8 100644 --- a/general_tests/tut_smgeneric_it_test.go +++ b/general_tests/tut_smgeneric_it_test.go @@ -114,7 +114,7 @@ func TestTutSMGCacheStats(t *testing.T) { } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 7, ActionPlans: 4, SharedGroups: 1, Aliases: 1, + expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, SharedGroups: 1, Aliases: 1, DerivedChargers: 1, LcrProfiles: 5, CdrStats: 6, Users: 3, LastLoadId: smgLoadInst.LoadId, LastLoadTime: smgLoadInst.LoadTime.Format(time.RFC3339)} var args utils.AttrCacheStats if err := tutSMGRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 5d4f08f58..e5f4e8b61 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -115,7 +115,7 @@ func TestTutLocalCacheStats(t *testing.T) { } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 7, ActionPlans: 4, SharedGroups: 1, Aliases: 1, + expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 4, RatingProfiles: 9, Actions: 8, ActionPlans: 4, SharedGroups: 1, Aliases: 1, DerivedChargers: 1, LcrProfiles: 5, CdrStats: 6, Users: 3, LastLoadId: loadInst.LoadId, LastLoadTime: loadInst.LoadTime.Format(time.RFC3339)} var args utils.AttrCacheStats if err := tutLocalRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 5f7359b1a..ee8b6e797 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -115,7 +115,7 @@ func (self *SMGeneric) sessionStart(evStart SMGenericEvent, connId string) error } } return nil, nil - }, time.Duration(3)*time.Second, sessionId) + }, time.Duration(2)*time.Second, sessionId) return err } @@ -206,6 +206,14 @@ func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([ return lcr.SuppliersSlice() } +// Called on session start +func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { + if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { + return nilDuration, err + } + return self.SessionUpdate(gev, clnt) +} + // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { @@ -238,14 +246,6 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim return evMaxUsage, nil } -// Called on session start -func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { - return nilDuration, err - } - return self.SessionUpdate(gev, clnt) -} - // Called on session end, should stop debit loop func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { @@ -261,34 +261,33 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { if sessionIDPrefix, err := gev.GetFieldAsString(utils.OriginIDPrefix); err == nil { // OriginIDPrefix is present, OriginID will not be anymore considered sessionIDs = self.getSessionIDsForPrefix(sessionIDPrefix) } - usage, err := gev.GetUsage(utils.META_DEFAULT) - if err != nil { - if err != utils.ErrNotFound { - return err + usage, errUsage := gev.GetUsage(utils.META_DEFAULT) + var lastUsed time.Duration + if errUsage != nil { + if errUsage != utils.ErrNotFound { + return errUsage } - lastUsed, err := gev.GetLastUsed(utils.META_DEFAULT) + var err error + lastUsed, err = gev.GetLastUsed(utils.META_DEFAULT) if err != nil { if err == utils.ErrNotFound { err = utils.ErrMandatoryIeMissing } return err } - var s *SMGSession - for _, sID := range sessionIDs { - for _, s = range self.getSession(sID) { - break - } - if s != nil { - break - } - } - if s == nil { - return nil - } - usage = s.TotalUsage() + lastUsed } var interimError error for _, sessionID := range sessionIDs { + if errUsage != nil { + var s *SMGSession + for _, s = range self.getSession(sessionID) { + break + } + if s == nil { + continue // No session active, will not be able to close it anyway + } + usage = s.TotalUsage() + lastUsed + } if err := self.sessionEnd(sessionID, usage); err != nil { interimError = err // Last error will be the one returned as API result } From 52ea74cf3b58322a44adc43894f8a4485d89a32a Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 31 Mar 2016 19:19:03 +0200 Subject: [PATCH 100/227] Diameter tests connecting over socket between components --- agents/dmtagent_it_test.go | 2 +- data/conf/samples/dmtagent/cgrates.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index ccaacbf1b..9826d75f1 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -208,7 +208,7 @@ func TestDmtAgentTPFromFolder(t *testing.T) { if err := apierRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &loadInst); err != nil { t.Error(err) } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups + time.Sleep(time.Duration(1000) * time.Millisecond) // Give time for scheduler to execute topups } func TestConnectDiameterClient(t *testing.T) { diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index 93fe0df45..6033f4bd2 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -47,8 +47,8 @@ "sm_generic": { "enabled": true, - "rater": "internal", - "cdrs": "internal", + "rater": "127.0.0.1:2013", + "cdrs": "127.0.0.1:2013", }, "diameter_agent": { From e6eb2439f158fd6e1df44b71175798d67fdf3c45 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 20:25:51 +0300 Subject: [PATCH 101/227] tests end session after no credit --- data/tariffplans/testtp/DerivedChargers.csv | 2 +- sessionmanager/data_it_test.go | 10 +++++----- sessionmanager/smgeneric.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data/tariffplans/testtp/DerivedChargers.csv b/data/tariffplans/testtp/DerivedChargers.csv index ff24c376d..5493da35d 100644 --- a/data/tariffplans/testtp/DerivedChargers.csv +++ b/data/tariffplans/testtp/DerivedChargers.csv @@ -3,4 +3,4 @@ *out,cgrates.org,call,dan,dan,,extra2,,,,,,^ivo,^ivo,,,,,,*default,*default,*default,*default *out,cgrates.org,call,dan,dan,,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,,,,*default,*default,*default,*default *out,cgrates.org,call,dan,*any,,extra1,,,,,,^rif2,^rif2,,,,,,*default,*default,*default,*default -*out,cgrates.org,call,1011,1011,GERMANY,extra1,,,+4915,,,,,,,,,,*default,*default,*default,*default \ No newline at end of file +*out,cgrates.org,call,1011,1011,GERMANY,extra1,,,,,,,,^+4915,,,,,*default,*default,*default,*default \ No newline at end of file diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index a31f1048d..08b701df8 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -349,10 +349,10 @@ func TestSMGDataDerivedChargingNoCredit(t *testing.T) { } var acnt *engine.Account attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1011"} - eAcntVal := 50000.000000 + eAcntVal := 50000.0 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + } else if acnt.BalanceMap[utils.VOICE].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.VOICE].GetTotalValue()) } smgEv := SMGenericEvent{ @@ -374,13 +374,13 @@ func TestSMGDataDerivedChargingNoCredit(t *testing.T) { if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { t.Error(err) } - if maxUsage != 100 { + if maxUsage != 0 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 50000.000000 + eAcntVal = 50000.0 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + } else if acnt.BalanceMap[utils.VOICE].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.VOICE].GetTotalValue()) } } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 3ebf05e63..64e839e75 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -245,7 +245,7 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time return nilDuration, err } d, err := self.SessionUpdate(gev, clnt) - if err != nil { + if err != nil || d == 0 { self.sessionEnd(gev.GetUUID(), 0) } return d, err From fae51f08413319cc7c53ad33808cf40c0e0a66ef Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 20:36:17 +0300 Subject: [PATCH 102/227] fix cache stats test --- apier/v1/apier_local_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 3f8a03b7a..cfa76e271 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1297,8 +1297,8 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { if rcvStats.Destinations != 5 || rcvStats.RatingPlans != 4 || rcvStats.RatingProfiles != 4 || - rcvStats.Actions != 7 || - rcvStats.DerivedChargers != 2 { + rcvStats.Actions != 8 || + rcvStats.DerivedChargers != 3 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) } } From ec3375d73aaad3b42dde7b6c16f931754d7bf463 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 21:34:51 +0300 Subject: [PATCH 103/227] test for wrong gob --- engine/responder.go | 10 ++-- engine/responder_test.go | 95 +++++++++++++++++++++++++++++++++++ engine/storage_utils.go | 2 +- sessionmanager/session.go | 2 +- sessionmanager/smg_session.go | 2 +- sessionmanager/smgeneric.go | 2 +- 6 files changed, 104 insertions(+), 9 deletions(-) diff --git a/engine/responder.go b/engine/responder.go index 1b211b3f5..2e1ff74e9 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -513,25 +513,25 @@ func (rs *Responder) ProcessCdr(cdr *CDR, reply *string) error { } func (rs *Responder) StoreSMCost(attrs AttrCDRSStoreSMCost, reply *string) error { - if item, err := rs.getCache().Get(utils.LOG_CALL_COST_CACHE_PREFIX + attrs.SMCost.CGRID); err == nil && item != nil { + if item, err := rs.getCache().Get(utils.LOG_CALL_COST_CACHE_PREFIX + attrs.Cost.CGRID); err == nil && item != nil { *reply = item.Value.(string) return item.Err } if rs.CdrSrv == nil { err := errors.New("CDR_SERVER_NOT_RUNNING") - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.Cost.CGRID, &cache2go.CacheItem{ Err: err, }) return err } - if err := rs.CdrSrv.StoreSMCost(attrs.SMCost, attrs.CheckDuplicate); err != nil { - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ + if err := rs.CdrSrv.StoreSMCost(attrs.Cost, attrs.CheckDuplicate); err != nil { + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.Cost.CGRID, &cache2go.CacheItem{ Err: err, }) return err } *reply = utils.OK - rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.SMCost.CGRID, &cache2go.CacheItem{ + rs.getCache().Cache(utils.LOG_CALL_COST_CACHE_PREFIX+attrs.Cost.CGRID, &cache2go.CacheItem{ Value: utils.OK, }) return nil diff --git a/engine/responder_test.go b/engine/responder_test.go index 34b573151..6822b1c05 100644 --- a/engine/responder_test.go +++ b/engine/responder_test.go @@ -18,6 +18,8 @@ along with this program. If not, see package engine import ( + "bytes" + "encoding/gob" "reflect" "testing" "time" @@ -538,3 +540,96 @@ func TestResponderGetLCR(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eQosLcr.SupplierCosts, lcrQ.SupplierCosts) } } + +func TestResponderGobSMCost(t *testing.T) { + attr := AttrCDRSStoreSMCost{ + Cost: &SMCost{ + CGRID: "b783a8bcaa356570436983cd8a0e6de4993f9ba6", + RunID: "*default", + OriginHost: "", + OriginID: "testdatagrp_grp1", + CostSource: "SMR", + Usage: 1536, + CostDetails: &CallCost{ + Direction: "*out", + Category: "generic", + Tenant: "cgrates.org", + Subject: "1001", + Account: "1001", + Destination: "data", + TOR: "*data", + Cost: 0, + Timespans: TimeSpans{&TimeSpan{ + TimeStart: time.Date(2016, 1, 5, 12, 30, 10, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 5, 12, 55, 46, 0, time.UTC), + Cost: 0, + RateInterval: &RateInterval{ + Timing: nil, + Rating: &RIRate{ + ConnectFee: 0, + RoundingMethod: "", + RoundingDecimals: 0, + MaxCost: 0, + MaxCostStrategy: "", + Rates: RateGroups{&Rate{ + GroupIntervalStart: 0, + Value: 0, + RateIncrement: 1 * time.Second, + RateUnit: 1 * time.Second, + }, + }, + }, + Weight: 0, + }, + DurationIndex: 0, + Increments: Increments{&Increment{ + Duration: 1 * time.Second, + Cost: 0, + BalanceInfo: &DebitInfo{ + Unit: &UnitInfo{ + UUID: "fa0aa280-2b76-4b5b-bb06-174f84b8c321", + ID: "", + Value: 100864, + DestinationID: "data", + Consumed: 1, + TOR: "*data", + RateInterval: nil, + }, + Monetary: nil, + AccountID: "cgrates.org:1001", + }, + CompressFactor: 1536, + }, + }, + RoundIncrement: nil, + MatchedSubject: "fa0aa280-2b76-4b5b-bb06-174f84b8c321", + MatchedPrefix: "data", + MatchedDestId: "*any", + RatingPlanId: "*none", + CompressFactor: 1, + }, + }, + RatedUsage: 1536, + }, + }, + CheckDuplicate: false, + } + + var network bytes.Buffer // Stand-in for a network connection + enc := gob.NewEncoder(&network) // Will write to network. + dec := gob.NewDecoder(&network) // Will read from network. + err := enc.Encode(attr) + if err != nil { + t.Error("encode error: ", err) + } + + // Decode (receive) and print the values. + var q AttrCDRSStoreSMCost + err = dec.Decode(&q) + if err != nil { + t.Error("decode error: ", err) + } + if !reflect.DeepEqual(attr, q) { + t.Error("wrong transmission") + } +} diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 5ec8f6f4e..464431a74 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -164,6 +164,6 @@ type SMCost struct { } type AttrCDRSStoreSMCost struct { - SMCost *SMCost + Cost *SMCost CheckDuplicate bool } diff --git a/sessionmanager/session.go b/sessionmanager/session.go index a6a5d1709..9a21d5984 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -248,7 +248,7 @@ func (s *Session) SaveOperations() { OriginID: s.eventStart.GetUUID(), CostDetails: firstCC, } - err := s.sessionManager.CdrSrv().StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply) + err := s.sessionManager.CdrSrv().StoreSMCost(engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply) // this is a protection against the case when the close event is missed for some reason // when the cdr arrives to cdrserver because our callcost is not there it will be rated // as postpaid. When the close event finally arives we have to refund everything diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 435842c86..bc71236dd 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -247,7 +247,7 @@ func (self *SMGSession) saveOperations(originID string) error { Usage: self.TotalUsage().Seconds(), CostDetails: firstCC, } - if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply); err != nil { + if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil { // this is a protection against the case when the close event is missed for some reason // when the cdr arrives to cdrserver because our callcost is not there it will be rated // as postpaid. When the close event finally arives we have to refund everything diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 64e839e75..508c6818e 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -393,7 +393,7 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu OriginID: gev.GetUUID(), CostDetails: cc, } - if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{SMCost: smCost, CheckDuplicate: true}, &reply); err != nil && err != utils.ErrExists { + if err := self.cdrsrv.StoreSMCost(engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil && err != utils.ErrExists { withErrors = true utils.Logger.Err(fmt.Sprintf(" Could not save CC: %+v, RunID: %s error: %s", cc, sR.DerivedCharger.RunID, err.Error())) } From c1837564633e348ed1784d1e3e11a16eb94d9ad3 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 31 Mar 2016 22:03:29 +0300 Subject: [PATCH 104/227] fix gob error --- apier/v1/cdrsv1.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apier/v1/cdrsv1.go b/apier/v1/cdrsv1.go index e96292369..4518026b4 100644 --- a/apier/v1/cdrsv1.go +++ b/apier/v1/cdrsv1.go @@ -59,8 +59,8 @@ func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { return nil } -func (self *CdrsV1) StoreSMCost(smCost *engine.SMCost, reply *string) error { - if err := self.CdrSrv.StoreSMCost(smCost, false); err != nil { +func (self *CdrsV1) StoreSMCost(attr *engine.AttrCDRSStoreSMCost, reply *string) error { + if err := self.CdrSrv.StoreSMCost(attr.Cost, attr.CheckDuplicate); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK From 1ad4633941fc0a4a7a38d333d83c05cdfdb35d07 Mon Sep 17 00:00:00 2001 From: rbarrabe Date: Fri, 1 Apr 2016 10:10:07 +0200 Subject: [PATCH 105/227] Update huawei.xml --- data/diameter/dict/huawei/huawei.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/diameter/dict/huawei/huawei.xml b/data/diameter/dict/huawei/huawei.xml index 8553ad05b..70e182ddd 100644 --- a/data/diameter/dict/huawei/huawei.xml +++ b/data/diameter/dict/huawei/huawei.xml @@ -133,7 +133,7 @@ - + From f49c1f694c9ea23ffd0f0dcd14d03a412df0c929 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 1 Apr 2016 13:25:54 +0300 Subject: [PATCH 106/227] LCR subject prefix matching fixes #388, fixes #394 --- cmd/cgr-engine/cgr-engine.go | 2 + config/config.go | 170 +++++++++++++++++---------------- config/config_defaults.go | 3 +- config/config_json_test.go | 2 +- config/libconfig_json.go | 17 ++-- data/conf/cgrates/cgrates.json | 3 +- engine/calldesc.go | 36 ++++--- engine/lcr_test.go | 18 ++++ engine/ratingprofile.go | 5 +- 9 files changed, 149 insertions(+), 107 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 1c1da74d7..7563fa0c9 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -668,6 +668,8 @@ func main() { } engine.SetRoundingDecimals(cfg.RoundingDecimals) + engine.SetRpSubjectPrefixMatching(cfg.RpSubjectPrefixMatching) + engine.SetLcrSubjectPrefixMatching(cfg.LcrSubjectPrefixMatching) stopHandled := false // Rpc/http server diff --git a/config/config.go b/config/config.go index 72a1bf98c..d2dbd6486 100644 --- a/config/config.go +++ b/config/config.go @@ -167,89 +167,90 @@ func NewCGRConfigFromFolder(cfgDir string) (*CGRConfig, error) { // Holds system configuration, defaults are overwritten with values from config file if found type CGRConfig struct { - TpDbType string - TpDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - TpDbPort string // The port to bind to. - TpDbName string // The name of the database to connect to. - TpDbUser string // The user to sign in as. - TpDbPass string // The user's password. - DataDbType string - DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - DataDbPort string // The port to bind to. - DataDbName string // The name of the database to connect to. - DataDbUser string // The user to sign in as. - DataDbPass string // The user's password. - LoadHistorySize int // Maximum number of records to archive in load history - StorDBType string // Should reflect the database type used to store logs - StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - StorDBPort string // Th e port to bind to. - StorDBName string // The name of the database to connect to. - StorDBUser string // The user to sign in as. - StorDBPass string // The user's password. - StorDBMaxOpenConns int // Maximum database connections opened - StorDBMaxIdleConns int // Maximum idle connections to keep opened - StorDBCDRSIndexes []string - DBDataEncoding string // The encoding used to store object data in strings: - RPCJSONListen string // RPC JSON listening address - RPCGOBListen string // RPC GOB listening address - HTTPListen string // HTTP listening address - DefaultReqType string // Use this request type if not defined on top - DefaultCategory string // set default type of record - DefaultTenant string // set default tenant - DefaultSubject string // set default rating subject, useful in case of fallback - DefaultTimezone string // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> - Reconnects int // number of recconect attempts in case of connection lost <-1 for infinite | nb> - ConnectAttempts int // number of initial connection attempts before giving up - ResponseCacheTTL time.Duration // the life span of a cached response - InternalTtl time.Duration // maximum duration to wait for internal connections before giving up - RoundingDecimals int // Number of decimals to round end prices at - HttpSkipTlsVerify bool // If enabled Http Client will accept any TLS certificate - TpExportPath string // Path towards export folder for offline Tariff Plans - HttpFailedDir string // Directory path where we store failed http requests - MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file - RaterEnabled bool // start standalone server (no balancer) - RaterBalancer string // balancer address host:port - RaterCdrStats string // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> - RaterHistoryServer string - RaterPubSubServer string - RaterUserServer string - RaterAliasesServer string - RpSubjectPrefixMatching bool // enables prefix matching for the rating profile subject - BalancerEnabled bool - SchedulerEnabled bool - CDRSEnabled bool // Enable CDR Server service - CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs - CDRSStoreCdrs bool // store cdrs in storDb - CDRSRater string // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> - CDRSPubSub string // address where to reach the pubsub service: <""|internal|x.y.z.y:1234> - CDRSUsers string // address where to reach the users service: <""|internal|x.y.z.y:1234> - CDRSAliases string // address where to reach the aliases service: <""|internal|x.y.z.y:1234> - CDRSStats string // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> - CDRSCdrReplication []*CdrReplicationCfg // Replicate raw CDRs to a number of servers - CDRStatsEnabled bool // Enable CDR Stats service - CDRStatsSaveInterval time.Duration // Save interval duration - CdreProfiles map[string]*CdreConfig - CdrcProfiles map[string]map[string]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath]map[instanceName]{Configs} - SmGenericConfig *SmGenericConfig - SmFsConfig *SmFsConfig // SM-FreeSWITCH configuration - SmKamConfig *SmKamConfig // SM-Kamailio Configuration - SmOsipsConfig *SmOsipsConfig // SM-OpenSIPS Configuration - diameterAgentCfg *DiameterAgentCfg // DiameterAgent configuration - HistoryServer string // Address where to reach the master history server: - HistoryServerEnabled bool // Starts History as server: . - HistoryDir string // Location on disk where to store history files. - HistorySaveInterval time.Duration // The timout duration between pubsub writes - PubSubServerEnabled bool // Starts PubSub as server: . - AliasesServerEnabled bool // Starts PubSub as server: . - UserServerEnabled bool // Starts User as server: - UserServerIndexes []string // List of user profile field indexes - MailerServer string // The server to use when sending emails out - MailerAuthUser string // Authenticate to email server using this user - MailerAuthPass string // Authenticate to email server with this password - MailerFromAddr string // From address used when sending emails out - DataFolderPath string // Path towards data folder, for tests internal usage, not loading out of .json options - sureTaxCfg *SureTaxCfg // Load here SureTax configuration, as pointer so we can have runtime reloads in the future - ConfigReloads map[string]chan struct{} // Signals to specific entities that a config reload should occur + TpDbType string + TpDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + TpDbPort string // The port to bind to. + TpDbName string // The name of the database to connect to. + TpDbUser string // The user to sign in as. + TpDbPass string // The user's password. + DataDbType string + DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + DataDbPort string // The port to bind to. + DataDbName string // The name of the database to connect to. + DataDbUser string // The user to sign in as. + DataDbPass string // The user's password. + LoadHistorySize int // Maximum number of records to archive in load history + StorDBType string // Should reflect the database type used to store logs + StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + StorDBPort string // Th e port to bind to. + StorDBName string // The name of the database to connect to. + StorDBUser string // The user to sign in as. + StorDBPass string // The user's password. + StorDBMaxOpenConns int // Maximum database connections opened + StorDBMaxIdleConns int // Maximum idle connections to keep opened + StorDBCDRSIndexes []string + DBDataEncoding string // The encoding used to store object data in strings: + RPCJSONListen string // RPC JSON listening address + RPCGOBListen string // RPC GOB listening address + HTTPListen string // HTTP listening address + DefaultReqType string // Use this request type if not defined on top + DefaultCategory string // set default type of record + DefaultTenant string // set default tenant + DefaultSubject string // set default rating subject, useful in case of fallback + DefaultTimezone string // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> + Reconnects int // number of recconect attempts in case of connection lost <-1 for infinite | nb> + ConnectAttempts int // number of initial connection attempts before giving up + ResponseCacheTTL time.Duration // the life span of a cached response + InternalTtl time.Duration // maximum duration to wait for internal connections before giving up + RoundingDecimals int // Number of decimals to round end prices at + HttpSkipTlsVerify bool // If enabled Http Client will accept any TLS certificate + TpExportPath string // Path towards export folder for offline Tariff Plans + HttpFailedDir string // Directory path where we store failed http requests + MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file + RaterEnabled bool // start standalone server (no balancer) + RaterBalancer string // balancer address host:port + RaterCdrStats string // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> + RaterHistoryServer string + RaterPubSubServer string + RaterUserServer string + RaterAliasesServer string + RpSubjectPrefixMatching bool // enables prefix matching for the rating profile subject + LcrSubjectPrefixMatching bool // enables prefix matching for the lcr subject + BalancerEnabled bool + SchedulerEnabled bool + CDRSEnabled bool // Enable CDR Server service + CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs + CDRSStoreCdrs bool // store cdrs in storDb + CDRSRater string // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> + CDRSPubSub string // address where to reach the pubsub service: <""|internal|x.y.z.y:1234> + CDRSUsers string // address where to reach the users service: <""|internal|x.y.z.y:1234> + CDRSAliases string // address where to reach the aliases service: <""|internal|x.y.z.y:1234> + CDRSStats string // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> + CDRSCdrReplication []*CdrReplicationCfg // Replicate raw CDRs to a number of servers + CDRStatsEnabled bool // Enable CDR Stats service + CDRStatsSaveInterval time.Duration // Save interval duration + CdreProfiles map[string]*CdreConfig + CdrcProfiles map[string]map[string]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath]map[instanceName]{Configs} + SmGenericConfig *SmGenericConfig + SmFsConfig *SmFsConfig // SM-FreeSWITCH configuration + SmKamConfig *SmKamConfig // SM-Kamailio Configuration + SmOsipsConfig *SmOsipsConfig // SM-OpenSIPS Configuration + diameterAgentCfg *DiameterAgentCfg // DiameterAgent configuration + HistoryServer string // Address where to reach the master history server: + HistoryServerEnabled bool // Starts History as server: . + HistoryDir string // Location on disk where to store history files. + HistorySaveInterval time.Duration // The timout duration between pubsub writes + PubSubServerEnabled bool // Starts PubSub as server: . + AliasesServerEnabled bool // Starts PubSub as server: . + UserServerEnabled bool // Starts User as server: + UserServerIndexes []string // List of user profile field indexes + MailerServer string // The server to use when sending emails out + MailerAuthUser string // Authenticate to email server using this user + MailerAuthPass string // Authenticate to email server with this password + MailerFromAddr string // From address used when sending emails out + DataFolderPath string // Path towards data folder, for tests internal usage, not loading out of .json options + sureTaxCfg *SureTaxCfg // Load here SureTax configuration, as pointer so we can have runtime reloads in the future + ConfigReloads map[string]chan struct{} // Signals to specific entities that a config reload should occur // Cache defaults loaded from json and needing clones dfltCdreProfile *CdreConfig // Default cdreConfig profile dfltCdrcProfile *CdrcConfig // Default cdrcConfig profile @@ -670,6 +671,9 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnRaterCfg.Rp_subject_prefix_matching != nil { self.RpSubjectPrefixMatching = *jsnRaterCfg.Rp_subject_prefix_matching } + if jsnRaterCfg.Lcr_subject_prefix_matching != nil { + self.LcrSubjectPrefixMatching = *jsnRaterCfg.Lcr_subject_prefix_matching + } } if jsnBalancerCfg != nil && jsnBalancerCfg.Enabled != nil { diff --git a/config/config_defaults.go b/config/config_defaults.go index f79ef663b..a0caa38b5 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -99,7 +99,8 @@ const CGRATES_CFG_JSON = ` "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> - "rp_subject_prefix_matching": false // enables prefix matching for the rating profile subject + "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject + "lcr_subject_prefix_matching": false // enables prefix matching for the lcr subject }, diff --git a/config/config_json_test.go b/config/config_json_test.go index 9238aae39..e80cb6aaa 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -129,7 +129,7 @@ func TestDfBalancerJsonCfg(t *testing.T) { func TestDfRaterJsonCfg(t *testing.T) { eCfg := &RaterJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats: utils.StringPointer(""), - Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer(""), Aliases: utils.StringPointer(""), Rp_subject_prefix_matching: utils.BoolPointer(false)} + Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer(""), Aliases: utils.StringPointer(""), Rp_subject_prefix_matching: utils.BoolPointer(false), Lcr_subject_prefix_matching: utils.BoolPointer(false)} if cfg, err := dfCgrJsonCfg.RaterJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { diff --git a/config/libconfig_json.go b/config/libconfig_json.go index bc94bca2d..e40dedcdd 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -64,14 +64,15 @@ type BalancerJsonCfg struct { // Rater config section type RaterJsonCfg struct { - Enabled *bool - Balancer *string - Cdrstats *string - Historys *string - Pubsubs *string - Aliases *string - Users *string - Rp_subject_prefix_matching *bool + Enabled *bool + Balancer *string + Cdrstats *string + Historys *string + Pubsubs *string + Aliases *string + Users *string + Rp_subject_prefix_matching *bool + Lcr_subject_prefix_matching *bool } // Scheduler config section diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index bc740467c..4e7790c06 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -77,7 +77,8 @@ // "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> // "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> // "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> -// "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject +// "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject, +// "lcr_subject_prefix_matching": false, // enables prefix matching for the lcr subject //}, diff --git a/engine/calldesc.go b/engine/calldesc.go index 9ab8a9373..fca3a671d 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -70,17 +70,18 @@ func init() { } var ( - ratingStorage RatingStorage - accountingStorage AccountingStorage - storageLogger LogStorage - cdrStorage CdrStorage - debitPeriod = 10 * time.Second - globalRoundingDecimals = 6 - historyScribe history.Scribe - pubSubServer rpcclient.RpcClientConnection - userService UserService - aliasService AliasService - rpSubjectPrefixMatching bool + ratingStorage RatingStorage + accountingStorage AccountingStorage + storageLogger LogStorage + cdrStorage CdrStorage + debitPeriod = 10 * time.Second + globalRoundingDecimals = 6 + historyScribe history.Scribe + pubSubServer rpcclient.RpcClientConnection + userService UserService + aliasService AliasService + rpSubjectPrefixMatching bool + lcrSubjectPrefixMatching bool ) // Exported method to set the storage getter. @@ -101,6 +102,10 @@ func SetRpSubjectPrefixMatching(flag bool) { rpSubjectPrefixMatching = flag } +func SetLcrSubjectPrefixMatching(flag bool) { + lcrSubjectPrefixMatching = flag +} + /* Sets the database for logging (can be de same as storage getter or different db) */ @@ -924,6 +929,15 @@ func (cd *CallDescriptor) GetLCRFromStorage() (*LCR, error) { utils.LCRKey(cd.Direction, utils.ANY, utils.ANY, utils.ANY, utils.ANY), utils.LCRKey(utils.ANY, utils.ANY, utils.ANY, utils.ANY, utils.ANY), } + if lcrSubjectPrefixMatching { + var partialSubjects []string + lenSubject := len(cd.Subject) + for i := 1; i < lenSubject; i++ { + partialSubjects = append(partialSubjects, utils.LCRKey(cd.Direction, cd.Tenant, cd.Category, cd.Account, cd.Subject[:lenSubject-i])) + } + // insert partialsubjects into keyVariants + keyVariants = append(keyVariants[:1], append(partialSubjects, keyVariants[1:]...)...) + } for _, key := range keyVariants { if lcr, err := ratingStorage.GetLCR(key, false); err != nil && err != utils.ErrNotFound { return nil, err diff --git a/engine/lcr_test.go b/engine/lcr_test.go index ddf112efb..49f201cff 100644 --- a/engine/lcr_test.go +++ b/engine/lcr_test.go @@ -210,6 +210,24 @@ func TestLcrGet(t *testing.T) { } } +func TestLcrGetPrefix(t *testing.T) { + lcrSubjectPrefixMatching = true + cd := &CallDescriptor{ + TimeStart: time.Date(2015, 04, 06, 17, 40, 0, 0, time.UTC), + TimeEnd: time.Date(2015, 04, 06, 17, 41, 0, 0, time.UTC), + Tenant: "cgrates.org", + Direction: "*in", + Category: "call", + Destination: "0723098765", + Account: "rif", + Subject: "rifus", + } + lcr, err := cd.GetLCR(nil, nil) + if err != nil || lcr == nil { + t.Errorf("Bad lcr: %+v, %v", lcr, err) + } +} + func TestLcrRequestAsCallDescriptor(t *testing.T) { sTime := time.Date(2015, 04, 06, 17, 40, 0, 0, time.UTC) callDur := time.Duration(1) * time.Minute diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index f0ebf0ac3..2560e96c0 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -256,8 +256,9 @@ func RatingProfileSubjectPrefixMatching(key string) (rp *RatingProfile, err erro lastIndex := strings.LastIndex(key, utils.CONCATENATED_KEY_SEP) baseKey := key[:lastIndex] subject := key[lastIndex:] - for i := 1; i < len(subject)-1; i++ { - if rp, err = ratingStorage.GetRatingProfile(baseKey+subject[:len(subject)-i], false); err == nil { + lenSubject := len(subject) + for i := 1; i < lenSubject-1; i++ { + if rp, err = ratingStorage.GetRatingProfile(baseKey+subject[:lenSubject-i], false); err == nil { return rp, err } } From 9d85826ed7122e5e3d9c6d4e8e2d7eb0daf3a19e Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 1 Apr 2016 13:29:53 +0200 Subject: [PATCH 107/227] Updating glide dependencies --- glide.lock | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/glide.lock b/glide.lock index fa156bc51..95e385375 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,6 @@ -hash: c4e3a1bdd7452ec3af195e09b8b3b1b9a61e36edfad557aeb01686706019c352 -updated: 2016-03-09T00:08:37.493018177+02:00 +hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 +updated: 2016-04-01T13:25:37.195540273+02:00 imports: -- name: github.com/cenkalti/hub - version: 57d753b5f4856e77b3cf8ecce78c97215a7d324d - name: github.com/cenkalti/rpc2 version: 2d1be381ce47537e9e076b2b76dc70933162e4e9 - name: github.com/cgrates/fsock @@ -23,22 +21,16 @@ imports: - diam/datatype - diam/dict - diam/sm - - diam/sm/smparser - - diam/sm/smpeer - name: github.com/go-sql-driver/mysql version: 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16 - name: github.com/gorhill/cronexpr version: a557574d6c024ed6e36acc8b610f5f211c91568a - name: github.com/jinzhu/gorm version: 2f7811c55f286c55cfc3a2aefb5c4049b9cd5214 -- name: github.com/jinzhu/inflection - version: 3272df6c21d04180007eb3349844c89a3856bc25 - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq version: 165a3529e799da61ab10faed1fabff3662d6193f - subpackages: - - oid - name: github.com/mediocregopher/radix.v2 version: 7bdaf7c45ec452ca691ab20535471e24460f0876 subpackages: @@ -54,13 +46,10 @@ imports: version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 subpackages: - websocket - - context - name: gopkg.in/fsnotify.v1 version: 875cf421b32f8f1b31bd43776297876d01542279 - name: gopkg.in/mgo.v2 version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson - - internal/sasl - - internal/scram devImports: [] From 5058bb1c0e9763766094c86a52b9f6804b26388e Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 1 Apr 2016 13:35:37 +0200 Subject: [PATCH 108/227] Reverting glide.lock --- glide.lock | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 95e385375..fa156bc51 100644 --- a/glide.lock +++ b/glide.lock @@ -1,6 +1,8 @@ -hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-01T13:25:37.195540273+02:00 +hash: c4e3a1bdd7452ec3af195e09b8b3b1b9a61e36edfad557aeb01686706019c352 +updated: 2016-03-09T00:08:37.493018177+02:00 imports: +- name: github.com/cenkalti/hub + version: 57d753b5f4856e77b3cf8ecce78c97215a7d324d - name: github.com/cenkalti/rpc2 version: 2d1be381ce47537e9e076b2b76dc70933162e4e9 - name: github.com/cgrates/fsock @@ -21,16 +23,22 @@ imports: - diam/datatype - diam/dict - diam/sm + - diam/sm/smparser + - diam/sm/smpeer - name: github.com/go-sql-driver/mysql version: 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16 - name: github.com/gorhill/cronexpr version: a557574d6c024ed6e36acc8b610f5f211c91568a - name: github.com/jinzhu/gorm version: 2f7811c55f286c55cfc3a2aefb5c4049b9cd5214 +- name: github.com/jinzhu/inflection + version: 3272df6c21d04180007eb3349844c89a3856bc25 - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq version: 165a3529e799da61ab10faed1fabff3662d6193f + subpackages: + - oid - name: github.com/mediocregopher/radix.v2 version: 7bdaf7c45ec452ca691ab20535471e24460f0876 subpackages: @@ -46,10 +54,13 @@ imports: version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 subpackages: - websocket + - context - name: gopkg.in/fsnotify.v1 version: 875cf421b32f8f1b31bd43776297876d01542279 - name: gopkg.in/mgo.v2 version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson + - internal/sasl + - internal/scram devImports: [] From cb80a24bac30a30671b1c9ed193946710bebbb4c Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 1 Apr 2016 16:27:50 +0200 Subject: [PATCH 109/227] Adding debug messages to SMG --- sessionmanager/smgeneric.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 508c6818e..08ac020b4 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -206,8 +206,23 @@ func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([ return lcr.SuppliersSlice() } +// Called on session start +func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { + utils.Logger.Debug(fmt.Sprintf("### SessionStart, gev: %+v, sessions: %+v", gev, self.getSessions())) + if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { + self.sessionEnd(gev.GetUUID(), 0) + return nilDuration, err + } + d, err := self.SessionUpdate(gev, clnt) + if err != nil || d == 0 { + self.sessionEnd(gev.GetUUID(), 0) + } + return d, err +} + // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { + utils.Logger.Debug(fmt.Sprintf("### SessionUpdate, gev: %+v, sessions: %+v", gev, self.getSessions())) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update @@ -238,21 +253,9 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim return evMaxUsage, nil } -// Called on session start -func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { - self.sessionEnd(gev.GetUUID(), 0) - return nilDuration, err - } - d, err := self.SessionUpdate(gev, clnt) - if err != nil || d == 0 { - self.sessionEnd(gev.GetUUID(), 0) - } - return d, err -} - // Called on session end, should stop debit loop func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { + utils.Logger.Debug(fmt.Sprintf("### SessionEnd, gev: %+v, sessions: %+v", gev, self.getSessions())) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update @@ -405,6 +408,7 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { + utils.Logger.Debug(fmt.Sprintf("### ProcessCdr, gev: %+v, sessions: %+v", gev, self.getSessions())) var reply string if err := self.cdrsrv.ProcessCdr(gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err From a2fd5116bea8f47db90f1f57b4e3bbd360c9950e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 00:10:19 +0300 Subject: [PATCH 110/227] started SetTriggers --- apier/v1/triggers.go | 143 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/apier/v1/triggers.go b/apier/v1/triggers.go index 642c137c7..23b22bd06 100644 --- a/apier/v1/triggers.go +++ b/apier/v1/triggers.go @@ -300,3 +300,146 @@ func (self *ApierV1) SetAccountActionTriggers(attr AttrSetAccountActionTriggers, *reply = utils.OK return nil } + +type AttrSetActionTriggers struct { + GroupID string + UniqueID string + ThresholdType *string + ThresholdValue *float64 + Recurrent *bool + MinSleep *string + ExpirationDate *string + ActivationDate *string + BalanceId *string + BalanceType *string + BalanceDirections *[]string + BalanceDestinationIds *[]string + BalanceWeight *float64 + BalanceExpirationDate *string + BalanceTimingTags *[]string + BalanceRatingSubject *string + BalanceCategories *[]string + BalanceSharedGroups *[]string + BalanceBlocker *bool + BalanceDisabled *bool + MinQueuedItems *int + ActionsId *string +} + +func (self *ApierV1) SetActionTriggers(attr AttrSetAccountActionTriggers, reply *string) error { + + if missing := utils.MissingStructFields(&attr, []string{"GroupID"}); len(missing) != 0 { + return utils.NewErrMandatoryIeMissing(missing...) + } + + atrs, err := self.RatingDb.GetActionTriggers(attr.GroupID) + if err != nil && err != utils.ErrNotFound { + *reply = err.Error() + return err + } + var newAtr *engine.ActionTrigger + if attr.UniqueID != "" { + //search for exiting one + for _, atr := range atrs { + if atr.UniqueID == attr.UniqueID { + newAtr = atr + break + } + } + } + + if newAtr == nil { + newAtr = &engine.ActionTrigger{} + atrs = append(atrs, newAtr) + } + + if attr.ThresholdType != nil { + newAtr.ThresholdType = *attr.ThresholdType + } + if attr.ThresholdValue != nil { + newAtr.ThresholdValue = *attr.ThresholdValue + } + if attr.Recurrent != nil { + newAtr.Recurrent = *attr.Recurrent + } + if attr.MinSleep != nil { + minSleep, err := utils.ParseDurationWithSecs(*attr.MinSleep) + if err != nil { + *reply = err.Error() + return err + } + newAtr.MinSleep = minSleep + } + if attr.ExpirationDate != nil { + expTime, err := utils.ParseTimeDetectLayout(*attr.ExpirationDate, self.Config.DefaultTimezone) + if err != nil { + *reply = err.Error() + return err + } + newAtr.ExpirationDate = expTime + } + if attr.ActivationDate != nil { + actTime, err := utils.ParseTimeDetectLayout(*attr.ActivationDate, self.Config.DefaultTimezone) + if err != nil { + *reply = err.Error() + return err + } + newAtr.ActivationDate = actTime + } + if attr.BalanceId != nil { + newAtr.Balance.ID = attr.BalanceId + } + if attr.BalanceType != nil { + newAtr.Balance.Type = attr.BalanceType + } + if attr.BalanceDirections != nil { + newAtr.Balance.Directions = utils.StringMapPointer(utils.NewStringMap(*attr.BalanceDirections...)) + } + if attr.BalanceDestinationIds != nil { + newAtr.Balance.DestinationIDs = utils.StringMapPointer(utils.NewStringMap(*attr.BalanceDestinationIds...)) + } + if attr.BalanceWeight != nil { + newAtr.Balance.Weight = attr.BalanceWeight + } + if attr.BalanceExpirationDate != nil { + balanceExpTime, err := utils.ParseDate(*attr.BalanceExpirationDate) + if err != nil { + *reply = err.Error() + return err + } + newAtr.Balance.ExpirationDate = &balanceExpTime + } + if attr.BalanceTimingTags != nil { + newAtr.Balance.TimingIDs = utils.StringMapPointer(utils.NewStringMap(*attr.BalanceTimingTags...)) + } + if attr.BalanceRatingSubject != nil { + newAtr.Balance.RatingSubject = attr.BalanceRatingSubject + } + if attr.BalanceCategories != nil { + newAtr.Balance.Categories = utils.StringMapPointer(utils.NewStringMap(*attr.BalanceCategories...)) + } + if attr.BalanceSharedGroups != nil { + newAtr.Balance.SharedGroups = utils.StringMapPointer(utils.NewStringMap(*attr.BalanceSharedGroups...)) + } + if attr.BalanceBlocker != nil { + newAtr.Balance.Blocker = attr.BalanceBlocker + } + if attr.BalanceDisabled != nil { + newAtr.Balance.Disabled = attr.BalanceDisabled + } + if attr.MinQueuedItems != nil { + newAtr.MinQueuedItems = *attr.MinQueuedItems + } + if attr.ActionsId != nil { + newAtr.ActionsID = *attr.ActionsId + } + + if err := self.RatingDb.SetActionTriggers(attr.GroupID, atrs); err != nil { + *reply = err.Error() + return err + } + //cache action triggers + self.RatingDb.CacheRatingPrefixValues(map[string][]string{utils.ACTION_TRIGGER_PREFIX: []string{utils.ACTION_TRIGGER_PREFIX + attr.GroupID}}) + *reply = utils.OK + return nil +} From 7b5091c7d82339aa4c344c0c87ee1337aca36df1 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 15:45:17 +0300 Subject: [PATCH 111/227] added SetActionTrigger, GetActionTriggers also added console commands an tests fixes #415 --- apier/v1/apier.go | 2 +- apier/v1/triggers.go | 77 ++++++++++++++----- ...{trigger_add.go => account_trigger_add.go} | 16 ++-- ...er_remove.go => account_trigger_remove.go} | 16 ++-- ...gger_reset.go => account_trigger_reset.go} | 16 ++-- console/account_trigger_set.go | 63 +++++++++++++++ console/trigger_set.go | 8 +- console/triggers.go | 66 ++++++++++++++++ engine/storage_mongo_datadb.go | 6 ++ general_tests/tp_it_test.go | 39 ++++++++++ 10 files changed, 259 insertions(+), 50 deletions(-) rename console/{trigger_add.go => account_trigger_add.go} (76%) rename console/{trigger_remove.go => account_trigger_remove.go} (75%) rename console/{trigger_reset.go => account_trigger_reset.go} (75%) create mode 100644 console/account_trigger_set.go create mode 100644 console/triggers.go diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 9afe5cf63..b6abf47f0 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -824,7 +824,7 @@ func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.Cach } if loadHistInsts, err := self.AccountDb.GetLoadHistory(1, false); err != nil || len(loadHistInsts) == 0 { if err != nil { // Not really an error here since we only count in cache - utils.Logger.Err(fmt.Sprintf("ApierV1.GetCacheStats, error on GetLoadHistory: %s", err.Error())) + utils.Logger.Warning(fmt.Sprintf("ApierV1.GetCacheStats, error on GetLoadHistory: %s", err.Error())) } cs.LastLoadId = utils.NOT_AVAILABLE cs.LastLoadTime = utils.NOT_AVAILABLE diff --git a/apier/v1/triggers.go b/apier/v1/triggers.go index 23b22bd06..b6c6a3f5f 100644 --- a/apier/v1/triggers.go +++ b/apier/v1/triggers.go @@ -175,7 +175,7 @@ type AttrSetAccountActionTriggers struct { MinSleep *string ExpirationDate *string ActivationDate *string - BalanceId *string + BalanceID *string BalanceType *string BalanceDirections *[]string BalanceDestinationIds *[]string @@ -188,7 +188,7 @@ type AttrSetAccountActionTriggers struct { BalanceBlocker *bool BalanceDisabled *bool MinQueuedItems *int - ActionsId *string + ActionsID *string } func (self *ApierV1) SetAccountActionTriggers(attr AttrSetAccountActionTriggers, reply *string) error { @@ -238,8 +238,9 @@ func (self *ApierV1) SetAccountActionTriggers(attr AttrSetAccountActionTriggers, } at.ActivationDate = actTime } - if attr.BalanceId != nil { - at.Balance.ID = attr.BalanceId + at.Balance = &engine.BalanceFilter{} + if attr.BalanceID != nil { + at.Balance.ID = attr.BalanceID } if attr.BalanceType != nil { at.Balance.Type = attr.BalanceType @@ -281,8 +282,8 @@ func (self *ApierV1) SetAccountActionTriggers(attr AttrSetAccountActionTriggers, if attr.MinQueuedItems != nil { at.MinQueuedItems = *attr.MinQueuedItems } - if attr.ActionsId != nil { - at.ActionsID = *attr.ActionsId + if attr.ActionsID != nil { + at.ActionsID = *attr.ActionsID } } @@ -301,7 +302,7 @@ func (self *ApierV1) SetAccountActionTriggers(attr AttrSetAccountActionTriggers, return nil } -type AttrSetActionTriggers struct { +type AttrSetActionTrigger struct { GroupID string UniqueID string ThresholdType *string @@ -310,7 +311,7 @@ type AttrSetActionTriggers struct { MinSleep *string ExpirationDate *string ActivationDate *string - BalanceId *string + BalanceID *string BalanceType *string BalanceDirections *[]string BalanceDestinationIds *[]string @@ -323,20 +324,16 @@ type AttrSetActionTriggers struct { BalanceBlocker *bool BalanceDisabled *bool MinQueuedItems *int - ActionsId *string + ActionsID *string } -func (self *ApierV1) SetActionTriggers(attr AttrSetAccountActionTriggers, reply *string) error { +func (self *ApierV1) SetActionTrigger(attr AttrSetActionTrigger, reply *string) error { if missing := utils.MissingStructFields(&attr, []string{"GroupID"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } - atrs, err := self.RatingDb.GetActionTriggers(attr.GroupID) - if err != nil && err != utils.ErrNotFound { - *reply = err.Error() - return err - } + atrs, _ := self.RatingDb.GetActionTriggers(attr.GroupID) var newAtr *engine.ActionTrigger if attr.UniqueID != "" { //search for exiting one @@ -352,6 +349,12 @@ func (self *ApierV1) SetActionTriggers(attr AttrSetAccountActionTriggers, reply newAtr = &engine.ActionTrigger{} atrs = append(atrs, newAtr) } + newAtr.ID = attr.GroupID + if attr.UniqueID != "" { + newAtr.UniqueID = attr.UniqueID + } else { + newAtr.UniqueID = utils.GenUUID() + } if attr.ThresholdType != nil { newAtr.ThresholdType = *attr.ThresholdType @@ -386,8 +389,9 @@ func (self *ApierV1) SetActionTriggers(attr AttrSetAccountActionTriggers, reply } newAtr.ActivationDate = actTime } - if attr.BalanceId != nil { - newAtr.Balance.ID = attr.BalanceId + newAtr.Balance = &engine.BalanceFilter{} + if attr.BalanceID != nil { + newAtr.Balance.ID = attr.BalanceID } if attr.BalanceType != nil { newAtr.Balance.Type = attr.BalanceType @@ -430,16 +434,47 @@ func (self *ApierV1) SetActionTriggers(attr AttrSetAccountActionTriggers, reply if attr.MinQueuedItems != nil { newAtr.MinQueuedItems = *attr.MinQueuedItems } - if attr.ActionsId != nil { - newAtr.ActionsID = *attr.ActionsId + if attr.ActionsID != nil { + newAtr.ActionsID = *attr.ActionsID } if err := self.RatingDb.SetActionTriggers(attr.GroupID, atrs); err != nil { *reply = err.Error() return err } - //cache action triggers - self.RatingDb.CacheRatingPrefixValues(map[string][]string{utils.ACTION_TRIGGER_PREFIX: []string{utils.ACTION_TRIGGER_PREFIX + attr.GroupID}}) + //no cache for action triggers *reply = utils.OK return nil } + +type AttrGetActionTriggers struct { + GroupIDs []string +} + +func (self *ApierV1) GetActionTriggers(attr AttrGetActionTriggers, atrs *engine.ActionTriggers) error { + var allAttrs engine.ActionTriggers + if len(attr.GroupIDs) > 0 { + for _, key := range attr.GroupIDs { + getAttrs, err := self.RatingDb.GetActionTriggers(key) + if err != nil { + return err + } + allAttrs = append(allAttrs, getAttrs...) + } + + } else { + keys, err := self.RatingDb.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX, true) + if err != nil { + return err + } + for _, key := range keys { + getAttrs, err := self.RatingDb.GetActionTriggers(key[len(utils.ACTION_TRIGGER_PREFIX):]) + if err != nil { + return err + } + allAttrs = append(allAttrs, getAttrs...) + } + } + *atrs = allAttrs + return nil +} diff --git a/console/trigger_add.go b/console/account_trigger_add.go similarity index 76% rename from console/trigger_add.go rename to console/account_trigger_add.go index bef41f910..3577aee9f 100644 --- a/console/trigger_add.go +++ b/console/account_trigger_add.go @@ -21,8 +21,8 @@ package console import "github.com/cgrates/cgrates/apier/v1" func init() { - c := &CmdAddTriggers{ - name: "triggers_add", + c := &CmdAccountAddTriggers{ + name: "account_triggers_add", rpcMethod: "ApierV1.AddAccountActionTriggers", rpcParams: &v1.AttrAddAccountActionTriggers{}, } @@ -31,33 +31,33 @@ func init() { } // Commander implementation -type CmdAddTriggers struct { +type CmdAccountAddTriggers struct { name string rpcMethod string rpcParams *v1.AttrAddAccountActionTriggers *CommandExecuter } -func (self *CmdAddTriggers) Name() string { +func (self *CmdAccountAddTriggers) Name() string { return self.name } -func (self *CmdAddTriggers) RpcMethod() string { +func (self *CmdAccountAddTriggers) RpcMethod() string { return self.rpcMethod } -func (self *CmdAddTriggers) RpcParams(reset bool) interface{} { +func (self *CmdAccountAddTriggers) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { self.rpcParams = &v1.AttrAddAccountActionTriggers{} } return self.rpcParams } -func (self *CmdAddTriggers) PostprocessRpcParams() error { +func (self *CmdAccountAddTriggers) PostprocessRpcParams() error { return nil } -func (self *CmdAddTriggers) RpcResult() interface{} { +func (self *CmdAccountAddTriggers) RpcResult() interface{} { var s string return &s } diff --git a/console/trigger_remove.go b/console/account_trigger_remove.go similarity index 75% rename from console/trigger_remove.go rename to console/account_trigger_remove.go index a460d8528..5d2739692 100644 --- a/console/trigger_remove.go +++ b/console/account_trigger_remove.go @@ -21,8 +21,8 @@ package console import "github.com/cgrates/cgrates/apier/v1" func init() { - c := &CmdRemoveTriggers{ - name: "triggers_remove", + c := &CmdAccountRemoveTriggers{ + name: "account_triggers_remove", rpcMethod: "ApierV1.RemoveAccountActionTriggers", rpcParams: &v1.AttrRemoveAccountActionTriggers{}, } @@ -31,33 +31,33 @@ func init() { } // Commander implementation -type CmdRemoveTriggers struct { +type CmdAccountRemoveTriggers struct { name string rpcMethod string rpcParams *v1.AttrRemoveAccountActionTriggers *CommandExecuter } -func (self *CmdRemoveTriggers) Name() string { +func (self *CmdAccountRemoveTriggers) Name() string { return self.name } -func (self *CmdRemoveTriggers) RpcMethod() string { +func (self *CmdAccountRemoveTriggers) RpcMethod() string { return self.rpcMethod } -func (self *CmdRemoveTriggers) RpcParams(reset bool) interface{} { +func (self *CmdAccountRemoveTriggers) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { self.rpcParams = &v1.AttrRemoveAccountActionTriggers{} } return self.rpcParams } -func (self *CmdRemoveTriggers) PostprocessRpcParams() error { +func (self *CmdAccountRemoveTriggers) PostprocessRpcParams() error { return nil } -func (self *CmdRemoveTriggers) RpcResult() interface{} { +func (self *CmdAccountRemoveTriggers) RpcResult() interface{} { var s string return &s } diff --git a/console/trigger_reset.go b/console/account_trigger_reset.go similarity index 75% rename from console/trigger_reset.go rename to console/account_trigger_reset.go index 0d5be9f24..d545361db 100644 --- a/console/trigger_reset.go +++ b/console/account_trigger_reset.go @@ -21,8 +21,8 @@ package console import "github.com/cgrates/cgrates/apier/v1" func init() { - c := &CmdResetTriggers{ - name: "triggers_reset", + c := &CmdAccountResetTriggers{ + name: "account_triggers_reset", rpcMethod: "ApierV1.ResetAccountActionTriggers", rpcParams: &v1.AttrRemoveAccountActionTriggers{}, } @@ -31,33 +31,33 @@ func init() { } // Commander implementation -type CmdResetTriggers struct { +type CmdAccountResetTriggers struct { name string rpcMethod string rpcParams *v1.AttrRemoveAccountActionTriggers *CommandExecuter } -func (self *CmdResetTriggers) Name() string { +func (self *CmdAccountResetTriggers) Name() string { return self.name } -func (self *CmdResetTriggers) RpcMethod() string { +func (self *CmdAccountResetTriggers) RpcMethod() string { return self.rpcMethod } -func (self *CmdResetTriggers) RpcParams(reset bool) interface{} { +func (self *CmdAccountResetTriggers) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { self.rpcParams = &v1.AttrRemoveAccountActionTriggers{} } return self.rpcParams } -func (self *CmdResetTriggers) PostprocessRpcParams() error { +func (self *CmdAccountResetTriggers) PostprocessRpcParams() error { return nil } -func (self *CmdResetTriggers) RpcResult() interface{} { +func (self *CmdAccountResetTriggers) RpcResult() interface{} { var s string return &s } diff --git a/console/account_trigger_set.go b/console/account_trigger_set.go new file mode 100644 index 000000000..84ed47079 --- /dev/null +++ b/console/account_trigger_set.go @@ -0,0 +1,63 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2012-2015 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package console + +import "github.com/cgrates/cgrates/apier/v1" + +func init() { + c := &CmdAccountSetTriggers{ + name: "account_triggers_set", + rpcMethod: "ApierV1.SetAccountActionTriggers", + rpcParams: &v1.AttrSetAccountActionTriggers{}, + } + commands[c.Name()] = c + c.CommandExecuter = &CommandExecuter{c} +} + +// Commander implementation +type CmdAccountSetTriggers struct { + name string + rpcMethod string + rpcParams *v1.AttrSetAccountActionTriggers + *CommandExecuter +} + +func (self *CmdAccountSetTriggers) Name() string { + return self.name +} + +func (self *CmdAccountSetTriggers) RpcMethod() string { + return self.rpcMethod +} + +func (self *CmdAccountSetTriggers) RpcParams(reset bool) interface{} { + if reset || self.rpcParams == nil { + self.rpcParams = &v1.AttrSetAccountActionTriggers{} + } + return self.rpcParams +} + +func (self *CmdAccountSetTriggers) PostprocessRpcParams() error { + return nil +} + +func (self *CmdAccountSetTriggers) RpcResult() interface{} { + var s string + return &s +} diff --git a/console/trigger_set.go b/console/trigger_set.go index 2e9095d95..e1a72dd67 100644 --- a/console/trigger_set.go +++ b/console/trigger_set.go @@ -23,8 +23,8 @@ import "github.com/cgrates/cgrates/apier/v1" func init() { c := &CmdSetTriggers{ name: "triggers_set", - rpcMethod: "ApierV1.SetAccountActionTriggers", - rpcParams: &v1.AttrSetAccountActionTriggers{}, + rpcMethod: "ApierV1.SetActionTrigger", + rpcParams: &v1.AttrSetActionTrigger{}, } commands[c.Name()] = c c.CommandExecuter = &CommandExecuter{c} @@ -34,7 +34,7 @@ func init() { type CmdSetTriggers struct { name string rpcMethod string - rpcParams *v1.AttrSetAccountActionTriggers + rpcParams *v1.AttrSetActionTrigger *CommandExecuter } @@ -48,7 +48,7 @@ func (self *CmdSetTriggers) RpcMethod() string { func (self *CmdSetTriggers) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - self.rpcParams = &v1.AttrSetAccountActionTriggers{} + self.rpcParams = &v1.AttrSetActionTrigger{} } return self.rpcParams } diff --git a/console/triggers.go b/console/triggers.go new file mode 100644 index 000000000..454d4994c --- /dev/null +++ b/console/triggers.go @@ -0,0 +1,66 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2012-2015 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package console + +import ( + "github.com/cgrates/cgrates/apier/v1" + "github.com/cgrates/cgrates/engine" +) + +func init() { + c := &CmdGetTriggers{ + name: "triggers", + rpcMethod: "ApierV1.GetActionTriggers", + rpcParams: &v1.AttrGetActionTriggers{}, + } + commands[c.Name()] = c + c.CommandExecuter = &CommandExecuter{c} +} + +// Commander implementation +type CmdGetTriggers struct { + name string + rpcMethod string + rpcParams *v1.AttrGetActionTriggers + *CommandExecuter +} + +func (self *CmdGetTriggers) Name() string { + return self.name +} + +func (self *CmdGetTriggers) RpcMethod() string { + return self.rpcMethod +} + +func (self *CmdGetTriggers) RpcParams(reset bool) interface{} { + if reset || self.rpcParams == nil { + self.rpcParams = &v1.AttrGetActionTriggers{} + } + return self.rpcParams +} + +func (self *CmdGetTriggers) PostprocessRpcParams() error { + return nil +} + +func (self *CmdGetTriggers) RpcResult() interface{} { + atr := engine.ActionTriggers{} + return &atr +} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 84503d7a1..51abc1989 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -310,6 +310,12 @@ func (ms *MongoStorage) GetKeysForPrefix(prefix string, skipCache bool) ([]strin result = append(result, utils.ACTION_PLAN_PREFIX+keyResult.Key) } return result, nil + case utils.ACTION_TRIGGER_PREFIX: + iter := ms.db.C(colAtr).Find(bson.M{"key": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"key": 1}).Iter() + for iter.Next(&keyResult) { + result = append(result, utils.ACTION_TRIGGER_PREFIX+keyResult.Key) + } + return result, nil case utils.ACCOUNT_PREFIX: iter := ms.db.C(colAcc).Find(bson.M{"id": bson.M{"$regex": bson.RegEx{Pattern: subject}}}).Select(bson.M{"id": 1}).Iter() for iter.Next(&idResult) { diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 3eef405e1..237015a7b 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/cgrates/cgrates/apier/v1" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" @@ -117,3 +118,41 @@ func TestTpBalanceCounter(t *testing.T) { t.Errorf("Calling ApierV2.GetBalance received: %s", utils.ToIJSON(acnt)) } } + +func TestTpActionTriggers(t *testing.T) { + if !*testIntegration { + return + } + var atrs engine.ActionTriggers + if err := tpRPC.Call("ApierV1.GetActionTriggers", v1.AttrGetActionTriggers{GroupIDs: []string{}}, &atrs); err != nil { + t.Error("Got error on ApierV1.GetActionTriggers: ", err.Error()) + } else if len(atrs) != 9 { + t.Errorf("Calling v1.GetActionTriggers got: %v", atrs) + } + var reply string + if err := tpRPC.Call("ApierV1.SetActionTrigger", v1.AttrSetActionTrigger{ + GroupID: "TestATR", + UniqueID: "Unique atr id", + BalanceID: utils.StringPointer("BID1"), + }, &reply); err != nil { + t.Error("Got error on ApierV1.SetActionTrigger: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling v1.SetActionTrigger got: %v", reply) + } + + if err := tpRPC.Call("ApierV1.GetActionTriggers", v1.AttrGetActionTriggers{GroupIDs: []string{}}, &atrs); err != nil { + t.Error("Got error on ApierV1.GetActionTriggers: ", err.Error()) + } else if len(atrs) != 10 { + t.Errorf("Calling v1.GetActionTriggers got: %v", atrs) + } + if err := tpRPC.Call("ApierV1.GetActionTriggers", v1.AttrGetActionTriggers{GroupIDs: []string{"TestATR"}}, &atrs); err != nil { + t.Error("Got error on ApierV1.GetActionTriggers: ", err.Error()) + } else if len(atrs) != 1 { + t.Errorf("Calling v1.GetActionTriggers got: %v", atrs) + } + if atrs[0].ID != "TestATR" || + atrs[0].UniqueID != "Unique atr id" || + *atrs[0].Balance.ID != "BID1" { + t.Error("Wrong action trigger set: ", utils.ToIJSON(atrs[0])) + } +} From 9520b09cd69a60f143329a959b4f4c5baeb7b902 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 16:10:37 +0300 Subject: [PATCH 112/227] added test for zero cost --- data/tariffplans/testtp/AccountActions.csv | 1 + data/tariffplans/testtp/DestinationRates.csv | 3 +- data/tariffplans/testtp/Rates.csv | 1 + data/tariffplans/testtp/RatingPlans.csv | 1 + data/tariffplans/testtp/RatingProfiles.csv | 1 + general_tests/tp_it_test.go | 33 +++++++++++++++++++- 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index d04a5bde4..fc5722aa3 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -7,3 +7,4 @@ cgrates.org,1005,PREPAID_10,STANDARD_TRIGGERS,, cgrates.org,1009,TEST_EXE,,, cgrates.org,1010,TEST_DATA_r,,true, cgrates.org,1011,TEST_VOICE,,, +cgrates.org,1012,PREPAID_10,,, diff --git a/data/tariffplans/testtp/DestinationRates.csv b/data/tariffplans/testtp/DestinationRates.csv index 9be903aef..2f51c5c18 100644 --- a/data/tariffplans/testtp/DestinationRates.csv +++ b/data/tariffplans/testtp/DestinationRates.csv @@ -3,4 +3,5 @@ DR_RETAIL,GERMANY,RT_1CENT,*up,4,0, DR_RETAIL,GERMANY_MOBILE,RT_1CENT,*up,4,0, DR_DATA_1,*any,RT_DATA_2c,*up,4,0, DR_SMS_1,*any,RT_SMS_5c,*up,4,0, -DR_DATA_r,DATA_DEST,RT_DATA_r,*up,5,0, \ No newline at end of file +DR_DATA_r,DATA_DEST,RT_DATA_r,*up,5,0, +DR_FREE,GERMANY,RT_ZERO,*middle,2,0, diff --git a/data/tariffplans/testtp/Rates.csv b/data/tariffplans/testtp/Rates.csv index 21c9abc4a..b8860104b 100644 --- a/data/tariffplans/testtp/Rates.csv +++ b/data/tariffplans/testtp/Rates.csv @@ -3,3 +3,4 @@ RT_1CENT,0,1,1s,1s,0s RT_DATA_2c,0,0.002,10,10,0 RT_SMS_5c,0,0.005,1,1,0 RT_DATA_r,0,0.1,1048576,10240,0 +RT_ZERO,0,0,1,1,0 diff --git a/data/tariffplans/testtp/RatingPlans.csv b/data/tariffplans/testtp/RatingPlans.csv index f7003dcf4..5be68df5d 100644 --- a/data/tariffplans/testtp/RatingPlans.csv +++ b/data/tariffplans/testtp/RatingPlans.csv @@ -3,3 +3,4 @@ RP_RETAIL,DR_RETAIL,ALWAYS,10 RP_DATA1,DR_DATA_1,ALWAYS,10 RP_SMS1,DR_SMS_1,ALWAYS,10 RP_DATAr,DR_DATA_r,ALWAYS,10 +RP_FREE,DR_FREE,ALWAYS,10 diff --git a/data/tariffplans/testtp/RatingProfiles.csv b/data/tariffplans/testtp/RatingProfiles.csv index ffa6bf2f6..3a6a28222 100644 --- a/data/tariffplans/testtp/RatingProfiles.csv +++ b/data/tariffplans/testtp/RatingProfiles.csv @@ -3,3 +3,4 @@ *out,cgrates.org,data,*any,2012-01-01T00:00:00Z,RP_DATA1,, *out,cgrates.org,sms,*any,2012-01-01T00:00:00Z,RP_SMS1,, *out,cgrates.org,data,datar,2016-01-01T00:00:00Z,RP_DATAr,, +*out,cgrates.org,call,free,2016-01-01T00:00:00Z,RP_FREE,, diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 237015a7b..cbea917f3 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -107,7 +107,7 @@ func TestTpBalanceCounter(t *testing.T) { var cc engine.CallCost if err := tpRPC.Call("Responder.Debit", cd, &cc); err != nil { t.Error("Got error on Responder.GetCost: ", err.Error()) - } else if cc.GetDuration() == 20 { + } else if cc.GetDuration() != 20*time.Second { t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc.GetDuration()) } var acnt *engine.Account @@ -156,3 +156,34 @@ func TestTpActionTriggers(t *testing.T) { t.Error("Wrong action trigger set: ", utils.ToIJSON(atrs[0])) } } + +func TestTpZeroCost(t *testing.T) { + if !*testIntegration { + return + } + tStart := time.Date(2016, 3, 31, 0, 0, 0, 0, time.UTC) + cd := engine.CallDescriptor{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "free", + Account: "1012", + Destination: "+49", + DurationIndex: 0, + TimeStart: tStart, + TimeEnd: tStart.Add(time.Duration(20) * time.Second), + } + var cc engine.CallCost + if err := tpRPC.Call("Responder.Debit", cd, &cc); err != nil { + t.Error("Got error on Responder.GetCost: ", err.Error()) + } else if cc.GetDuration() != 20*time.Second { + t.Errorf("Calling Responder.MaxDebit got callcost: %v", utils.ToIJSON(cc)) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1012"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } else if acnt.BalanceMap[utils.MONETARY][0].Value != 11.0 { + t.Errorf("Calling ApierV2.GetAccount received: %s", utils.ToIJSON(acnt)) + } +} From bec7f70fd3f1a1fed1d551422f899f3d57ca71cd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 16:15:45 +0300 Subject: [PATCH 113/227] fix apier integration test --- apier/v1/apier_local_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index cfa76e271..002d95e43 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1295,8 +1295,8 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) } else { if rcvStats.Destinations != 5 || - rcvStats.RatingPlans != 4 || - rcvStats.RatingProfiles != 4 || + rcvStats.RatingPlans != 5 || + rcvStats.RatingProfiles != 5 || rcvStats.Actions != 8 || rcvStats.DerivedChargers != 3 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) From d8a5dff8ddfff882c008c283dd72f9bfb36fe87c Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 23:28:23 +0300 Subject: [PATCH 114/227] rsync and tmux for devel docker --- data/docker/devel/Dockerfile | 5 ++++- data/docker/devel/start.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/data/docker/devel/Dockerfile b/data/docker/devel/Dockerfile index 47e251317..ea0a8b803 100644 --- a/data/docker/devel/Dockerfile +++ b/data/docker/devel/Dockerfile @@ -17,7 +17,7 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 RUN echo 'deb http://repo.mongodb.org/apt/debian wheezy/mongodb-org/3.2 main' | tee '/etc/apt/sources.list.d/mongodb-org-3.2.list' # install dependencies -RUN apt-get -y update && apt-get -y install git bzr mercurial redis-server mysql-server python-pycurl python-mysqldb postgresql postgresql-client sudo wget freeswitch-meta-vanilla vim zsh mongodb-org +RUN apt-get -y update && apt-get -y install git bzr mercurial redis-server mysql-server python-pycurl python-mysqldb postgresql postgresql-client sudo wget freeswitch-meta-vanilla vim zsh mongodb-org tmux rsyslog # add mongo conf COPY mongod.conf /etc/mongod.conf @@ -34,6 +34,9 @@ RUN GOROOT=/root/go GOPATH=/root/code /root/go/bin/go get github.com/Masterminds #install oh-my-zsh RUN TERM=xterm sh -c "$(wget https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh -O -)"; exit 0 +# change shell for tmux +RUN chsh -s /usr/bin/zsh + # cleanup RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/data/docker/devel/start.sh b/data/docker/devel/start.sh index ac7992663..0a4a75591 100755 --- a/data/docker/devel/start.sh +++ b/data/docker/devel/start.sh @@ -2,6 +2,7 @@ sed -i 's/127.0.0.1/0.0.0.0/g' /etc/redis/redis.conf /etc/mysql/my.cnf echo 'host all all 0.0.0.0/32 md5'>>/etc/postgresql/9.4/main/pg_hba.conf +/etc/init.d/rsyslog start /etc/init.d/mysql start /etc/init.d/postgresql start /etc/init.d/redis-server start From 8bdfc8803e6259579816911428c5c3b835c4967b Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 12:48:19 +0300 Subject: [PATCH 115/227] update glide on get --- packages/squeeze/rules | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/squeeze/rules b/packages/squeeze/rules index b813db9c6..58007a4bf 100755 --- a/packages/squeeze/rules +++ b/packages/squeeze/rules @@ -5,7 +5,6 @@ export DH_VERBOSE=1 export GOPATH=$(CURDIR) -export GO15VENDOREXPERIMENT=1 PKGDIR=debian/cgrates SRCDIR=src/github.com/cgrates/cgrates @@ -24,7 +23,7 @@ binary-arch: clean dh_installdirs mkdir -p src/github.com/cgrates ln -sf $(CURDIR) src/github.com/cgrates - go get -v github.com/Masterminds/glide + go get -u -v github.com/Masterminds/glide $(GOPATH)/bin/glide install exec $(CURDIR)/build.sh mkdir -p $(PKGDIR)/usr/bin From a9352129d6373ce4eed0d05419c8cdb5ae808b74 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 13:37:05 +0300 Subject: [PATCH 116/227] glide install force --- packages/squeeze/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/squeeze/rules b/packages/squeeze/rules index 58007a4bf..34115a611 100755 --- a/packages/squeeze/rules +++ b/packages/squeeze/rules @@ -24,7 +24,7 @@ binary-arch: clean mkdir -p src/github.com/cgrates ln -sf $(CURDIR) src/github.com/cgrates go get -u -v github.com/Masterminds/glide - $(GOPATH)/bin/glide install + $(GOPATH)/bin/glide install --force exec $(CURDIR)/build.sh mkdir -p $(PKGDIR)/usr/bin cp $(GOPATH)/bin/cgr-* $(PKGDIR)/usr/bin/ From a307dc3ce07336d1a791f2c38c3ad6b2d3fcf01b Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 4 Apr 2016 13:02:06 +0200 Subject: [PATCH 117/227] Support for latest gorm --- apier/v2/cdrs_mysql_local_test.go | 1 + data/conf/samples/dmtagent/cgrates.json | 4 ++-- data/storage/mysql/create_cdrs_tables.sql | 10 +++++----- data/storage/postgres/create_cdrs_tables.sql | 6 +++--- engine/cdrs.go | 2 +- engine/models.go | 4 ++-- engine/storage_sql.go | 20 ++++++++++---------- glide.lock | 17 +++-------------- 8 files changed, 27 insertions(+), 37 deletions(-) diff --git a/apier/v2/cdrs_mysql_local_test.go b/apier/v2/cdrs_mysql_local_test.go index bbbed922e..60fddfc70 100644 --- a/apier/v2/cdrs_mysql_local_test.go +++ b/apier/v2/cdrs_mysql_local_test.go @@ -247,6 +247,7 @@ func TestV2CDRsMySQLRateWithoutTP(t *testing.T) { if !*testLocal { return } + //"d32a571d7bcbc6700fd35c1c0c5c6f458a62e260" rawCdrCGRID := utils.Sha1("bbb1", time.Date(2015, 11, 21, 10, 47, 24, 0, time.UTC).String()) // Rate the injected CDR, should not rate it since we have no TP loaded attrs := utils.AttrRateCdrs{CgrIds: []string{rawCdrCGRID}} diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index 6033f4bd2..93fe0df45 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -47,8 +47,8 @@ "sm_generic": { "enabled": true, - "rater": "127.0.0.1:2013", - "cdrs": "127.0.0.1:2013", + "rater": "internal", + "cdrs": "internal", }, "diameter_agent": { diff --git a/data/storage/mysql/create_cdrs_tables.sql b/data/storage/mysql/create_cdrs_tables.sql index 2bd6d9a59..c6e86d496 100644 --- a/data/storage/mysql/create_cdrs_tables.sql +++ b/data/storage/mysql/create_cdrs_tables.sql @@ -29,9 +29,9 @@ CREATE TABLE cdrs ( cost DECIMAL(20,4) NOT NULL, cost_details text, extra_info text, - created_at TIMESTAMP, - updated_at TIMESTAMP, - deleted_at TIMESTAMP, + created_at TIMESTAMP NULL, + updated_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, PRIMARY KEY (id), UNIQUE KEY cdrrun (cgrid, run_id, origin_id) ); @@ -46,8 +46,8 @@ CREATE TABLE sm_costs ( cost_source varchar(64) NOT NULL, `usage` DECIMAL(30,9) NOT NULL, cost_details text, - created_at TIMESTAMP, - deleted_at TIMESTAMP, + created_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, PRIMARY KEY (`id`), UNIQUE KEY costid (cgrid,run_id), KEY origin_idx (origin_host, origin_id), diff --git a/data/storage/postgres/create_cdrs_tables.sql b/data/storage/postgres/create_cdrs_tables.sql index a44c24520..ed9751694 100644 --- a/data/storage/postgres/create_cdrs_tables.sql +++ b/data/storage/postgres/create_cdrs_tables.sql @@ -30,8 +30,8 @@ CREATE TABLE cdrs ( cost_details jsonb, extra_info text, created_at TIMESTAMP, - updated_at TIMESTAMP, - deleted_at TIMESTAMP, + updated_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, UNIQUE (cgrid, run_id, origin_id) ); ; @@ -50,7 +50,7 @@ CREATE TABLE sm_costs ( usage NUMERIC(30,9) NOT NULL, cost_details jsonb, created_at TIMESTAMP, - deleted_at TIMESTAMP, + deleted_at TIMESTAMP NULL, UNIQUE (cgrid, run_id) ); DROP INDEX IF EXISTS cgrid_smcost_idx; diff --git a/engine/cdrs.go b/engine/cdrs.go index dd164fe03..9d43aa862 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -375,7 +375,7 @@ func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { } return cdrsRated, nil } - if err != nil && (err == gorm.RecordNotFound || err == mgov2.ErrNotFound) { //calculate CDR as for pseudoprepaid + if err != nil && (err == gorm.ErrRecordNotFound || err == mgov2.ErrNotFound) { //calculate CDR as for pseudoprepaid utils.Logger.Warning(fmt.Sprintf(" WARNING: Could not find CallCostLog for cgrid: %s, source: %s, runid: %s, will recalculate", cdr.CGRID, utils.SESSION_MANAGER_SOURCE, cdr.RunID)) qryCC, err = self.getCostFromRater(cdr) } diff --git a/engine/models.go b/engine/models.go index e5c686892..127c04757 100644 --- a/engine/models.go +++ b/engine/models.go @@ -440,7 +440,7 @@ type TBLCDRs struct { ExtraInfo string CreatedAt time.Time UpdatedAt time.Time - DeletedAt time.Time + DeletedAt *time.Time } func (t TBLCDRs) TableName() string { @@ -457,7 +457,7 @@ type TBLSMCosts struct { Usage float64 CostDetails string CreatedAt time.Time - DeletedAt time.Time + DeletedAt *time.Time } func (t TBLSMCosts) TableName() string { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 07b878a37..ddb363aec 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -34,7 +34,7 @@ import ( type SQLStorage struct { Db *sql.DB - db gorm.DB + db *gorm.DB } func (self *SQLStorage) Close() { @@ -675,8 +675,8 @@ func (self *SQLStorage) SetCDR(cdr *CDR, allowUpdate bool) error { return saved.Error } tx = self.db.Begin() - updated := tx.Model(TBLCDRs{}).Where(&TBLCDRs{Cgrid: cdr.CGRID, RunID: cdr.RunID}).Updates( - &TBLCDRs{ + updated := tx.Model(&TBLCDRs{}).Where(&TBLCDRs{Cgrid: cdr.CGRID, RunID: cdr.RunID, OriginID: cdr.OriginID}).Updates( + TBLCDRs{ OriginHost: cdr.OriginHost, Source: cdr.Source, OriginID: cdr.OriginID, @@ -718,9 +718,6 @@ func (self *SQLStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, q := self.db.Table(utils.TBL_CDRS).Select("*") if qryFltr.Unscoped { q = q.Unscoped() - } else { - // Query filter - q = q.Where("(deleted_at IS NULL OR deleted_at <= '0001-01-02')") // Soft deletes } // Add filters, use in to replace the high number of ORs if len(qryFltr.CGRIDs) != 0 { @@ -957,12 +954,15 @@ func (self *SQLStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, q.Find(&results) for _, result := range results { - var extraFieldsMp map[string]string - if err := json.Unmarshal([]byte(result.ExtraFields), &extraFieldsMp); err != nil { - return nil, 0, fmt.Errorf("JSON unmarshal error for cgrid: %s, runid: %v, error: %s", result.Cgrid, result.RunID, err.Error()) + extraFieldsMp := make(map[string]string) + if result.ExtraFields != "" { + if err := json.Unmarshal([]byte(result.ExtraFields), &extraFieldsMp); err != nil { + utils.Logger.Debug(fmt.Sprintf("Unmarshall json for result: %+v, error: %v", result.ExtraFields, err)) + return nil, 0, fmt.Errorf("JSON unmarshal error for cgrid: %s, runid: %v, error: %s", result.Cgrid, result.RunID, err.Error()) + } } var callCost CallCost - if len(result.CostDetails) != 0 { + if result.CostDetails != "" { if err := json.Unmarshal([]byte(result.CostDetails), &callCost); err != nil { return nil, 0, fmt.Errorf("JSON unmarshal callcost error for cgrid: %s, runid: %v, error: %s", result.Cgrid, result.RunID, err.Error()) } diff --git a/glide.lock b/glide.lock index fa156bc51..bc1e4e4f1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,6 @@ -hash: c4e3a1bdd7452ec3af195e09b8b3b1b9a61e36edfad557aeb01686706019c352 -updated: 2016-03-09T00:08:37.493018177+02:00 +hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 +updated: 2016-04-01T13:47:06.215526502+02:00 imports: -- name: github.com/cenkalti/hub - version: 57d753b5f4856e77b3cf8ecce78c97215a7d324d - name: github.com/cenkalti/rpc2 version: 2d1be381ce47537e9e076b2b76dc70933162e4e9 - name: github.com/cgrates/fsock @@ -23,22 +21,16 @@ imports: - diam/datatype - diam/dict - diam/sm - - diam/sm/smparser - - diam/sm/smpeer - name: github.com/go-sql-driver/mysql version: 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16 - name: github.com/gorhill/cronexpr version: a557574d6c024ed6e36acc8b610f5f211c91568a - name: github.com/jinzhu/gorm - version: 2f7811c55f286c55cfc3a2aefb5c4049b9cd5214 -- name: github.com/jinzhu/inflection - version: 3272df6c21d04180007eb3349844c89a3856bc25 + version: 2530dcbccd9c9ff7ce5a903e01bbf8601b726112 - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq version: 165a3529e799da61ab10faed1fabff3662d6193f - subpackages: - - oid - name: github.com/mediocregopher/radix.v2 version: 7bdaf7c45ec452ca691ab20535471e24460f0876 subpackages: @@ -54,13 +46,10 @@ imports: version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 subpackages: - websocket - - context - name: gopkg.in/fsnotify.v1 version: 875cf421b32f8f1b31bd43776297876d01542279 - name: gopkg.in/mgo.v2 version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson - - internal/sasl - - internal/scram devImports: [] From 16753edf5082edc6c2bf7e0364608c9bf81d60a3 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 4 Apr 2016 13:04:26 +0200 Subject: [PATCH 118/227] Updating glide.lock --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index bc1e4e4f1..77e0b037e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-01T13:47:06.215526502+02:00 +updated: 2016-04-04T13:02:20.098184491+02:00 imports: - name: github.com/cenkalti/rpc2 version: 2d1be381ce47537e9e076b2b76dc70933162e4e9 From 3ba9a2d0f270d930d88d208cc22f2cafd5c709d8 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 15:24:55 +0300 Subject: [PATCH 119/227] more smg session debug logging --- sessionmanager/smg_session.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index bc71236dd..57c2c8268 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -79,6 +79,8 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { // Attempts to debit a duration, returns maximum duration which can be debitted or error func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.Duration, error) { requestedDuration := dur + //utils.Logger.Debug(fmt.Sprintf("InitDur: %f, lastUsed: %f", requestedDuration.Seconds(), lastUsed.Seconds())) + //utils.Logger.Debug(fmt.Sprintf("TotalUsage: %f, extraDuration: %f", self.totalUsage.Seconds(), self.extraDuration.Seconds())) self.totalUsage += lastUsed // Should reflect the total usage so far if lastUsed > 0 { self.extraDuration = self.lastUsage - lastUsed @@ -109,6 +111,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.TimeEnd = cc.GetEndTime() // set debited timeEnd // update call duration with real debited duration ccDuration := cc.GetDuration() + utils.Logger.Debug(fmt.Sprintf("CCDur: %f", ccDuration.Seconds())) if ccDuration != dur { self.extraDuration = ccDuration - dur } @@ -121,8 +124,10 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.lastUsage = initialExtraDuration + ccDuration if ccDuration >= dur { // we got what we asked to be debited + //utils.Logger.Debug(fmt.Sprintf("returning normal: %f", requestedDuration.Seconds())) return requestedDuration, nil } + //utils.Logger.Debug(fmt.Sprintf("returning initialExtra: %f + ccDuration: %f", initialExtraDuration.Seconds(), ccDuration.Seconds())) return initialExtraDuration + ccDuration, nil } From 6419e0a36afd8e825aa2ffc681753746df5f9a61 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 4 Apr 2016 15:12:33 +0200 Subject: [PATCH 120/227] Temporary add inflection to glide lock file --- glide.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glide.lock b/glide.lock index 77e0b037e..2a8b4eeaa 100644 --- a/glide.lock +++ b/glide.lock @@ -52,4 +52,6 @@ imports: version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson +- name: github.com/jinzhu/inflection + version: 3272df6c21d04180007eb3349844c89a3856bc25 devImports: [] From 922b5509c6fa7787f15e494cb4fbfa9dea07bdaf Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 4 Apr 2016 15:14:26 +0200 Subject: [PATCH 121/227] Removing some Debug messages --- engine/storage_sql.go | 1 - sessionmanager/smg_session.go | 2 +- sessionmanager/smgeneric.go | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/engine/storage_sql.go b/engine/storage_sql.go index ddb363aec..5c8ab5528 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -957,7 +957,6 @@ func (self *SQLStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR, extraFieldsMp := make(map[string]string) if result.ExtraFields != "" { if err := json.Unmarshal([]byte(result.ExtraFields), &extraFieldsMp); err != nil { - utils.Logger.Debug(fmt.Sprintf("Unmarshall json for result: %+v, error: %v", result.ExtraFields, err)) return nil, 0, fmt.Errorf("JSON unmarshal error for cgrid: %s, runid: %v, error: %s", result.Cgrid, result.RunID, err.Error()) } } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 57c2c8268..05cc38605 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -111,7 +111,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.cd.TimeEnd = cc.GetEndTime() // set debited timeEnd // update call duration with real debited duration ccDuration := cc.GetDuration() - utils.Logger.Debug(fmt.Sprintf("CCDur: %f", ccDuration.Seconds())) + //utils.Logger.Debug(fmt.Sprintf("CCDur: %f", ccDuration.Seconds())) if ccDuration != dur { self.extraDuration = ccDuration - dur } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 08ac020b4..dd5e27fd5 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -208,7 +208,6 @@ func (self *SMGeneric) GetLcrSuppliers(gev SMGenericEvent, clnt *rpc2.Client) ([ // Called on session start func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - utils.Logger.Debug(fmt.Sprintf("### SessionStart, gev: %+v, sessions: %+v", gev, self.getSessions())) if err := self.sessionStart(gev, getClientConnId(clnt)); err != nil { self.sessionEnd(gev.GetUUID(), 0) return nilDuration, err @@ -222,7 +221,6 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - utils.Logger.Debug(fmt.Sprintf("### SessionUpdate, gev: %+v, sessions: %+v", gev, self.getSessions())) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update @@ -255,7 +253,6 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim // Called on session end, should stop debit loop func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { - utils.Logger.Debug(fmt.Sprintf("### SessionEnd, gev: %+v, sessions: %+v", gev, self.getSessions())) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update @@ -408,7 +405,6 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { - utils.Logger.Debug(fmt.Sprintf("### ProcessCdr, gev: %+v, sessions: %+v", gev, self.getSessions())) var reply string if err := self.cdrsrv.ProcessCdr(gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err From 14ac02f6845abafaf5b360b2d160db45991cb309 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 4 Apr 2016 16:20:10 +0200 Subject: [PATCH 122/227] Diameter - consider smallest MaxUsage returned --- agents/dmtagent.go | 15 ++++++++++++--- agents/dmtagent_it_test.go | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/agents/dmtagent.go b/agents/dmtagent.go index bce2fa6db..7c78d79bd 100644 --- a/agents/dmtagent.go +++ b/agents/dmtagent.go @@ -69,7 +69,7 @@ func (self *DiameterAgent) handlers() diam.Handler { return dSM } -func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestProcessor, cca *CCA) (bool, error) { +func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestProcessor, processorVars map[string]string, cca *CCA) (bool, error) { passesAllFilters := true for _, fldFilter := range reqProcessor.RequestFilter { if passes, _ := passesFieldFilter(ccr.diamMessage, fldFilter, nil); !passes { @@ -122,7 +122,6 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro } } var maxUsage float64 - processorVars := make(map[string]string) processorVars[CGRResultCode] = strconv.Itoa(diam.Success) processorVars[CGRError] = "" if reqProcessor.DryRun { // DryRun does not send over network @@ -166,6 +165,15 @@ func (self DiameterAgent) processCCR(ccr *CCR, reqProcessor *config.DARequestPro processorVars[CGRResultCode] = strconv.Itoa(DiameterRatingFailed) } } + if maxUsage < 0 { + maxUsage = 0 + } + if prevMaxUsageStr, hasKey := processorVars[CGRMaxUsage]; hasKey { + prevMaxUsage, _ := strconv.ParseFloat(prevMaxUsageStr, 64) + if prevMaxUsage < maxUsage { + maxUsage = prevMaxUsage + } + } processorVars[CGRMaxUsage] = strconv.FormatFloat(maxUsage, 'f', -1, 64) } if err := messageSetAVPsWithPath(cca.diamMessage, []interface{}{"Result-Code"}, processorVars[CGRResultCode], @@ -191,8 +199,9 @@ func (self *DiameterAgent) handleCCR(c diam.Conn, m *diam.Message) { } cca := NewBareCCAFromCCR(ccr, self.cgrCfg.DiameterAgentCfg().OriginHost, self.cgrCfg.DiameterAgentCfg().OriginRealm) var processed, lclProcessed bool + processorVars := make(map[string]string) // Shared between processors for _, reqProcessor := range self.cgrCfg.DiameterAgentCfg().RequestProcessors { - lclProcessed, err = self.processCCR(ccr, reqProcessor, cca) + lclProcessed, err = self.processCCR(ccr, reqProcessor, processorVars, cca) if lclProcessed { // Process local so we don't overwrite globally processed = lclProcessed } diff --git a/agents/dmtagent_it_test.go b/agents/dmtagent_it_test.go index 9826d75f1..b91891107 100644 --- a/agents/dmtagent_it_test.go +++ b/agents/dmtagent_it_test.go @@ -626,7 +626,7 @@ func TestDmtAgentSendCCRSimpaEvent(t *testing.T) { }), diam.NewAVP(29000, avp.Mbit, 2011, &diam.GroupedAVP{ // MC-Information AVP: []*diam.AVP{ - diam.NewAVP(29938, avp.Mbit, 2011, datatype.OctetString("0x38924012914528")), // HighLayerCharacteristics + diam.NewAVP(20938, avp.Mbit, 2011, datatype.OctetString("0x38924012914528")), // HighLayerCharacteristics diam.NewAVP(29002, avp.Mbit, 2011, datatype.UTF8String("12928471313847173")), // MC-Service-Id diam.NewAVP(29003, avp.Mbit, 2011, datatype.UTF8String("SPV123456012123")), // TransparentData diam.NewAVP(1201, avp.Mbit, 10415, &diam.GroupedAVP{ // MC-Information @@ -641,7 +641,7 @@ func TestDmtAgentSendCCRSimpaEvent(t *testing.T) { if err := dmtClient.SendMessage(ccr); err != nil { t.Error(err) } - time.Sleep(time.Duration(100) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) msg := dmtClient.ReceivedMessage() // Discard the received message so we can test next one if msg == nil { t.Fatal("No message returned") From bfcf26b09f0d5d887183e6768e2fd119960edbeb Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 18:55:09 +0300 Subject: [PATCH 123/227] fix glide compilation problems --- glide.lock | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 2a8b4eeaa..540f03ecf 100644 --- a/glide.lock +++ b/glide.lock @@ -1,6 +1,8 @@ hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-04T13:02:20.098184491+02:00 +updated: 2016-04-04T18:52:04.065644968+03:00 imports: +- name: github.com/cenkalti/hub + version: b864404b5f990410d56858a1b0a6fac23a85443f - name: github.com/cenkalti/rpc2 version: 2d1be381ce47537e9e076b2b76dc70933162e4e9 - name: github.com/cgrates/fsock @@ -21,16 +23,22 @@ imports: - diam/datatype - diam/dict - diam/sm + - diam/sm/smparser + - diam/sm/smpeer - name: github.com/go-sql-driver/mysql version: 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16 - name: github.com/gorhill/cronexpr version: a557574d6c024ed6e36acc8b610f5f211c91568a - name: github.com/jinzhu/gorm version: 2530dcbccd9c9ff7ce5a903e01bbf8601b726112 +- name: github.com/jinzhu/inflection + version: 3272df6c21d04180007eb3349844c89a3856bc25 - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq version: 165a3529e799da61ab10faed1fabff3662d6193f + subpackages: + - oid - name: github.com/mediocregopher/radix.v2 version: 7bdaf7c45ec452ca691ab20535471e24460f0876 subpackages: @@ -46,12 +54,13 @@ imports: version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 subpackages: - websocket + - context - name: gopkg.in/fsnotify.v1 version: 875cf421b32f8f1b31bd43776297876d01542279 - name: gopkg.in/mgo.v2 version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson -- name: github.com/jinzhu/inflection - version: 3272df6c21d04180007eb3349844c89a3856bc25 + - internal/sasl + - internal/scram devImports: [] From 6d8dff42430fc321989c6b2a12e8152141a55754 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 21:14:44 +0300 Subject: [PATCH 124/227] fix and test for stats purge fixes #384 --- engine/stats_queue.go | 13 ++++++++----- engine/stats_test.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/engine/stats_queue.go b/engine/stats_queue.go index bfc07356d..d69ffd8dc 100644 --- a/engine/stats_queue.go +++ b/engine/stats_queue.go @@ -184,16 +184,19 @@ func (sq *StatsQueue) purgeObsoleteCdrs() { } } if sq.conf.TimeWindow > 0 { + var index int for i, cdr := range sq.Cdrs { if time.Now().Sub(cdr.SetupTime) > sq.conf.TimeWindow { sq.removeFromMetrics(cdr) continue - } else { - if i > 0 { - sq.Cdrs = sq.Cdrs[i:] - } - break } + index = i + break + } + if index > 0 { + sq.Cdrs = sq.Cdrs[index:] + } else { + sq.Cdrs = make([]*QCdr, 0) } } } diff --git a/engine/stats_test.go b/engine/stats_test.go index def739f13..c08374ee5 100644 --- a/engine/stats_test.go +++ b/engine/stats_test.go @@ -458,3 +458,25 @@ func TestStatsSaveRestoreQeue(t *testing.T) { t.Errorf("Expecting %+v got: %+v", sq.Cdrs[0], recovered.Cdrs[0]) } } + +func TestStatsPurge(t *testing.T) { + sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, TimeWindow: 30 * time.Minute}) + cdr := &CDR{ + AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + Usage: 10 * time.Second, + Cost: 1, + } + sq.AppendCDR(cdr) + cdr.Cost = 2 + sq.AppendCDR(cdr) + cdr.Cost = 3 + sq.AppendCDR(cdr) + s := sq.GetStats() + if s[ASR] != -1 || + s[ACD] != -1 || + s[TCD] != -1 || + s[ACC] != -1 || + s[TCC] != -1 { + t.Errorf("Error getting stats: %+v", s) + } +} From 28f409934ce4f56c4a5358fa76df905a7ef6b2e2 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 21:25:36 +0300 Subject: [PATCH 125/227] test for queue length purge --- engine/stats_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/engine/stats_test.go b/engine/stats_test.go index c08374ee5..e74d8d86e 100644 --- a/engine/stats_test.go +++ b/engine/stats_test.go @@ -459,7 +459,7 @@ func TestStatsSaveRestoreQeue(t *testing.T) { } } -func TestStatsPurge(t *testing.T) { +func TestStatsPurgeTime(t *testing.T) { sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, TimeWindow: 30 * time.Minute}) cdr := &CDR{ AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), @@ -480,3 +480,25 @@ func TestStatsPurge(t *testing.T) { t.Errorf("Error getting stats: %+v", s) } } + +func TestStatsPurgeLength(t *testing.T) { + sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, QueueLength: 1}) + cdr := &CDR{ + AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + Usage: 10 * time.Second, + Cost: 1, + } + sq.AppendCDR(cdr) + cdr.Cost = 2 + sq.AppendCDR(cdr) + cdr.Cost = 3 + sq.AppendCDR(cdr) + s := sq.GetStats() + if s[ASR] != 100 || + s[ACD] != 10 || + s[TCD] != 10 || + s[ACC] != 3 || + s[TCC] != 3 { + t.Errorf("Error getting stats: %+v", s) + } +} From c85f3bdf23c1a7e6644197cfa9098db4423c3d40 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 4 Apr 2016 21:43:37 +0300 Subject: [PATCH 126/227] another cdrstats purge fix --- engine/stats_queue.go | 14 ++++++++------ engine/stats_test.go | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/engine/stats_queue.go b/engine/stats_queue.go index d69ffd8dc..28c39cbb6 100644 --- a/engine/stats_queue.go +++ b/engine/stats_queue.go @@ -184,19 +184,21 @@ func (sq *StatsQueue) purgeObsoleteCdrs() { } } if sq.conf.TimeWindow > 0 { - var index int + index := -1 for i, cdr := range sq.Cdrs { if time.Now().Sub(cdr.SetupTime) > sq.conf.TimeWindow { sq.removeFromMetrics(cdr) + index = i continue } - index = i break } - if index > 0 { - sq.Cdrs = sq.Cdrs[index:] - } else { - sq.Cdrs = make([]*QCdr, 0) + if index != -1 { + if index > 0 { + sq.Cdrs = sq.Cdrs[index:] + } else { + sq.Cdrs = make([]*QCdr, 0) + } } } } diff --git a/engine/stats_test.go b/engine/stats_test.go index e74d8d86e..a35c98e57 100644 --- a/engine/stats_test.go +++ b/engine/stats_test.go @@ -227,7 +227,7 @@ func TestStatsAppendCdr(t *testing.T) { if len(cdrStats.queues) != 2 || len(cdrStats.queues["CDRST1"].Cdrs) != 0 || len(cdrStats.queues["CDRST2"].Cdrs) != 1 { - t.Error("Error appending cdr to queue: ", len(cdrStats.queues)) + t.Error("Error appending cdr to queue: ", utils.ToIJSON(cdrStats.queues)) } } From 653b61e807c9d637197221b20c85a13d9691bfc7 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 5 Apr 2016 13:15:35 +0300 Subject: [PATCH 127/227] added EventTime to cdr stats --- engine/stats_queue.go | 18 ++++++++---- engine/stats_test.go | 64 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/engine/stats_queue.go b/engine/stats_queue.go index 28c39cbb6..6260fd1ee 100644 --- a/engine/stats_queue.go +++ b/engine/stats_queue.go @@ -56,6 +56,7 @@ var METRIC_TRIGGER_MAP = map[string]string{ type QCdr struct { SetupTime time.Time AnswerTime time.Time + EventTime time.Time Pdd time.Duration Usage time.Duration Cost float64 @@ -111,15 +112,19 @@ func (sq *StatsQueue) Load(saved *StatsQueue) { } } -func (sq *StatsQueue) AppendCDR(cdr *CDR) { +func (sq *StatsQueue) AppendCDR(cdr *CDR) *QCdr { sq.mux.Lock() defer sq.mux.Unlock() + var qcdr *QCdr if sq.conf.AcceptCdr(cdr) { - sq.appendQcdr(sq.simplifyCdr(cdr), true) + qcdr = sq.simplifyCdr(cdr) + sq.appendQcdr(qcdr, true) } + return qcdr } func (sq *StatsQueue) appendQcdr(qcdr *QCdr, runTrigger bool) { + qcdr.EventTime = time.Now() //used for TimeWindow sq.Cdrs = append(sq.Cdrs, qcdr) sq.addToMetrics(qcdr) sq.purgeObsoleteCdrs() @@ -151,6 +156,7 @@ func (sq *StatsQueue) appendQcdr(qcdr *QCdr, runTrigger bool) { } func (sq *StatsQueue) addToMetrics(cdr *QCdr) { + //log.Print("AddToMetrics: " + utils.ToIJSON(cdr)) for _, metric := range sq.metrics { metric.AddCdr(cdr) } @@ -186,16 +192,16 @@ func (sq *StatsQueue) purgeObsoleteCdrs() { if sq.conf.TimeWindow > 0 { index := -1 for i, cdr := range sq.Cdrs { - if time.Now().Sub(cdr.SetupTime) > sq.conf.TimeWindow { + if time.Now().Sub(cdr.EventTime) > sq.conf.TimeWindow { sq.removeFromMetrics(cdr) index = i continue } break } - if index != -1 { - if index > 0 { - sq.Cdrs = sq.Cdrs[index:] + if index > -1 { + if index < len(sq.Cdrs)-1 { + sq.Cdrs = sq.Cdrs[index+1:] } else { sq.Cdrs = make([]*QCdr, 0) } diff --git a/engine/stats_test.go b/engine/stats_test.go index a35c98e57..c618fb373 100644 --- a/engine/stats_test.go +++ b/engine/stats_test.go @@ -35,6 +35,7 @@ func TestStatsQueueInit(t *testing.T) { func TestStatsValue(t *testing.T) { sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}}) cdr := &CDR{ + SetupTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), Usage: 10 * time.Second, Cost: 1, @@ -459,18 +460,16 @@ func TestStatsSaveRestoreQeue(t *testing.T) { } } -func TestStatsPurgeTime(t *testing.T) { +func TestStatsPurgeTimeOne(t *testing.T) { sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, TimeWindow: 30 * time.Minute}) cdr := &CDR{ + SetupTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), Usage: 10 * time.Second, Cost: 1, } - sq.AppendCDR(cdr) - cdr.Cost = 2 - sq.AppendCDR(cdr) - cdr.Cost = 3 - sq.AppendCDR(cdr) + qcdr := sq.AppendCDR(cdr) + qcdr.EventTime = qcdr.SetupTime s := sq.GetStats() if s[ASR] != -1 || s[ACD] != -1 || @@ -481,9 +480,62 @@ func TestStatsPurgeTime(t *testing.T) { } } +func TestStatsPurgeTime(t *testing.T) { + sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, TimeWindow: 30 * time.Minute}) + cdr := &CDR{ + SetupTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + Usage: 10 * time.Second, + Cost: 1, + } + qcdr := sq.AppendCDR(cdr) + qcdr.EventTime = qcdr.SetupTime + cdr.Cost = 2 + qcdr = sq.AppendCDR(cdr) + qcdr.EventTime = qcdr.SetupTime + cdr.Cost = 3 + qcdr = sq.AppendCDR(cdr) + qcdr.EventTime = qcdr.SetupTime + s := sq.GetStats() + if s[ASR] != -1 || + s[ACD] != -1 || + s[TCD] != -1 || + s[ACC] != -1 || + s[TCC] != -1 { + t.Errorf("Error getting stats: %+v", s) + } +} + +func TestStatsPurgeTimeFirst(t *testing.T) { + sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, TimeWindow: 30 * time.Minute}) + cdr := &CDR{ + SetupTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + Usage: 10 * time.Second, + Cost: 1, + } + qcdr := sq.AppendCDR(cdr) + cdr.Cost = 2 + cdr.SetupTime = time.Date(2024, 7, 14, 14, 25, 0, 0, time.UTC) + cdr.AnswerTime = time.Date(2024, 7, 14, 14, 25, 0, 0, time.UTC) + qcdr.EventTime = qcdr.SetupTime + sq.AppendCDR(cdr) + cdr.Cost = 3 + sq.AppendCDR(cdr) + s := sq.GetStats() + if s[ASR] != 100 || + s[ACD] != 10 || + s[TCD] != 20 || + s[ACC] != 2.5 || + s[TCC] != 5 { + t.Errorf("Error getting stats: %+v", s) + } +} + func TestStatsPurgeLength(t *testing.T) { sq := NewStatsQueue(&CdrStats{Metrics: []string{ASR, ACD, TCD, ACC, TCC}, QueueLength: 1}) cdr := &CDR{ + SetupTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), AnswerTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), Usage: 10 * time.Second, Cost: 1, From 23eaaa0bc7a8aed6e5815679be3bfc117fda0182 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 5 Apr 2016 17:44:29 +0300 Subject: [PATCH 128/227] Added BalanceValue template field to cdrlog fixes #419 --- engine/account.go | 3 ++- engine/action.go | 15 ++++++++++---- engine/actions_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/engine/account.go b/engine/account.go index 45497e32f..b356a8ab3 100644 --- a/engine/account.go +++ b/engine/account.go @@ -190,6 +190,7 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { b.SubstractValue(bClone.GetValue()) b.dirty = true found = true + a.balanceValue = b.GetValue() } } // if it is not found then we add it to the list @@ -207,7 +208,7 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { } } bClone.dirty = true // Mark the balance as dirty since we have modified and it should be checked by action triggers - + a.balanceValue = bClone.GetValue() bClone.Uuid = utils.GenUUID() // alway overwrite the uuid for consistency // load ValueFactor if defined in extra parametrs if a.ExtraParameters != "" { diff --git a/engine/action.go b/engine/action.go index 07152a58d..62ca439dd 100644 --- a/engine/action.go +++ b/engine/action.go @@ -45,6 +45,7 @@ type Action struct { ExpirationString string // must stay as string because it can have relative values like 1month Weight float64 Balance *BalanceFilter + balanceValue float64 // balance value after action execution, used with cdrlog } const ( @@ -181,6 +182,8 @@ func parseTemplateValue(rsrFlds utils.RSRFields, acnt *Account, action *Action) parsedValue += rsrFld.ParseValue(action.Id) case "ActionType": parsedValue += rsrFld.ParseValue(action.ActionType) + case "ActionValue": + parsedValue += rsrFld.ParseValue(strconv.FormatFloat(b.GetValue(), 'f', -1, 64)) case "BalanceType": parsedValue += rsrFld.ParseValue(action.Balance.GetType()) case "BalanceUUID": @@ -188,7 +191,7 @@ func parseTemplateValue(rsrFlds utils.RSRFields, acnt *Account, action *Action) case "BalanceID": parsedValue += rsrFld.ParseValue(b.ID) case "BalanceValue": - parsedValue += rsrFld.ParseValue(strconv.FormatFloat(b.GetValue(), 'f', -1, 64)) + parsedValue += rsrFld.ParseValue(strconv.FormatFloat(action.balanceValue, 'f', -1, 64)) case "DestinationIDs": parsedValue += rsrFld.ParseValue(b.DestinationIDs.String()) case "ExtraParameters": @@ -215,7 +218,7 @@ func cdrLogAction(acc *Account, sq *StatsQueueTriggered, a *Action, acs Actions) utils.TENANT: utils.ParseRSRFieldsMustCompile(utils.TENANT, utils.INFIELD_SEP), utils.ACCOUNT: utils.ParseRSRFieldsMustCompile(utils.ACCOUNT, utils.INFIELD_SEP), utils.SUBJECT: utils.ParseRSRFieldsMustCompile(utils.ACCOUNT, utils.INFIELD_SEP), - utils.COST: utils.ParseRSRFieldsMustCompile("BalanceValue", utils.INFIELD_SEP), + utils.COST: utils.ParseRSRFieldsMustCompile("ActionValue", utils.INFIELD_SEP), } template := make(map[string]string) @@ -329,7 +332,9 @@ func topupResetAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actio } c := a.Clone() genericMakeNegative(c) - return genericDebit(ub, c, true) + err = genericDebit(ub, c, true) + a.balanceValue = c.balanceValue + return } func topupAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) (err error) { @@ -338,7 +343,9 @@ func topupAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) ( } c := a.Clone() genericMakeNegative(c) - return genericDebit(ub, c, false) + err = genericDebit(ub, c, false) + a.balanceValue = c.balanceValue + return } func debitResetAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) (err error) { diff --git a/engine/actions_test.go b/engine/actions_test.go index 1fea3f47d..bead47620 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2097,6 +2097,53 @@ func TestActionExpNoExp(t *testing.T) { } } +func TestActionCdrlogBalanceValue(t *testing.T) { + err := accountingStorage.SetAccount(&Account{ + ID: "cgrates.org:bv", + BalanceMap: map[string]Balances{ + utils.MONETARY: Balances{&Balance{ + Value: 10, + }}, + }, + }) + if err != nil { + t.Error("Error setting account: ", err) + } + at := &ActionTiming{ + accountIDs: utils.StringMap{"cgrates.org:bv": true}, + Timing: &RateInterval{}, + actions: []*Action{ + &Action{ + ActionType: TOPUP, + Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer(utils.MONETARY)}, + }, + &Action{ + ActionType: DEBIT, + Balance: &BalanceFilter{Value: utils.Float64Pointer(2.1), Type: utils.StringPointer(utils.MONETARY)}, + }, + &Action{ + ActionType: CDRLOG, + ExtraParameters: `{"BalanceValue":"BalanceValue"}`, + }, + }, + } + err = at.Execute() + acc, err := accountingStorage.GetAccount("cgrates.org:bv") + if err != nil || acc == nil { + t.Error("Error getting account: ", acc, err) + } + if acc.BalanceMap[utils.MONETARY][0].Value != 9 { + t.Errorf("Transaction didn't work: %v", acc.BalanceMap[utils.MONETARY][0].Value) + } + cdrs := make([]*CDR, 0) + json.Unmarshal([]byte(at.actions[2].ExpirationString), &cdrs) + if len(cdrs) != 2 || + cdrs[0].ExtraFields["BalanceValue"] != "11.1" || + cdrs[1].ExtraFields["BalanceValue"] != "9" { + t.Errorf("Wrong cdrlogs: %+v", cdrs[1]) + } +} + /**************** Benchmarks ********************************/ func BenchmarkUUID(b *testing.B) { From f3f257eff11c2be6965ea7871c0e92f11e53094a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 22:53:47 +0300 Subject: [PATCH 129/227] balance destinationIDs exceptions --- data/tariffplans/testtp/AccountActions.csv | 1 + data/tariffplans/testtp/ActionPlans.csv | 1 + data/tariffplans/testtp/Actions.csv | 3 +- engine/account.go | 19 +++++++++---- general_tests/tp_it_test.go | 31 ++++++++++++++++++++ utils/map.go | 12 ++++++-- utils/map_test.go | 33 ++++++++++++++++++++++ 7 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 utils/map_test.go diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index fc5722aa3..917d8d0b9 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -8,3 +8,4 @@ cgrates.org,1009,TEST_EXE,,, cgrates.org,1010,TEST_DATA_r,,true, cgrates.org,1011,TEST_VOICE,,, cgrates.org,1012,PREPAID_10,,, +cgrates.org,1013,TEST_NEG,,, \ No newline at end of file diff --git a/data/tariffplans/testtp/ActionPlans.csv b/data/tariffplans/testtp/ActionPlans.csv index 45e11a89d..54731428a 100644 --- a/data/tariffplans/testtp/ActionPlans.csv +++ b/data/tariffplans/testtp/ActionPlans.csv @@ -4,3 +4,4 @@ PREPAID_10,BONUS_1,ASAP,10 TEST_EXE,TOPUP_EXE,ALWAYS,10 TEST_DATA_r,TOPUP_DATA_r,ASAP,10 TEST_VOICE,TOPUP_VOICE,ASAP,10 +TEST_NEG,TOPUP_NEG,ASAP,10 diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index e85a4781e..98a48be14 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -7,4 +7,5 @@ CDRST_LOG,*log,,,,,,,,,,,,,,false,false,10 TOPUP_EXE,*topup,,,,*monetary,*out,,*any,,,*unlimited,,5,10,false,false,10 TOPUP_DATA_r,*topup,,,,*monetary,*out,,DATA_DEST,,,*unlimited,,5000000,10,false,false,10 TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 -TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 \ No newline at end of file +TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 +TOPUP_NEG,*topup,,,,*voice,*out,,GERMANY;!GERMANY_MOBILE,*zero1m,,*unlimited,,100,10,false,false,10 diff --git a/engine/account.go b/engine/account.go index b356a8ab3..ca02216d3 100644 --- a/engine/account.go +++ b/engine/account.go @@ -306,14 +306,21 @@ func (ub *Account) getBalancesForPrefix(prefix, category, direction, tor string, if x, err := cache2go.Get(utils.DESTINATION_PREFIX + p); err == nil { destIds := x.(map[interface{}]struct{}) for dId, _ := range destIds { - if b.DestinationIDs[dId.(string)] == true { - b.precision = len(p) - usefulBalances = append(usefulBalances, b) - break + includeDest, found := b.DestinationIDs[dId.(string)] + utils.Logger.Debug("DID: " + dId.(string)) + if found { + if includeDest { + b.precision = len(p) + usefulBalances = append(usefulBalances, b) + break + } else { // the balance had !, so now equals false => exclude balance + b.precision = 1 // fake to exit the outer loop + break + } } - if b.precision > 0 { + /*if b.precision > 0 { break - } + }*/ } } if b.precision > 0 { diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index cbea917f3..1ad58d833 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -187,3 +187,34 @@ func TestTpZeroCost(t *testing.T) { t.Errorf("Calling ApierV2.GetAccount received: %s", utils.ToIJSON(acnt)) } } + +func TestTpZeroNegativeCost(t *testing.T) { + if !*testIntegration { + return + } + tStart := time.Date(2016, 3, 31, 0, 0, 0, 0, time.UTC) + cd := engine.CallDescriptor{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "free", + Account: "1013", + Destination: "+4915", + DurationIndex: 0, + TimeStart: tStart, + TimeEnd: tStart.Add(time.Duration(20) * time.Second), + } + var cc engine.CallCost + if err := tpRPC.Call("Responder.Debit", cd, &cc); err != nil { + t.Error("Got error on Responder.GetCost: ", err.Error()) + } else if cc.GetDuration() != 20*time.Second { + t.Errorf("Calling Responder.MaxDebit got callcost: %v", utils.ToIJSON(cc)) + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1013"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } else if acnt.BalanceMap[utils.VOICE][0].Value != 100.0 { + t.Errorf("Calling ApierV2.GetAccount received: %s", utils.ToIJSON(acnt)) + } +} diff --git a/utils/map.go b/utils/map.go index 0bf197a7a..f9f63df76 100644 --- a/utils/map.go +++ b/utils/map.go @@ -67,7 +67,11 @@ func NewStringMap(s ...string) StringMap { for _, v := range s { v = strings.TrimSpace(v) if v != "" { - result[v] = true + if strings.HasPrefix(v, "!") { + result[v[1:]] = false + } else { + result[v] = true + } } } return result @@ -128,7 +132,11 @@ func StringMapFromSlice(s []string) StringMap { for _, v := range s { v = strings.TrimSpace(v) if v != "" { - result[v] = true + if strings.HasPrefix(v, "!") { + result[v[1:]] = false + } else { + result[v] = true + } } } return result diff --git a/utils/map_test.go b/utils/map_test.go new file mode 100644 index 000000000..a8d629492 --- /dev/null +++ b/utils/map_test.go @@ -0,0 +1,33 @@ +package utils + +import "testing" + +func TestStringMapParse(t *testing.T) { + sm := ParseStringMap("1;2;3;4") + if len(sm) != 4 { + t.Error("Error pasring map: ", sm) + } +} + +func TestStringMapParseNegative(t *testing.T) { + sm := ParseStringMap("1;2;!3;4") + if len(sm) != 4 { + t.Error("Error pasring map: ", sm) + } + if sm["3"] != false { + t.Error("Error parsing negative: ", sm) + } +} + +func TestStringMapCompare(t *testing.T) { + sm := ParseStringMap("1;2;!3;4") + if include, found := sm["2"]; include != true && found != true { + t.Error("Error detecting positive: ", sm) + } + if include, found := sm["3"]; include != false && found != true { + t.Error("Error detecting negative: ", sm) + } + if include, found := sm["5"]; include != false && found != false { + t.Error("Error detecting missing: ", sm) + } +} From cc390042292bd4b8e3a909a2d87f22f31a16570c Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 2 Apr 2016 22:59:59 +0300 Subject: [PATCH 130/227] removed logging --- engine/account.go | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/account.go b/engine/account.go index ca02216d3..b49ef4edd 100644 --- a/engine/account.go +++ b/engine/account.go @@ -307,7 +307,6 @@ func (ub *Account) getBalancesForPrefix(prefix, category, direction, tor string, destIds := x.(map[interface{}]struct{}) for dId, _ := range destIds { includeDest, found := b.DestinationIDs[dId.(string)] - utils.Logger.Debug("DID: " + dId.(string)) if found { if includeDest { b.precision = len(p) From 0989b71fb018779b5d91322b4203c4fcfb1acc5d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 5 Apr 2016 18:03:46 +0300 Subject: [PATCH 131/227] fix apier tests --- apier/v1/apier_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 002d95e43..5c2504cec 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1297,7 +1297,7 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { if rcvStats.Destinations != 5 || rcvStats.RatingPlans != 5 || rcvStats.RatingProfiles != 5 || - rcvStats.Actions != 8 || + rcvStats.Actions != 9 || rcvStats.DerivedChargers != 3 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) } From e5c05ff192c4bf2cda0b8c3087ee4e46c83acd66 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 6 Apr 2016 10:59:05 +0200 Subject: [PATCH 132/227] Diameter - pass empty AVP as ^$ in filter --- agents/libdmt.go | 5 +++++ agents/libdmt_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/agents/libdmt.go b/agents/libdmt.go index f3ee75dee..3467b980c 100644 --- a/agents/libdmt.go +++ b/agents/libdmt.go @@ -287,6 +287,11 @@ func passesFieldFilter(m *diam.Message, fieldFilter *utils.RSRField, processorVa if err != nil { return false, 0 } + if len(avps) == 0 { // No AVP found in request, treat it same as empty + if fieldFilter.FilterPasses("") { + return true, -1 + } + } for avpIdx, avpVal := range avps { // First match wins due to index if fieldFilter.FilterPasses(avpValAsString(avpVal)) { return true, avpIdx diff --git a/agents/libdmt_test.go b/agents/libdmt_test.go index 5bb1d7c01..fcddb1ce1 100644 --- a/agents/libdmt_test.go +++ b/agents/libdmt_test.go @@ -473,3 +473,10 @@ func TestCCRAsSMGenericEvent(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eSMGEv, rSMGEv) } } + +func TestPassesFieldFilter(t *testing.T) { + m := diam.NewRequest(diam.CreditControl, 4, nil) // Multiple-Services-Credit-Control>Rating-Group + if pass, _ := passesFieldFilter(m, utils.ParseRSRFieldsMustCompile("Multiple-Services-Credit-Control>Rating-Group(^$)", utils.INFIELD_SEP)[0], nil); !pass { + t.Error("Does not pass") + } +} From 411fcf9007a6f067a8a982555f9f9224921c5995 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 6 Apr 2016 21:18:59 +0200 Subject: [PATCH 133/227] Possible regression fix for recalculating prepaid CDRs without SMCost --- engine/cdrs.go | 8 +-- general_tests/tutorial_local_test.go | 97 +++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/engine/cdrs.go b/engine/cdrs.go index 9d43aa862..d2ec7c348 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -29,8 +29,6 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/cgrates/rpcclient" - "github.com/jinzhu/gorm" - mgov2 "gopkg.in/mgo.v2" ) var cdrServer *CdrServer // Share the server so we can use it in http handlers @@ -360,7 +358,9 @@ func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { if err == nil && len(smCosts) != 0 { break } - time.Sleep(delay()) + if i != 3 { + time.Sleep(delay()) + } } if len(smCosts) != 0 { // Cost retrieved from SMCost table for _, smCost := range smCosts { @@ -375,7 +375,7 @@ func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { } return cdrsRated, nil } - if err != nil && (err == gorm.ErrRecordNotFound || err == mgov2.ErrNotFound) { //calculate CDR as for pseudoprepaid + if len(smCosts) == 0 { //calculate CDR as for pseudoprepaid utils.Logger.Warning(fmt.Sprintf(" WARNING: Could not find CallCostLog for cgrid: %s, source: %s, runid: %s, will recalculate", cdr.CGRID, utils.SESSION_MANAGER_SOURCE, cdr.RunID)) qryCC, err = self.getCostFromRater(cdr) } diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index e5f4e8b61..6ed3076cd 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -716,7 +716,7 @@ func TestTutLocalCostErrors(t *testing.T) { } // Make sure queueids were created -func TestTutFsCallsCdrStats(t *testing.T) { +func TestTutLocalCdrStats(t *testing.T) { if !*testLocal { return } @@ -1290,6 +1290,101 @@ func TestTutLocalCdrStatsAfter(t *testing.T) { } */ +func TestTutLocalPrepaidCDRWithSMCost(t *testing.T) { + if !*testLocal { + return + } + cdr := &engine.CDR{CGRID: utils.Sha1("testprepaid1", time.Date(2016, 4, 6, 13, 29, 24, 0, time.UTC).String()), + ToR: utils.VOICE, OriginID: "testprepaid1", OriginHost: "192.168.1.1", Source: "TEST_PREPAID_CDR_SMCOST1", RequestType: utils.META_PREPAID, + Direction: utils.OUT, Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1003", + SetupTime: time.Date(2016, 4, 6, 13, 29, 24, 0, time.UTC), AnswerTime: time.Date(2016, 4, 6, 13, 30, 0, 0, time.UTC), + Usage: time.Duration(90) * time.Second, Supplier: "suppl1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}} + smCost := &engine.SMCost{CGRID: cdr.CGRID, + RunID: utils.META_DEFAULT, + OriginHost: cdr.OriginHost, + OriginID: cdr.OriginID, + CostSource: "TestTutLocalPrepaidCDRWithSMCost", + Usage: cdr.Usage.Seconds(), + CostDetails: &engine.CallCost{ + Direction: utils.OUT, + Destination: "1003", + Timespans: []*engine.TimeSpan{ + &engine.TimeSpan{ + TimeStart: time.Date(2016, 4, 6, 13, 30, 0, 0, time.UTC).Local(), // MongoDB saves timestamps in local timezone + TimeEnd: time.Date(2016, 4, 6, 13, 31, 30, 0, time.UTC).Local(), + DurationIndex: 0, + RateInterval: &engine.RateInterval{ + Rating: &engine.RIRate{Rates: engine.RateGroups{ + &engine.Rate{GroupIntervalStart: 0, Value: 0.01, RateIncrement: 10 * time.Second, RateUnit: time.Second}}}}, + }, + }, + TOR: utils.VOICE, + }, + } + var reply string + if err := tutLocalRpc.Call("CdrsV2.StoreSMCost", &engine.AttrCDRSStoreSMCost{Cost: smCost}, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for CDR to be processed + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, CGRIDs: []string{cdr.CGRID}} + if err := tutLocalRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].OriginID != cdr.OriginID { + t.Errorf("Unexpected OriginID for Cdr received: %+v", cdrs[0]) + } + if cdrs[0].Cost != 0.9 { + t.Errorf("Unexpected Cost for Cdr received: %+v", cdrs[0]) + } + } +} + +func TestTutLocalPrepaidCDRWithoutSMCost(t *testing.T) { + if !*testLocal { + return + } + cdr := &engine.CDR{CGRID: utils.Sha1("testprepaid2", time.Date(2016, 4, 6, 13, 29, 24, 0, time.UTC).String()), + ToR: utils.VOICE, OriginID: "testprepaid2", OriginHost: "192.168.1.1", Source: "TEST_PREPAID_CDR_NO_SMCOST1", RequestType: utils.META_PREPAID, + Direction: utils.OUT, Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1003", + SetupTime: time.Date(2016, 4, 6, 13, 29, 24, 0, time.UTC), AnswerTime: time.Date(2016, 4, 6, 13, 30, 0, 0, time.UTC), + Usage: time.Duration(90) * time.Second, Supplier: "suppl1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}} + var reply string + if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + /* + time.Sleep(time.Duration(7000) * time.Millisecond) // Give time for CDR to be processed + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, CGRIDs: []string{cdr.CGRID}} + if err := tutLocalRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].OriginID != cdr.OriginID { + t.Errorf("Unexpected OriginID for Cdr received: %+v", cdrs[0]) + } + if cdrs[0].Cost != 0.9 { + t.Errorf("Unexpected Cost for Cdr received: %+v", cdrs[0]) + } + } + */ +} + func TestTutLocalStopCgrEngine(t *testing.T) { if !*testLocal { return From b176511212ab4c1233fb250bd419e4dca0d247c4 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 7 Apr 2016 20:20:34 +0300 Subject: [PATCH 134/227] updated dependecies --- glide.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/glide.lock b/glide.lock index 540f03ecf..6f3370ee8 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-04T18:52:04.065644968+03:00 +updated: 2016-04-07T20:19:19.216414077+03:00 imports: - name: github.com/cenkalti/hub version: b864404b5f990410d56858a1b0a6fac23a85443f @@ -12,7 +12,7 @@ imports: - name: github.com/cgrates/osipsdagram version: 3d6beed663452471dec3ca194137a30d379d9e8f - name: github.com/cgrates/rpcclient - version: 79661b1e514823a9ac93b2b9e97e037ee190ba47 + version: 9302178591f1b3b5460c36a4ddae395f0f5397c5 - name: github.com/DisposaBoy/JsonConfigReader version: 33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4 - name: github.com/fiorix/go-diameter @@ -26,17 +26,17 @@ imports: - diam/sm/smparser - diam/sm/smpeer - name: github.com/go-sql-driver/mysql - version: 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16 + version: 1421caf44f6464fd2ee8de694c7508ee13f92964 - name: github.com/gorhill/cronexpr - version: a557574d6c024ed6e36acc8b610f5f211c91568a + version: f0984319b44273e83de132089ae42b1810f4933b - name: github.com/jinzhu/gorm - version: 2530dcbccd9c9ff7ce5a903e01bbf8601b726112 + version: 5174cc5c242a728b435ea2be8a2f7f998e15429b - name: github.com/jinzhu/inflection version: 3272df6c21d04180007eb3349844c89a3856bc25 - name: github.com/kr/pty version: f7ee69f31298ecbe5d2b349c711e2547a617d398 - name: github.com/lib/pq - version: 165a3529e799da61ab10faed1fabff3662d6193f + version: 3cd0097429be7d611bb644ef85b42bfb102ceea4 subpackages: - oid - name: github.com/mediocregopher/radix.v2 @@ -45,20 +45,20 @@ imports: - pool - redis - name: github.com/peterh/liner - version: ad1edfd30321d8f006ccf05f1e0524adeb943060 + version: 49ca65981c3cd7db64145977af1d186e9d317afa - name: github.com/ugorji/go - version: 187fa0f8af224437e08ecb3f208c4d1a94859a61 + version: a396ed22fc049df733440d90efe17475e3929ccb subpackages: - codec - name: golang.org/x/net - version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 + version: e45385e9b226f570b1f086bf287b25d3d4117776 subpackages: - websocket - context - name: gopkg.in/fsnotify.v1 version: 875cf421b32f8f1b31bd43776297876d01542279 - name: gopkg.in/mgo.v2 - version: d90005c5262a3463800497ea5a89aed5fe22c886 + version: b6e2fa371e64216a45e61072a96d4e3859f169da subpackages: - bson - internal/sasl From 155168406ccb0b6a92d0208c36da9c4da29dd985 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 8 Apr 2016 00:33:21 +0300 Subject: [PATCH 135/227] added session ttl to smgeneric --- config/config_defaults.go | 1 + config/config_json_test.go | 1 + config/libconfig_json.go | 1 + config/smconfig.go | 6 +++ data/conf/cgrates/cgrates.json | 1 + data/conf/samples/smg/cgrates.json | 1 + sessionmanager/data_it_test.go | 49 ++++++++++++++++++++++++ sessionmanager/smgeneric.go | 61 ++++++++++++++++++++++++------ 8 files changed, 109 insertions(+), 12 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index a0caa38b5..9d1cc2d87 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -210,6 +210,7 @@ const CGRATES_CFG_JSON = ` "debit_interval": "0s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this "max_call_duration": "3h", // maximum call duration a prepaid call can last + "session_ttl": "0s", // time after a session with no updates is terminated }, diff --git a/config/config_json_test.go b/config/config_json_test.go index e80cb6aaa..b7bc3c0c8 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -338,6 +338,7 @@ func TestSmGenericJsonCfg(t *testing.T) { Debit_interval: utils.StringPointer("0s"), Min_call_duration: utils.StringPointer("0s"), Max_call_duration: utils.StringPointer("3h"), + Session_ttl: utils.StringPointer("0s"), } if cfg, err := dfCgrJsonCfg.SmGenericJsonCfg(); err != nil { t.Error(err) diff --git a/config/libconfig_json.go b/config/libconfig_json.go index e40dedcdd..2dca787a4 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -174,6 +174,7 @@ type SmGenericJsonCfg struct { Debit_interval *string Min_call_duration *string Max_call_duration *string + Session_ttl *string } // SM-FreeSWITCH config section diff --git a/config/smconfig.go b/config/smconfig.go index 50e503bce..e828c76a0 100644 --- a/config/smconfig.go +++ b/config/smconfig.go @@ -79,6 +79,7 @@ type SmGenericConfig struct { DebitInterval time.Duration MinCallDuration time.Duration MaxCallDuration time.Duration + SessionTTL time.Duration } func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { @@ -113,6 +114,11 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { return err } } + if jsnCfg.Session_ttl != nil { + if self.SessionTTL, err = utils.ParseDurationWithSecs(*jsnCfg.Session_ttl); err != nil { + return err + } + } return nil } diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 4e7790c06..87cc9a860 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -187,6 +187,7 @@ // "debit_interval": "0s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this // "max_call_duration": "3h", // maximum call duration a prepaid call can last +// "session_ttl": "0s", // time after a session with no updates is terminated //}, diff --git a/data/conf/samples/smg/cgrates.json b/data/conf/samples/smg/cgrates.json index 0cbfc28c2..d6aaf2a6e 100644 --- a/data/conf/samples/smg/cgrates.json +++ b/data/conf/samples/smg/cgrates.json @@ -27,6 +27,7 @@ "enabled": true, "rater": "internal", "cdrs": "internal", + "session_ttl": "10ms", }, } diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 08b701df8..75e927719 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -384,3 +384,52 @@ func TestSMGDataDerivedChargingNoCredit(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.VOICE].GetTotalValue()) } } + +func TestSMGDataTTLExpired(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} + eAcntVal := 49999897600.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "1048576", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998842880.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + time.Sleep(50 * time.Millisecond) + eAcntVal = 49999897600.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } +} diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index dd5e27fd5..1dcbe7508 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -35,24 +35,47 @@ var ErrPartiallyExecuted = errors.New("Partially executed") func NewSMGeneric(cgrCfg *config.CGRConfig, rater engine.Connector, cdrsrv engine.Connector, timezone string, extconns *SMGExternalConnections) *SMGeneric { gsm := &SMGeneric{cgrCfg: cgrCfg, rater: rater, cdrsrv: cdrsrv, extconns: extconns, timezone: timezone, - sessions: make(map[string][]*SMGSession), sessionsMux: new(sync.Mutex), guard: engine.Guardian} + sessions: make(map[string][]*SMGSession), sessionTerminators: make(map[string]*smgSessionTerminator), sessionsMux: new(sync.RWMutex), guard: engine.Guardian} return gsm } type SMGeneric struct { - cgrCfg *config.CGRConfig // Separate from smCfg since there can be multiple - rater engine.Connector - cdrsrv engine.Connector - timezone string - sessions map[string][]*SMGSession //Group sessions per sessionId, multiple runs based on derived charging - extconns *SMGExternalConnections // Reference towards external connections manager - sessionsMux *sync.Mutex // Locks sessions map - guard *engine.GuardianLock // Used to lock on uuid + cgrCfg *config.CGRConfig // Separate from smCfg since there can be multiple + rater engine.Connector + cdrsrv engine.Connector + timezone string + sessions map[string][]*SMGSession //Group sessions per sessionId, multiple runs based on derived charging + sessionTerminators map[string]*smgSessionTerminator // terminate and cleanup the session if timer expires + extconns *SMGExternalConnections // Reference towards external connections manager + sessionsMux *sync.RWMutex // Locks sessions map + guard *engine.GuardianLock // Used to lock on uuid +} +type smgSessionTerminator struct { + timer *time.Timer + endChan chan bool } func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { self.sessionsMux.Lock() self.sessions[uuid] = append(self.sessions[uuid], s) + if self.cgrCfg.SmGenericConfig.SessionTTL > 0 { + if _, found := self.sessionTerminators[uuid]; !found { + timer := time.NewTimer(self.cgrCfg.SmGenericConfig.SessionTTL) + endChan := make(chan bool, 1) + go func() { + select { + case <-timer.C: + self.sessionEnd(uuid, 0) + case <-endChan: + timer.Stop() + } + }() + self.sessionTerminators[uuid] = &smgSessionTerminator{ + timer: timer, + endChan: endChan, + } + } + } self.sessionsMux.Unlock() } @@ -60,10 +83,14 @@ func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { func (self *SMGeneric) unindexSession(uuid string) bool { self.sessionsMux.Lock() defer self.sessionsMux.Unlock() - if _, hasIt := self.sessions[uuid]; !hasIt { + if _, found := self.sessions[uuid]; !found { return false } delete(self.sessions, uuid) + if st, found := self.sessionTerminators[uuid]; found { + st.endChan <- true + delete(self.sessionTerminators, uuid) + } return true } @@ -88,11 +115,20 @@ func (self *SMGeneric) getSessionIDsForPrefix(prefix string) []string { // Returns sessions/derived for a specific uuid func (self *SMGeneric) getSession(uuid string) []*SMGSession { - self.sessionsMux.Lock() - defer self.sessionsMux.Unlock() + self.sessionsMux.RLock() + defer self.sessionsMux.RUnlock() return self.sessions[uuid] } +// Updates the timer for the session to a new ttl +func (self *SMGeneric) resetTerminatorTimer(uuid string) { + self.sessionsMux.RLock() + defer self.sessionsMux.RUnlock() + if st, found := self.sessionTerminators[uuid]; found { + st.timer.Reset(self.cgrCfg.SmGenericConfig.SessionTTL) + } +} + // Handle a new session, pass the connectionId so we can communicate on disconnect request func (self *SMGeneric) sessionStart(evStart SMGenericEvent, connId string) error { sessionId := evStart.GetUUID() @@ -221,6 +257,7 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { + self.resetTerminatorTimer(gev.GetUUID()) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update From 7afc0de334f5e8210fcddda9c4787458e66ff9e7 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 8 Apr 2016 00:49:00 +0300 Subject: [PATCH 136/227] possible quick fix for #422 --- apier/v1/accounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index c6c161331..add9e9b97 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -567,7 +567,7 @@ func (self *ApierV1) RemoveBalances(attr *AttrSetBalance, reply *string) error { at := &engine.ActionTiming{} at.SetAccountIDs(utils.StringMap{accID: true}) a := &engine.Action{ - ActionType: engine.SET_BALANCE, + ActionType: engine.REMOVE_BALANCE, Balance: &engine.BalanceFilter{ Uuid: attr.BalanceUUID, ID: attr.BalanceID, From 2063d1bf410289282c0720679beac3e9ec2f0907 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 8 Apr 2016 07:46:21 +0200 Subject: [PATCH 137/227] Sample SMFS running as standalone --- data/conf/samples/smfs/smfs.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 data/conf/samples/smfs/smfs.json diff --git a/data/conf/samples/smfs/smfs.json b/data/conf/samples/smfs/smfs.json new file mode 100644 index 000000000..5176b1802 --- /dev/null +++ b/data/conf/samples/smfs/smfs.json @@ -0,0 +1,21 @@ +{ + +// Real-time Charging System for Telecom & ISP environments +// Copyright (C) ITsysCOM GmbH +// +// This file contains the default configuration hardcoded into CGRateS. +// This is what you get when you load CGRateS with an empty configuration file. + + +"sm_freeswitch": { + "enabled": true, // starts SessionManager service: + "rater": "127.0.0.1:2013", // address where to reach the Rater <""|internal|127.0.0.1:2013> + "cdrs": "127.0.0.1:2013", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> + "debit_interval": "5s", // interval to perform debits on. + "channel_sync_interval": "10s", + "connections":[ // instantiate connections to multiple FreeSWITCH servers + {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 15} + ], +}, + +} \ No newline at end of file From 0c3a07392e59d9ed0d7349227df20bf0422b0522 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 8 Apr 2016 10:48:43 +0200 Subject: [PATCH 138/227] ParseDate properly parsing RFC 3339 --- utils/coreutils.go | 2 +- utils/utils_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/utils/coreutils.go b/utils/coreutils.go index 0f05b1044..40697a0ea 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -195,7 +195,7 @@ func ParseDate(date string) (expDate time.Time, err error) { expDate = time.Now().AddDate(1, 0, 0) // add one year case date == "*month_end": expDate = GetEndOfMonth(time.Now()) - case strings.HasSuffix(date, "Z"): + case strings.HasSuffix(date, "Z") || strings.Index(date, "+") != -1: // Allow both Z and +hh:mm format expDate, err = time.Parse(time.RFC3339, date) default: unix, err := strconv.ParseInt(date, 10, 64) diff --git a/utils/utils_test.go b/utils/utils_test.go index 1edd0ada0..d33674aa8 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -156,7 +156,20 @@ func TestParseTimeDetectLayout(t *testing.T) { if err == nil { t.Errorf("Expecting error") } + tmStr = "2016-04-01T02:00:00+02:00" + expectedTime = time.Date(2016, 4, 1, 0, 0, 0, 0, time.UTC) + tm, err = ParseTimeDetectLayout(tmStr, "") + if err != nil { + t.Error(err) + } else if !tm.Equal(expectedTime) { + t.Errorf("Unexpected time parsed: %v, expecting: %v", tm, expectedTime) + } + _, err = ParseTimeDetectLayout(tmStr[1:], "") + if err == nil { + t.Errorf("Expecting error") + } sqlTmStr := "2013-12-30 15:00:01" + expectedTime = time.Date(2013, 12, 30, 15, 0, 1, 0, time.UTC) sqlTm, err := ParseTimeDetectLayout(sqlTmStr, "") if err != nil { t.Error(err) @@ -291,6 +304,11 @@ func TestParseDateRFC3339(t *testing.T) { if err != nil || !date.Equal(expected) { t.Error("error parsing date: ", expected.Sub(date)) } + date, err = ParseDate("2016-04-01T02:00:00+02:00") + expected = time.Date(2016, 4, 1, 0, 0, 0, 0, time.UTC) + if err != nil || !date.Equal(expected) { + t.Errorf("Expecting: %v, received: %v", expected, date) + } } func TestMissingStructFieldsCorrect(t *testing.T) { From 318ceed9c993a3816689889716330f053ca17735 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 11 Apr 2016 08:48:13 +0200 Subject: [PATCH 139/227] Adding InstanceID to status command, rpcclient integration tests with strategy first --- config/config.go | 2 + data/conf/samples/multiral1/cgrates.json | 17 +++ data/conf/samples/multiral2/cgrates.json | 17 +++ engine/responder.go | 1 + general_tests/rpcclient_it_test.go | 154 +++++++++++++++++++++++ general_tests/tutorial_local_test.go | 4 +- utils/consts.go | 1 + 7 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 data/conf/samples/multiral1/cgrates.json create mode 100644 data/conf/samples/multiral2/cgrates.json create mode 100644 general_tests/rpcclient_it_test.go diff --git a/config/config.go b/config/config.go index 6f21b4f7f..39963358a 100644 --- a/config/config.go +++ b/config/config.go @@ -60,6 +60,7 @@ func SetCgrConfig(cfg *CGRConfig) { func NewDefaultCGRConfig() (*CGRConfig, error) { cfg := new(CGRConfig) + cfg.InstanceID = utils.GenUUID() cfg.DataFolderPath = "/usr/share/cgrates/" cfg.SmGenericConfig = new(SmGenericConfig) cfg.SmFsConfig = new(SmFsConfig) @@ -167,6 +168,7 @@ func NewCGRConfigFromFolder(cfgDir string) (*CGRConfig, error) { // Holds system configuration, defaults are overwritten with values from config file if found type CGRConfig struct { + InstanceID string // Identifier for this engine instance TpDbType string TpDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. TpDbPort string // The port to bind to. diff --git a/data/conf/samples/multiral1/cgrates.json b/data/conf/samples/multiral1/cgrates.json new file mode 100644 index 000000000..310b3861d --- /dev/null +++ b/data/conf/samples/multiral1/cgrates.json @@ -0,0 +1,17 @@ +{ +// CGRateS Configuration file +// +// Used for multiple RAL configuration tests +// Starts rater, scheduler + +"listen": { + "rpc_json": ":2012", // RPC JSON listening address + "rpc_gob": ":2013", // RPC GOB listening address + "http": ":2080", // HTTP listening address +}, + +"rater": { + "enabled": true, // enable Rater service: +}, + +} \ No newline at end of file diff --git a/data/conf/samples/multiral2/cgrates.json b/data/conf/samples/multiral2/cgrates.json new file mode 100644 index 000000000..8f9270ae6 --- /dev/null +++ b/data/conf/samples/multiral2/cgrates.json @@ -0,0 +1,17 @@ +{ +// CGRateS Configuration file +// +// Used for multiple RAL configuration tests +// Starts RAL + +"listen": { + "rpc_json": ":12012", // RPC JSON listening address + "rpc_gob": ":12013", // RPC GOB listening address + "http": ":12080", // HTTP listening address +}, + +"rater": { + "enabled": true, // enable Rater service: +}, + +} \ No newline at end of file diff --git a/engine/responder.go b/engine/responder.go index e43c65040..4dd81c66c 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -551,6 +551,7 @@ func (rs *Responder) Status(arg string, reply *map[string]interface{}) (err erro memstats := new(runtime.MemStats) runtime.ReadMemStats(memstats) response := make(map[string]interface{}) + response[utils.InstanceID] = config.CgrConfig().InstanceID if rs.Bal != nil { response["Raters"] = rs.Bal.GetClientAddresses() } diff --git a/general_tests/rpcclient_it_test.go b/general_tests/rpcclient_it_test.go new file mode 100644 index 000000000..987b30662 --- /dev/null +++ b/general_tests/rpcclient_it_test.go @@ -0,0 +1,154 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can Storagetribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITH*out ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package general_tests + +import ( + "os/exec" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" +) + +var rpcITCfgPath1, rpcITCfgPath2 string +var rpcITCfg1, rpcITCfg2 *config.CGRConfig +var rpcRAL1, rpcRAL2 *rpcclient.RpcClient +var rpcPoolFirst *rpcclient.RpcClientPool +var ral1, ral2 *exec.Cmd +var err error +var ral1ID, ral2ID string + +func TestRPCITInitCfg(t *testing.T) { + if !*testIntegration { + return + } + rpcITCfgPath1 = path.Join(*dataDir, "conf", "samples", "multiral1") + rpcITCfgPath2 = path.Join(*dataDir, "conf", "samples", "multiral2") + // Init config first + rpcITCfg1, err = config.NewCGRConfigFromFolder(rpcITCfgPath1) + if err != nil { + t.Error(err) + } + rpcITCfg2, err = config.NewCGRConfigFromFolder(rpcITCfgPath2) + if err != nil { + t.Error(err) + } +} + +func TestRPCITStartEngine(t *testing.T) { + if !*testIntegration { + return + } + if ral1, err = engine.StopStartEngine(rpcITCfgPath1, *waitRater); err != nil { + t.Fatal(err) + } + if ral2, err = engine.StartEngine(rpcITCfgPath2, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func TestRPCITRpcConnPool(t *testing.T) { + if !*testIntegration { + return + } + rpcPoolFirst = rpcclient.NewRpcClientPool(rpcclient.POOL_FIRST) + rpcRAL1, err = rpcclient.NewRpcClient("tcp", rpcITCfg1.RPCJSONListen, 3, 1, rpcclient.JSON_RPC, nil) + if err != nil { + t.Fatal(err) + } + rpcPoolFirst.AddClient(rpcRAL1) + rpcRAL2, err = rpcclient.NewRpcClient("tcp", rpcITCfg2.RPCJSONListen, 3, 1, rpcclient.JSON_RPC, nil) + if err != nil { + t.Fatal(err) + } + rpcPoolFirst.AddClient(rpcRAL2) +} + +// Connect rpc client to rater +func TestRPCITStatusFirstInitial(t *testing.T) { + if !*testIntegration { + return + } + var status map[string]interface{} + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { + t.Error(err) + } else if status[utils.InstanceID].(string) == "" { + t.Error("Empty InstanceID received") + } else { + ral1ID = status[utils.InstanceID].(string) + } + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { // Make sure second time we land on the same instance + t.Error(err) + } else if status[utils.InstanceID].(string) != ral1ID { + t.Errorf("Expecting: %s, received: %s", ral1ID, status[utils.InstanceID].(string)) + } +} + +// Connect rpc client to rater +func TestRPCITStatusFirstFailover(t *testing.T) { + if !*testIntegration { + return + } + if err := ral1.Process.Kill(); err != nil { // Kill the first RAL + t.Error(err) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) + var status map[string]interface{} + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { + t.Error(err) + } else if status[utils.InstanceID].(string) == "" { + t.Error("Empty InstanceID received") + } else { + ral1ID = status[utils.InstanceID].(string) + } + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { // Make sure second time we land on the same instance + t.Error(err) + } else if status[utils.InstanceID].(string) != ral1ID { + t.Errorf("Expecting: %s, received: %s", ral1ID, status[utils.InstanceID].(string)) + } else { + ral2ID = status[utils.InstanceID].(string) + } +} + +func TestRPCITStatusFirstFailback(t *testing.T) { + if !*testIntegration { + return + } + if ral1, err = engine.StartEngine(rpcITCfgPath1, *waitRater); err != nil { + t.Fatal(err) + } + var status map[string]interface{} + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { + t.Error(err) + } else if status[utils.InstanceID].(string) == ral2ID { + t.Error("Should receive new ID") + } else { + ral1ID = status[utils.InstanceID].(string) + } + if err := rpcPoolFirst.Call("Responder.Status", "", &status); err != nil { // Make sure second time we land on the same instance + t.Error(err) + } else if status[utils.InstanceID].(string) != ral1ID { + t.Errorf("Expecting: %s, received: %s", ral1ID, status[utils.InstanceID].(string)) + } +} diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 244c5e794..17d986ea9 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -427,7 +427,7 @@ func TestTutLocalMaxDebit(t *testing.T) { TimeStart: tStart, TimeEnd: tStart.Add(time.Duration(120) * time.Second), } - cd.CgrId = "1" + cd.CgrID = "1" if err := tutLocalRpc.Call("Responder.MaxDebit", cd, &cc); err != nil { t.Error("Got error on Responder.GetCost: ", err.Error()) } else if cc.GetDuration() == 120 { @@ -444,7 +444,7 @@ func TestTutLocalMaxDebit(t *testing.T) { TimeStart: tStart, TimeEnd: tStart.Add(time.Duration(120) * time.Second), } - cd.CgrId = "2" + cd.CgrID = "2" if err := tutLocalRpc.Call("Responder.MaxDebit", cd, &cc); err != nil { t.Error("Got error on Responder.GetCost: ", err.Error()) } else if cc.GetDuration() != time.Duration(62)*time.Second { // We have as strategy *dsconnect diff --git a/utils/consts.go b/utils/consts.go index 0bacf323e..f3b9b5b5d 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -278,6 +278,7 @@ const ( UpdatedAt = "UpdatedAt" HandlerArgSep = "|" FlagForceDuration = "fd" + InstanceID = "InstanceID" ) var ( From df7e92268b937eca0079fcad7e8142449fde28bd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 11 Apr 2016 11:27:32 +0300 Subject: [PATCH 140/227] updated dependencies (for rpcclient) --- glide.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/glide.lock b/glide.lock index 6f3370ee8..50c7300b5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-07T20:19:19.216414077+03:00 +updated: 2016-04-11T11:24:07.55158903+03:00 imports: - name: github.com/cenkalti/hub version: b864404b5f990410d56858a1b0a6fac23a85443f @@ -12,7 +12,7 @@ imports: - name: github.com/cgrates/osipsdagram version: 3d6beed663452471dec3ca194137a30d379d9e8f - name: github.com/cgrates/rpcclient - version: 9302178591f1b3b5460c36a4ddae395f0f5397c5 + version: 64c9ff2b700d5a97a969a59d648bd75d09274143 - name: github.com/DisposaBoy/JsonConfigReader version: 33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4 - name: github.com/fiorix/go-diameter @@ -26,7 +26,7 @@ imports: - diam/sm/smparser - diam/sm/smpeer - name: github.com/go-sql-driver/mysql - version: 1421caf44f6464fd2ee8de694c7508ee13f92964 + version: 7ebe0a500653eeb1859664bed5e48dec1e164e73 - name: github.com/gorhill/cronexpr version: f0984319b44273e83de132089ae42b1810f4933b - name: github.com/jinzhu/gorm From 5a102384e3f9ccb7efad0d0c666e1bdb76f2c91d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 11 Apr 2016 13:17:32 +0300 Subject: [PATCH 141/227] format status memory sizes --- engine/responder.go | 4 ++-- utils/coreutils.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/engine/responder.go b/engine/responder.go index 2e1ff74e9..f0499c1f5 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -592,8 +592,8 @@ func (rs *Responder) Status(arg string, reply *map[string]interface{}) (err erro if rs.Bal != nil { response["Raters"] = rs.Bal.GetClientAddresses() } - response["memstat"] = memstats.HeapAlloc / 1024 - response["footprint"] = memstats.Sys / 1024 + response["memstat"] = utils.SizeFmt(float64(memstats.HeapAlloc), "") + response["footprint"] = utils.SizeFmt(float64(memstats.Sys), "") *reply = response return } diff --git a/utils/coreutils.go b/utils/coreutils.go index 40697a0ea..35453b5d5 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -533,3 +533,17 @@ func GetEndOfMonth(ref time.Time) time.Time { eom := time.Date(year, month, 1, 0, 0, 0, 0, ref.Location()) return eom.Add(-time.Second) } + +// formats number in K,M,G, etc. +func SizeFmt(num float64, suffix string) string { + if suffix == "" { + suffix = "B" + } + for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} { + if math.Abs(num) < 1024.0 { + return fmt.Sprintf("%3.1f%s%s", num, unit, suffix) + } + num /= 1024.0 + } + return fmt.Sprintf("%.1f%s%s", num, "Yi", suffix) +} From d40a9193e6538bbeccea4122d0418ccb4c4651ec Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 11 Apr 2016 20:31:01 +0200 Subject: [PATCH 142/227] SMG SessionTimers considering LastUsed, generating also CDR --- sessionmanager/data_it_test.go | 2 +- sessionmanager/smgeneric.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 75e927719..6944fdd8b 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -426,7 +426,7 @@ func TestSMGDataTTLExpired(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } time.Sleep(50 * time.Millisecond) - eAcntVal = 49999897600.000000 + eAcntVal = 49999887360.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 1dcbe7508..fd086efbf 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -65,7 +65,12 @@ func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { go func() { select { case <-timer.C: - self.sessionEnd(uuid, 0) + totalUsage := s.TotalUsage() + self.cgrCfg.SmGenericConfig.SessionTTL + self.sessionEnd(uuid, totalUsage) + cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) + cdr.Usage = totalUsage + var reply string + self.cdrsrv.ProcessCdr(cdr, &reply) case <-endChan: timer.Stop() } From 769960bf2f01f5d8971e83fb719814f1b5eea815 Mon Sep 17 00:00:00 2001 From: rbarrabe Date: Tue, 12 Apr 2016 11:35:15 +0200 Subject: [PATCH 143/227] Update huawei.xml --- data/diameter/dict/huawei/huawei.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/data/diameter/dict/huawei/huawei.xml b/data/diameter/dict/huawei/huawei.xml index 70e182ddd..4f48257d3 100644 --- a/data/diameter/dict/huawei/huawei.xml +++ b/data/diameter/dict/huawei/huawei.xml @@ -36,6 +36,15 @@ + + + + + + + + + From ea0f6745d0cac3f26f37875b4a504ce7f4213891 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 12 Apr 2016 19:36:40 +0300 Subject: [PATCH 144/227] derived charging exclude destination feature --- engine/handler_derivedcharging.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/engine/handler_derivedcharging.go b/engine/handler_derivedcharging.go index 52f48eb03..e845ac1dd 100644 --- a/engine/handler_derivedcharging.go +++ b/engine/handler_derivedcharging.go @@ -50,12 +50,10 @@ func DerivedChargersMatchesDest(dcs *utils.DerivedChargers, dest string) bool { for _, p := range utils.SplitPrefix(dest, MIN_PREFIX_MATCH) { if x, err := cache2go.Get(utils.DESTINATION_PREFIX + p); err == nil { destIds := x.(map[interface{}]struct{}) - for value := range dcs.DestinationIDs { - for idId := range destIds { - dId := idId.(string) - if value == dId { - return true - } + for dId := range destIds { + includeDest, found := dcs.DestinationIDs[dId.(string)] + if found { + return includeDest } } } From cbaeb9262664364e8fb03f637c78f2264e788e69 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 12 Apr 2016 19:48:01 +0300 Subject: [PATCH 145/227] dc exclude destination test --- engine/handler_derivedcharging_test.go | 18 ++++++++++++++++++ engine/history_test.go | 1 + engine/loader_csv_test.go | 3 ++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/engine/handler_derivedcharging_test.go b/engine/handler_derivedcharging_test.go index 8a2c98f54..bb793b9ba 100644 --- a/engine/handler_derivedcharging_test.go +++ b/engine/handler_derivedcharging_test.go @@ -119,3 +119,21 @@ func TestHandleDeivedChargersMatchDestNatRet(t *testing.T) { t.Error("Derived charger failed to match dest") } } + +func TestHandleDeivedChargersMatchDestSpec(t *testing.T) { + dcs := &utils.DerivedChargers{ + DestinationIDs: utils.NewStringMap("NAT", "SPEC"), + } + if !DerivedChargersMatchesDest(dcs, "0723045326") { + t.Error("Derived charger failed to match dest") + } +} + +func TestHandleDeivedChargersMatchDestNegativeSpec(t *testing.T) { + dcs := &utils.DerivedChargers{ + DestinationIDs: utils.NewStringMap("NAT", "!SPEC"), + } + if DerivedChargersMatchesDest(dcs, "0723045326") { + t.Error("Derived charger failed to match dest") + } +} diff --git a/engine/history_test.go b/engine/history_test.go index 3e51c5a4a..e3d797e4b 100644 --- a/engine/history_test.go +++ b/engine/history_test.go @@ -47,6 +47,7 @@ func TestHistoryDestinations(t *testing.T) { {"Id":"PSTN_71","Prefixes":["+4971"]}, {"Id":"PSTN_72","Prefixes":["+4972"]}, {"Id":"RET","Prefixes":["0723","0724"]}, +{"Id":"SPEC","Prefixes":["0723045"]}, {"Id":"URG","Prefixes":["112"]}` if !strings.Contains(buf.String(), expected) { t.Error("Error in destination history content:", buf.String()) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 357f56fb8..d26ebbcc6 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -42,6 +42,7 @@ NAT,0723 NAT,+49 RET,0723 RET,0724 +SPEC,0723045 PSTN_71,+4971 PSTN_72,+4972 PSTN_70,+4970 @@ -319,7 +320,7 @@ func init() { } func TestLoadDestinations(t *testing.T) { - if len(csvr.destinations) != 12 { + if len(csvr.destinations) != 13 { t.Error("Failed to load destinations: ", len(csvr.destinations)) } for _, d := range csvr.destinations { From bafa9d9802e0198d28b78bb924d3b7997abfc4df Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 12 Apr 2016 22:33:54 +0300 Subject: [PATCH 146/227] improve rating selection with splitting on days --- engine/calldesc.go | 21 ++++++++++++++++++-- engine/calldesc_test.go | 37 ++++++++++++++++++++++++++++++++++++ engine/ratingplan.go | 7 +------ engine/ratingprofile_test.go | 35 ++++++++++++++++++++++++++++++++++ engine/timespans.go | 22 +++++++++++++++++++++ 5 files changed, 114 insertions(+), 8 deletions(-) diff --git a/engine/calldesc.go b/engine/calldesc.go index fca3a671d..d6b98f57e 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -362,6 +362,7 @@ func (cd *CallDescriptor) splitInTimeSpans() (timespans []*TimeSpan) { // split on rating plans afterStart, afterEnd := false, false //optimization for multiple activation periods for _, rp := range cd.RatingInfos { + //log.Print("RP: ", utils.ToJSON(rp)) if !afterStart && !afterEnd && rp.ActivationTime.Before(cd.TimeStart) { firstSpan.setRatingInfo(rp) } else { @@ -369,7 +370,6 @@ func (cd *CallDescriptor) splitInTimeSpans() (timespans []*TimeSpan) { for i := 0; i < len(timespans); i++ { newTs := timespans[i].SplitByRatingPlan(rp) if newTs != nil { - //log.Print("NEW TS", newTs.TimeStart) timespans = append(timespans, newTs) } else { afterEnd = true @@ -379,7 +379,22 @@ func (cd *CallDescriptor) splitInTimeSpans() (timespans []*TimeSpan) { } } } - // utils.Logger.Debug(fmt.Sprintf("After SplitByRatingPlan: %+v", timespans)) + //log.Printf("After SplitByRatingPlan: %+v", utils.ToJSON(timespans)) + // split on days + for i := 0; i < len(timespans); i++ { + rp := timespans[i].ratingInfo + newTs := timespans[i].SplitByDay() + if newTs != nil { + //log.Print("NEW TS: ", newTs.TimeStart, newTs.TimeEnd) + newTs.setRatingInfo(rp) + // insert the new timespan + index := i + 1 + timespans = append(timespans, nil) + copy(timespans[index+1:], timespans[index:]) + timespans[index] = newTs + } + } + //log.Printf("After SplitByDay: %+v", utils.ToJSON(timespans)) // split on rate intervals for i := 0; i < len(timespans); i++ { //log.Printf("==============%v==================", i) @@ -388,6 +403,7 @@ func (cd *CallDescriptor) splitInTimeSpans() (timespans []*TimeSpan) { // utils.Logger.Debug(fmt.Sprintf("rp: %+v", rp)) //timespans[i].RatingPlan = nil rateIntervals := rp.SelectRatingIntevalsForTimespan(timespans[i]) + //log.Print("RIs: ", utils.ToJSON(rateIntervals)) /*for _, interval := range rp.RateIntervals { if !timespans[i].hasBetterRateIntervalThan(interval) { timespans[i].SetRateInterval(interval) @@ -520,6 +536,7 @@ func (cd *CallDescriptor) getCost() (*CallCost, error) { cd.TOR = utils.VOICE } err := cd.LoadRatingPlans() + //log.Print("RI: ", utils.ToJSON(cd.RatingInfos)) if err != nil { //utils.Logger.Err(fmt.Sprintf("error getting cost for key <%s>: %s", cd.GetKey(cd.Subject), err.Error())) return &CallCost{Cost: -1}, err diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index ddde0471f..607a97a23 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -326,6 +326,43 @@ func TestGetCostRatingPlansAndRatingIntervalsMore(t *testing.T) { } } +func TestGetCostRatingPlansAndRatingIntervalsMoreDays(t *testing.T) { + t1 := time.Date(2012, time.February, 20, 9, 50, 0, 0, time.UTC) + t2 := time.Date(2012, time.February, 23, 18, 10, 0, 0, time.UTC) + cd := &CallDescriptor{Direction: "*out", Category: "0", Tenant: "CUSTOMER_1", Subject: "rif:from:tm", Destination: "49178", TimeStart: t1, TimeEnd: t2, LoopIndex: 0, DurationIndex: t2.Sub(t1)} + result, _ := cd.GetCost() + if len(result.Timespans) != 8 || + !result.Timespans[0].TimeEnd.Equal(result.Timespans[1].TimeStart) || + !result.Timespans[1].TimeEnd.Equal(result.Timespans[2].TimeStart) || + !result.Timespans[2].TimeEnd.Equal(result.Timespans[3].TimeStart) || + !result.Timespans[3].TimeEnd.Equal(result.Timespans[4].TimeStart) || + !result.Timespans[4].TimeEnd.Equal(result.Timespans[5].TimeStart) || + !result.Timespans[5].TimeEnd.Equal(result.Timespans[6].TimeStart) || + !result.Timespans[6].TimeEnd.Equal(result.Timespans[7].TimeStart) { + for _, ts := range result.Timespans { + t.Logf("TS %+v", ts) + } + t.Errorf("Expected %+v was %+v", 4, len(result.Timespans)) + } +} + +func TestGetCostRatingPlansAndRatingIntervalsMoreDaysWeekend(t *testing.T) { + t1 := time.Date(2012, time.February, 24, 9, 50, 0, 0, time.UTC) + t2 := time.Date(2012, time.February, 27, 18, 10, 0, 0, time.UTC) + cd := &CallDescriptor{Direction: "*out", Category: "0", Tenant: "CUSTOMER_1", Subject: "rif:from:tm", Destination: "49178", TimeStart: t1, TimeEnd: t2, LoopIndex: 0, DurationIndex: t2.Sub(t1)} + result, _ := cd.GetCost() + if len(result.Timespans) != 5 || + !result.Timespans[0].TimeEnd.Equal(result.Timespans[1].TimeStart) || + !result.Timespans[1].TimeEnd.Equal(result.Timespans[2].TimeStart) || + !result.Timespans[2].TimeEnd.Equal(result.Timespans[3].TimeStart) || + !result.Timespans[3].TimeEnd.Equal(result.Timespans[4].TimeStart) { + for _, ts := range result.Timespans { + t.Logf("TS %+v", ts) + } + t.Errorf("Expected %+v was %+v", 4, len(result.Timespans)) + } +} + func TestGetCostRateGroups(t *testing.T) { t1 := time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC) t2 := time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC) diff --git a/engine/ratingplan.go b/engine/ratingplan.go index 3f1214477..fd902ed8e 100644 --- a/engine/ratingplan.go +++ b/engine/ratingplan.go @@ -59,12 +59,7 @@ func (rp *RatingPlan) RateIntervalList(dId string) RateIntervalList { return ril } -/* -type xCachedRatingPlan struct { - rp *RatingPlan - *cache2go.XEntry -} -*/ +// no sorter because it's sorted with RateIntervalTimeSorter /* Adds one ore more intervals to the internal interval list only if it is not allready in the list. diff --git a/engine/ratingprofile_test.go b/engine/ratingprofile_test.go index 4f929029a..100aa0630 100644 --- a/engine/ratingprofile_test.go +++ b/engine/ratingprofile_test.go @@ -250,6 +250,41 @@ func TestRatingProfileRIforTSMidnight(t *testing.T) { } } +func TestRatingProfileYearMonthDay(t *testing.T) { + ri := &RatingInfo{ + RateIntervals: RateIntervalList{ + &RateInterval{ + Timing: &RITiming{ + StartTime: "09:00:00", + }, + }, + &RateInterval{ + Timing: &RITiming{ + StartTime: "00:00:00", + }, + }, + &RateInterval{ + Timing: &RITiming{ + Years: utils.Years{2016}, + Months: utils.Months{1}, + MonthDays: utils.MonthDays{6, 7}, + WeekDays: utils.WeekDays{}, + StartTime: "19:00:00", + }, + }, + }, + } + ts := &TimeSpan{ + TimeStart: time.Date(2016, 1, 6, 23, 40, 0, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 7, 1, 1, 30, 0, time.UTC), + } + rIntervals := ri.SelectRatingIntevalsForTimespan(ts) + if len(rIntervals) != 1 || + rIntervals[0].Timing.StartTime != "19:00:00" { + t.Error("Wrong interval list: ", utils.ToIJSON(rIntervals)) + } +} + func TestRatingProfileSubjectPrefixMatching(t *testing.T) { rpSubjectPrefixMatching = true rp, err := RatingProfileSubjectPrefixMatching("*out:cgrates.org:data:rif") diff --git a/engine/timespans.go b/engine/timespans.go index 8a716f307..366de1d3f 100644 --- a/engine/timespans.go +++ b/engine/timespans.go @@ -612,6 +612,28 @@ func (ts *TimeSpan) SplitByRatingPlan(rp *RatingInfo) (newTs *TimeSpan) { return } +// Splits the given timespan on activation period's activation time. +func (ts *TimeSpan) SplitByDay() (newTs *TimeSpan) { + if ts.TimeStart.Day() == ts.TimeEnd.Day() { + return + } + splitDate := ts.TimeStart.AddDate(0, 0, 1) + splitDate = time.Date(splitDate.Year(), splitDate.Month(), splitDate.Day(), 0, 0, 0, 0, splitDate.Location()) + if splitDate == ts.TimeEnd { // the end date time was actually 00:00:00 + return + } + newTs = &TimeSpan{ + TimeStart: splitDate, + TimeEnd: ts.TimeEnd, + } + newTs.copyRatingInfo(ts) + newTs.DurationIndex = ts.DurationIndex + ts.TimeEnd = splitDate + ts.SetNewDurationIndex(newTs) + // Logger.Debug(fmt.Sprintf("RP SPLITTING: %+v %+v", ts, newTs)) + return +} + // Returns the starting time of this timespan func (ts *TimeSpan) GetGroupStart() time.Duration { s := ts.DurationIndex - ts.GetDuration() From 6cf36e5be0c5867a409bab25a71db39683f7e30f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 12 Apr 2016 23:17:01 +0300 Subject: [PATCH 147/227] better rate intervals sorting according to weight should solve the issue #424 --- engine/rateinterval.go | 20 ++----- engine/ratingprofile.go | 10 +++- engine/ratingprofile_test.go | 102 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 18 deletions(-) diff --git a/engine/rateinterval.go b/engine/rateinterval.go index 0ddcb241d..b03af1b94 100644 --- a/engine/rateinterval.go +++ b/engine/rateinterval.go @@ -360,23 +360,6 @@ func (ri *RateInterval) GetMaxCost() (float64, string) { // Structure to store intervals according to weight type RateIntervalList []*RateInterval -func (il RateIntervalList) Len() int { - return len(il) -} - -func (il RateIntervalList) Swap(i, j int) { - il[i], il[j] = il[j], il[i] -} - -// we need higher weights earlyer in the list -func (il RateIntervalList) Less(j, i int) bool { - return il[i].Weight < il[j].Weight //|| il[i].Timing.StartTime > il[j].Timing.StartTime -} - -func (il RateIntervalList) Sort() { - sort.Sort(il) -} - // Structure to store intervals according to weight type RateIntervalTimeSorter struct { referenceTime time.Time @@ -393,6 +376,9 @@ func (il *RateIntervalTimeSorter) Swap(i, j int) { // we need higher weights earlyer in the list func (il *RateIntervalTimeSorter) Less(j, i int) bool { + if il.ris[i].Weight < il.ris[j].Weight { + return il.ris[i].Weight < il.ris[j].Weight + } t1 := il.ris[i].Timing.getLeftMargin(il.referenceTime) t2 := il.ris[j].Timing.getLeftMargin(il.referenceTime) return t1.After(t2) diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 2560e96c0..e72665a22 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -92,28 +92,36 @@ type RatingInfo struct { // SelectRatingIntevalsForTimespan orders rate intervals in time preserving only those which aply to the specified timestamp func (ri RatingInfo) SelectRatingIntevalsForTimespan(ts *TimeSpan) (result RateIntervalList) { - ri.RateIntervals.Sort() sorter := &RateIntervalTimeSorter{referenceTime: ts.TimeStart, ris: ri.RateIntervals} rateIntervals := sorter.Sort() // get the rating interval closest to begining of timespan var delta time.Duration = -1 var bestRateIntervalIndex int + var bestIntervalWeight float64 for index, rateInterval := range rateIntervals { if !rateInterval.Contains(ts.TimeStart, false) { continue } + if rateInterval.Weight < bestIntervalWeight { + break // don't consider lower weights' + } startTime := rateInterval.Timing.getLeftMargin(ts.TimeStart) tmpDelta := ts.TimeStart.Sub(startTime) if (startTime.Before(ts.TimeStart) || startTime.Equal(ts.TimeStart)) && (delta == -1 || tmpDelta < delta) { bestRateIntervalIndex = index + bestIntervalWeight = rateInterval.Weight delta = tmpDelta } } result = append(result, rateIntervals[bestRateIntervalIndex]) // check if later rating intervals influence this timespan + //log.Print("RIS: ", utils.ToIJSON(rateIntervals)) for i := bestRateIntervalIndex + 1; i < len(rateIntervals); i++ { + if rateIntervals[i].Weight < bestIntervalWeight { + break // don't consider lower weights' + } startTime := rateIntervals[i].Timing.getLeftMargin(ts.TimeStart) if startTime.Before(ts.TimeEnd) { result = append(result, rateIntervals[i]) diff --git a/engine/ratingprofile_test.go b/engine/ratingprofile_test.go index 100aa0630..08cfecacd 100644 --- a/engine/ratingprofile_test.go +++ b/engine/ratingprofile_test.go @@ -285,6 +285,108 @@ func TestRatingProfileYearMonthDay(t *testing.T) { } } +func TestRatingProfileWeighted(t *testing.T) { + ri := &RatingInfo{ + RateIntervals: RateIntervalList{ + &RateInterval{ + Timing: &RITiming{ + StartTime: "09:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + StartTime: "00:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + StartTime: "19:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + Years: utils.Years{2016}, + Months: utils.Months{1}, + MonthDays: utils.MonthDays{6}, + WeekDays: utils.WeekDays{}, + StartTime: "00:00:00", + }, + Weight: 11, + }, + }, + } + ts := &TimeSpan{ + TimeStart: time.Date(2016, 1, 6, 23, 40, 0, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 6, 23, 45, 30, 0, time.UTC), + } + rIntervals := ri.SelectRatingIntevalsForTimespan(ts) + if len(rIntervals) != 1 || + rIntervals[0].Timing.StartTime != "00:00:00" || + rIntervals[0].Weight != 11 { + t.Error("Wrong interval list: ", utils.ToIJSON(rIntervals)) + } +} + +func TestRatingProfileWeightedMultiple(t *testing.T) { + ri := &RatingInfo{ + RateIntervals: RateIntervalList{ + &RateInterval{ + Timing: &RITiming{ + StartTime: "09:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + StartTime: "00:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + StartTime: "19:00:00", + }, + Weight: 10, + }, + &RateInterval{ + Timing: &RITiming{ + Years: utils.Years{2016}, + Months: utils.Months{1}, + MonthDays: utils.MonthDays{6}, + WeekDays: utils.WeekDays{}, + StartTime: "00:00:00", + }, + Weight: 11, + }, + &RateInterval{ + Timing: &RITiming{ + Years: utils.Years{2016}, + Months: utils.Months{1}, + MonthDays: utils.MonthDays{6}, + WeekDays: utils.WeekDays{}, + StartTime: "18:00:00", + }, + Weight: 11, + }, + }, + } + ts := &TimeSpan{ + TimeStart: time.Date(2016, 1, 6, 17, 40, 0, 0, time.UTC), + TimeEnd: time.Date(2016, 1, 6, 23, 45, 30, 0, time.UTC), + } + rIntervals := ri.SelectRatingIntevalsForTimespan(ts) + if len(rIntervals) != 2 || + rIntervals[0].Timing.StartTime != "00:00:00" || + rIntervals[0].Weight != 11 || + rIntervals[1].Timing.StartTime != "18:00:00" || + rIntervals[1].Weight != 11 { + t.Error("Wrong interval list: ", utils.ToIJSON(rIntervals)) + } +} + func TestRatingProfileSubjectPrefixMatching(t *testing.T) { rpSubjectPrefixMatching = true rp, err := RatingProfileSubjectPrefixMatching("*out:cgrates.org:data:rif") From 6f686fd26a01c165962b32a4aa133158ab7a08c8 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 13 Apr 2016 09:38:48 +0300 Subject: [PATCH 148/227] updated dockerfiole to go 1.6.1 --- data/docker/devel/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/docker/devel/Dockerfile b/data/docker/devel/Dockerfile index ea0a8b803..5e4917784 100644 --- a/data/docker/devel/Dockerfile +++ b/data/docker/devel/Dockerfile @@ -26,7 +26,7 @@ COPY mongod.conf /etc/mongod.conf RUN useradd -c CGRateS -d /var/run/cgrates -s /bin/false -r cgrates # install golang -RUN wget -qO- https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz | tar xzf - -C /root/ +RUN wget -qO- https://storage.googleapis.com/golang/go1.6.1.linux-amd64.tar.gz | tar xzf - -C /root/ #install glide RUN GOROOT=/root/go GOPATH=/root/code /root/go/bin/go get github.com/Masterminds/glide From c9c49015b989b1d838d54af2489874a56fcb8213 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 13 Apr 2016 09:39:03 +0300 Subject: [PATCH 149/227] better check of 0h --- engine/rateinterval.go | 2 +- engine/timespans.go | 6 ++---- utils/coreutils.go | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/engine/rateinterval.go b/engine/rateinterval.go index b03af1b94..0b0b13ed2 100644 --- a/engine/rateinterval.go +++ b/engine/rateinterval.go @@ -289,7 +289,7 @@ Returns true if the received time result inside the interval */ func (i *RateInterval) Contains(t time.Time, endTime bool) bool { if endTime { - if t.Hour() == 0 && t.Minute() == 0 && t.Second() == 0 { // back one second to 23:59:59 + if utils.TimeIs0h(t) { // back one second to 23:59:59 t = t.Add(-1 * time.Second) } } diff --git a/engine/timespans.go b/engine/timespans.go index 366de1d3f..33bc79e8d 100644 --- a/engine/timespans.go +++ b/engine/timespans.go @@ -614,14 +614,12 @@ func (ts *TimeSpan) SplitByRatingPlan(rp *RatingInfo) (newTs *TimeSpan) { // Splits the given timespan on activation period's activation time. func (ts *TimeSpan) SplitByDay() (newTs *TimeSpan) { - if ts.TimeStart.Day() == ts.TimeEnd.Day() { + if ts.TimeStart.Day() == ts.TimeEnd.Day() || utils.TimeIs0h(ts.TimeEnd) { return } + splitDate := ts.TimeStart.AddDate(0, 0, 1) splitDate = time.Date(splitDate.Year(), splitDate.Month(), splitDate.Day(), 0, 0, 0, 0, splitDate.Location()) - if splitDate == ts.TimeEnd { // the end date time was actually 00:00:00 - return - } newTs = &TimeSpan{ TimeStart: splitDate, TimeEnd: ts.TimeEnd, diff --git a/utils/coreutils.go b/utils/coreutils.go index 35453b5d5..461fb2d98 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -547,3 +547,7 @@ func SizeFmt(num float64, suffix string) string { } return fmt.Sprintf("%.1f%s%s", num, "Yi", suffix) } + +func TimeIs0h(t time.Time) bool { + return t.Hour() == 0 && t.Minute() == 0 && t.Second() == 0 +} From 77753ffc640abfa9bc5eeb32677781680a70acaa Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 13 Apr 2016 18:29:02 +0300 Subject: [PATCH 150/227] do not split data on days --- engine/calldesc.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/engine/calldesc.go b/engine/calldesc.go index d6b98f57e..150563676 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -378,20 +378,20 @@ func (cd *CallDescriptor) splitInTimeSpans() (timespans []*TimeSpan) { } } } - } - //log.Printf("After SplitByRatingPlan: %+v", utils.ToJSON(timespans)) - // split on days - for i := 0; i < len(timespans); i++ { - rp := timespans[i].ratingInfo - newTs := timespans[i].SplitByDay() - if newTs != nil { - //log.Print("NEW TS: ", newTs.TimeStart, newTs.TimeEnd) - newTs.setRatingInfo(rp) - // insert the new timespan - index := i + 1 - timespans = append(timespans, nil) - copy(timespans[index+1:], timespans[index:]) - timespans[index] = newTs + //log.Printf("After SplitByRatingPlan: %+v", utils.ToJSON(timespans)) + // split on days + for i := 0; i < len(timespans); i++ { + rp := timespans[i].ratingInfo + newTs := timespans[i].SplitByDay() + if newTs != nil { + //log.Print("NEW TS: ", newTs.TimeStart, newTs.TimeEnd) + newTs.setRatingInfo(rp) + // insert the new timespan + index := i + 1 + timespans = append(timespans, nil) + copy(timespans[index+1:], timespans[index:]) + timespans[index] = newTs + } } } //log.Printf("After SplitByDay: %+v", utils.ToJSON(timespans)) From 538aba2293deb602b389d32d0244f795b4f38da2 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 13 Apr 2016 23:29:47 +0300 Subject: [PATCH 151/227] improved one integration test --- general_tests/tp_it_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 1ad58d833..cb66dca3e 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -161,6 +161,12 @@ func TestTpZeroCost(t *testing.T) { if !*testIntegration { return } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1012"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } + balanceValueBefore := acnt.BalanceMap[utils.MONETARY][0].Value tStart := time.Date(2016, 3, 31, 0, 0, 0, 0, time.UTC) cd := engine.CallDescriptor{ Direction: "*out", @@ -175,15 +181,13 @@ func TestTpZeroCost(t *testing.T) { } var cc engine.CallCost if err := tpRPC.Call("Responder.Debit", cd, &cc); err != nil { - t.Error("Got error on Responder.GetCost: ", err.Error()) + t.Error("Got error on Responder.Debit: ", err.Error()) } else if cc.GetDuration() != 20*time.Second { t.Errorf("Calling Responder.MaxDebit got callcost: %v", utils.ToIJSON(cc)) } - var acnt *engine.Account - attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1012"} if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error("Got error on ApierV2.GetAccount: ", err.Error()) - } else if acnt.BalanceMap[utils.MONETARY][0].Value != 11.0 { + } else if acnt.BalanceMap[utils.MONETARY][0].Value != balanceValueBefore { t.Errorf("Calling ApierV2.GetAccount received: %s", utils.ToIJSON(acnt)) } } From 4107ac76d0d9d9c02c68c5edabaa8e4c341fcff7 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 14 Apr 2016 19:17:47 +0200 Subject: [PATCH 152/227] SMG - SessionTTLUsage and SessionTTLLastUsed support --- config/config_defaults.go | 4 +- config/libconfig_json.go | 18 ++++---- config/smconfig.go | 32 +++++++++---- sessionmanager/smgeneric.go | 92 +++++++++++++++++++++++++++---------- utils/consts.go | 3 ++ 5 files changed, 109 insertions(+), 40 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 9d1cc2d87..8f1d96149 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -210,7 +210,9 @@ const CGRATES_CFG_JSON = ` "debit_interval": "0s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this "max_call_duration": "3h", // maximum call duration a prepaid call can last - "session_ttl": "0s", // time after a session with no updates is terminated + "session_ttl": "0s", // time after a session with no updates is terminated, not defined by default + //"session_ttl_last_used": "", // tweak LastUsed for sessions timing-out, not defined by default + //"session_ttl_usage": "", // tweak Usage for sessions timing-out, not defined by default }, diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 2dca787a4..82d9469f3 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -167,14 +167,16 @@ type CdrcJsonCfg struct { // SM-Generic config section type SmGenericJsonCfg struct { - Enabled *bool - Listen_bijson *string - Rater *string - Cdrs *string - Debit_interval *string - Min_call_duration *string - Max_call_duration *string - Session_ttl *string + Enabled *bool + Listen_bijson *string + Rater *string + Cdrs *string + Debit_interval *string + Min_call_duration *string + Max_call_duration *string + Session_ttl *string + Session_ttl_last_used *string + Session_ttl_usage *string } // SM-FreeSWITCH config section diff --git a/config/smconfig.go b/config/smconfig.go index e828c76a0..fe883576c 100644 --- a/config/smconfig.go +++ b/config/smconfig.go @@ -72,14 +72,16 @@ func (self *FsConnConfig) loadFromJsonCfg(jsnCfg *FsConnJsonCfg) error { } type SmGenericConfig struct { - Enabled bool - ListenBijson string - HaRater []*HaPoolConfig - HaCdrs []*HaPoolConfig - DebitInterval time.Duration - MinCallDuration time.Duration - MaxCallDuration time.Duration - SessionTTL time.Duration + Enabled bool + ListenBijson string + HaRater []*HaPoolConfig + HaCdrs []*HaPoolConfig + DebitInterval time.Duration + MinCallDuration time.Duration + MaxCallDuration time.Duration + SessionTTL time.Duration + SessionTTLLastUsed *time.Duration + SessionTTLUsage *time.Duration } func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { @@ -119,6 +121,20 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { return err } } + if jsnCfg.Session_ttl_last_used != nil { + if sessionTTLLastUsed, err := utils.ParseDurationWithSecs(*jsnCfg.Session_ttl_last_used); err != nil { + return err + } else { + self.SessionTTLLastUsed = &sessionTTLLastUsed + } + } + if jsnCfg.Session_ttl_usage != nil { + if sessionTTLUsage, err := utils.ParseDurationWithSecs(*jsnCfg.Session_ttl_usage); err != nil { + return err + } else { + self.SessionTTLUsage = &sessionTTLUsage + } + } return nil } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index fd086efbf..68afa1ea5 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -51,34 +51,75 @@ type SMGeneric struct { guard *engine.GuardianLock // Used to lock on uuid } type smgSessionTerminator struct { - timer *time.Timer - endChan chan bool + timer *time.Timer + endChan chan bool + ttl time.Duration + ttlLastUsed *time.Duration + ttlUsage *time.Duration +} + +// Updates the timer for the session to a new ttl and terminate info +func (self *SMGeneric) resetTerminatorTimer(uuid string, ttl time.Duration, ttlLastUsed, ttlUsage *time.Duration) { + self.sessionsMux.RLock() + defer self.sessionsMux.RUnlock() + if st, found := self.sessionTerminators[uuid]; found { + if ttl != 0 { + st.ttl = ttl + } + if ttlLastUsed != nil { + st.ttlLastUsed = ttlLastUsed + } + if ttlUsage != nil { + st.ttlUsage = ttlUsage + } + st.timer.Reset(st.ttl) + } +} + +// Called when a session timeouts +func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { + totalSessionUsage := s.TotalUsage() + tmtr.ttl + if tmtr.ttlUsage != nil { + totalSessionUsage = *tmtr.ttlUsage + } + if totalSessionUsage > s.TotalUsage() { + evUpdate := s.eventStart + diffSessionUsage := totalSessionUsage - s.TotalUsage() + evUpdate[utils.USAGE] = diffSessionUsage.Seconds() // Debit additionally + if tmtr.ttlLastUsed != nil { + evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() + } + self.SessionUpdate(evUpdate, nil) + } + self.sessionEnd(s.eventStart.GetUUID(), totalSessionUsage) + cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) + cdr.Usage = totalSessionUsage + var reply string + self.cdrsrv.ProcessCdr(cdr, &reply) } func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { self.sessionsMux.Lock() self.sessions[uuid] = append(self.sessions[uuid], s) - if self.cgrCfg.SmGenericConfig.SessionTTL > 0 { + if self.cgrCfg.SmGenericConfig.SessionTTL != 0 { if _, found := self.sessionTerminators[uuid]; !found { timer := time.NewTimer(self.cgrCfg.SmGenericConfig.SessionTTL) endChan := make(chan bool, 1) + terminator := &smgSessionTerminator{ + timer: timer, + endChan: endChan, + ttl: self.cgrCfg.SmGenericConfig.SessionTTL, + } + self.sessionTerminators[uuid] = terminator go func() { select { case <-timer.C: - totalUsage := s.TotalUsage() + self.cgrCfg.SmGenericConfig.SessionTTL - self.sessionEnd(uuid, totalUsage) - cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) - cdr.Usage = totalUsage - var reply string - self.cdrsrv.ProcessCdr(cdr, &reply) + self.ttlTerminate(s, terminator) case <-endChan: timer.Stop() } }() - self.sessionTerminators[uuid] = &smgSessionTerminator{ - timer: timer, - endChan: endChan, - } + } } self.sessionsMux.Unlock() @@ -125,15 +166,6 @@ func (self *SMGeneric) getSession(uuid string) []*SMGSession { return self.sessions[uuid] } -// Updates the timer for the session to a new ttl -func (self *SMGeneric) resetTerminatorTimer(uuid string) { - self.sessionsMux.RLock() - defer self.sessionsMux.RUnlock() - if st, found := self.sessionTerminators[uuid]; found { - st.timer.Reset(self.cgrCfg.SmGenericConfig.SessionTTL) - } -} - // Handle a new session, pass the connectionId so we can communicate on disconnect request func (self *SMGeneric) sessionStart(evStart SMGenericEvent, connId string) error { sessionId := evStart.GetUUID() @@ -262,7 +294,21 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - self.resetTerminatorTimer(gev.GetUUID()) + ttl := time.Duration(0) + if ttlStr, err := gev.GetFieldAsString(utils.SessionTTL); err == nil { + ttl, _ = utils.ParseDurationWithSecs(ttlStr) + } + var ttlLastUsed *time.Duration + if ttlLastUsedStr, err := gev.GetFieldAsString(utils.SessionTTLLastUsed); err == nil { + ttlLastUsedParsed, _ := utils.ParseDurationWithSecs(ttlLastUsedStr) + ttlLastUsed = &ttlLastUsedParsed + } + var ttlUsage *time.Duration + if ttlUsageStr, err := gev.GetFieldAsString(utils.SessionTTLUsage); err == nil { + ttlUsageParsed, _ := utils.ParseDurationWithSecs(ttlUsageStr) + ttlUsage = &ttlUsageParsed + } + self.resetTerminatorTimer(gev.GetUUID(), ttl, ttlLastUsed, ttlUsage) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update diff --git a/utils/consts.go b/utils/consts.go index 0bacf323e..b83e10808 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -278,6 +278,9 @@ const ( UpdatedAt = "UpdatedAt" HandlerArgSep = "|" FlagForceDuration = "fd" + SessionTTL = "SessionTTL" + SessionTTLLastUsed = "SessionTTLLastUsed" + SessionTTLUsage = "SessionTTLUsage" ) var ( From 3d1c0c0007b8e90b48199320ab0877482a4a2514 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 14 Apr 2016 19:51:42 +0200 Subject: [PATCH 153/227] SMGeneric SessionTTL - apply update even if Usage is 0 --- sessionmanager/smgeneric.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 68afa1ea5..4ca71f974 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -82,15 +82,16 @@ func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { if tmtr.ttlUsage != nil { totalSessionUsage = *tmtr.ttlUsage } - if totalSessionUsage > s.TotalUsage() { - evUpdate := s.eventStart - diffSessionUsage := totalSessionUsage - s.TotalUsage() - evUpdate[utils.USAGE] = diffSessionUsage.Seconds() // Debit additionally - if tmtr.ttlLastUsed != nil { - evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() - } - self.SessionUpdate(evUpdate, nil) + diffSessionUsage := totalSessionUsage - s.TotalUsage() + evUpdate := s.eventStart + evUpdate[utils.USAGE] = 0.0 + if diffSessionUsage > 0 { + evUpdate[utils.USAGE] = diffSessionUsage.Seconds() } + if tmtr.ttlLastUsed != nil { + evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() + } + self.SessionUpdate(evUpdate, nil) self.sessionEnd(s.eventStart.GetUUID(), totalSessionUsage) cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) cdr.Usage = totalSessionUsage From 0e59798fb552be327bc28c1ef324a3182a1862ad Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 14 Apr 2016 20:07:22 +0200 Subject: [PATCH 154/227] SMGeneric SessionTTLUsed should be considered in Update --- sessionmanager/smgeneric.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 4ca71f974..cd07feac5 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -78,14 +78,11 @@ func (self *SMGeneric) resetTerminatorTimer(uuid string, ttl time.Duration, ttlL // Called when a session timeouts func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { - totalSessionUsage := s.TotalUsage() + tmtr.ttl - if tmtr.ttlUsage != nil { - totalSessionUsage = *tmtr.ttlUsage - } - diffSessionUsage := totalSessionUsage - s.TotalUsage() evUpdate := s.eventStart evUpdate[utils.USAGE] = 0.0 - if diffSessionUsage > 0 { + totalSessionUsage := s.TotalUsage() + tmtr.ttl + if tmtr.ttlUsage != nil { + totalSessionUsage = s.TotalUsage() + *tmtr.ttlUsage evUpdate[utils.USAGE] = diffSessionUsage.Seconds() } if tmtr.ttlLastUsed != nil { From c48a4c4b1981149f5844f589f6bd1f465c8705dc Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 14 Apr 2016 20:15:15 +0200 Subject: [PATCH 155/227] Build fix --- sessionmanager/smgeneric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index cd07feac5..4e8b33730 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -83,7 +83,7 @@ func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { totalSessionUsage := s.TotalUsage() + tmtr.ttl if tmtr.ttlUsage != nil { totalSessionUsage = s.TotalUsage() + *tmtr.ttlUsage - evUpdate[utils.USAGE] = diffSessionUsage.Seconds() + evUpdate[utils.USAGE] = tmtr.ttlUsage.Seconds() } if tmtr.ttlLastUsed != nil { evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() From b025e5d6612aa26e6a111abd3e2513c0656b6285 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 14 Apr 2016 21:55:02 +0300 Subject: [PATCH 156/227] another session ttl test --- sessionmanager/data_it_test.go | 77 ++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 6944fdd8b..34601788d 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -433,3 +433,80 @@ func TestSMGDataTTLExpired(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } } + +func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} + eAcntVal := 49999887360.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "1048576", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998832640.000000 //1054720 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "20000", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49998812160.000000 // 20480 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + time.Sleep(50 * time.Millisecond) + eAcntVal = 49999866880.000000 //-1054720 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } +} From 93d204622c24061b1397a028f03838b05be3a644 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 14 Apr 2016 22:10:39 +0300 Subject: [PATCH 157/227] voice call ttl test --- sessionmanager/smg_it_test.go | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index ba131c5e0..e42be764a 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -604,3 +604,80 @@ func TestSMGLastUsedNotFixed(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } } + +func TestSMGSessionTTL(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} + eAcntVal := 5.590000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "2m", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 4.190020 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.VOICE, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1001", + utils.SUBJECT: "1001", + utils.DESTINATION: "1006", + utils.CATEGORY: "call", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "2m", + utils.LastUsed: "30s", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 120 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 4.090030 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } + + time.Sleep(50 * time.Millisecond) + + eAcntVal = 4.390000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } +} From dc117b0995883eb3c50e4d40509141d2becffc2b Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 15 Apr 2016 09:21:41 +0200 Subject: [PATCH 158/227] Fix for SMG - SessionTTL --- sessionmanager/smg_it_test.go | 26 ++++++++++++++++++++------ sessionmanager/smgeneric.go | 4 ++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index e42be764a..cc4ddda11 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -618,13 +618,13 @@ func TestSMGSessionTTL(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } smgEv := SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", + utils.EVENT_NAME: "TEST_EVENT_SESSION_TTL", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", - utils.DESTINATION: "1006", + utils.DESTINATION: "1008", utils.CATEGORY: "call", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, @@ -646,13 +646,13 @@ func TestSMGSessionTTL(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } smgEv = SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", + utils.EVENT_NAME: "TEST_EVENT_SESSION_TTL", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", - utils.DESTINATION: "1006", + utils.DESTINATION: "1008", utils.CATEGORY: "call", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, @@ -680,4 +680,18 @@ func TestSMGSessionTTL(t *testing.T) { } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, DestinationPrefixes: []string{"1008"}} + if err := smgRPC.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Usage != "30.01" { + t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) + } + if cdrs[0].Cost != 1.2 { + t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) + } + } } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 4e8b33730..5c4089e99 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -78,9 +78,9 @@ func (self *SMGeneric) resetTerminatorTimer(uuid string, ttl time.Duration, ttlL // Called when a session timeouts func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { - evUpdate := s.eventStart - evUpdate[utils.USAGE] = 0.0 totalSessionUsage := s.TotalUsage() + tmtr.ttl + evUpdate := s.eventStart + evUpdate[utils.USAGE] = tmtr.ttl.Seconds() if tmtr.ttlUsage != nil { totalSessionUsage = s.TotalUsage() + *tmtr.ttlUsage evUpdate[utils.USAGE] = tmtr.ttlUsage.Seconds() From b81eeca2012f17c51e10b3d31248669c06d1aa81 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 15 Apr 2016 10:00:46 +0200 Subject: [PATCH 159/227] SMG SessionTTL, totalUsage in end event fix --- sessionmanager/smgeneric.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 5c4089e99..137e75733 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -78,20 +78,18 @@ func (self *SMGeneric) resetTerminatorTimer(uuid string, ttl time.Duration, ttlL // Called when a session timeouts func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { - totalSessionUsage := s.TotalUsage() + tmtr.ttl evUpdate := s.eventStart evUpdate[utils.USAGE] = tmtr.ttl.Seconds() if tmtr.ttlUsage != nil { - totalSessionUsage = s.TotalUsage() + *tmtr.ttlUsage evUpdate[utils.USAGE] = tmtr.ttlUsage.Seconds() } if tmtr.ttlLastUsed != nil { evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() } self.SessionUpdate(evUpdate, nil) - self.sessionEnd(s.eventStart.GetUUID(), totalSessionUsage) + self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) - cdr.Usage = totalSessionUsage + cdr.Usage = s.TotalUsage() var reply string self.cdrsrv.ProcessCdr(cdr, &reply) } From 1f13daa31c7a0a8540322edb8f729bce43f2130d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 15 Apr 2016 16:21:14 +0300 Subject: [PATCH 160/227] all tests passing on session ttl --- sessionmanager/data_it_test.go | 16 ++++++++-------- sessionmanager/smg_it_test.go | 10 +++++----- sessionmanager/smg_session.go | 17 ++++++++++++----- sessionmanager/smgeneric.go | 8 ++++++-- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 34601788d..dd21c6c8c 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -115,7 +115,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998945280.000000 + eAcntVal = 49998945280.000000 //1054720 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -141,7 +141,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998924800.000000 + eAcntVal = 49998924800.000000 //20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -164,7 +164,7 @@ func TestSMGDataLastUsedData(t *testing.T) { if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { t.Error(err) } - eAcntVal = 49999979520.000000 + eAcntVal = 49999979520.000000 //20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -426,7 +426,7 @@ func TestSMGDataTTLExpired(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } time.Sleep(50 * time.Millisecond) - eAcntVal = 49999887360.000000 + eAcntVal = 49998842880.000000 //1054720 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -440,7 +440,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { } var acnt *engine.Account attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} - eAcntVal := 49999887360.000000 + eAcntVal := 49998842880.000000 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -468,7 +468,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998832640.000000 //1054720 + eAcntVal = 49997788160.000000 //1054720 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -495,7 +495,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { if maxUsage != 1.048576e+06 { t.Error("Bad max usage: ", maxUsage) } - eAcntVal = 49998812160.000000 // 20480 + eAcntVal = 49997767680.000000 // 20480 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { @@ -503,7 +503,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { } time.Sleep(50 * time.Millisecond) - eAcntVal = 49999866880.000000 //-1054720 + eAcntVal = 49997767680.000000 //0 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index cc4ddda11..6d810f82e 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -674,7 +674,7 @@ func TestSMGSessionTTL(t *testing.T) { time.Sleep(50 * time.Millisecond) - eAcntVal = 4.390000 + eAcntVal = 4.089900 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { @@ -687,11 +687,11 @@ func TestSMGSessionTTL(t *testing.T) { } else if len(cdrs) != 1 { t.Error("Unexpected number of CDRs returned: ", len(cdrs)) } else { - if cdrs[0].Usage != "30.01" { - t.Errorf("Unexpected CDR Usage received, cdr: %+v ", cdrs[0]) + if cdrs[0].Usage != "150" { + t.Errorf("Unexpected CDR Usage received, cdr: %v %+v ", cdrs[0].Usage, cdrs[0]) } - if cdrs[0].Cost != 1.2 { - t.Errorf("Unexpected CDR Cost received, cdr: %+v ", cdrs[0]) + if cdrs[0].Cost != 1.5 { + t.Errorf("Unexpected CDR Cost received, cdr: %v %+v ", cdrs[0].Cost, cdrs[0]) } } } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 05cc38605..9553c5fbb 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -54,7 +54,7 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { return default: } - if maxDebit, err := self.debit(debitInterval, nilDuration); err != nil { + if maxDebit, err := self.debit(debitInterval, nil); err != nil { utils.Logger.Err(fmt.Sprintf(" Could not complete debit opperation on session: %s, error: %s", self.eventStart.GetUUID(), err.Error())) disconnectReason := SYSTEM_ERROR if err.Error() == utils.ErrUnauthorizedDestination.Error() { @@ -77,14 +77,19 @@ func (self *SMGSession) debitLoop(debitInterval time.Duration) { } // Attempts to debit a duration, returns maximum duration which can be debitted or error -func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.Duration, error) { +func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time.Duration, error) { requestedDuration := dur //utils.Logger.Debug(fmt.Sprintf("InitDur: %f, lastUsed: %f", requestedDuration.Seconds(), lastUsed.Seconds())) //utils.Logger.Debug(fmt.Sprintf("TotalUsage: %f, extraDuration: %f", self.totalUsage.Seconds(), self.extraDuration.Seconds())) - self.totalUsage += lastUsed // Should reflect the total usage so far - if lastUsed > 0 { - self.extraDuration = self.lastUsage - lastUsed + if lastUsed != nil { + self.extraDuration = self.lastUsage - *lastUsed //utils.Logger.Debug(fmt.Sprintf("ExtraDuration LastUsed: %f", self.extraDuration.Seconds())) + if *lastUsed != self.lastUsage { + // total usage correction + self.totalUsage -= self.lastUsage + self.totalUsage += *lastUsed + //utils.Logger.Debug(fmt.Sprintf("Correction: %f", self.totalUsage.Seconds())) + } } // apply correction from previous run if self.extraDuration < dur { @@ -122,6 +127,8 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed time.Duration) (time.D self.sessionCds = append(self.sessionCds, self.cd.Clone()) self.callCosts = append(self.callCosts, cc) self.lastUsage = initialExtraDuration + ccDuration + self.totalUsage += self.lastUsage + //utils.Logger.Debug(fmt.Sprintf("TotalUsage: %f", self.totalUsage.Seconds())) if ccDuration >= dur { // we got what we asked to be debited //utils.Logger.Debug(fmt.Sprintf("returning normal: %f", requestedDuration.Seconds())) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 137e75733..d0a60bceb 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -314,10 +314,14 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim return nilDuration, err } } + var lastUsed *time.Duration evLastUsed, err := gev.GetLastUsed(utils.META_DEFAULT) if err != nil && err != utils.ErrNotFound { return nilDuration, err } + if err == nil { + lastUsed = &evLastUsed + } evMaxUsage, err := gev.GetMaxUsage(utils.META_DEFAULT, self.cgrCfg.MaxCallDuration) if err != nil { if err == utils.ErrNotFound { @@ -326,7 +330,7 @@ func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (tim return nilDuration, err } for _, s := range self.getSession(gev.GetUUID()) { - if maxDur, err := s.debit(evMaxUsage, evLastUsed); err != nil { + if maxDur, err := s.debit(evMaxUsage, lastUsed); err != nil { return nilDuration, err } else if maxDur < evMaxUsage { evMaxUsage = maxDur @@ -375,7 +379,7 @@ func (self *SMGeneric) SessionEnd(gev SMGenericEvent, clnt *rpc2.Client) error { if s == nil { continue // No session active, will not be able to close it anyway } - usage = s.TotalUsage() + lastUsed + usage = s.TotalUsage() - s.lastUsage + lastUsed } if err := self.sessionEnd(sessionID, usage); err != nil { interimError = err // Last error will be the one returned as API result From 9933a03c79a99659beec078e121b9e121ccc5e7b Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 18 Apr 2016 12:20:19 +0200 Subject: [PATCH 161/227] SMGeneric - Consider SessionTTL settings in init event --- sessionmanager/smg_event.go | 48 +++++++++++++++++++++++++++++++++++++ sessionmanager/smgeneric.go | 30 +++++++++-------------- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/sessionmanager/smg_event.go b/sessionmanager/smg_event.go index 890e9a1c9..2d6f7855f 100644 --- a/sessionmanager/smg_event.go +++ b/sessionmanager/smg_event.go @@ -177,6 +177,54 @@ func (self SMGenericEvent) GetLastUsed(fieldName string) (time.Duration, error) return utils.ParseDurationWithSecs(result) } +// GetSessionTTL retrieves SessionTTL setting out of SMGenericEvent +func (self SMGenericEvent) GetSessionTTL() time.Duration { + valIf, hasVal := self[utils.SessionTTL] + if !hasVal { + return time.Duration(0) + } + ttlStr, converted := utils.ConvertIfaceToString(valIf) + if !converted { + return time.Duration(0) + } + ttl, _ := utils.ParseDurationWithSecs(ttlStr) + return ttl +} + +// GetSessionTTLLastUsed retrieves SessionTTLLastUsed setting out of SMGenericEvent +func (self SMGenericEvent) GetSessionTTLLastUsed() *time.Duration { + valIf, hasVal := self[utils.SessionTTLLastUsed] + if !hasVal { + return nil + } + ttlStr, converted := utils.ConvertIfaceToString(valIf) + if !converted { + return nil + } + if ttl, err := utils.ParseDurationWithSecs(ttlStr); err != nil { + return nil + } else { + return &ttl + } +} + +// GetSessionTTLUsage retrieves SessionTTLUsage setting out of SMGenericEvent +func (self SMGenericEvent) GetSessionTTLUsage() *time.Duration { + valIf, hasVal := self[utils.SessionTTLUsage] + if !hasVal { + return nil + } + ttlStr, converted := utils.ConvertIfaceToString(valIf) + if !converted { + return nil + } + if ttl, err := utils.ParseDurationWithSecs(ttlStr); err != nil { + return nil + } else { + return &ttl + } +} + func (self SMGenericEvent) GetMaxUsage(fieldName string, cfgMaxUsage time.Duration) (time.Duration, error) { if fieldName == utils.META_DEFAULT { fieldName = utils.USAGE diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index d0a60bceb..05a46177d 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -99,12 +99,18 @@ func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { self.sessions[uuid] = append(self.sessions[uuid], s) if self.cgrCfg.SmGenericConfig.SessionTTL != 0 { if _, found := self.sessionTerminators[uuid]; !found { - timer := time.NewTimer(self.cgrCfg.SmGenericConfig.SessionTTL) + ttl := self.cgrCfg.SmGenericConfig.SessionTTL + if ttlEv := s.eventStart.GetSessionTTL(); ttlEv != 0 { + ttl = ttlEv + } + timer := time.NewTimer(ttl) endChan := make(chan bool, 1) terminator := &smgSessionTerminator{ - timer: timer, - endChan: endChan, - ttl: self.cgrCfg.SmGenericConfig.SessionTTL, + timer: timer, + endChan: endChan, + ttl: ttl, + ttlLastUsed: s.eventStart.GetSessionTTLLastUsed(), + ttlUsage: s.eventStart.GetSessionTTLUsage(), } self.sessionTerminators[uuid] = terminator go func() { @@ -290,21 +296,7 @@ func (self *SMGeneric) SessionStart(gev SMGenericEvent, clnt *rpc2.Client) (time // Execute debits for usage/maxUsage func (self *SMGeneric) SessionUpdate(gev SMGenericEvent, clnt *rpc2.Client) (time.Duration, error) { - ttl := time.Duration(0) - if ttlStr, err := gev.GetFieldAsString(utils.SessionTTL); err == nil { - ttl, _ = utils.ParseDurationWithSecs(ttlStr) - } - var ttlLastUsed *time.Duration - if ttlLastUsedStr, err := gev.GetFieldAsString(utils.SessionTTLLastUsed); err == nil { - ttlLastUsedParsed, _ := utils.ParseDurationWithSecs(ttlLastUsedStr) - ttlLastUsed = &ttlLastUsedParsed - } - var ttlUsage *time.Duration - if ttlUsageStr, err := gev.GetFieldAsString(utils.SessionTTLUsage); err == nil { - ttlUsageParsed, _ := utils.ParseDurationWithSecs(ttlUsageStr) - ttlUsage = &ttlUsageParsed - } - self.resetTerminatorTimer(gev.GetUUID(), ttl, ttlLastUsed, ttlUsage) + self.resetTerminatorTimer(gev.GetUUID(), gev.GetSessionTTL(), gev.GetSessionTTLLastUsed(), gev.GetSessionTTLUsage()) if initialID, err := gev.GetFieldAsString(utils.InitialOriginID); err == nil { err := self.sessionRelocate(gev.GetUUID(), initialID) if err == utils.ErrNotFound { // Session was already relocated, create a new session with this update From 69ece5cb9467e1bd82cc84340278ca343a00260e Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 18 Apr 2016 17:34:21 +0200 Subject: [PATCH 162/227] Added OriginID filter in SMGenericV1.ActiveSessions, disabled sessionEnd in SMGeneric.ttlTerminate for testing purposes, testing total usage in active sessions --- apier/v1/smgenericv1.go | 3 + sessionmanager/session.go | 4 +- sessionmanager/smg_it_test.go | 101 ++++++++++++++++++++++------------ sessionmanager/smg_session.go | 5 +- sessionmanager/smgeneric.go | 3 +- utils/apitpdata.go | 1 + 6 files changed, 75 insertions(+), 42 deletions(-) diff --git a/apier/v1/smgenericv1.go b/apier/v1/smgenericv1.go index eed4310fb..1460bfaa2 100644 --- a/apier/v1/smgenericv1.go +++ b/apier/v1/smgenericv1.go @@ -97,6 +97,9 @@ func (self *SMGenericV1) ActiveSessions(attrs utils.AttrSMGGetActiveSessions, re if attrs.ToR != nil && *attrs.ToR != as.TOR { continue } + if attrs.OriginID != nil && *attrs.OriginID != as.OriginID { + continue + } if attrs.RunID != nil && *attrs.RunID != as.RunId { continue } diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 9a21d5984..43f18b1ac 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -272,7 +272,7 @@ func (s *Session) AsActiveSessions() []*ActiveSession { aSession := &ActiveSession{ CgrId: s.eventStart.GetCgrId(s.sessionManager.Timezone()), TOR: utils.VOICE, - AccId: s.eventStart.GetUUID(), + OriginID: s.eventStart.GetUUID(), CdrHost: s.eventStart.GetOriginatorIP(utils.META_DEFAULT), CdrSource: "FS_" + s.eventStart.GetName(), ReqType: s.eventStart.GetReqType(utils.META_DEFAULT), @@ -309,7 +309,7 @@ func (s *Session) AsActiveSessions() []*ActiveSession { type ActiveSession struct { CgrId string TOR string // type of record, meta-field, should map to one of the TORs hardcoded inside the server <*voice|*data|*sms|*generic> - AccId string // represents the unique accounting id given by the telecom switch generating the CDR + OriginID string // represents the unique accounting id given by the telecom switch generating the CDR CdrHost string // represents the IP address of the host generating the CDR (automatically populated by the server) CdrSource string // formally identifies the source of the CDR (free form field) ReqType string // matching the supported request types by the **CGRateS**, accepted values are hardcoded in the server . diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 6d810f82e..1b44df9e7 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -40,7 +40,7 @@ var daCfg *config.CGRConfig var smgRPC *rpc.Client var err error -func TestSMGInitCfg(t *testing.T) { +func TestSMGVoiceInitCfg(t *testing.T) { if !*testIntegration { return } @@ -56,7 +56,7 @@ func TestSMGInitCfg(t *testing.T) { } // Remove data in both rating and accounting db -func TestSMGResetDataDb(t *testing.T) { +func TestSMGVoiceResetDataDb(t *testing.T) { if !*testIntegration { return } @@ -66,7 +66,7 @@ func TestSMGResetDataDb(t *testing.T) { } // Wipe out the cdr database -func TestSMGResetStorDb(t *testing.T) { +func TestSMGVoiceResetStorDb(t *testing.T) { if !*testIntegration { return } @@ -76,7 +76,7 @@ func TestSMGResetStorDb(t *testing.T) { } // Start CGR Engine -func TestSMGStartEngine(t *testing.T) { +func TestSMGVoiceStartEngine(t *testing.T) { if !*testIntegration { return } @@ -86,7 +86,7 @@ func TestSMGStartEngine(t *testing.T) { } // Connect rpc client to rater -func TestSMGApierRpcConn(t *testing.T) { +func TestSMGVoiceApierRpcConn(t *testing.T) { if !*testIntegration { return } @@ -98,7 +98,7 @@ func TestSMGApierRpcConn(t *testing.T) { } // Load the tariff plan, creating accounts and their balances -func TestSMGTPFromFolder(t *testing.T) { +func TestSMGVoiceTPFromFolder(t *testing.T) { if !*testIntegration { return } @@ -110,7 +110,7 @@ func TestSMGTPFromFolder(t *testing.T) { time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups } -func TestSMGMonetaryRefund(t *testing.T) { +func TestSMGVoiceMonetaryRefund(t *testing.T) { if !*testIntegration { return } @@ -171,7 +171,7 @@ func TestSMGMonetaryRefund(t *testing.T) { } } -func TestSMGVoiceRefund(t *testing.T) { +func TestSMGVoiceVoiceRefund(t *testing.T) { if !*testIntegration { return } @@ -232,7 +232,7 @@ func TestSMGVoiceRefund(t *testing.T) { } } -func TestSMGMixedRefund(t *testing.T) { +func TestSMGVoiceMixedRefund(t *testing.T) { if !*testIntegration { return } @@ -306,7 +306,7 @@ func TestSMGMixedRefund(t *testing.T) { t.Logf("After voice: %f", acnt.BalanceMap[utils.VOICE].GetTotalValue()) } -func TestSMGLastUsed(t *testing.T) { +func TestSMGVoiceLastUsed(t *testing.T) { if !*testIntegration { return } @@ -321,7 +321,7 @@ func TestSMGLastUsed(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -349,7 +349,7 @@ func TestSMGLastUsed(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -375,7 +375,7 @@ func TestSMGLastUsed(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -401,7 +401,7 @@ func TestSMGLastUsed(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "12350", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -423,7 +423,7 @@ func TestSMGLastUsed(t *testing.T) { } } -func TestSMGLastUsedEnd(t *testing.T) { +func TestSMGVoiceLastUsedEnd(t *testing.T) { if !*testIntegration { return } @@ -514,7 +514,7 @@ func TestSMGLastUsedEnd(t *testing.T) { } } -func TestSMGLastUsedNotFixed(t *testing.T) { +func TestSMGVoiceLastUsedNotFixed(t *testing.T) { if !*testIntegration { return } @@ -605,7 +605,7 @@ func TestSMGLastUsedNotFixed(t *testing.T) { } } -func TestSMGSessionTTL(t *testing.T) { +func TestSMGVoiceSessionTTL(t *testing.T) { if !*testIntegration { return } @@ -620,7 +620,7 @@ func TestSMGSessionTTL(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT_SESSION_TTL", utils.TOR: utils.VOICE, - utils.ACCID: "12350", + utils.ACCID: "12360", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -639,6 +639,14 @@ func TestSMGSessionTTL(t *testing.T) { if maxUsage != 120 { t.Error("Bad max usage: ", maxUsage) } + var aSessions []*ActiveSession + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 { + t.Errorf("Unexpected number of sessions received: %+v", aSessions) + } else if aSessions[0].Usage != time.Duration(120)*time.Second { + t.Errorf("Expecting 2m, received usage: %v", aSessions[0].Usage) + } eAcntVal = 4.190020 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) @@ -648,7 +656,7 @@ func TestSMGSessionTTL(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT_SESSION_TTL", utils.TOR: utils.VOICE, - utils.ACCID: "12350", + utils.ACCID: "12360", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -665,6 +673,13 @@ func TestSMGSessionTTL(t *testing.T) { if maxUsage != 120 { t.Error("Bad max usage: ", maxUsage) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 { + t.Errorf("Unexpected number of sessions received: %+v", aSessions) + } else if aSessions[0].Usage != time.Duration(150)*time.Second { + t.Errorf("Expecting 2m30s, received usage: %v", aSessions[0].Usage) + } eAcntVal = 4.090030 if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) @@ -673,25 +688,39 @@ func TestSMGSessionTTL(t *testing.T) { } time.Sleep(50 * time.Millisecond) - - eAcntVal = 4.089900 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + } else if len(aSessions) != 1 { + t.Errorf("Unexpected number of sessions received: %+v", aSessions) + } else if aSessions[0].Usage != time.Duration(150)*time.Second+time.Duration(10)*time.Millisecond { + t.Errorf("Expecting 2m30s10ms, received usage: %v", aSessions[0].Usage) } - var cdrs []*engine.ExternalCDR - req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, DestinationPrefixes: []string{"1008"}} - if err := smgRPC.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Usage != "150" { - t.Errorf("Unexpected CDR Usage received, cdr: %v %+v ", cdrs[0].Usage, cdrs[0]) + + /* + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 0 { + t.Errorf("Unexpected number of sessions received: %+v", aSessions) } - if cdrs[0].Cost != 1.5 { - t.Errorf("Unexpected CDR Cost received, cdr: %v %+v ", cdrs[0].Cost, cdrs[0]) + eAcntVal = 4.089900 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } - } + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, DestinationPrefixes: []string{"1008"}} + if err := smgRPC.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Usage != "150" { + t.Errorf("Unexpected CDR Usage received, cdr: %v %+v ", cdrs[0].Usage, cdrs[0]) + } + if cdrs[0].Cost != 1.5 { + t.Errorf("Unexpected CDR Cost received, cdr: %v %+v ", cdrs[0].Cost, cdrs[0]) + } + } + */ } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 9553c5fbb..73a2cc751 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -279,13 +279,12 @@ func (self *SMGSession) TotalUsage() time.Duration { func (self *SMGSession) AsActiveSession(timezone string) *ActiveSession { sTime, _ := self.eventStart.GetSetupTime(utils.META_DEFAULT, timezone) aTime, _ := self.eventStart.GetAnswerTime(utils.META_DEFAULT, timezone) - usage, _ := self.eventStart.GetUsage(utils.META_DEFAULT) pdd, _ := self.eventStart.GetPdd(utils.META_DEFAULT) aSession := &ActiveSession{ CgrId: self.eventStart.GetCgrId(timezone), TOR: utils.VOICE, RunId: self.runId, - AccId: self.eventStart.GetUUID(), + OriginID: self.eventStart.GetUUID(), CdrHost: self.eventStart.GetOriginatorIP(utils.META_DEFAULT), CdrSource: self.eventStart.GetCdrSource(), ReqType: self.eventStart.GetReqType(utils.META_DEFAULT), @@ -297,7 +296,7 @@ func (self *SMGSession) AsActiveSession(timezone string) *ActiveSession { Destination: self.eventStart.GetDestination(utils.META_DEFAULT), SetupTime: sTime, AnswerTime: aTime, - Usage: usage, + Usage: self.TotalUsage(), Pdd: pdd, ExtraFields: self.eventStart.GetExtraFields(), Supplier: self.eventStart.GetSupplier(utils.META_DEFAULT), diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 05a46177d..970923895 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -87,11 +87,12 @@ func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() } self.SessionUpdate(evUpdate, nil) - self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) + /*self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) cdr.Usage = s.TotalUsage() var reply string self.cdrsrv.ProcessCdr(cdr, &reply) + */ } func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 56dcd006f..62f38434b 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1160,6 +1160,7 @@ type AliasValue struct { // AttrSMGGetActiveSessions will filter returned sessions by SMGenericV1 type AttrSMGGetActiveSessions struct { ToR *string + OriginID *string RunID *string RequestType *string Tenant *string From 2c118eb954fbbc5de0d0442215634940f0cfcab9 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 18 Apr 2016 18:06:15 +0200 Subject: [PATCH 163/227] Simplified SMGeneric.ttlTerminate to avoid loops via updateSession --- data/conf/samples/smg/cgrates.json | 2 +- sessionmanager/smg_it_test.go | 6 +++--- sessionmanager/smgeneric.go | 10 ++++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/data/conf/samples/smg/cgrates.json b/data/conf/samples/smg/cgrates.json index d6aaf2a6e..82ef9fa86 100644 --- a/data/conf/samples/smg/cgrates.json +++ b/data/conf/samples/smg/cgrates.json @@ -27,7 +27,7 @@ "enabled": true, "rater": "internal", "cdrs": "internal", - "session_ttl": "10ms", + "session_ttl": "50ms", }, } diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 1b44df9e7..31d3b9410 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -687,13 +687,13 @@ func TestSMGVoiceSessionTTL(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } - time.Sleep(50 * time.Millisecond) + time.Sleep(100 * time.Millisecond) if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { t.Error(err) } else if len(aSessions) != 1 { t.Errorf("Unexpected number of sessions received: %+v", aSessions) - } else if aSessions[0].Usage != time.Duration(150)*time.Second+time.Duration(10)*time.Millisecond { - t.Errorf("Expecting 2m30s10ms, received usage: %v", aSessions[0].Usage) + } else if aSessions[0].Usage != time.Duration(150)*time.Second+time.Duration(50)*time.Millisecond { + t.Errorf("Expecting 2m30s50ms, received usage: %v", aSessions[0].Usage) } /* diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 970923895..4c2fbaae5 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -78,15 +78,13 @@ func (self *SMGeneric) resetTerminatorTimer(uuid string, ttl time.Duration, ttlL // Called when a session timeouts func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { - evUpdate := s.eventStart - evUpdate[utils.USAGE] = tmtr.ttl.Seconds() + debitUsage := tmtr.ttl if tmtr.ttlUsage != nil { - evUpdate[utils.USAGE] = tmtr.ttlUsage.Seconds() + debitUsage = *tmtr.ttlUsage } - if tmtr.ttlLastUsed != nil { - evUpdate[utils.LastUsed] = tmtr.ttlLastUsed.Seconds() + for _, s := range self.getSession(s.eventStart.GetUUID()) { + s.debit(debitUsage, tmtr.ttlLastUsed) } - self.SessionUpdate(evUpdate, nil) /*self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) cdr.Usage = s.TotalUsage() From 8a09ed24a986a3f0b7a4073e7355b653de09ca04 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Apr 2016 19:52:33 +0300 Subject: [PATCH 164/227] changed total usage formula --- sessionmanager/smg_session.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 73a2cc751..8f0d98c6b 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -41,8 +41,9 @@ type SMGSession struct { sessionCds []*engine.CallDescriptor callCosts []*engine.CallCost extraDuration time.Duration // keeps the current duration debited on top of what heas been asked - lastUsage time.Duration - totalUsage time.Duration + lastUsage time.Duration // last requested Duration + lastDebit time.Duration // last real debited duration + totalUsage time.Duration // sum of lastUsage } // Called in case of automatic debits @@ -82,7 +83,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time. //utils.Logger.Debug(fmt.Sprintf("InitDur: %f, lastUsed: %f", requestedDuration.Seconds(), lastUsed.Seconds())) //utils.Logger.Debug(fmt.Sprintf("TotalUsage: %f, extraDuration: %f", self.totalUsage.Seconds(), self.extraDuration.Seconds())) if lastUsed != nil { - self.extraDuration = self.lastUsage - *lastUsed + self.extraDuration = self.lastDebit - *lastUsed //utils.Logger.Debug(fmt.Sprintf("ExtraDuration LastUsed: %f", self.extraDuration.Seconds())) if *lastUsed != self.lastUsage { // total usage correction @@ -109,7 +110,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time. self.cd.DurationIndex += dur cc := &engine.CallCost{} if err := self.rater.MaxDebit(self.cd, cc); err != nil { - self.lastUsage = 0 + self.lastDebit = 0 return 0, err } // cd corrections @@ -120,13 +121,18 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time. if ccDuration != dur { self.extraDuration = ccDuration - dur } + if ccDuration >= dur { + self.lastUsage = requestedDuration + } else { + self.lastUsage = ccDuration + } self.cd.DurationIndex -= dur self.cd.DurationIndex += ccDuration self.cd.MaxCostSoFar += cc.Cost self.cd.LoopIndex += 1 self.sessionCds = append(self.sessionCds, self.cd.Clone()) self.callCosts = append(self.callCosts, cc) - self.lastUsage = initialExtraDuration + ccDuration + self.lastDebit = initialExtraDuration + ccDuration self.totalUsage += self.lastUsage //utils.Logger.Debug(fmt.Sprintf("TotalUsage: %f", self.totalUsage.Seconds())) From 1f647e4ada65dcc3e196c13d84253f285b415f73 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 18 Apr 2016 20:03:53 +0200 Subject: [PATCH 165/227] SMGeneric SessionTTL back into code --- sessionmanager/smg_it_test.go | 48 ++++++++++++----------------------- sessionmanager/smgeneric.go | 3 +-- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 31d3b9410..85796545b 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -686,41 +686,25 @@ func TestSMGVoiceSessionTTL(t *testing.T) { } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } - time.Sleep(100 * time.Millisecond) - if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { + eAcntVal = 4.0565 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) - } else if len(aSessions) != 1 { - t.Errorf("Unexpected number of sessions received: %+v", aSessions) - } else if aSessions[0].Usage != time.Duration(150)*time.Second+time.Duration(50)*time.Millisecond { - t.Errorf("Expecting 2m30s50ms, received usage: %v", aSessions[0].Usage) + } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) } - - /* - if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{RunID: utils.StringPointer(utils.META_DEFAULT), OriginID: utils.StringPointer("12360")}, &aSessions); err != nil { - t.Error(err) - } else if len(aSessions) != 0 { - t.Errorf("Unexpected number of sessions received: %+v", aSessions) + var cdrs []*engine.ExternalCDR + req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, DestinationPrefixes: []string{"1008"}} + if err := smgRPC.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Usage != "150.05" { + t.Errorf("Unexpected CDR Usage received, cdr: %v %+v ", cdrs[0].Usage, cdrs[0]) } - eAcntVal = 4.089900 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.MONETARY].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.MONETARY].GetTotalValue()) + if cdrs[0].Cost != 1.5333 { + t.Errorf("Unexpected CDR Cost received, cdr: %v %+v ", cdrs[0].Cost, cdrs[0]) } - var cdrs []*engine.ExternalCDR - req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, DestinationPrefixes: []string{"1008"}} - if err := smgRPC.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Usage != "150" { - t.Errorf("Unexpected CDR Usage received, cdr: %v %+v ", cdrs[0].Usage, cdrs[0]) - } - if cdrs[0].Cost != 1.5 { - t.Errorf("Unexpected CDR Cost received, cdr: %v %+v ", cdrs[0].Cost, cdrs[0]) - } - } - */ + } } diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 4c2fbaae5..a3dbd8789 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -85,12 +85,11 @@ func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { for _, s := range self.getSession(s.eventStart.GetUUID()) { s.debit(debitUsage, tmtr.ttlLastUsed) } - /*self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) + self.sessionEnd(s.eventStart.GetUUID(), s.TotalUsage()) cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) cdr.Usage = s.TotalUsage() var reply string self.cdrsrv.ProcessCdr(cdr, &reply) - */ } func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { From 81d8715463dd8f81fdfa7b6aedc0973b735b611f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 8 Apr 2016 13:42:28 +0300 Subject: [PATCH 166/227] started *cgr_rpc action --- engine/action.go | 19 +++++++++++++++++++ engine/actions_test.go | 23 +++++++++++++++++++---- utils/struct.go | 23 +++++++++++++++++++++++ utils/struct_test.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/engine/action.go b/engine/action.go index 62ca439dd..3aedbf153 100644 --- a/engine/action.go +++ b/engine/action.go @@ -74,6 +74,7 @@ const ( CDRLOG = "*cdrlog" SET_DDESTINATIONS = "*set_ddestinations" TRANSFER_MONETARY_DEFAULT = "*transfer_monetary_default" + CGR_RPC = "*cgr_rpc" ) func (a *Action) Clone() *Action { @@ -140,6 +141,8 @@ func getActionFunc(typ string) (actionTypeFunc, bool) { return setBalanceAction, true case TRANSFER_MONETARY_DEFAULT: return transferMonetaryDefaultAction, true + case CGR_RPC: + return cgrRPCAction, true } return nil, false } @@ -647,6 +650,22 @@ func transferMonetaryDefaultAction(acc *Account, sq *StatsQueueTriggered, a *Act return nil } +type RPCRequest struct { + Server string + Transport string + Attempts int + Async bool + Arg map[string]interface{} +} + +func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error { + rpcRequest := RPCRequest{} + if err := json.Unmarshal([]byte(a.ExtraParameters), &rpcRequest); err != nil { + return err + } + return nil +} + // Structure to store actions according to weight type Actions []*Action diff --git a/engine/actions_test.go b/engine/actions_test.go index bead47620..9b3dc8f22 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2102,6 +2102,8 @@ func TestActionCdrlogBalanceValue(t *testing.T) { ID: "cgrates.org:bv", BalanceMap: map[string]Balances{ utils.MONETARY: Balances{&Balance{ + ID: "*default", + Uuid: "25a02c82-f09f-4c6e-bacf-8ed4b076475a", Value: 10, }}, }, @@ -2114,16 +2116,29 @@ func TestActionCdrlogBalanceValue(t *testing.T) { Timing: &RateInterval{}, actions: []*Action{ &Action{ + Id: "RECUR_FOR_V3HSILLMILLD1G", ActionType: TOPUP, - Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{ + ID: utils.StringPointer("*default"), + Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), + Value: utils.Float64Pointer(1.1), + Type: utils.StringPointer(utils.MONETARY), + }, }, &Action{ + Id: "RECUR_FOR_V3HSILLMILLD5G", ActionType: DEBIT, - Balance: &BalanceFilter{Value: utils.Float64Pointer(2.1), Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{ + ID: utils.StringPointer("*default"), + Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), + Value: utils.Float64Pointer(2.1), + Type: utils.StringPointer(utils.MONETARY), + }, }, &Action{ + Id: "c", ActionType: CDRLOG, - ExtraParameters: `{"BalanceValue":"BalanceValue"}`, + ExtraParameters: `{"BalanceID":"BalanceID","BalanceUUID":"BalanceUUID","ActionID":"ActionID","BalanceValue":"BalanceValue"}`, }, }, } @@ -2140,7 +2155,7 @@ func TestActionCdrlogBalanceValue(t *testing.T) { if len(cdrs) != 2 || cdrs[0].ExtraFields["BalanceValue"] != "11.1" || cdrs[1].ExtraFields["BalanceValue"] != "9" { - t.Errorf("Wrong cdrlogs: %+v", cdrs[1]) + t.Errorf("Wrong cdrlogs: %", utils.ToIJSON(cdrs)) } } diff --git a/utils/struct.go b/utils/struct.go index 0b16caeef..3da80ec84 100644 --- a/utils/struct.go +++ b/utils/struct.go @@ -18,6 +18,7 @@ along with this program. If not, see package utils import ( + "errors" "reflect" "strconv" "strings" @@ -171,6 +172,28 @@ func FromMapStringString(m map[string]string, in interface{}) { return } +func FromMapStringInterface(m map[string]interface{}, in interface{}) error { + v := reflect.ValueOf(in) + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + for fieldName, fieldValue := range m { + field := v.FieldByName(fieldName) + if field.IsValid() { + if !field.IsValid() || !field.CanSet() { + continue + } + structFieldType := field.Type() + val := reflect.ValueOf(fieldValue) + if structFieldType != val.Type() { + return errors.New("Provided value type didn't match obj field type") + } + field.Set(val) + } + } + return nil +} + // Update struct with map fields, returns not matching map keys, s is a struct to be updated func UpdateStructWithStrMap(s interface{}, m map[string]string) []string { notMatched := []string{} diff --git a/utils/struct_test.go b/utils/struct_test.go index 50371ea36..74450a7cb 100644 --- a/utils/struct_test.go +++ b/utils/struct_test.go @@ -84,3 +84,32 @@ func TestStructExtraFields(t *testing.T) { t.Errorf("expected: %v got: %v", ts.ExtraFields, efMap) } } + +func TestStructFromMapStringInterface(t *testing.T) { + ts := &struct { + Name string + Class *string + List []string + Elements struct { + Type string + Value float64 + } + }{} + s := "test2" + m := map[string]interface{}{ + "Name": "test1", + "Class": &s, + "List": []string{"test3", "test4"}, + "Elements": struct { + Type string + Value float64 + }{ + Type: "test5", + Value: 9.8, + }, + } + if err := FromMapStringInterface(m, ts); err != nil { + t.Logf("ts: %+v", ToJSON(ts)) + t.Error("Error converting map to struct: ", err) + } +} From c1bc21b996e67f23c051e5e44f329ad4f5bc0d56 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Apr 2016 13:27:57 +0300 Subject: [PATCH 167/227] added method --- engine/action.go | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/action.go b/engine/action.go index 3aedbf153..a7f9cdb5e 100644 --- a/engine/action.go +++ b/engine/action.go @@ -653,6 +653,7 @@ func transferMonetaryDefaultAction(acc *Account, sq *StatsQueueTriggered, a *Act type RPCRequest struct { Server string Transport string + Method string Attempts int Async bool Arg map[string]interface{} From 75a565a6584d8ed2f83d3c2d0d400eb8725c3f2f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Apr 2016 18:35:42 +0300 Subject: [PATCH 168/227] started cgr_rpc --- utils/rpc_object_test.go | 21 +++++++++++++++++++++ utils/rpc_objects.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 utils/rpc_object_test.go create mode 100644 utils/rpc_objects.go diff --git a/utils/rpc_object_test.go b/utils/rpc_object_test.go new file mode 100644 index 000000000..8cdf368c3 --- /dev/null +++ b/utils/rpc_object_test.go @@ -0,0 +1,21 @@ +package utils + +import "testing" + +type RpcStruct struct{} + +func (rpc *RpcStruct) Hopa(normal string, out *float64) error { + return nil +} + +func TestRPCObjectSimple(t *testing.T) { + + RegisterRpcObject("", &RpcStruct{}) + if len(RpcObjects) != 1 { + t.Errorf("error registering rpc object: %v", RpcObjects) + } + x, found := RpcObjects["RpcStruct.Hopa"] + if found { + t.Errorf("error getting rpcobject: %v (%+v)", RpcObjects, x) + } +} diff --git a/utils/rpc_objects.go b/utils/rpc_objects.go new file mode 100644 index 000000000..6aef9be2e --- /dev/null +++ b/utils/rpc_objects.go @@ -0,0 +1,38 @@ +package utils + +import "reflect" + +var RpcObjects map[string]interface{} + +type RpcObject struct { + Object interface{} + InParam interface{} + OutParam interface{} +} + +func init() { + RpcObjects = make(map[string]interface{}) +} + +func RegisterRpcObject(name string, rpcObject interface{}) { + objType := reflect.TypeOf(rpcObject) + if name == "" { + val := reflect.ValueOf(rpcObject) + name = objType.Name() + if val.Kind() == reflect.Ptr { + name = objType.Elem().Name() + } + } + for i := 0; i < objType.NumMethod(); i++ { + method := objType.Method(i) + methodType := method.Type + if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) + RpcObjects[name+"."+method.Name] = &RpcObject{ + Object: objType, + InParam: reflect.New(methodType.In(1)).Elem().Interface(), + OutParam: reflect.New(methodType.In(2)).Elem().Interface(), + } + } + + } +} From d158fcec80f0a028d9c6c8dceb326de4f336910a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Apr 2016 19:07:37 +0300 Subject: [PATCH 169/227] work in proggress --- engine/action.go | 14 ++++++++++---- utils/rpc_object_test.go | 13 ++++++++++--- utils/rpc_objects.go | 4 ++-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/engine/action.go b/engine/action.go index a7f9cdb5e..4018a7c1c 100644 --- a/engine/action.go +++ b/engine/action.go @@ -32,6 +32,7 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) /* @@ -651,19 +652,24 @@ func transferMonetaryDefaultAction(acc *Account, sq *StatsQueueTriggered, a *Act } type RPCRequest struct { - Server string + Address string Transport string Method string Attempts int Async bool - Arg map[string]interface{} + Param string } func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error { - rpcRequest := RPCRequest{} - if err := json.Unmarshal([]byte(a.ExtraParameters), &rpcRequest); err != nil { + req := RPCRequest{} + if err := json.Unmarshal([]byte(a.ExtraParameters), &req); err != nil { return err } + client, err := rpcclient.NewRpcClient(req.Method, req.Address, req.Attempts, 0, req.Transport, nil) + if err != nil { + return nil, err + } + client.Call() return nil } diff --git a/utils/rpc_object_test.go b/utils/rpc_object_test.go index 8cdf368c3..f42ea882a 100644 --- a/utils/rpc_object_test.go +++ b/utils/rpc_object_test.go @@ -8,14 +8,21 @@ func (rpc *RpcStruct) Hopa(normal string, out *float64) error { return nil } -func TestRPCObjectSimple(t *testing.T) { +func (rpc *RpcStruct) Tropa(pointer *string, out *float64) error { + return nil +} +func TestRPCObjectPointer(t *testing.T) { RegisterRpcObject("", &RpcStruct{}) - if len(RpcObjects) != 1 { + if len(RpcObjects) != 2 { t.Errorf("error registering rpc object: %v", RpcObjects) } x, found := RpcObjects["RpcStruct.Hopa"] - if found { + if !found { + t.Errorf("error getting rpcobject: %v (%+v)", RpcObjects, x) + } + x, found = RpcObjects["RpcStruct.Tropa"] + if !found { t.Errorf("error getting rpcobject: %v (%+v)", RpcObjects, x) } } diff --git a/utils/rpc_objects.go b/utils/rpc_objects.go index 6aef9be2e..22028d274 100644 --- a/utils/rpc_objects.go +++ b/utils/rpc_objects.go @@ -29,8 +29,8 @@ func RegisterRpcObject(name string, rpcObject interface{}) { if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) RpcObjects[name+"."+method.Name] = &RpcObject{ Object: objType, - InParam: reflect.New(methodType.In(1)).Elem().Interface(), - OutParam: reflect.New(methodType.In(2)).Elem().Interface(), + InParam: reflect.New(methodType.In(1)).Interface(), + OutParam: reflect.New(methodType.In(2).Elem()).Interface(), } } From b198d5fc181288c141e57afc06b298ba486373fa Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 18 Apr 2016 23:51:32 +0300 Subject: [PATCH 170/227] first *rpc_cgr tests --- engine/action.go | 22 ++++++++++++++---- engine/actions_test.go | 38 ++++++++++++++++++++++++++++++ utils/rpc_object_test.go | 28 ---------------------- utils/rpc_objects.go | 38 ------------------------------ utils/rpc_params.go | 50 ++++++++++++++++++++++++++++++++++++++++ utils/rpc_params_test.go | 32 +++++++++++++++++++++++++ 6 files changed, 138 insertions(+), 70 deletions(-) delete mode 100644 utils/rpc_object_test.go delete mode 100644 utils/rpc_objects.go create mode 100644 utils/rpc_params.go create mode 100644 utils/rpc_params_test.go diff --git a/engine/action.go b/engine/action.go index 4018a7c1c..0d4e51fe4 100644 --- a/engine/action.go +++ b/engine/action.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/smtp" "path" "reflect" @@ -665,12 +666,25 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti if err := json.Unmarshal([]byte(a.ExtraParameters), &req); err != nil { return err } - client, err := rpcclient.NewRpcClient(req.Method, req.Address, req.Attempts, 0, req.Transport, nil) + log.Printf("REQ: %+v", req) + params, err := utils.GetRpcParams(req.Method) if err != nil { - return nil, err + return err } - client.Call() - return nil + var client rpcclient.RpcClientConnection + if req.Address != utils.INTERNAL { + if client, err = rpcclient.NewRpcClient(req.Method, req.Address, req.Attempts, 0, req.Transport, nil); err != nil { + return err + } + } else { + client = params.Object + } + + in, out := params.InParam, params.OutParam + if err := json.Unmarshal([]byte(req.Param), &in); err != nil { + return err + } + return client.Call(req.Method, in, out) } // Structure to store actions according to weight diff --git a/engine/actions_test.go b/engine/actions_test.go index 9b3dc8f22..738a25d72 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2159,6 +2159,44 @@ func TestActionCdrlogBalanceValue(t *testing.T) { } } +type TestRPCParameters struct { + status string +} + +type Attr struct { + Name string + Surname string + Age float64 +} + +func (trpcp *TestRPCParameters) Hopa(in Attr, out *float64) error { + trpcp.status = utils.OK + return nil +} + +func (trpcp *TestRPCParameters) Call(string, interface{}, interface{}) error { + return nil +} + +func TestCgrRpcAction(t *testing.T) { + trpcp := &TestRPCParameters{} + utils.RegisterRpcParams("", trpcp) + a := &Action{ + ExtraParameters: `{"Address": "internal", + "Transport": "*gob", + "Method": "TestRPCParameters.Hopa", + "Attempts":1, + "Async" :false, + "Param": "{\"Name\":\"n\", \"Surname\":\"s\", \"Age\":10.2}"}`, + } + if err := cgrRPCAction(nil, nil, a, nil); err != nil { + t.Error("error executing cgr action: ", err) + } + if trpcp.status != utils.OK { + t.Error("RPC not called!") + } +} + /**************** Benchmarks ********************************/ func BenchmarkUUID(b *testing.B) { diff --git a/utils/rpc_object_test.go b/utils/rpc_object_test.go deleted file mode 100644 index f42ea882a..000000000 --- a/utils/rpc_object_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package utils - -import "testing" - -type RpcStruct struct{} - -func (rpc *RpcStruct) Hopa(normal string, out *float64) error { - return nil -} - -func (rpc *RpcStruct) Tropa(pointer *string, out *float64) error { - return nil -} - -func TestRPCObjectPointer(t *testing.T) { - RegisterRpcObject("", &RpcStruct{}) - if len(RpcObjects) != 2 { - t.Errorf("error registering rpc object: %v", RpcObjects) - } - x, found := RpcObjects["RpcStruct.Hopa"] - if !found { - t.Errorf("error getting rpcobject: %v (%+v)", RpcObjects, x) - } - x, found = RpcObjects["RpcStruct.Tropa"] - if !found { - t.Errorf("error getting rpcobject: %v (%+v)", RpcObjects, x) - } -} diff --git a/utils/rpc_objects.go b/utils/rpc_objects.go deleted file mode 100644 index 22028d274..000000000 --- a/utils/rpc_objects.go +++ /dev/null @@ -1,38 +0,0 @@ -package utils - -import "reflect" - -var RpcObjects map[string]interface{} - -type RpcObject struct { - Object interface{} - InParam interface{} - OutParam interface{} -} - -func init() { - RpcObjects = make(map[string]interface{}) -} - -func RegisterRpcObject(name string, rpcObject interface{}) { - objType := reflect.TypeOf(rpcObject) - if name == "" { - val := reflect.ValueOf(rpcObject) - name = objType.Name() - if val.Kind() == reflect.Ptr { - name = objType.Elem().Name() - } - } - for i := 0; i < objType.NumMethod(); i++ { - method := objType.Method(i) - methodType := method.Type - if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) - RpcObjects[name+"."+method.Name] = &RpcObject{ - Object: objType, - InParam: reflect.New(methodType.In(1)).Interface(), - OutParam: reflect.New(methodType.In(2).Elem()).Interface(), - } - } - - } -} diff --git a/utils/rpc_params.go b/utils/rpc_params.go new file mode 100644 index 000000000..ffd667b98 --- /dev/null +++ b/utils/rpc_params.go @@ -0,0 +1,50 @@ +package utils + +import ( + "reflect" + + "github.com/cgrates/rpcclient" +) + +var rpcParamsMap map[string]*RpcParams + +type RpcParams struct { + Object rpcclient.RpcClientConnection + InParam interface{} + OutParam interface{} +} + +func init() { + rpcParamsMap = make(map[string]*RpcParams) +} + +func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { + objType := reflect.TypeOf(obj) + if name == "" { + val := reflect.ValueOf(obj) + name = objType.Name() + if val.Kind() == reflect.Ptr { + name = objType.Elem().Name() + } + } + for i := 0; i < objType.NumMethod(); i++ { + method := objType.Method(i) + methodType := method.Type + if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) + rpcParamsMap[name+"."+method.Name] = &RpcParams{ + Object: obj, + InParam: reflect.New(methodType.In(1)).Interface(), + OutParam: reflect.New(methodType.In(2).Elem()).Interface(), + } + } + + } +} + +func GetRpcParams(method string) (*RpcParams, error) { + x, found := rpcParamsMap[method] + if !found { + return nil, ErrNotFound + } + return x, nil +} diff --git a/utils/rpc_params_test.go b/utils/rpc_params_test.go new file mode 100644 index 000000000..a500fd3c2 --- /dev/null +++ b/utils/rpc_params_test.go @@ -0,0 +1,32 @@ +package utils + +import "testing" + +type RpcStruct struct{} + +func (rpc *RpcStruct) Hopa(normal string, out *float64) error { + return nil +} + +func (rpc *RpcStruct) Tropa(pointer *string, out *float64) error { + return nil +} + +func (rpc *RpcStruct) Call(string, interface{}, interface{}) error { + return nil +} + +func TestRPCObjectPointer(t *testing.T) { + RegisterRpcParams("", &RpcStruct{}) + if len(rpcParamsMap) != 2 { + t.Errorf("error registering rpc object: %v", rpcParamsMap) + } + x, found := rpcParamsMap["RpcStruct.Hopa"] + if !found { + t.Errorf("error getting rpcobject: %v (%+v)", rpcParamsMap, x) + } + x, found = rpcParamsMap["RpcStruct.Tropa"] + if !found { + t.Errorf("error getting rpcobject: %v (%+v)", rpcParamsMap, x) + } +} From f08524837a5b8c52e5571d92a1f5aa73ea188021 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 19 Apr 2016 09:57:19 +0300 Subject: [PATCH 171/227] testing *cgr_rpc --- engine/action.go | 4 +++- engine/actions_test.go | 29 +++++++++++++++++++++++++++-- utils/rpc_params.go | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/engine/action.go b/engine/action.go index 0d4e51fe4..8aeeb5676 100644 --- a/engine/action.go +++ b/engine/action.go @@ -666,7 +666,6 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti if err := json.Unmarshal([]byte(a.ExtraParameters), &req); err != nil { return err } - log.Printf("REQ: %+v", req) params, err := utils.GetRpcParams(req.Method) if err != nil { return err @@ -681,9 +680,12 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti } in, out := params.InParam, params.OutParam + log.Print("IN: ", reflect.TypeOf(in)) + log.Print(req.Param) if err := json.Unmarshal([]byte(req.Param), &in); err != nil { return err } + log.Print("IN: ", reflect.TypeOf(in)) return client.Call(req.Method, in, out) } diff --git a/engine/actions_test.go b/engine/actions_test.go index 738a25d72..02e941e8f 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "reflect" + "strings" "testing" "time" @@ -2174,8 +2175,32 @@ func (trpcp *TestRPCParameters) Hopa(in Attr, out *float64) error { return nil } -func (trpcp *TestRPCParameters) Call(string, interface{}, interface{}) error { - return nil +func (trpcp *TestRPCParameters) Call(serviceMethod string, args interface{}, reply interface{}) error { + parts := strings.Split(serviceMethod, ".") + if len(parts) != 2 { + return utils.ErrNotImplemented + } + // get method + method := reflect.ValueOf(trpcp).MethodByName(parts[1]) + if !method.IsValid() { + return utils.ErrNotImplemented + } + + // construct the params + params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} + + ret := method.Call(params) + if len(ret) != 1 { + return utils.ErrServerError + } + if ret[0].Interface() == nil { + return nil + } + err, ok := ret[0].Interface().(error) + if !ok { + return utils.ErrServerError + } + return err } func TestCgrRpcAction(t *testing.T) { diff --git a/utils/rpc_params.go b/utils/rpc_params.go index ffd667b98..21a4aa5f4 100644 --- a/utils/rpc_params.go +++ b/utils/rpc_params.go @@ -33,7 +33,7 @@ func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) rpcParamsMap[name+"."+method.Name] = &RpcParams{ Object: obj, - InParam: reflect.New(methodType.In(1)).Interface(), + InParam: reflect.Zero(methodType.In(1)).Interface(), OutParam: reflect.New(methodType.In(2).Elem()).Interface(), } } From c174605ab8277fd02f14c2ed5720757bb8cb2109 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 19 Apr 2016 18:14:44 +0300 Subject: [PATCH 172/227] passing call test --- engine/action.go | 7 ++----- utils/rpc_params.go | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/engine/action.go b/engine/action.go index 8aeeb5676..35834d55b 100644 --- a/engine/action.go +++ b/engine/action.go @@ -22,7 +22,6 @@ import ( "encoding/json" "errors" "fmt" - "log" "net/smtp" "path" "reflect" @@ -680,12 +679,10 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti } in, out := params.InParam, params.OutParam - log.Print("IN: ", reflect.TypeOf(in)) - log.Print(req.Param) - if err := json.Unmarshal([]byte(req.Param), &in); err != nil { + x := in + if err := json.Unmarshal([]byte(req.Param), &x); err != nil { return err } - log.Print("IN: ", reflect.TypeOf(in)) return client.Call(req.Method, in, out) } diff --git a/utils/rpc_params.go b/utils/rpc_params.go index 21a4aa5f4..9eaecba1a 100644 --- a/utils/rpc_params.go +++ b/utils/rpc_params.go @@ -33,7 +33,7 @@ func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) rpcParamsMap[name+"."+method.Name] = &RpcParams{ Object: obj, - InParam: reflect.Zero(methodType.In(1)).Interface(), + InParam: (reflect.New(methodType.In(1)).Elem()).Interface(), OutParam: reflect.New(methodType.In(2).Elem()).Interface(), } } From da1a9b344cf8c8cfd37248e8db2dda6c88771c27 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 20 Apr 2016 00:39:14 +0300 Subject: [PATCH 173/227] integration test for *cgr_rpc --- apier/v1/apier.go | 5 +++-- apier/v1/apier_local_test.go | 2 +- data/tariffplans/testtp/Actions.csv | 1 + engine/action.go | 20 ++++++++++++++++---- engine/actions_test.go | 2 +- general_tests/tp_it_test.go | 19 ++++++++++++++++++- utils/rpc_params.go | 4 ++-- utils/rpc_params_test.go | 22 +++++++++++++++++++--- utils/struct.go | 21 +++++++++++++++++++++ 9 files changed, 82 insertions(+), 14 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index b6abf47f0..e8eaa63ed 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -103,11 +103,12 @@ func (self *ApierV1) GetRatingPlan(rplnId string, reply *engine.RatingPlan) erro } func (self *ApierV1) ExecuteAction(attr *utils.AttrExecuteAction, reply *string) error { - accID := utils.AccountKey(attr.Tenant, attr.Account) at := &engine.ActionTiming{ ActionsID: attr.ActionsId, } - at.SetAccountIDs(utils.StringMap{accID: true}) + if attr.Tenant != "" && attr.Account != "" { + at.SetAccountIDs(utils.StringMap{utils.AccountKey(attr.Tenant, attr.Account): true}) + } if err := at.Execute(); err != nil { *reply = err.Error() return err diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 5c2504cec..124b6a984 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1297,7 +1297,7 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { if rcvStats.Destinations != 5 || rcvStats.RatingPlans != 5 || rcvStats.RatingProfiles != 5 || - rcvStats.Actions != 9 || + rcvStats.Actions != 10 || rcvStats.DerivedChargers != 3 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) } diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 98a48be14..1525522f2 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -9,3 +9,4 @@ TOPUP_DATA_r,*topup,,,,*monetary,*out,,DATA_DEST,,,*unlimited,,5000000,10,false, TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 TOPUP_NEG,*topup,,,,*voice,*out,,GERMANY;!GERMANY_MOBILE,*zero1m,,*unlimited,,100,10,false,false,10 +RPC,*cgr_rpc,"{""Address"": ""localhost:2013"",""Transport"":""*gob"",""Method"":""ApierV2.SetAccount"",""Attempts"":1,""Async"" :false,""Param"":{""Account"":""rpc"",""Tenant"":""cgrates.org""}}",,,,,,,,,,,,,,, diff --git a/engine/action.go b/engine/action.go index 35834d55b..84669ff1f 100644 --- a/engine/action.go +++ b/engine/action.go @@ -657,7 +657,7 @@ type RPCRequest struct { Method string Attempts int Async bool - Param string + Param map[string]interface{} } func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error { @@ -677,13 +677,25 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti } else { client = params.Object } + if client == nil { + return utils.ErrServerError + } in, out := params.InParam, params.OutParam - x := in - if err := json.Unmarshal([]byte(req.Param), &x); err != nil { + p, err := utils.FromMapStringInterfaceValue(req.Param, in) + if err != nil { return err } - return client.Call(req.Method, in, out) + if !req.Async { + err = client.Call(req.Method, p, out) + utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %+v err: %v", out, err)) + return err + } + go func() { + err := client.Call(req.Method, p, out) + utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %+v err: %v", out, err)) + }() + return nil } // Structure to store actions according to weight diff --git a/engine/actions_test.go b/engine/actions_test.go index 02e941e8f..f9297d25a 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2212,7 +2212,7 @@ func TestCgrRpcAction(t *testing.T) { "Method": "TestRPCParameters.Hopa", "Attempts":1, "Async" :false, - "Param": "{\"Name\":\"n\", \"Surname\":\"s\", \"Age\":10.2}"}`, + "Param": {"Name":"n", "Surname":"s", "Age":10.2}}`, } if err := cgrRPCAction(nil, nil, a, nil); err != nil { t.Error("error executing cgr action: ", err) diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index cb66dca3e..8165ed19c 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -214,7 +214,7 @@ func TestTpZeroNegativeCost(t *testing.T) { } else if cc.GetDuration() != 20*time.Second { t.Errorf("Calling Responder.MaxDebit got callcost: %v", utils.ToIJSON(cc)) } - var acnt *engine.Account + var acnt engine.Account attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1013"} if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error("Got error on ApierV2.GetAccount: ", err.Error()) @@ -222,3 +222,20 @@ func TestTpZeroNegativeCost(t *testing.T) { t.Errorf("Calling ApierV2.GetAccount received: %s", utils.ToIJSON(acnt)) } } + +func TestTpExecuteActionCgrRpc(t *testing.T) { + if !*testIntegration { + return + } + var reply string + if err := tpRPC.Call("ApierV2.ExecuteAction", utils.AttrExecuteAction{ActionsId: "RPC"}, &reply); err != nil { + t.Error("Got error on ApierV2.ExecuteAction: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ExecuteAction got reply: %s", reply) + } + var acnt engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "rpc"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } +} diff --git a/utils/rpc_params.go b/utils/rpc_params.go index 9eaecba1a..69020fdec 100644 --- a/utils/rpc_params.go +++ b/utils/rpc_params.go @@ -10,7 +10,7 @@ var rpcParamsMap map[string]*RpcParams type RpcParams struct { Object rpcclient.RpcClientConnection - InParam interface{} + InParam reflect.Value OutParam interface{} } @@ -33,7 +33,7 @@ func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) rpcParamsMap[name+"."+method.Name] = &RpcParams{ Object: obj, - InParam: (reflect.New(methodType.In(1)).Elem()).Interface(), + InParam: reflect.New(methodType.In(1)), OutParam: reflect.New(methodType.In(2).Elem()).Interface(), } } diff --git a/utils/rpc_params_test.go b/utils/rpc_params_test.go index a500fd3c2..aac04ecdc 100644 --- a/utils/rpc_params_test.go +++ b/utils/rpc_params_test.go @@ -4,11 +4,17 @@ import "testing" type RpcStruct struct{} -func (rpc *RpcStruct) Hopa(normal string, out *float64) error { +type Attr struct { + Name string + Surname string + Age float64 +} + +func (rpc *RpcStruct) Hopa(normal Attr, out *float64) error { return nil } -func (rpc *RpcStruct) Tropa(pointer *string, out *float64) error { +func (rpc *RpcStruct) Tropa(pointer *Attr, out *float64) error { return nil } @@ -25,8 +31,18 @@ func TestRPCObjectPointer(t *testing.T) { if !found { t.Errorf("error getting rpcobject: %v (%+v)", rpcParamsMap, x) } - x, found = rpcParamsMap["RpcStruct.Tropa"] + a := x.InParam + if v, err := FromMapStringInterfaceValue(map[string]interface{}{"Name": "a", "Surname": "b", "Age": 10.2}, a); err != nil || v.(Attr).Name != "a" || v.(Attr).Surname != "b" || v.(Attr).Age != 10.2 { + t.Errorf("error converting to struct: %+v (%v)", v, err) + } + //TODO: make pointer in arguments usable + /*x, found = rpcParamsMap["RpcStruct.Tropa"] if !found { t.Errorf("error getting rpcobject: %v (%+v)", rpcParamsMap, x) } + b := x.InParam + log.Printf("T: %+v", b) + if v, err := FromMapStringInterfaceValue(map[string]interface{}{"Name": "a", "Surname": "b", "Age": 10.2}, b); err != nil || v.(Attr).Name != "a" || v.(Attr).Surname != "b" || v.(Attr).Age != 10.2 { + t.Errorf("error converting to struct: %+v (%v)", v, err) + }*/ } diff --git a/utils/struct.go b/utils/struct.go index 3da80ec84..e6f20fff3 100644 --- a/utils/struct.go +++ b/utils/struct.go @@ -194,6 +194,27 @@ func FromMapStringInterface(m map[string]interface{}, in interface{}) error { return nil } +func FromMapStringInterfaceValue(m map[string]interface{}, v reflect.Value) (interface{}, error) { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + for fieldName, fieldValue := range m { + field := v.FieldByName(fieldName) + if field.IsValid() { + if !field.IsValid() || !field.CanSet() { + continue + } + structFieldType := field.Type() + val := reflect.ValueOf(fieldValue) + if structFieldType != val.Type() { + return nil, errors.New("Provided value type didn't match obj field type") + } + field.Set(val) + } + } + return v.Interface(), nil +} + // Update struct with map fields, returns not matching map keys, s is a struct to be updated func UpdateStructWithStrMap(s interface{}, m map[string]string) []string { notMatched := []string{} From 671f35bbcac18ad97cc787e2bc0b3f8e189d02b9 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 20 Apr 2016 13:24:09 +0200 Subject: [PATCH 174/227] Refactored configuration file to include cluster connections everywhere, renamed old rater into RALs, fixes #389 --- config/cdrcconfig.go | 16 +- config/cfg_data.json | 10 +- config/cfg_data2.json | 4 +- config/config.go | 300 ++++++++++++++++++++----------------- config/config_defaults.go | 133 ++++++++-------- config/config_json.go | 8 +- config/config_json_test.go | 95 ++++++------ config/config_test.go | 12 +- config/configcdrc_test.go | 8 +- config/libconfig.go | 4 +- config/libconfig_json.go | 49 +++--- config/smconfig.go | 122 +++++++-------- config/smconfig_test.go | 12 +- utils/consts.go | 1 + 14 files changed, 408 insertions(+), 366 deletions(-) diff --git a/config/cdrcconfig.go b/config/cdrcconfig.go index 00005be51..abac4b187 100644 --- a/config/cdrcconfig.go +++ b/config/cdrcconfig.go @@ -27,7 +27,7 @@ import ( type CdrcConfig struct { Enabled bool // Enable/Disable the profile DryRun bool // Do not post CDRs to the server - Cdrs string // The address where CDRs can be reached + CdrsConns []*HaPoolConfig // The address where CDRs can be reached CdrFormat string // The type of CDR file to process FieldSeparator rune // The separator to use when reading csvs DataUsageMultiplyFactor float64 // Conversion factor for data usage @@ -57,8 +57,12 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { if jsnCfg.Dry_run != nil { self.DryRun = *jsnCfg.Dry_run } - if jsnCfg.Cdrs != nil { - self.Cdrs = *jsnCfg.Cdrs + if jsnCfg.Cdrs_conns != nil { + self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { + self.CdrsConns[idx] = NewDfltHaPoolConfig() + self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) + } } if jsnCfg.Cdr_format != nil { self.CdrFormat = *jsnCfg.Cdr_format @@ -126,7 +130,11 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { func (self *CdrcConfig) Clone() *CdrcConfig { clnCdrc := new(CdrcConfig) clnCdrc.Enabled = self.Enabled - clnCdrc.Cdrs = self.Cdrs + clnCdrc.CdrsConns = make([]*HaPoolConfig, len(self.CdrsConns)) + for idx, cdrConn := range self.CdrsConns { + clonedVal := *cdrConn + clnCdrc.CdrsConns[idx] = &clonedVal + } clnCdrc.CdrFormat = self.CdrFormat clnCdrc.FieldSeparator = self.FieldSeparator clnCdrc.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor diff --git a/config/cfg_data.json b/config/cfg_data.json index f5eb4a811..000854998 100644 --- a/config/cfg_data.json +++ b/config/cfg_data.json @@ -7,14 +7,14 @@ // This is what you get when you load CGRateS with an empty configuration file. "general": { - "default_reqtype": "*pseudoprepaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> + "default_request_type": "*pseudoprepaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> }, "cdrs": { "enabled": true, // start the CDR Server service: }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, @@ -42,9 +42,9 @@ "sm_freeswitch": { "enabled": true, // starts SessionManager service: - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 5}, - {"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5}, + "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers + {"address": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 5}, + {"address": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5}, ], }, diff --git a/config/cfg_data2.json b/config/cfg_data2.json index 67b846168..dc001f760 100644 --- a/config/cfg_data2.json +++ b/config/cfg_data2.json @@ -22,8 +22,8 @@ "sm_freeswitch": { "enabled": true, // starts SessionManager service: - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5}, + "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers + {"address": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5}, ], }, diff --git a/config/config.go b/config/config.go index 39963358a..b605cfe35 100644 --- a/config/config.go +++ b/config/config.go @@ -86,8 +86,8 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { } cfg.dfltCdreProfile = cfg.CdreProfiles[utils.META_DEFAULT].Clone() // So default will stay unique, will have nil pointer in case of no defaults loaded which is an extra check cfg.dfltCdrcProfile = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT].Clone() - dfltFsConnConfig = cfg.SmFsConfig.Connections[0] // We leave it crashing here on purpose if no Connection defaults defined - dfltKamConnConfig = cfg.SmKamConfig.Connections[0] + dfltFsConnConfig = cfg.SmFsConfig.EventSocketConns[0] // We leave it crashing here on purpose if no Connection defaults defined + dfltKamConnConfig = cfg.SmKamConfig.EvapiConns[0] if err := cfg.checkConfigSanity(); err != nil { return nil, err } @@ -191,31 +191,30 @@ type CGRConfig struct { StorDBMaxOpenConns int // Maximum database connections opened StorDBMaxIdleConns int // Maximum idle connections to keep opened StorDBCDRSIndexes []string - DBDataEncoding string // The encoding used to store object data in strings: - RPCJSONListen string // RPC JSON listening address - RPCGOBListen string // RPC GOB listening address - HTTPListen string // HTTP listening address - DefaultReqType string // Use this request type if not defined on top - DefaultCategory string // set default type of record - DefaultTenant string // set default tenant - DefaultSubject string // set default rating subject, useful in case of fallback - DefaultTimezone string // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> - Reconnects int // number of recconect attempts in case of connection lost <-1 for infinite | nb> - ConnectAttempts int // number of initial connection attempts before giving up - ResponseCacheTTL time.Duration // the life span of a cached response - InternalTtl time.Duration // maximum duration to wait for internal connections before giving up - RoundingDecimals int // Number of decimals to round end prices at - HttpSkipTlsVerify bool // If enabled Http Client will accept any TLS certificate - TpExportPath string // Path towards export folder for offline Tariff Plans - HttpFailedDir string // Directory path where we store failed http requests - MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file - RaterEnabled bool // start standalone server (no balancer) - RaterBalancer string // balancer address host:port - RaterCdrStats string // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> - RaterHistoryServer string - RaterPubSubServer string - RaterUserServer string - RaterAliasesServer string + DBDataEncoding string // The encoding used to store object data in strings: + RPCJSONListen string // RPC JSON listening address + RPCGOBListen string // RPC GOB listening address + HTTPListen string // HTTP listening address + DefaultReqType string // Use this request type if not defined on top + DefaultCategory string // set default type of record + DefaultTenant string // set default tenant + DefaultTimezone string // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> + Reconnects int // number of recconect attempts in case of connection lost <-1 for infinite | nb> + ConnectAttempts int // number of initial connection attempts before giving up + ResponseCacheTTL time.Duration // the life span of a cached response + InternalTtl time.Duration // maximum duration to wait for internal connections before giving up + RoundingDecimals int // Number of decimals to round end prices at + HttpSkipTlsVerify bool // If enabled Http Client will accept any TLS certificate + TpExportPath string // Path towards export folder for offline Tariff Plans + HttpFailedDir string // Directory path where we store failed http requests + MaxCallDuration time.Duration // The maximum call duration (used by responder when querying DerivedCharging) // ToDo: export it in configuration file + RALsEnabled bool // start standalone server (no balancer) + RALsBalancer string // balancer address host:port + RALsCDRStatSConns []*HaPoolConfig // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> + RALsHistorySConns []*HaPoolConfig + RALsPubSubSConns []*HaPoolConfig + RALsUserSConns []*HaPoolConfig + RALsAliasSConns []*HaPoolConfig RpSubjectPrefixMatching bool // enables prefix matching for the rating profile subject LcrSubjectPrefixMatching bool // enables prefix matching for the lcr subject BalancerEnabled bool @@ -234,9 +233,9 @@ type CGRConfig struct { CdreProfiles map[string]*CdreConfig CdrcProfiles map[string]map[string]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath]map[instanceName]{Configs} SmGenericConfig *SmGenericConfig - SmFsConfig *SmFsConfig // SM-FreeSWITCH configuration + SmFsConfig *SmFsConfig // SMFreeSWITCH configuration SmKamConfig *SmKamConfig // SM-Kamailio Configuration - SmOsipsConfig *SmOsipsConfig // SM-OpenSIPS Configuration + SmOsipsConfig *SmOsipsConfig // SMOpenSIPS Configuration diameterAgentCfg *DiameterAgentCfg // DiameterAgent configuration HistoryServer string // Address where to reach the master history server: HistoryServerEnabled bool // Starts History as server: . @@ -260,64 +259,77 @@ type CGRConfig struct { func (self *CGRConfig) checkConfigSanity() error { // Rater checks - if self.RaterEnabled { - if self.RaterBalancer == utils.INTERNAL && !self.BalancerEnabled { + if self.RALsEnabled { + if self.RALsBalancer == utils.MetaInternal && !self.BalancerEnabled { return errors.New("Balancer not enabled but requested by Rater component.") } - if self.RaterCdrStats == utils.INTERNAL && !self.CDRStatsEnabled { - return errors.New("CDRStats not enabled but requested by Rater component.") + for _, connCfg := range self.RALsCDRStatSConns { + if connCfg.Address == utils.MetaInternal && !self.CDRStatsEnabled { + return errors.New("CDRStats not enabled but requested by Rater component.") + } } - if self.RaterHistoryServer == utils.INTERNAL && !self.HistoryServerEnabled { - return errors.New("History server not enabled but requested by Rater component.") + for _, connCfg := range self.RALsHistorySConns { + if connCfg.Address == utils.MetaInternal && !self.HistoryServerEnabled { + return errors.New("History server not enabled but requested by Rater component.") + } } - if self.RaterPubSubServer == utils.INTERNAL && !self.PubSubServerEnabled { - return errors.New("PubSub server not enabled but requested by Rater component.") + for _, connCfg := range self.RALsPubSubSConns { + if connCfg.Address == utils.MetaInternal && !self.PubSubServerEnabled { + return errors.New("PubSub server not enabled but requested by Rater component.") + } } - if self.RaterAliasesServer == utils.INTERNAL && !self.AliasesServerEnabled { - return errors.New("Aliases server not enabled but requested by Rater component.") + for _, connCfg := range self.RALsAliasSConns { + if connCfg.Address == utils.MetaInternal && !self.AliasesServerEnabled { + return errors.New("Alias server not enabled but requested by Rater component.") + } } - if self.RaterUserServer == utils.INTERNAL && !self.UserServerEnabled { - return errors.New("Users service not enabled but requested by Rater component.") + for _, connCfg := range self.RALsUserSConns { + if connCfg.Address == utils.MetaInternal && !self.UserServerEnabled { + return errors.New("User service not enabled but requested by Rater component.") + } } } // CDRServer checks if self.CDRSEnabled { for _, cdrsRaterConn := range self.CDRSRaterConns { - if cdrsRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by CDRS component.") + if cdrsRaterConn.Address == utils.MetaInternal && !self.RALsEnabled { + return errors.New("RALs not enabled but requested by CDRS component.") } } for _, connCfg := range self.CDRSPubSubSConns { - if connCfg.Server == utils.INTERNAL && !self.PubSubServerEnabled { + if connCfg.Address == utils.MetaInternal && !self.PubSubServerEnabled { return errors.New("PubSubS not enabled but requested by CDRS component.") } } for _, connCfg := range self.CDRSUserSConns { - if connCfg.Server == utils.INTERNAL && !self.UserServerEnabled { + if connCfg.Address == utils.MetaInternal && !self.UserServerEnabled { return errors.New("UserS not enabled but requested by CDRS component.") } } for _, connCfg := range self.CDRSAliaseSConns { - if connCfg.Server == utils.INTERNAL && !self.AliasesServerEnabled { + if connCfg.Address == utils.MetaInternal && !self.AliasesServerEnabled { return errors.New("AliaseS not enabled but requested by CDRS component.") } } for _, connCfg := range self.CDRSStatSConns { - if connCfg.Server == utils.INTERNAL && !self.CDRStatsEnabled { + if connCfg.Address == utils.MetaInternal && !self.CDRStatsEnabled { return errors.New("CDRStatS not enabled but requested by CDRS component.") } } } // CDRC sanity checks for _, cdrcCfgs := range self.CdrcProfiles { - for _, cdrcInst := range cdrcCfgs { + for instID, cdrcInst := range cdrcCfgs { if !cdrcInst.Enabled { continue } - if len(cdrcInst.Cdrs) == 0 { - return errors.New("CdrC enabled but no CDRS defined!") - } else if cdrcInst.Cdrs == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced from CDRC") + if len(cdrcInst.CdrsConns) == 0 { + return fmt.Errorf(" Instance: %s, CdrC enabled but no CDRS defined!", instID) + } + for _, conn := range cdrcInst.CdrsConns { + if conn.Address == utils.MetaInternal && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced from CDRC") + } } if len(cdrcInst.ContentFields) == 0 { return errors.New("CdrC enabled but no fields to be processed defined!") @@ -333,86 +345,87 @@ func (self *CGRConfig) checkConfigSanity() error { } } } - // SM-Generic checks + // SMGeneric checks if self.SmGenericConfig.Enabled { - if len(self.SmGenericConfig.RaterConns) == 0 { - return errors.New("Rater definition is mandatory!") + if len(self.SmGenericConfig.RALsConns) == 0 { + return errors.New(" RALs definition is mandatory!") } - if len(self.SmGenericConfig.CdrsConns) == 0 { - return errors.New("Cdrs definition is mandatory!") - } - for _, smgRaterConn := range self.SmGenericConfig.RaterConns { - if smgRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-Generic component.") + for _, smgRALsConn := range self.SmGenericConfig.RALsConns { + if smgRALsConn.Address == utils.MetaInternal && !self.RALsEnabled { + return errors.New(" RALs not enabled but requested by SMGeneric component.") } } - for _, smgCDRSConn := range self.SmGenericConfig.CdrsConns { - if smgCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-Generic component") + if len(self.SmGenericConfig.CDRsConns) == 0 { + return errors.New(" CDRs definition is mandatory!") + } + for _, smgCDRSConn := range self.SmGenericConfig.CDRsConns { + if smgCDRSConn.Address == utils.MetaInternal && !self.CDRSEnabled { + return errors.New(" CDRS not enabled but referenced by SMGeneric component") } } } - // SM-FreeSWITCH checks + // SMFreeSWITCH checks if self.SmFsConfig.Enabled { - if len(self.SmFsConfig.RaterConns) == 0 { - return errors.New("Rater definition is mandatory!") + if len(self.SmFsConfig.RALsConns) == 0 { + return errors.New(" RALs definition is mandatory!") } - if len(self.SmFsConfig.CdrsConns) == 0 { - return errors.New("CDRS definition is mandatory!") - } - for _, smFSRaterConn := range self.SmFsConfig.RaterConns { - if smFSRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-FreeSWITCH component.") + for _, smFSRaterConn := range self.SmFsConfig.RALsConns { + if smFSRaterConn.Address == utils.MetaInternal && !self.RALsEnabled { + return errors.New(" RALs not enabled but requested by SMFreeSWITCH component.") } } - for _, smFSCDRSConn := range self.SmFsConfig.CdrsConns { - if smFSCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-FreeSWITCH component") + if len(self.SmFsConfig.CDRsConns) == 0 { + return errors.New(" CDRS definition is mandatory!") + } + for _, smFSCDRSConn := range self.SmFsConfig.CDRsConns { + if smFSCDRSConn.Address == utils.MetaInternal && !self.CDRSEnabled { + return errors.New("CDRS not enabled but referenced by SMFreeSWITCH component") } } } // SM-Kamailio checks if self.SmKamConfig.Enabled { - if len(self.SmKamConfig.RaterConns) == 0 { + if len(self.SmKamConfig.RALsConns) == 0 { return errors.New("Rater definition is mandatory!") } - if len(self.SmKamConfig.CdrsConns) == 0 { - return errors.New("Cdrs definition is mandatory!") - } - for _, smKamRaterConn := range self.SmKamConfig.RaterConns { - if smKamRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { + for _, smKamRaterConn := range self.SmKamConfig.RALsConns { + if smKamRaterConn.Address == utils.MetaInternal && !self.RALsEnabled { return errors.New("Rater not enabled but requested by SM-Kamailio component.") } } - for _, smKamCDRSConn := range self.SmKamConfig.CdrsConns { - if smKamCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { + if len(self.SmKamConfig.CDRsConns) == 0 { + return errors.New("Cdrs definition is mandatory!") + } + for _, smKamCDRSConn := range self.SmKamConfig.CDRsConns { + if smKamCDRSConn.Address == utils.MetaInternal && !self.CDRSEnabled { return errors.New("CDRS not enabled but referenced by SM-Kamailio component") } } } - // SM-OpenSIPS checks + // SMOpenSIPS checks if self.SmOsipsConfig.Enabled { - if len(self.SmOsipsConfig.RaterConns) == 0 { - return errors.New("Rater definition is mandatory!") + if len(self.SmOsipsConfig.RALsConns) == 0 { + return errors.New(" Rater definition is mandatory!") } - if len(self.SmOsipsConfig.CdrsConns) == 0 { - return errors.New("Cdrs definition is mandatory!") - } - for _, smOsipsRaterConn := range self.SmOsipsConfig.RaterConns { - if smOsipsRaterConn.Server == utils.INTERNAL && !self.RaterEnabled { - return errors.New("Rater not enabled but requested by SM-OpenSIPS component.") + for _, smOsipsRaterConn := range self.SmOsipsConfig.RALsConns { + if smOsipsRaterConn.Address == utils.MetaInternal && !self.RALsEnabled { + return errors.New(" RALs not enabled but requested by SMOpenSIPS component.") } } - for _, smOsipsCDRSConn := range self.SmOsipsConfig.CdrsConns { - if smOsipsCDRSConn.Server == utils.INTERNAL && !self.CDRSEnabled { - return errors.New("CDRS not enabled but referenced by SM-OpenSIPS component") + if len(self.SmOsipsConfig.CDRsConns) == 0 { + return errors.New(" CDRs definition is mandatory!") + } + + for _, smOsipsCDRSConn := range self.SmOsipsConfig.CDRsConns { + if smOsipsCDRSConn.Address == utils.MetaInternal && !self.CDRSEnabled { + return errors.New(" CDRS not enabled but referenced by SMOpenSIPS component") } } } // DAgent checks if self.diameterAgentCfg.Enabled { for _, daSMGConn := range self.diameterAgentCfg.SMGenericConns { - if daSMGConn.Server == utils.INTERNAL && !self.SmGenericConfig.Enabled { + if daSMGConn.Address == utils.MetaInternal && !self.SmGenericConfig.Enabled { return errors.New("SMGeneric not enabled but referenced by DiameterAgent component") } } @@ -454,7 +467,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { return err } - jsnRaterCfg, err := jsnCfg.RaterJsonCfg() + jsnRALsCfg, err := jsnCfg.RalsJsonCfg() if err != nil { return err } @@ -556,8 +569,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnTpDbCfg.Db_user != nil { self.TpDbUser = *jsnTpDbCfg.Db_user } - if jsnTpDbCfg.Db_passwd != nil { - self.TpDbPass = *jsnTpDbCfg.Db_passwd + if jsnTpDbCfg.Db_password != nil { + self.TpDbPass = *jsnTpDbCfg.Db_password } } @@ -577,8 +590,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnDataDbCfg.Db_user != nil { self.DataDbUser = *jsnDataDbCfg.Db_user } - if jsnDataDbCfg.Db_passwd != nil { - self.DataDbPass = *jsnDataDbCfg.Db_passwd + if jsnDataDbCfg.Db_password != nil { + self.DataDbPass = *jsnDataDbCfg.Db_password } if jsnDataDbCfg.Load_history_size != nil { self.LoadHistorySize = *jsnDataDbCfg.Load_history_size @@ -601,8 +614,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnStorDbCfg.Db_user != nil { self.StorDBUser = *jsnStorDbCfg.Db_user } - if jsnStorDbCfg.Db_passwd != nil { - self.StorDBPass = *jsnStorDbCfg.Db_passwd + if jsnStorDbCfg.Db_password != nil { + self.StorDBPass = *jsnStorDbCfg.Db_password } if jsnStorDbCfg.Max_open_conns != nil { self.StorDBMaxOpenConns = *jsnStorDbCfg.Max_open_conns @@ -619,8 +632,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnGeneralCfg.Dbdata_encoding != nil { self.DBDataEncoding = *jsnGeneralCfg.Dbdata_encoding } - if jsnGeneralCfg.Default_reqtype != nil { - self.DefaultReqType = *jsnGeneralCfg.Default_reqtype + if jsnGeneralCfg.Default_request_type != nil { + self.DefaultReqType = *jsnGeneralCfg.Default_request_type } if jsnGeneralCfg.Default_category != nil { self.DefaultCategory = *jsnGeneralCfg.Default_category @@ -628,9 +641,6 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnGeneralCfg.Default_tenant != nil { self.DefaultTenant = *jsnGeneralCfg.Default_tenant } - if jsnGeneralCfg.Default_subject != nil { - self.DefaultSubject = *jsnGeneralCfg.Default_subject - } if jsnGeneralCfg.Connect_attempts != nil { self.ConnectAttempts = *jsnGeneralCfg.Connect_attempts } @@ -676,33 +686,53 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { } } - if jsnRaterCfg != nil { - if jsnRaterCfg.Enabled != nil { - self.RaterEnabled = *jsnRaterCfg.Enabled + if jsnRALsCfg != nil { + if jsnRALsCfg.Enabled != nil { + self.RALsEnabled = *jsnRALsCfg.Enabled } - if jsnRaterCfg.Balancer != nil { - self.RaterBalancer = *jsnRaterCfg.Balancer + if jsnRALsCfg.Balancer != nil { + self.RALsBalancer = *jsnRALsCfg.Balancer } - if jsnRaterCfg.Cdrstats != nil { - self.RaterCdrStats = *jsnRaterCfg.Cdrstats + if jsnRALsCfg.Cdrstats_conns != nil { + self.RALsCDRStatSConns = make([]*HaPoolConfig, len(*jsnRALsCfg.Cdrstats_conns)) + for idx, jsnHaCfg := range *jsnRALsCfg.Cdrstats_conns { + self.RALsCDRStatSConns[idx] = NewDfltHaPoolConfig() + self.RALsCDRStatSConns[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnRaterCfg.Historys != nil { - self.RaterHistoryServer = *jsnRaterCfg.Historys + if jsnRALsCfg.Historys_conns != nil { + self.RALsHistorySConns = make([]*HaPoolConfig, len(*jsnRALsCfg.Historys_conns)) + for idx, jsnHaCfg := range *jsnRALsCfg.Historys_conns { + self.RALsHistorySConns[idx] = NewDfltHaPoolConfig() + self.RALsHistorySConns[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnRaterCfg.Pubsubs != nil { - self.RaterPubSubServer = *jsnRaterCfg.Pubsubs + if jsnRALsCfg.Pubsubs_conns != nil { + self.RALsPubSubSConns = make([]*HaPoolConfig, len(*jsnRALsCfg.Pubsubs_conns)) + for idx, jsnHaCfg := range *jsnRALsCfg.Pubsubs_conns { + self.RALsPubSubSConns[idx] = NewDfltHaPoolConfig() + self.RALsPubSubSConns[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnRaterCfg.Aliases != nil { - self.RaterAliasesServer = *jsnRaterCfg.Aliases + if jsnRALsCfg.Aliases_conns != nil { + self.RALsAliasSConns = make([]*HaPoolConfig, len(*jsnRALsCfg.Aliases_conns)) + for idx, jsnHaCfg := range *jsnRALsCfg.Aliases_conns { + self.RALsAliasSConns[idx] = NewDfltHaPoolConfig() + self.RALsAliasSConns[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnRaterCfg.Users != nil { - self.RaterUserServer = *jsnRaterCfg.Users + if jsnRALsCfg.Users_conns != nil { + self.RALsUserSConns = make([]*HaPoolConfig, len(*jsnRALsCfg.Users_conns)) + for idx, jsnHaCfg := range *jsnRALsCfg.Users_conns { + self.RALsUserSConns[idx] = NewDfltHaPoolConfig() + self.RALsUserSConns[idx].loadFromJsonCfg(jsnHaCfg) + } } - if jsnRaterCfg.Rp_subject_prefix_matching != nil { - self.RpSubjectPrefixMatching = *jsnRaterCfg.Rp_subject_prefix_matching + if jsnRALsCfg.Rp_subject_prefix_matching != nil { + self.RpSubjectPrefixMatching = *jsnRALsCfg.Rp_subject_prefix_matching } - if jsnRaterCfg.Lcr_subject_prefix_matching != nil { - self.LcrSubjectPrefixMatching = *jsnRaterCfg.Lcr_subject_prefix_matching + if jsnRALsCfg.Lcr_subject_prefix_matching != nil { + self.LcrSubjectPrefixMatching = *jsnRALsCfg.Lcr_subject_prefix_matching } } @@ -726,9 +756,9 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnCdrsCfg.Store_cdrs != nil { self.CDRSStoreCdrs = *jsnCdrsCfg.Store_cdrs } - if jsnCdrsCfg.Rater_conns != nil { - self.CDRSRaterConns = make([]*HaPoolConfig, len(*jsnCdrsCfg.Rater_conns)) - for idx, jsnHaCfg := range *jsnCdrsCfg.Rater_conns { + if jsnCdrsCfg.Rals_conns != nil { + self.CDRSRaterConns = make([]*HaPoolConfig, len(*jsnCdrsCfg.Rals_conns)) + for idx, jsnHaCfg := range *jsnCdrsCfg.Rals_conns { self.CDRSRaterConns[idx] = NewDfltHaPoolConfig() self.CDRSRaterConns[idx].loadFromJsonCfg(jsnHaCfg) } @@ -768,8 +798,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if rplJsonCfg.Transport != nil { self.CDRSCdrReplication[idx].Transport = *rplJsonCfg.Transport } - if rplJsonCfg.Server != nil { - self.CDRSCdrReplication[idx].Server = *rplJsonCfg.Server + if rplJsonCfg.Address != nil { + self.CDRSCdrReplication[idx].Address = *rplJsonCfg.Address } if rplJsonCfg.Synchronous != nil { self.CDRSCdrReplication[idx].Synchronous = *rplJsonCfg.Synchronous @@ -907,8 +937,8 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnMailerCfg.Auth_user != nil { self.MailerAuthUser = *jsnMailerCfg.Auth_user } - if jsnMailerCfg.Auth_passwd != nil { - self.MailerAuthPass = *jsnMailerCfg.Auth_passwd + if jsnMailerCfg.Auth_password != nil { + self.MailerAuthPass = *jsnMailerCfg.Auth_password } if jsnMailerCfg.From_address != nil { self.MailerFromAddr = *jsnMailerCfg.From_address diff --git a/config/config_defaults.go b/config/config_defaults.go index 8dd2b2596..f5ada7074 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -29,14 +29,13 @@ const CGRATES_CFG_JSON = ` "general": { "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate - "rounding_decimals": 5, // system level precision for floats + "rounding_decimals": 5, // system level precision for floats "dbdata_encoding": "msgpack", // encoding used to store object data in strings: "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans "http_failed_dir": "/var/log/cgrates/http_failed", // directory path where we store failed http requests - "default_reqtype": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> - "default_category": "call", // default Type of Record to consider when missing from requests - "default_tenant": "cgrates.org", // default Tenant to consider when missing from requests - "default_subject": "cgrates", // default rating Subject to consider when missing from requests + "default_request_type": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> + "default_category": "call", // default category to consider when missing from requests + "default_tenant": "cgrates.org", // default tenant to consider when missing from requests "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> "connect_attempts": 3, // initial server connect attempts "reconnects": -1, // number of retries in case of connection lost @@ -58,7 +57,7 @@ const CGRATES_CFG_JSON = ` "db_port": 6379, // port to reach the tariffplan_db "db_name": "10", // tariffplan_db name to connect to "db_user": "", // sername to use when connecting to tariffplan_db - "db_passwd": "", // password to use when connecting to tariffplan_db + "db_password": "", // password to use when connecting to tariffplan_db }, @@ -68,7 +67,7 @@ const CGRATES_CFG_JSON = ` "db_port": 6379, // data_db port to reach the database "db_name": "11", // data_db database name to connect to "db_user": "", // username to use when connecting to data_db - "db_passwd": "", // password to use when connecting to data_db + "db_password": "", // password to use when connecting to data_db "load_history_size": 10, // Number of records in the load history }, @@ -79,7 +78,7 @@ const CGRATES_CFG_JSON = ` "db_port": 3306, // the port to reach the stordb "db_name": "cgrates", // stor database name "db_user": "cgrates", // username to use when connecting to stordb - "db_passwd": "CGRateS.org", // password to use when connecting to stordb + "db_password": "CGRateS.org", // password to use when connecting to stordb "max_open_conns": 100, // maximum database connections opened "max_idle_conns": 10, // maximum database connections idle "cdrs_indexes": [], // indexes on cdrs table to speed up queries, used only in case of mongo @@ -91,16 +90,16 @@ const CGRATES_CFG_JSON = ` }, -"rater": { +"rals": { "enabled": false, // enable Rater service: - "balancer": "", // register to balancer as worker: <""|internal|x.y.z.y:1234> - "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234> - "historys": "", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> - "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> - "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject - "lcr_subject_prefix_matching": false // enables prefix matching for the lcr subject + "balancer": "", // register to balancer as worker: <""|*internal|x.y.z.y:1234> + "cdrstats_conns": [], // address where to reach the cdrstats service, empty to disable stats functionality: <""|*internal|x.y.z.y:1234> + "historys_conns": [], // address where to reach the history service, empty to disable history functionality: <""|*internal|x.y.z.y:1234> + "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> + "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> + "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> + "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject + "lcr_subject_prefix_matching": false // enables prefix matching for the lcr subject }, @@ -113,13 +112,13 @@ const CGRATES_CFG_JSON = ` "enabled": false, // start the CDR Server service: "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs "store_cdrs": true, // store cdrs in storDb - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> ], - "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> - "cdrstats_conns": [], // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> + "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> + "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> + "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> + "cdrstats_conns": [], // address where to reach the cdrstats service, empty to disable stats functionality<""|*internal|x.y.z.y:1234> "cdr_replication":[] // replicate the raw CDR to a number of servers }, @@ -171,7 +170,9 @@ const CGRATES_CFG_JSON = ` "*default": { "enabled": false, // enable CDR client functionality "dry_run": false, // do not send the CDRs to CDRS, just parse them - "cdrs": "internal", // address where to reach CDR server. + "cdrs_conns": [ + {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> + ], "cdr_format": "csv", // CDR file format "field_separator": ",", // separator used in case of csv files "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> @@ -207,61 +208,61 @@ const CGRATES_CFG_JSON = ` "sm_generic": { "enabled": false, // starts SessionManager service: "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> - ], + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> + ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing - ], + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> + ], "debit_interval": "0s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this "max_call_duration": "3h", // maximum call duration a prepaid call can last - "session_ttl": "0s", // time after a session with no updates is terminated, not defined by default - //"session_ttl_last_used": "", // tweak LastUsed for sessions timing-out, not defined by default - //"session_ttl_usage": "", // tweak Usage for sessions timing-out, not defined by default + "session_ttl": "0s", // time after a session with no updates is terminated, not defined by default + //"session_ttl_last_used": "", // tweak LastUsed for sessions timing-out, not defined by default + //"session_ttl_usage": "", // tweak Usage for sessions timing-out, not defined by default }, "sm_freeswitch": { "enabled": false, // starts SessionManager service: - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> ], - "create_cdr": false, // create CDR out of events and sends them to CDRS component - "extra_fields": [], // extra fields to store in auth/CDRs when creating them - "debit_interval": "10s", // interval to perform debits on. - "min_call_duration": "0s", // only authorize calls with allowed duration higher than this - "max_call_duration": "3h", // maximum call duration a prepaid call can last - "min_dur_low_balance": "5s", // threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval) - "low_balance_ann_file": "", // file to be played when low balance is reached for prepaid calls - "empty_balance_context": "", // if defined, prepaid calls will be transfered to this context on empty balance - "empty_balance_ann_file": "", // file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined) - "subscribe_park": true, // subscribe via fsock to receive park events - "channel_sync_interval": "5m", // sync channels with freeswitch regularly - "max_wait_connection": "2s", // maximum duration to wait for a connection to be retrieved from the pool - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} + "create_cdr": false, // create CDR out of events and sends them to CDRS component + "extra_fields": [], // extra fields to store in auth/CDRs when creating them + "debit_interval": "10s", // interval to perform debits on. + "min_call_duration": "0s", // only authorize calls with allowed duration higher than this + "max_call_duration": "3h", // maximum call duration a prepaid call can last + "min_dur_low_balance": "5s", // threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval) + "low_balance_ann_file": "", // file to be played when low balance is reached for prepaid calls + "empty_balance_context": "", // if defined, prepaid calls will be transfered to this context on empty balance + "empty_balance_ann_file": "", // file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined) + "subscribe_park": true, // subscribe via fsock to receive park events + "channel_sync_interval": "5m", // sync channels with freeswitch regularly + "max_wait_connection": "2s", // maximum duration to wait for a connection to be retrieved from the pool + "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers + {"address": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} ], }, "sm_kamailio": { - "enabled": false, // starts SessionManager service: - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + "enabled": false, // starts SessionManager service: + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> ], - "create_cdr": false, // create CDR out of events and sends them to CDRS component - "debit_interval": "10s", // interval to perform debits on. - "min_call_duration": "0s", // only authorize calls with allowed duration higher than this - "max_call_duration": "3h", // maximum call duration a prepaid call can last - "connections":[ // instantiate connections to multiple Kamailio servers - {"evapi_addr": "127.0.0.1:8448", "reconnects": 5} + "create_cdr": false, // create CDR out of events and sends them to CDRS component + "debit_interval": "10s", // interval to perform debits on. + "min_call_duration": "0s", // only authorize calls with allowed duration higher than this + "max_call_duration": "3h", // maximum call duration a prepaid call can last + "evapi_conns":[ // instantiate connections to multiple Kamailio servers + {"address": "127.0.0.1:8448", "reconnects": 5} ], }, @@ -269,11 +270,11 @@ const CGRATES_CFG_JSON = ` "sm_opensips": { "enabled": false, // starts SessionManager service: "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> ], "reconnects": 5, // number of reconnects if connection is lost "create_cdr": false, // create CDR out of events and sends it to CDRS component @@ -290,9 +291,9 @@ const CGRATES_CFG_JSON = ` "listen": "127.0.0.1:3868", // address where to listen for diameter requests "dictionaries_dir": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load "sm_generic_conns": [ - {"server": "internal"} // connection towards SMG component for session management - ], - "pubsubs": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> + {"address": "*internal"} // connection towards SMG component for session management + ], + "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> "create_cdr": true, // create CDR out of CCR terminate and send it to SMG component "debit_interval": "5m", // interval for CCR updates "timezone": "", // timezone for timestamps where not specified, empty for general defaults <""|UTC|Local|$IANA_TZ_DB> @@ -359,7 +360,7 @@ const CGRATES_CFG_JSON = ` "mailer": { "server": "localhost", // the server to use when sending emails out "auth_user": "cgrates", // authenticate to email server using this user - "auth_passwd": "CGRateS.org", // authenticate to email server with this password + "auth_password": "CGRateS.org", // authenticate to email server with this password "from_address": "cgr-mailer@localhost.localdomain" // from address used when sending emails out }, diff --git a/config/config_json.go b/config/config_json.go index ff7dd366d..72b12a949 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -33,7 +33,7 @@ const ( DATADB_JSN = "data_db" STORDB_JSN = "stor_db" BALANCER_JSN = "balancer" - RATER_JSN = "rater" + RALS_JSN = "rals" SCHEDULER_JSN = "scheduler" CDRS_JSN = "cdrs" MEDIATOR_JSN = "mediator" @@ -128,12 +128,12 @@ func (self CgrJsonCfg) BalancerJsonCfg() (*BalancerJsonCfg, error) { return cfg, nil } -func (self CgrJsonCfg) RaterJsonCfg() (*RaterJsonCfg, error) { - rawCfg, hasKey := self[RATER_JSN] +func (self CgrJsonCfg) RalsJsonCfg() (*RalsJsonCfg, error) { + rawCfg, hasKey := self[RALS_JSN] if !hasKey { return nil, nil } - cfg := new(RaterJsonCfg) + cfg := new(RalsJsonCfg) if err := json.Unmarshal(*rawCfg, cfg); err != nil { return nil, err } diff --git a/config/config_json_test.go b/config/config_json_test.go index 532282535..c6de54cdd 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -44,10 +44,9 @@ func TestDfGeneralJsonCfg(t *testing.T) { Dbdata_encoding: utils.StringPointer("msgpack"), Tpexport_dir: utils.StringPointer("/var/log/cgrates/tpe"), Http_failed_dir: utils.StringPointer("/var/log/cgrates/http_failed"), - Default_reqtype: utils.StringPointer(utils.META_RATED), + Default_request_type: utils.StringPointer(utils.META_RATED), Default_category: utils.StringPointer("call"), Default_tenant: utils.StringPointer("cgrates.org"), - Default_subject: utils.StringPointer("cgrates"), Default_timezone: utils.StringPointer("Local"), Connect_attempts: utils.IntPointer(3), Reconnects: utils.IntPointer(-1), @@ -74,12 +73,12 @@ func TestDfListenJsonCfg(t *testing.T) { func TestDfDbJsonCfg(t *testing.T) { eCfg := &DbJsonCfg{ - Db_type: utils.StringPointer("redis"), - Db_host: utils.StringPointer("127.0.0.1"), - Db_port: utils.IntPointer(6379), - Db_name: utils.StringPointer("10"), - Db_user: utils.StringPointer(""), - Db_passwd: utils.StringPointer(""), + Db_type: utils.StringPointer("redis"), + Db_host: utils.StringPointer("127.0.0.1"), + Db_port: utils.IntPointer(6379), + Db_name: utils.StringPointer("10"), + Db_user: utils.StringPointer(""), + Db_password: utils.StringPointer(""), } if cfg, err := dfCgrJsonCfg.DbJsonCfg(TPDB_JSN); err != nil { t.Error(err) @@ -92,7 +91,7 @@ func TestDfDbJsonCfg(t *testing.T) { Db_port: utils.IntPointer(6379), Db_name: utils.StringPointer("11"), Db_user: utils.StringPointer(""), - Db_passwd: utils.StringPointer(""), + Db_password: utils.StringPointer(""), Load_history_size: utils.IntPointer(10), } if cfg, err := dfCgrJsonCfg.DbJsonCfg(DATADB_JSN); err != nil { @@ -106,7 +105,7 @@ func TestDfDbJsonCfg(t *testing.T) { Db_port: utils.IntPointer(3306), Db_name: utils.StringPointer("cgrates"), Db_user: utils.StringPointer("cgrates"), - Db_passwd: utils.StringPointer("CGRateS.org"), + Db_password: utils.StringPointer("CGRateS.org"), Max_open_conns: utils.IntPointer(100), Max_idle_conns: utils.IntPointer(10), Cdrs_indexes: utils.StringSlicePointer([]string{}), @@ -127,10 +126,11 @@ func TestDfBalancerJsonCfg(t *testing.T) { } } -func TestDfRaterJsonCfg(t *testing.T) { - eCfg := &RaterJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats: utils.StringPointer(""), - Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer(""), Aliases: utils.StringPointer(""), Rp_subject_prefix_matching: utils.BoolPointer(false), Lcr_subject_prefix_matching: utils.BoolPointer(false)} - if cfg, err := dfCgrJsonCfg.RaterJsonCfg(); err != nil { +func TestDfRalsJsonCfg(t *testing.T) { + eCfg := &RalsJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats_conns: &[]*HaPoolJsonCfg{}, + Historys_conns: &[]*HaPoolJsonCfg{}, Pubsubs_conns: &[]*HaPoolJsonCfg{}, Users_conns: &[]*HaPoolJsonCfg{}, Aliases_conns: &[]*HaPoolJsonCfg{}, + Rp_subject_prefix_matching: utils.BoolPointer(false), Lcr_subject_prefix_matching: utils.BoolPointer(false)} + if cfg, err := dfCgrJsonCfg.RalsJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { t.Errorf("Received: %+v", cfg) @@ -151,9 +151,9 @@ func TestDfCdrsJsonCfg(t *testing.T) { Enabled: utils.BoolPointer(false), Extra_fields: utils.StringSlicePointer([]string{}), Store_cdrs: utils.BoolPointer(true), - Rater_conns: &[]*HaPoolJsonCfg{ + Rals_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer("*internal"), }}, Pubsubs_conns: &[]*HaPoolJsonCfg{}, Users_conns: &[]*HaPoolJsonCfg{}, @@ -304,9 +304,11 @@ func TestDfCdrcJsonCfg(t *testing.T) { } eCfg := map[string]*CdrcJsonCfg{ "*default": &CdrcJsonCfg{ - Enabled: utils.BoolPointer(false), - Dry_run: utils.BoolPointer(false), - Cdrs: utils.StringPointer("internal"), + Enabled: utils.BoolPointer(false), + Dry_run: utils.BoolPointer(false), + Cdrs_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{ + Address: utils.StringPointer(utils.MetaInternal), + }}, Cdr_format: utils.StringPointer("csv"), Field_separator: utils.StringPointer(","), Timezone: utils.StringPointer(""), @@ -336,13 +338,13 @@ func TestSmGenericJsonCfg(t *testing.T) { eCfg := &SmGenericJsonCfg{ Enabled: utils.BoolPointer(false), Listen_bijson: utils.StringPointer("127.0.0.1:2014"), - Rater_conns: &[]*HaPoolJsonCfg{ + Rals_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Debit_interval: utils.StringPointer("0s"), Min_call_duration: utils.StringPointer("0s"), @@ -359,13 +361,13 @@ func TestSmGenericJsonCfg(t *testing.T) { func TestSmFsJsonCfg(t *testing.T) { eCfg := &SmFsJsonCfg{ Enabled: utils.BoolPointer(false), - Rater_conns: &[]*HaPoolJsonCfg{ + Rals_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Create_cdr: utils.BoolPointer(false), Extra_fields: utils.StringSlicePointer([]string{}), @@ -379,9 +381,9 @@ func TestSmFsJsonCfg(t *testing.T) { Subscribe_park: utils.BoolPointer(true), Channel_sync_interval: utils.StringPointer("5m"), Max_wait_connection: utils.StringPointer("2s"), - Connections: &[]*FsConnJsonCfg{ + Event_socket_conns: &[]*FsConnJsonCfg{ &FsConnJsonCfg{ - Server: utils.StringPointer("127.0.0.1:8021"), + Address: utils.StringPointer("127.0.0.1:8021"), Password: utils.StringPointer("ClueCon"), Reconnects: utils.IntPointer(5), }}, @@ -396,21 +398,21 @@ func TestSmFsJsonCfg(t *testing.T) { func TestSmKamJsonCfg(t *testing.T) { eCfg := &SmKamJsonCfg{ Enabled: utils.BoolPointer(false), - Rater_conns: &[]*HaPoolJsonCfg{ + Rals_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Create_cdr: utils.BoolPointer(false), Debit_interval: utils.StringPointer("10s"), Min_call_duration: utils.StringPointer("0s"), Max_call_duration: utils.StringPointer("3h"), - Connections: &[]*KamConnJsonCfg{ + Evapi_conns: &[]*KamConnJsonCfg{ &KamConnJsonCfg{ - Evapi_addr: utils.StringPointer("127.0.0.1:8448"), + Address: utils.StringPointer("127.0.0.1:8448"), Reconnects: utils.IntPointer(5), }, }, @@ -426,13 +428,13 @@ func TestSmOsipsJsonCfg(t *testing.T) { eCfg := &SmOsipsJsonCfg{ Enabled: utils.BoolPointer(false), Listen_udp: utils.StringPointer("127.0.0.1:2020"), - Rater_conns: &[]*HaPoolJsonCfg{ + Rals_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Cdrs_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Create_cdr: utils.BoolPointer(false), Debit_interval: utils.StringPointer("10s"), @@ -455,7 +457,7 @@ func TestDiameterAgentJsonCfg(t *testing.T) { Dictionaries_dir: utils.StringPointer("/usr/share/cgrates/diameter/dict/"), Sm_generic_conns: &[]*HaPoolJsonCfg{ &HaPoolJsonCfg{ - Server: utils.StringPointer("internal"), + Address: utils.StringPointer(utils.MetaInternal), }}, Pubsub_conns: nil, Create_cdr: utils.BoolPointer(true), @@ -567,10 +569,10 @@ func TestDfUserServJsonCfg(t *testing.T) { func TestDfMailerJsonCfg(t *testing.T) { eCfg := &MailerJsonCfg{ - Server: utils.StringPointer("localhost"), - Auth_user: utils.StringPointer("cgrates"), - Auth_passwd: utils.StringPointer("CGRateS.org"), - From_address: utils.StringPointer("cgr-mailer@localhost.localdomain"), + Server: utils.StringPointer("localhost"), + Auth_user: utils.StringPointer("cgrates"), + Auth_password: utils.StringPointer("CGRateS.org"), + From_address: utils.StringPointer("cgr-mailer@localhost.localdomain"), } if cfg, err := dfCgrJsonCfg.MailerJsonCfg(); err != nil { t.Error(err) @@ -620,11 +622,11 @@ func TestNewCgrJsonCfgFromFile(t *testing.T) { if err != nil { t.Error(err) } - eCfg := &GeneralJsonCfg{Default_reqtype: utils.StringPointer(utils.META_PSEUDOPREPAID)} + eCfg := &GeneralJsonCfg{Default_request_type: utils.StringPointer(utils.META_PSEUDOPREPAID)} if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, gCfg) { - t.Error("Received: ", gCfg) + t.Errorf("Expecting: %+v, received: ", eCfg, gCfg) } cdrFields := []*CdrFieldJsonCfg{ &CdrFieldJsonCfg{Field_id: utils.StringPointer(utils.TOR), Value: utils.StringPointer("~7:s/^(voice|data|sms|mms|generic)$/*$1/")}, @@ -651,18 +653,19 @@ func TestNewCgrJsonCfgFromFile(t *testing.T) { if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfgCdrc, cfg) { - t.Error("Received: ", utils.ToIJSON(cfg["CDRC-CSV2"])) + key := "CDRC-CSV2" + t.Errorf("Expecting:\n %+v\n received:\n %+v\n", utils.ToIJSON(eCfgCdrc[key]), utils.ToIJSON(cfg[key])) } eCfgSmFs := &SmFsJsonCfg{ Enabled: utils.BoolPointer(true), - Connections: &[]*FsConnJsonCfg{ + Event_socket_conns: &[]*FsConnJsonCfg{ &FsConnJsonCfg{ - Server: utils.StringPointer("1.2.3.4:8021"), + Address: utils.StringPointer("1.2.3.4:8021"), Password: utils.StringPointer("ClueCon"), Reconnects: utils.IntPointer(5), }, &FsConnJsonCfg{ - Server: utils.StringPointer("2.3.4.5:8021"), + Address: utils.StringPointer("2.3.4.5:8021"), Password: utils.StringPointer("ClueCon"), Reconnects: utils.IntPointer(5), }, diff --git a/config/config_test.go b/config/config_test.go index d9f3ff7ee..204aaa8b9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,18 +39,18 @@ func TestLoadCgrCfgWithDefaults(t *testing.T) { { "sm_freeswitch": { "enabled": true, // starts SessionManager service: - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 3}, - {"server": "1.2.3.5:8021", "password": "ClueCon", "reconnects": 5} + "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers + {"address": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 3}, + {"address": "1.2.3.5:8021", "password": "ClueCon", "reconnects": 5} ], }, }` eCgrCfg, _ := NewDefaultCGRConfig() eCgrCfg.SmFsConfig.Enabled = true - eCgrCfg.SmFsConfig.Connections = []*FsConnConfig{ - &FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 3}, - &FsConnConfig{Server: "1.2.3.5:8021", Password: "ClueCon", Reconnects: 5}, + eCgrCfg.SmFsConfig.EventSocketConns = []*FsConnConfig{ + &FsConnConfig{Address: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 3}, + &FsConnConfig{Address: "1.2.3.5:8021", Password: "ClueCon", Reconnects: 5}, } if cgrCfg, err := NewCGRConfigFromJsonStringWithDefaults(JSN_CFG); err != nil { t.Error(err) diff --git a/config/configcdrc_test.go b/config/configcdrc_test.go index 621a07b70..497f4bfd4 100644 --- a/config/configcdrc_test.go +++ b/config/configcdrc_test.go @@ -37,7 +37,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = map[string]*CdrcConfig{ "*default": &CdrcConfig{ Enabled: false, - Cdrs: "internal", + CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, @@ -82,7 +82,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = map[string]*CdrcConfig{ "CDRC-CSV1": &CdrcConfig{ Enabled: true, - Cdrs: "internal", + CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, @@ -125,7 +125,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = map[string]*CdrcConfig{ "CDRC-CSV2": &CdrcConfig{ Enabled: true, - Cdrs: "internal", + CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 0.000976563, @@ -148,7 +148,7 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = map[string]*CdrcConfig{ "CDRC-CSV3": &CdrcConfig{ Enabled: true, - Cdrs: "internal", + CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 1024, diff --git a/config/libconfig.go b/config/libconfig.go index af5dfb6c4..1471fdaa0 100644 --- a/config/libconfig.go +++ b/config/libconfig.go @@ -26,12 +26,12 @@ import ( type CdrReplicationCfg struct { Transport string - Server string + Address string Synchronous bool Attempts int // Number of attempts if not success CdrFilter utils.RSRFields // Only replicate if the filters here are matching } func (rplCfg CdrReplicationCfg) FallbackFileName() string { - return fmt.Sprintf("cdr_%s_%s_%s.form", rplCfg.Transport, url.QueryEscape(rplCfg.Server), utils.GenUUID()) + return fmt.Sprintf("cdr_%s_%s_%s.form", rplCfg.Transport, url.QueryEscape(rplCfg.Address), utils.GenUUID()) } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index a688896d4..8f8ea6c17 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -25,10 +25,9 @@ type GeneralJsonCfg struct { Dbdata_encoding *string Tpexport_dir *string Http_failed_dir *string - Default_reqtype *string + Default_request_type *string Default_category *string Default_tenant *string - Default_subject *string Default_timezone *string Connect_attempts *int Reconnects *int @@ -50,7 +49,7 @@ type DbJsonCfg struct { Db_port *int Db_name *string Db_user *string - Db_passwd *string + Db_password *string Max_open_conns *int // Used only in case of storDb Max_idle_conns *int Load_history_size *int // Used in case of dataDb to limit the length of the loads history @@ -63,14 +62,14 @@ type BalancerJsonCfg struct { } // Rater config section -type RaterJsonCfg struct { +type RalsJsonCfg struct { Enabled *bool Balancer *string - Cdrstats *string - Historys *string - Pubsubs *string - Aliases *string - Users *string + Cdrstats_conns *[]*HaPoolJsonCfg + Historys_conns *[]*HaPoolJsonCfg + Pubsubs_conns *[]*HaPoolJsonCfg + Aliases_conns *[]*HaPoolJsonCfg + Users_conns *[]*HaPoolJsonCfg Rp_subject_prefix_matching *bool Lcr_subject_prefix_matching *bool } @@ -85,7 +84,7 @@ type CdrsJsonCfg struct { Enabled *bool Extra_fields *[]string Store_cdrs *bool - Rater_conns *[]*HaPoolJsonCfg + Rals_conns *[]*HaPoolJsonCfg Pubsubs_conns *[]*HaPoolJsonCfg Users_conns *[]*HaPoolJsonCfg Aliases_conns *[]*HaPoolJsonCfg @@ -95,7 +94,7 @@ type CdrsJsonCfg struct { type CdrReplicationJsonCfg struct { Transport *string - Server *string + Address *string Synchronous *bool Attempts *int Cdr_filter *string @@ -146,7 +145,7 @@ type CdreJsonCfg struct { type CdrcJsonCfg struct { Enabled *bool Dry_run *bool - Cdrs *string + Cdrs_conns *[]*HaPoolJsonCfg Cdr_format *string Field_separator *string Timezone *string @@ -169,7 +168,7 @@ type CdrcJsonCfg struct { type SmGenericJsonCfg struct { Enabled *bool Listen_bijson *string - Rater_conns *[]*HaPoolJsonCfg + Rals_conns *[]*HaPoolJsonCfg Cdrs_conns *[]*HaPoolJsonCfg Debit_interval *string Min_call_duration *string @@ -182,7 +181,7 @@ type SmGenericJsonCfg struct { // SM-FreeSWITCH config section type SmFsJsonCfg struct { Enabled *bool - Rater_conns *[]*HaPoolJsonCfg + Rals_conns *[]*HaPoolJsonCfg Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Extra_fields *[]string @@ -196,17 +195,17 @@ type SmFsJsonCfg struct { Subscribe_park *bool Channel_sync_interval *string Max_wait_connection *string - Connections *[]*FsConnJsonCfg + Event_socket_conns *[]*FsConnJsonCfg } // Represents one connection instance towards a rater/cdrs server type HaPoolJsonCfg struct { - Server *string + Address *string } // Represents one connection instance towards FreeSWITCH type FsConnJsonCfg struct { - Server *string + Address *string Password *string Reconnects *int } @@ -214,18 +213,18 @@ type FsConnJsonCfg struct { // SM-Kamailio config section type SmKamJsonCfg struct { Enabled *bool - Rater_conns *[]*HaPoolJsonCfg + Rals_conns *[]*HaPoolJsonCfg Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string Min_call_duration *string Max_call_duration *string - Connections *[]*KamConnJsonCfg + Evapi_conns *[]*KamConnJsonCfg } // Represents one connection instance towards Kamailio type KamConnJsonCfg struct { - Evapi_addr *string + Address *string Reconnects *int } @@ -233,7 +232,7 @@ type KamConnJsonCfg struct { type SmOsipsJsonCfg struct { Enabled *bool Listen_udp *string - Rater_conns *[]*HaPoolJsonCfg + Rals_conns *[]*HaPoolJsonCfg Cdrs_conns *[]*HaPoolJsonCfg Create_cdr *bool Debit_interval *string @@ -305,10 +304,10 @@ type UserServJsonCfg struct { // Mailer config section type MailerJsonCfg struct { - Server *string - Auth_user *string - Auth_passwd *string - From_address *string + Server *string + Auth_user *string + Auth_password *string + From_address *string } // SureTax config section diff --git a/config/smconfig.go b/config/smconfig.go index e03b79e28..a1113d7a2 100644 --- a/config/smconfig.go +++ b/config/smconfig.go @@ -35,15 +35,15 @@ func NewDfltHaPoolConfig() *HaPoolConfig { // One connection to Rater type HaPoolConfig struct { - Server string + Address string } func (self *HaPoolConfig) loadFromJsonCfg(jsnCfg *HaPoolJsonCfg) error { if jsnCfg == nil { return nil } - if jsnCfg.Server != nil { - self.Server = *jsnCfg.Server + if jsnCfg.Address != nil { + self.Address = *jsnCfg.Address } return nil } @@ -59,7 +59,7 @@ func NewDfltFsConnConfig() *FsConnConfig { // One connection to FreeSWITCH server type FsConnConfig struct { - Server string + Address string Password string Reconnects int } @@ -68,8 +68,8 @@ func (self *FsConnConfig) loadFromJsonCfg(jsnCfg *FsConnJsonCfg) error { if jsnCfg == nil { return nil } - if jsnCfg.Server != nil { - self.Server = *jsnCfg.Server + if jsnCfg.Address != nil { + self.Address = *jsnCfg.Address } if jsnCfg.Password != nil { self.Password = *jsnCfg.Password @@ -83,8 +83,8 @@ func (self *FsConnConfig) loadFromJsonCfg(jsnCfg *FsConnJsonCfg) error { type SmGenericConfig struct { Enabled bool ListenBijson string - RaterConns []*HaPoolConfig - CdrsConns []*HaPoolConfig + RALsConns []*HaPoolConfig + CDRsConns []*HaPoolConfig DebitInterval time.Duration MinCallDuration time.Duration MaxCallDuration time.Duration @@ -104,18 +104,18 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { if jsnCfg.Listen_bijson != nil { self.ListenBijson = *jsnCfg.Listen_bijson } - if jsnCfg.Rater_conns != nil { - self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) - for idx, jsnHaCfg := range *jsnCfg.Rater_conns { - self.RaterConns[idx] = NewDfltHaPoolConfig() - self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rals_conns != nil { + self.RALsConns = make([]*HaPoolConfig, len(*jsnCfg.Rals_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rals_conns { + self.RALsConns[idx] = NewDfltHaPoolConfig() + self.RALsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Cdrs_conns != nil { - self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + self.CDRsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { - self.CdrsConns[idx] = NewDfltHaPoolConfig() - self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) + self.CDRsConns[idx] = NewDfltHaPoolConfig() + self.CDRsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Debit_interval != nil { @@ -157,8 +157,8 @@ func (self *SmGenericConfig) loadFromJsonCfg(jsnCfg *SmGenericJsonCfg) error { type SmFsConfig struct { Enabled bool - RaterConns []*HaPoolConfig - CdrsConns []*HaPoolConfig + RALsConns []*HaPoolConfig + CDRsConns []*HaPoolConfig CreateCdr bool ExtraFields []*utils.RSRField DebitInterval time.Duration @@ -171,7 +171,7 @@ type SmFsConfig struct { SubscribePark bool ChannelSyncInterval time.Duration MaxWaitConnection time.Duration - Connections []*FsConnConfig + EventSocketConns []*FsConnConfig } func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error { @@ -182,18 +182,18 @@ func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Rater_conns != nil { - self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) - for idx, jsnHaCfg := range *jsnCfg.Rater_conns { - self.RaterConns[idx] = NewDfltHaPoolConfig() - self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rals_conns != nil { + self.RALsConns = make([]*HaPoolConfig, len(*jsnCfg.Rals_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rals_conns { + self.RALsConns[idx] = NewDfltHaPoolConfig() + self.RALsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Cdrs_conns != nil { - self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + self.CDRsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { - self.CdrsConns[idx] = NewDfltHaPoolConfig() - self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) + self.CDRsConns[idx] = NewDfltHaPoolConfig() + self.CDRsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { @@ -246,11 +246,11 @@ func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error { return err } } - if jsnCfg.Connections != nil { - self.Connections = make([]*FsConnConfig, len(*jsnCfg.Connections)) - for idx, jsnConnCfg := range *jsnCfg.Connections { - self.Connections[idx] = NewDfltFsConnConfig() - self.Connections[idx].loadFromJsonCfg(jsnConnCfg) + if jsnCfg.Event_socket_conns != nil { + self.EventSocketConns = make([]*FsConnConfig, len(*jsnCfg.Event_socket_conns)) + for idx, jsnConnCfg := range *jsnCfg.Event_socket_conns { + self.EventSocketConns[idx] = NewDfltFsConnConfig() + self.EventSocketConns[idx].loadFromJsonCfg(jsnConnCfg) } } return nil @@ -267,7 +267,7 @@ func NewDfltKamConnConfig() *KamConnConfig { // Represents one connection instance towards Kamailio type KamConnConfig struct { - EvapiAddr string + Address string Reconnects int } @@ -275,8 +275,8 @@ func (self *KamConnConfig) loadFromJsonCfg(jsnCfg *KamConnJsonCfg) error { if jsnCfg == nil { return nil } - if jsnCfg.Evapi_addr != nil { - self.EvapiAddr = *jsnCfg.Evapi_addr + if jsnCfg.Address != nil { + self.Address = *jsnCfg.Address } if jsnCfg.Reconnects != nil { self.Reconnects = *jsnCfg.Reconnects @@ -287,13 +287,13 @@ func (self *KamConnConfig) loadFromJsonCfg(jsnCfg *KamConnJsonCfg) error { // SM-Kamailio config section type SmKamConfig struct { Enabled bool - RaterConns []*HaPoolConfig - CdrsConns []*HaPoolConfig + RALsConns []*HaPoolConfig + CDRsConns []*HaPoolConfig CreateCdr bool DebitInterval time.Duration MinCallDuration time.Duration MaxCallDuration time.Duration - Connections []*KamConnConfig + EvapiConns []*KamConnConfig } func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error { @@ -304,18 +304,18 @@ func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error { if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } - if jsnCfg.Rater_conns != nil { - self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) - for idx, jsnHaCfg := range *jsnCfg.Rater_conns { - self.RaterConns[idx] = NewDfltHaPoolConfig() - self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rals_conns != nil { + self.RALsConns = make([]*HaPoolConfig, len(*jsnCfg.Rals_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rals_conns { + self.RALsConns[idx] = NewDfltHaPoolConfig() + self.RALsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Cdrs_conns != nil { - self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + self.CDRsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { - self.CdrsConns[idx] = NewDfltHaPoolConfig() - self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) + self.CDRsConns[idx] = NewDfltHaPoolConfig() + self.CDRsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { @@ -336,11 +336,11 @@ func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error { return err } } - if jsnCfg.Connections != nil { - self.Connections = make([]*KamConnConfig, len(*jsnCfg.Connections)) - for idx, jsnConnCfg := range *jsnCfg.Connections { - self.Connections[idx] = NewDfltKamConnConfig() - self.Connections[idx].loadFromJsonCfg(jsnConnCfg) + if jsnCfg.Evapi_conns != nil { + self.EvapiConns = make([]*KamConnConfig, len(*jsnCfg.Evapi_conns)) + for idx, jsnConnCfg := range *jsnCfg.Evapi_conns { + self.EvapiConns[idx] = NewDfltKamConnConfig() + self.EvapiConns[idx].loadFromJsonCfg(jsnConnCfg) } } return nil @@ -366,8 +366,8 @@ func (self *OsipsConnConfig) loadFromJsonCfg(jsnCfg *OsipsConnJsonCfg) error { type SmOsipsConfig struct { Enabled bool ListenUdp string - RaterConns []*HaPoolConfig - CdrsConns []*HaPoolConfig + RALsConns []*HaPoolConfig + CDRsConns []*HaPoolConfig CreateCdr bool DebitInterval time.Duration MinCallDuration time.Duration @@ -384,18 +384,18 @@ func (self *SmOsipsConfig) loadFromJsonCfg(jsnCfg *SmOsipsJsonCfg) error { if jsnCfg.Listen_udp != nil { self.ListenUdp = *jsnCfg.Listen_udp } - if jsnCfg.Rater_conns != nil { - self.RaterConns = make([]*HaPoolConfig, len(*jsnCfg.Rater_conns)) - for idx, jsnHaCfg := range *jsnCfg.Rater_conns { - self.RaterConns[idx] = NewDfltHaPoolConfig() - self.RaterConns[idx].loadFromJsonCfg(jsnHaCfg) + if jsnCfg.Rals_conns != nil { + self.RALsConns = make([]*HaPoolConfig, len(*jsnCfg.Rals_conns)) + for idx, jsnHaCfg := range *jsnCfg.Rals_conns { + self.RALsConns[idx] = NewDfltHaPoolConfig() + self.RALsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Cdrs_conns != nil { - self.CdrsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) + self.CDRsConns = make([]*HaPoolConfig, len(*jsnCfg.Cdrs_conns)) for idx, jsnHaCfg := range *jsnCfg.Cdrs_conns { - self.CdrsConns[idx] = NewDfltHaPoolConfig() - self.CdrsConns[idx].loadFromJsonCfg(jsnHaCfg) + self.CDRsConns[idx] = NewDfltHaPoolConfig() + self.CDRsConns[idx].loadFromJsonCfg(jsnHaCfg) } } if jsnCfg.Create_cdr != nil { diff --git a/config/smconfig_test.go b/config/smconfig_test.go index 4276f9089..c279de16f 100644 --- a/config/smconfig_test.go +++ b/config/smconfig_test.go @@ -29,14 +29,14 @@ func TesSmFsConfigLoadFromJsonCfg(t *testing.T) { Enabled: utils.BoolPointer(true), Create_cdr: utils.BoolPointer(true), Subscribe_park: utils.BoolPointer(true), - Connections: &[]*FsConnJsonCfg{ + Event_socket_conns: &[]*FsConnJsonCfg{ &FsConnJsonCfg{ - Server: utils.StringPointer("1.2.3.4:8021"), + Address: utils.StringPointer("1.2.3.4:8021"), Password: utils.StringPointer("ClueCon"), Reconnects: utils.IntPointer(5), }, &FsConnJsonCfg{ - Server: utils.StringPointer("2.3.4.5:8021"), + Address: utils.StringPointer("2.3.4.5:8021"), Password: utils.StringPointer("ClueCon"), Reconnects: utils.IntPointer(5), }, @@ -45,9 +45,9 @@ func TesSmFsConfigLoadFromJsonCfg(t *testing.T) { eSmFsConfig := &SmFsConfig{Enabled: true, CreateCdr: true, SubscribePark: true, - Connections: []*FsConnConfig{ - &FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5}, - &FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5}, + EventSocketConns: []*FsConnConfig{ + &FsConnConfig{Address: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5}, + &FsConnConfig{Address: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5}, }, } smFsCfg := new(SmFsConfig) diff --git a/utils/consts.go b/utils/consts.go index 7c7552f87..4c81a53c8 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -144,6 +144,7 @@ const ( DRYRUN = "dry_run" META_COMBIMED = "*combimed" INTERNAL = "internal" + MetaInternal = "*internal" ZERO_RATING_SUBJECT_PREFIX = "*zero" OK = "OK" CDRE_FIXED_WIDTH = "fwv" From 5c9f48e8974e93cd25bb815e64f8b21faab8cb03 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 20 Apr 2016 14:30:47 +0300 Subject: [PATCH 175/227] integration tests passing --- cmd/cgr-engine/rater.go | 10 ++++++++++ engine/action.go | 12 ++++-------- utils/rpc_params.go | 13 ++++++++----- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 41535636d..22d5570d0 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -247,5 +247,15 @@ func startRater(internalRaterChan chan *engine.Responder, cacheDoneChan chan str server.RpcRegister(responder) server.RpcRegister(apierRpcV1) server.RpcRegister(apierRpcV2) + + utils.RegisterRpcParams("", &engine.Stats{}) + utils.RegisterRpcParams("", &history.FileScribe{}) + utils.RegisterRpcParams("", &engine.PubSub{}) + utils.RegisterRpcParams("", &engine.AliasHandler{}) + utils.RegisterRpcParams("", &engine.UserMap{}) + utils.RegisterRpcParams("", responder) + utils.RegisterRpcParams("", apierRpcV1) + utils.RegisterRpcParams("", apierRpcV2) + internalRaterChan <- responder // Rater done } diff --git a/engine/action.go b/engine/action.go index 84669ff1f..71a0fdf05 100644 --- a/engine/action.go +++ b/engine/action.go @@ -671,16 +671,12 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti } var client rpcclient.RpcClientConnection if req.Address != utils.INTERNAL { - if client, err = rpcclient.NewRpcClient(req.Method, req.Address, req.Attempts, 0, req.Transport, nil); err != nil { + if client, err = rpcclient.NewRpcClient("tcp", req.Address, req.Attempts, 0, req.Transport, nil); err != nil { return err } } else { - client = params.Object + client = params.Object.(rpcclient.RpcClientConnection) } - if client == nil { - return utils.ErrServerError - } - in, out := params.InParam, params.OutParam p, err := utils.FromMapStringInterfaceValue(req.Param, in) if err != nil { @@ -688,12 +684,12 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti } if !req.Async { err = client.Call(req.Method, p, out) - utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %+v err: %v", out, err)) + utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %s err: %v", utils.ToJSON(out), err)) return err } go func() { err := client.Call(req.Method, p, out) - utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %+v err: %v", out, err)) + utils.Logger.Info(fmt.Sprintf("<*cgr_rpc> result: %s err: %v", utils.ToJSON(out), err)) }() return nil } diff --git a/utils/rpc_params.go b/utils/rpc_params.go index 69020fdec..c48e84085 100644 --- a/utils/rpc_params.go +++ b/utils/rpc_params.go @@ -2,14 +2,12 @@ package utils import ( "reflect" - - "github.com/cgrates/rpcclient" ) var rpcParamsMap map[string]*RpcParams type RpcParams struct { - Object rpcclient.RpcClientConnection + Object interface{} InParam reflect.Value OutParam interface{} } @@ -18,7 +16,7 @@ func init() { rpcParamsMap = make(map[string]*RpcParams) } -func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { +func RegisterRpcParams(name string, obj interface{}) { objType := reflect.TypeOf(obj) if name == "" { val := reflect.ValueOf(obj) @@ -31,10 +29,14 @@ func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { method := objType.Method(i) methodType := method.Type if methodType.NumIn() == 3 { // if it has three parameters (one is self and two are rpc params) + out := methodType.In(2) + if out.Kind() == reflect.Ptr { + out = out.Elem() + } rpcParamsMap[name+"."+method.Name] = &RpcParams{ Object: obj, InParam: reflect.New(methodType.In(1)), - OutParam: reflect.New(methodType.In(2).Elem()).Interface(), + OutParam: reflect.New(out).Interface(), } } @@ -42,6 +44,7 @@ func RegisterRpcParams(name string, obj rpcclient.RpcClientConnection) { } func GetRpcParams(method string) (*RpcParams, error) { + Logger.Info("REGISTERED: " + ToJSON(rpcParamsMap)) x, found := rpcParamsMap[method] if !found { return nil, ErrNotFound From 31ea08df333796c01d38311274e40570e84c95be Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 20 Apr 2016 14:38:28 +0300 Subject: [PATCH 176/227] correct names for registered objects --- cmd/cgr-engine/rater.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 22d5570d0..946c3b61f 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -249,10 +249,16 @@ func startRater(internalRaterChan chan *engine.Responder, cacheDoneChan chan str server.RpcRegister(apierRpcV2) utils.RegisterRpcParams("", &engine.Stats{}) - utils.RegisterRpcParams("", &history.FileScribe{}) - utils.RegisterRpcParams("", &engine.PubSub{}) - utils.RegisterRpcParams("", &engine.AliasHandler{}) - utils.RegisterRpcParams("", &engine.UserMap{}) + utils.RegisterRpcParams("", &v1.CDRStatsV1{}) + utils.RegisterRpcParams("ScribeV1", &history.FileScribe{}) + utils.RegisterRpcParams("PubSubV1", &engine.PubSub{}) + utils.RegisterRpcParams("AliasesV1", &engine.AliasHandler{}) + utils.RegisterRpcParams("UsersV1", &engine.UserMap{}) + utils.RegisterRpcParams("UsersV1", &engine.UserMap{}) + utils.RegisterRpcParams("", &v1.CdrsV1{}) + utils.RegisterRpcParams("", &v2.CdrsV2{}) + utils.RegisterRpcParams("", &v1.SessionManagerV1{}) + utils.RegisterRpcParams("", &v1.SMGenericV1{}) utils.RegisterRpcParams("", responder) utils.RegisterRpcParams("", apierRpcV1) utils.RegisterRpcParams("", apierRpcV2) From aa28ef31ce8d7e12c5458455f439a22bbade7773 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 20 Apr 2016 15:18:55 +0300 Subject: [PATCH 177/227] fix cross loading integration tests --- data/tariffplans/testtp/AccountActions.csv | 3 ++- data/tariffplans/testtp/ActionPlans.csv | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index 917d8d0b9..850d03f77 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -8,4 +8,5 @@ cgrates.org,1009,TEST_EXE,,, cgrates.org,1010,TEST_DATA_r,,true, cgrates.org,1011,TEST_VOICE,,, cgrates.org,1012,PREPAID_10,,, -cgrates.org,1013,TEST_NEG,,, \ No newline at end of file +cgrates.org,1013,TEST_NEG,,, +cgrates.org,1014,TEST_RPC,,, \ No newline at end of file diff --git a/data/tariffplans/testtp/ActionPlans.csv b/data/tariffplans/testtp/ActionPlans.csv index 54731428a..897278d0e 100644 --- a/data/tariffplans/testtp/ActionPlans.csv +++ b/data/tariffplans/testtp/ActionPlans.csv @@ -5,3 +5,4 @@ TEST_EXE,TOPUP_EXE,ALWAYS,10 TEST_DATA_r,TOPUP_DATA_r,ASAP,10 TEST_VOICE,TOPUP_VOICE,ASAP,10 TEST_NEG,TOPUP_NEG,ASAP,10 +TEST_RPC,RPC,ALWAYS,10 \ No newline at end of file From 4f399e7683fb5cb37c92a1f06dd1ac515b34c76e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 21 Apr 2016 15:16:13 +0300 Subject: [PATCH 178/227] renamed Param to params --- data/tariffplans/testtp/Actions.csv | 2 +- engine/action.go | 4 ++-- engine/actions_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 1525522f2..680866202 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -9,4 +9,4 @@ TOPUP_DATA_r,*topup,,,,*monetary,*out,,DATA_DEST,,,*unlimited,,5000000,10,false, TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,false,false,10 TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 TOPUP_NEG,*topup,,,,*voice,*out,,GERMANY;!GERMANY_MOBILE,*zero1m,,*unlimited,,100,10,false,false,10 -RPC,*cgr_rpc,"{""Address"": ""localhost:2013"",""Transport"":""*gob"",""Method"":""ApierV2.SetAccount"",""Attempts"":1,""Async"" :false,""Param"":{""Account"":""rpc"",""Tenant"":""cgrates.org""}}",,,,,,,,,,,,,,, +RPC,*cgr_rpc,"{""Address"": ""localhost:2013"",""Transport"":""*gob"",""Method"":""ApierV2.SetAccount"",""Attempts"":1,""Async"" :false,""Params"":{""Account"":""rpc"",""Tenant"":""cgrates.org""}}",,,,,,,,,,,,,,, diff --git a/engine/action.go b/engine/action.go index 71a0fdf05..de95875c4 100644 --- a/engine/action.go +++ b/engine/action.go @@ -657,7 +657,7 @@ type RPCRequest struct { Method string Attempts int Async bool - Param map[string]interface{} + Params map[string]interface{} } func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Actions) error { @@ -678,7 +678,7 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti client = params.Object.(rpcclient.RpcClientConnection) } in, out := params.InParam, params.OutParam - p, err := utils.FromMapStringInterfaceValue(req.Param, in) + p, err := utils.FromMapStringInterfaceValue(req.Params, in) if err != nil { return err } diff --git a/engine/actions_test.go b/engine/actions_test.go index f9297d25a..33d683ca5 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2212,7 +2212,7 @@ func TestCgrRpcAction(t *testing.T) { "Method": "TestRPCParameters.Hopa", "Attempts":1, "Async" :false, - "Param": {"Name":"n", "Surname":"s", "Age":10.2}}`, + "Params": {"Name":"n", "Surname":"s", "Age":10.2}}`, } if err := cgrRPCAction(nil, nil, a, nil); err != nil { t.Error("error executing cgr action: ", err) From a706e5b0e6287f220bcea77e6e2725b3ea09ae50 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 21 Apr 2016 15:52:37 +0200 Subject: [PATCH 179/227] Updating cgrates.json config --- data/conf/cgrates/cgrates.json | 632 +++++++++++++++++---------------- 1 file changed, 324 insertions(+), 308 deletions(-) diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 3319bf458..b585790bc 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -6,355 +6,371 @@ // This file contains the default configuration hardcoded into CGRateS. // This is what you get when you load CGRateS with an empty configuration file. -//"general": { -// "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate -// "rounding_decimals": 5, // system level precision for floats -// "dbdata_encoding": "msgpack", // encoding used to store object data in strings: -// "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans -// "http_failed_dir": "/var/log/cgrates/http_failed", // directory path where we store failed http requests -// "default_reqtype": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> -// "default_category": "call", // default Type of Record to consider when missing from requests -// "default_tenant": "cgrates.org", // default Tenant to consider when missing from requests -// "default_subject": "cgrates", // default rating Subject to consider when missing from requests -// "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> -// "connect_attempts": 3, // initial server connect attempts -// "reconnects": -1, // number of retries in case of connection lost -// "response_cache_ttl": "3s", // the life span of a cached response -// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up -//}, +// "general": { +// "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate +// "rounding_decimals": 5, // system level precision for floats +// "dbdata_encoding": "msgpack", // encoding used to store object data in strings: +// "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans +// "http_failed_dir": "/var/log/cgrates/http_failed", // directory path where we store failed http requests +// "default_request_type": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> +// "default_category": "call", // default category to consider when missing from requests +// "default_tenant": "cgrates.org", // default tenant to consider when missing from requests +// "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> +// "connect_attempts": 3, // initial server connect attempts +// "reconnects": -1, // number of retries in case of connection lost +// "response_cache_ttl": "3s", // the life span of a cached response +// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up +// }, -//"listen": { -// "rpc_json": "127.0.0.1:2012", // RPC JSON listening address -// "rpc_gob": "127.0.0.1:2013", // RPC GOB listening address -// "http": "127.0.0.1:2080", // HTTP listening address -//}, +// "listen": { +// "rpc_json": "127.0.0.1:2012", // RPC JSON listening address +// "rpc_gob": "127.0.0.1:2013", // RPC GOB listening address +// "http": "127.0.0.1:2080", // HTTP listening address +// }, -//"tariffplan_db": { // database used to store active tariff plan configuration -// "db_type": "redis", // tariffplan_db type: -// "db_host": "127.0.0.1", // tariffplan_db host address -// "db_port": 6379, // port to reach the tariffplan_db -// "db_name": "10", // tariffplan_db name to connect to -// "db_user": "", // sername to use when connecting to tariffplan_db -// "db_passwd": "", // password to use when connecting to tariffplan_db -//}, +// "tariffplan_db": { // database used to store active tariff plan configuration +// "db_type": "redis", // tariffplan_db type: +// "db_host": "127.0.0.1", // tariffplan_db host address +// "db_port": 6379, // port to reach the tariffplan_db +// "db_name": "10", // tariffplan_db name to connect to +// "db_user": "", // sername to use when connecting to tariffplan_db +// "db_password": "", // password to use when connecting to tariffplan_db +// }, -//"data_db": { // database used to store runtime data (eg: accounts, cdr stats) -// "db_type": "redis", // data_db type: -// "db_host": "127.0.0.1", // data_db host address -// "db_port": 6379, // data_db port to reach the database -// "db_name": "11", // data_db database name to connect to -// "db_user": "", // username to use when connecting to data_db -// "db_passwd": "", // password to use when connecting to data_db -// "load_history_size": 10, // Number of records in the load history -//}, +// "data_db": { // database used to store runtime data (eg: accounts, cdr stats) +// "db_type": "redis", // data_db type: +// "db_host": "127.0.0.1", // data_db host address +// "db_port": 6379, // data_db port to reach the database +// "db_name": "11", // data_db database name to connect to +// "db_user": "", // username to use when connecting to data_db +// "db_password": "", // password to use when connecting to data_db +// "load_history_size": 10, // Number of records in the load history +// }, -//"stor_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mysql", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 3306, // the port to reach the stordb -// "db_name": "cgrates", // stor database name -// "db_user": "cgrates", // username to use when connecting to stordb -// "db_passwd": "CGRateS.org", // password to use when connecting to stordb -// "max_open_conns": 100, // maximum database connections opened -// "max_idle_conns": 10, // maximum database connections idle -//}, +// "stor_db": { // database used to store offline tariff plans and CDRs +// "db_type": "mysql", // stor database type to use: +// "db_host": "127.0.0.1", // the host to connect to +// "db_port": 3306, // the port to reach the stordb +// "db_name": "cgrates", // stor database name +// "db_user": "cgrates", // username to use when connecting to stordb +// "db_password": "CGRateS.org", // password to use when connecting to stordb +// "max_open_conns": 100, // maximum database connections opened +// "max_idle_conns": 10, // maximum database connections idle +// "cdrs_indexes": [], // indexes on cdrs table to speed up queries, used only in case of mongo +// }, -//"balancer": { -// "enabled": false, // start Balancer service: -//}, +// "balancer": { +// "enabled": false, // start Balancer service: +// }, -//"rater": { -// "enabled": false, // enable Rater service: -// "balancer": "", // register to balancer as worker: <""|internal|x.y.z.y:1234> -// "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234> -// "historys": "", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> -// "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> -// "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> -// "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> -// "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject, -// "lcr_subject_prefix_matching": false, // enables prefix matching for the lcr subject -//}, +// "rals": { +// "enabled": false, // enable Rater service: +// "balancer": "", // register to balancer as worker: <""|*internal|x.y.z.y:1234> +// "cdrstats_conns": [], // address where to reach the cdrstats service, empty to disable stats functionality: <""|*internal|x.y.z.y:1234> +// "historys_conns": [], // address where to reach the history service, empty to disable history functionality: <""|*internal|x.y.z.y:1234> +// "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> +// "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> +// "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> +// "rp_subject_prefix_matching": false, // enables prefix matching for the rating profile subject +// "lcr_subject_prefix_matching": false // enables prefix matching for the lcr subject +// }, -//"scheduler": { -// "enabled": false, // start Scheduler service: -//}, +// "scheduler": { +// "enabled": false, // start Scheduler service: +// }, -//"cdrs": { -// "enabled": false, // start the CDR Server service: -// "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs -// "store_cdrs": true, // store cdrs in storDb - // "rater_conns": [ - // {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - //], -// "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> -// "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> -// "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> -// "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> -// "cdr_replication":[], // replicate the raw CDR to a number of servers -//}, +// "cdrs": { +// "enabled": false, // start the CDR Server service: +// "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs +// "store_cdrs": true, // store cdrs in storDb +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> +// ], +// "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> +// "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> +// "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> +// "cdrstats_conns": [], // address where to reach the cdrstats service, empty to disable stats functionality<""|*internal|x.y.z.y:1234> +// "cdr_replication":[] // replicate the raw CDR to a number of servers +// }, -//"cdrstats": { -// "enabled": false, // starts the cdrstats service: -// "save_interval": "1m", // interval to save changed stats into dataDb storage -//}, +// "cdrstats": { +// "enabled": false, // starts the cdrstats service: +// "save_interval": "1m", // interval to save changed stats into dataDb storage +// }, -//"cdre": { -// "*default": { -// "cdr_format": "csv", // exported CDRs format -// "field_separator": ",", -// "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) -// "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) -// "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) -// "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT -// "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding -// "cost_shift_digits": 0, // shift digits in the cost on export (eg: convert from EUR to cents) -// "mask_destination_id": "MASKED_DESTINATIONS", // destination id containing called addresses to be masked on export -// "mask_length": 0, // length of the destination suffix to be masked -// "export_dir": "/var/log/cgrates/cdre", // path where the exported CDRs will be placed -// "header_fields": [], // template of the exported header fields -// "content_fields": [ // template of the exported content fields -// {"tag": "CgrId", "field_id": "CgrId", "type": "*composed", "value": "CgrId"}, -// {"tag":"RunId", "field_id": "MediationRunId", "type": "*composed", "value": "MediationRunId"}, -// {"tag":"Tor", "field_id": "TOR", "type": "*composed", "value": "TOR"}, -// {"tag":"AccId", "field_id": "AccId", "type": "*composed", "value": "AccId"}, -// {"tag":"ReqType", "field_id": "ReqType", "type": "*composed", "value": "ReqType"}, -// {"tag":"Direction", "field_id": "Direction", "type": "*composed", "value": "Direction"}, -// {"tag":"Tenant", "field_id": "Tenant", "type": "*composed", "value": "Tenant"}, -// {"tag":"Category", "field_id": "Category", "type": "*composed", "value": "Category"}, -// {"tag":"Account", "field_id": "Account", "type": "*composed", "value": "Account"}, -// {"tag":"Subject", "field_id": "Subject", "type": "*composed", "value": "Subject"}, -// {"tag":"Destination", "field_id": "Destination", "type": "*composed", "value": "Destination"}, -// {"tag":"SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"tag":"AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"tag":"Usage", "field_id": "Usage", "type": "*composed", "value": "Usage"}, -// {"tag":"Cost", "field_id": "Cost", "type": "*composed", "value": "Cost"}, -// ], -// "trailer_fields": [], // template of the exported trailer fields -// } -//}, +// "cdre": { +// "*default": { +// "cdr_format": "csv", // exported CDRs format +// "field_separator": ",", +// "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) +// "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) +// "mms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from MMS unit to call duration in some billing systems) +// "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) +// "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT +// "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding +// "cost_shift_digits": 0, // shift digits in the cost on export (eg: convert from EUR to cents) +// "mask_destination_id": "MASKED_DESTINATIONS", // destination id containing called addresses to be masked on export +// "mask_length": 0, // length of the destination suffix to be masked +// "export_folder": "/var/log/cgrates/cdre", // path where the exported CDRs will be placed +// "header_fields": [], // template of the exported header fields +// "content_fields": [ // template of the exported content fields +// {"tag": "CGRID", "field_id": "CGRID", "type": "*composed", "value": "CGRID"}, +// {"tag":"RunID", "field_id": "RunID", "type": "*composed", "value": "RunID"}, +// {"tag":"TOR", "field_id": "ToR", "type": "*composed", "value": "ToR"}, +// {"tag":"OriginID", "field_id": "OriginID", "type": "*composed", "value": "OriginID"}, +// {"tag":"RequestType", "field_id": "RequestType", "type": "*composed", "value": "RequestType"}, +// {"tag":"Direction", "field_id": "Direction", "type": "*composed", "value": "Direction"}, +// {"tag":"Tenant", "field_id": "Tenant", "type": "*composed", "value": "Tenant"}, +// {"tag":"Category", "field_id": "Category", "type": "*composed", "value": "Category"}, +// {"tag":"Account", "field_id": "Account", "type": "*composed", "value": "Account"}, +// {"tag":"Subject", "field_id": "Subject", "type": "*composed", "value": "Subject"}, +// {"tag":"Destination", "field_id": "Destination", "type": "*composed", "value": "Destination"}, +// {"tag":"SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, +// {"tag":"AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, +// {"tag":"Usage", "field_id": "Usage", "type": "*composed", "value": "Usage"}, +// {"tag":"Cost", "field_id": "Cost", "type": "*composed", "value": "Cost"}, +// ], +// "trailer_fields": [], // template of the exported trailer fields +// } +// }, -//"cdrc": { -// "*default": { -// "enabled": false, // enable CDR client functionality -// "dry_run": false, // do not send the CDRs to CDRS, just parse them -// "cdrs": "internal", // address where to reach CDR server. -// "cdr_format": "csv", // CDR file format -// "field_separator": ",", // separator used in case of csv files -// "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> -// "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify -// "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited -// "data_usage_multiply_factor": 1024, // conversion factor for data usage -// "cdr_in_dir": "/var/log/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored -// "cdr_out_dir": "/var/log/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved -// "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records -// "cdr_source_id": "freeswitch_csv", // free form field, tag identifying the source of the CDRs within CDRS database -// "cdr_filter": "", // filter CDR records to import -// "continue_on_success": false, // continue to the next template if executed -// "partial_record_cache": "10s", // duration to cache partial records when not pairing -// "header_fields": [], // template of the import header fields -// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value -// {"tag": "tor", "field_id": "TOR", "type": "*composed", "value": "2", "mandatory": true}, -// {"tag": "accid", "field_id": "AccId", "type": "*composed", "value": "3", "mandatory": true}, -// {"tag": "reqtype", "field_id": "ReqType", "type": "*composed", "value": "4", "mandatory": true}, -// {"tag": "direction", "field_id": "Direction", "type": "*composed", "value": "5", "mandatory": true}, -// {"tag": "tenant", "field_id": "Tenant", "type": "*composed", "value": "6", "mandatory": true}, -// {"tag": "category", "field_id": "Category", "type": "*composed", "value": "7", "mandatory": true}, -// {"tag": "account", "field_id": "Account", "type": "*composed", "value": "8", "mandatory": true}, -// {"tag": "subject", "field_id": "Subject", "type": "*composed", "value": "9", "mandatory": true}, -// {"tag": "destination", "field_id": "Destination", "type": "*composed", "value": "10", "mandatory": true}, -// {"tag": "setup_time", "field_id": "SetupTime", "type": "*composed", "value": "11", "mandatory": true}, -// {"tag": "answer_time", "field_id": "AnswerTime", "type": "*composed", "value": "12", "mandatory": true}, -// {"tag": "usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, -// ], -// "trailer_fields": [], // template of the import trailer fields -// } -//}, +// "cdrc": { +// "*default": { +// "enabled": false, // enable CDR client functionality +// "dry_run": false, // do not send the CDRs to CDRS, just parse them +// "cdrs_conns": [ +// {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> +// ], +// "cdr_format": "csv", // CDR file format +// "field_separator": ",", // separator used in case of csv files +// "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> +// "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify +// "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited +// "data_usage_multiply_factor": 1024, // conversion factor for data usage +// "cdr_in_dir": "/var/log/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored +// "cdr_out_dir": "/var/log/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved +// "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records +// "cdr_source_id": "freeswitch_csv", // free form field, tag identifying the source of the CDRs within CDRS database +// "cdr_filter": "", // filter CDR records to import +// "continue_on_success": false, // continue to the next template if executed +// "partial_record_cache": "10s", // duration to cache partial records when not pairing +// "header_fields": [], // template of the import header fields +// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value +// {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "2", "mandatory": true}, +// {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "3", "mandatory": true}, +// {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "4", "mandatory": true}, +// {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "5", "mandatory": true}, +// {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "6", "mandatory": true}, +// {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "7", "mandatory": true}, +// {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "8", "mandatory": true}, +// {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "9", "mandatory": true}, +// {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "10", "mandatory": true}, +// {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "11", "mandatory": true}, +// {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "12", "mandatory": true}, +// {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, +// ], +// "trailer_fields": [], // template of the import trailer fields +// } +// }, -//"sm_generic": { -// "enabled": false, // starts SessionManager service: -// "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_conns": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "debit_interval": "0s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "session_ttl": "0s", // time after a session with no updates is terminated -//}, +// "sm_generic": { +// "enabled": false, // starts SessionManager service: +// "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], +// "cdrs_conns": [ +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], +// "debit_interval": "0s", // interval to perform debits on. +// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this +// "max_call_duration": "3h", // maximum call duration a prepaid call can last +// "session_ttl": "0s", // time after a session with no updates is terminated, not defined by default + //"session_ttl_last_used": "", // tweak LastUsed for sessions timing-out, not defined by default + //"session_ttl_usage": "", // tweak Usage for sessions timing-out, not defined by default +// }, -//"sm_freeswitch": { -// "enabled": false, // starts SessionManager service: -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_conns": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "extra_fields": [], // extra fields to store in auth/CDRs when creating them -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "min_dur_low_balance": "5s", // threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval) -// "low_balance_ann_file": "", // file to be played when low balance is reached for prepaid calls -// "empty_balance_context": "", // if defined, prepaid calls will be transfered to this context on empty balance -// "empty_balance_ann_file": "", // file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined) -// "subscribe_park": true, // subscribe via fsock to receive park events -// "channel_sync_interval": "5m", // sync channels with freeswitch regularly -// "connections":[ // instantiate connections to multiple FreeSWITCH servers -// {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} -// ], -//}, +// "sm_freeswitch": { +// "enabled": false, // starts SessionManager service: +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], +// "cdrs_conns": [ +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], +// "create_cdr": false, // create CDR out of events and sends them to CDRS component +// "extra_fields": [], // extra fields to store in auth/CDRs when creating them +// "debit_interval": "10s", // interval to perform debits on. +// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this +// "max_call_duration": "3h", // maximum call duration a prepaid call can last +// "min_dur_low_balance": "5s", // threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval) +// "low_balance_ann_file": "", // file to be played when low balance is reached for prepaid calls +// "empty_balance_context": "", // if defined, prepaid calls will be transfered to this context on empty balance +// "empty_balance_ann_file": "", // file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined) +// "subscribe_park": true, // subscribe via fsock to receive park events +// "channel_sync_interval": "5m", // sync channels with freeswitch regularly +// "max_wait_connection": "2s", // maximum duration to wait for a connection to be retrieved from the pool +// "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers +// {"address": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} +// ], +// }, -//"sm_kamailio": { -// "enabled": false, // starts SessionManager service: -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_conns": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "connections":[ // instantiate connections to multiple Kamailio servers -// {"evapi_addr": "127.0.0.1:8448", "reconnects": 5} -// ], -//}, +// "sm_kamailio": { +// "enabled": false, // starts SessionManager service: +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], +// "cdrs_conns": [ +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], +// "create_cdr": false, // create CDR out of events and sends them to CDRS component +// "debit_interval": "10s", // interval to perform debits on. +// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this +// "max_call_duration": "3h", // maximum call duration a prepaid call can last +// "evapi_conns":[ // instantiate connections to multiple Kamailio servers +// {"address": "127.0.0.1:8448", "reconnects": 5} +// ], +// }, -//"sm_opensips": { -// "enabled": false, // starts SessionManager service: -// "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_conns": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "reconnects": 5, // number of reconnects if connection is lost -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "events_subscribe_interval": "60s", // automatic events subscription to OpenSIPS, 0 to disable it -// "mi_addr": "127.0.0.1:8020", // address where to reach OpenSIPS MI to send session disconnects -//}, +// "sm_opensips": { +// "enabled": false, // starts SessionManager service: +// "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], +// "cdrs_conns": [ +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], +// "reconnects": 5, // number of reconnects if connection is lost +// "create_cdr": false, // create CDR out of events and sends it to CDRS component +// "debit_interval": "10s", // interval to perform debits on. +// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this +// "max_call_duration": "3h", // maximum call duration a prepaid call can last +// "events_subscribe_interval": "60s", // automatic events subscription to OpenSIPS, 0 to disable it +// "mi_addr": "127.0.0.1:8020", // address where to reach OpenSIPS MI to send session disconnects +// }, -//"diameter_agent": { -// "enabled": false, // enables the diameter agent: -// "listen": "127.0.0.1:3868", // address where to listen for diameter requests -// "dictionaries_dir": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load -// "sm_generic": [ -// {"server":"internal"} // connection towards SMG component for session management -// ] -// "timezone": "", // timezone for timestamps where not specified, empty for general defaults <""|UTC|Local|$IANA_TZ_DB> -// "origin_host": "CGR-DA", // diameter Origin-Host AVP used in replies -// "origin_realm": "cgrates.org", // diameter Origin-Realm AVP used in replies -// "vendor_id": 0, // diameter Vendor-Id AVP used in replies -// "product_name": "CGRateS", // diameter Product-Name AVP used in replies -// "request_processors": [ -// { -// "id": "*default", // formal identifier of this processor -// "dry_run": false, // do not send the CDRs to CDRS, just parse them -// "request_filter": "Subscription-Id>Subscription-Id-Type(0)", // filter requests processed by this processor -// "continue_on_success": false, // continue to the next template if executed -// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value -// {"tag": "tor", "field_id": "TOR", "type": "*composed", "value": "^*voice", "mandatory": true}, -// {"tag": "accid", "field_id": "AccId", "type": "*composed", "value": "Session-Id", "mandatory": true}, -// {"tag": "reqtype", "field_id": "ReqType", "type": "*composed", "value": "^*users", "mandatory": true}, -// {"tag": "direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, -// {"tag": "tenant", "field_id": "Tenant", "type": "*composed", "value": "^*users", "mandatory": true}, -// {"tag": "category", "field_id": "Category", "type": "*composed", "value": "^call_;~Service-Information>IN-Information>Calling-Vlr-Number:s/^$/33000/;~Service-Information>IN-Information>Calling-Vlr-Number:s/^(\\d{5})/${1}/", "mandatory": true}, -// {"tag": "account", "field_id": "Account", "type": "*composed", "value": "^*users", "mandatory": true}, -// {"tag": "subject", "field_id": "Subject", "type": "*composed", "value": "^*users", "mandatory": true}, -// {"tag": "destination", "field_id": "Destination", "type": "*composed", "value": "Service-Information>IN-Information>Real-Called-Number", "mandatory": true}, -// {"tag": "setup_time", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, -// {"tag": "answer_time", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, -// {"tag": "usage", "field_id": "Usage", "type": "*composed", "value": "Requested-Service-Unit>CC-Time", "mandatory": true}, -// {"tag": "subscriber_id", "field_id": "SubscriberId", "type": "*composed", "value": "Subscription-Id>Subscription-Id-Data", "mandatory": true}, -// ], -// }, -// ], -//}, +// "diameter_agent": { +// "enabled": false, // enables the diameter agent: +// "listen": "127.0.0.1:3868", // address where to listen for diameter requests +// "dictionaries_dir": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load +// "sm_generic_conns": [ +// {"address": "*internal"} // connection towards SMG component for session management +// ], +// "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> +// "create_cdr": true, // create CDR out of CCR terminate and send it to SMG component +// "debit_interval": "5m", // interval for CCR updates +// "timezone": "", // timezone for timestamps where not specified, empty for general defaults <""|UTC|Local|$IANA_TZ_DB> +// "dialect": "huawei", // the diameter dialect used in the communication, supported: +// "origin_host": "CGR-DA", // diameter Origin-Host AVP used in replies +// "origin_realm": "cgrates.org", // diameter Origin-Realm AVP used in replies +// "vendor_id": 0, // diameter Vendor-Id AVP used in replies +// "product_name": "CGRateS", // diameter Product-Name AVP used in replies +// "request_processors": [ +// { +// "id": "*default", // formal identifier of this processor +// "dry_run": false, // do not send the events to SMG, just log them +// "publish_event": false, // if enabled, it will publish internal event to pubsub +// "request_filter": "Subscription-Id>Subscription-Id-Type(0)", // filter requests processed by this processor +// "flags": [], // flags to influence processing behavior +// "continue_on_success": false, // continue to the next template if executed +// "append_cca": true, // when continuing will append cca fields to the previous ones +// "ccr_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value +// {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*voice", "mandatory": true}, +// {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "Session-Id", "mandatory": true}, +// {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "^*users", "mandatory": true}, +// {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, +// {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "^*users", "mandatory": true}, +// {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^call", "mandatory": true}, +// {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "^*users", "mandatory": true}, +// {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "^*users", "mandatory": true}, +// {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "Service-Information>IN-Information>Real-Called-Number", "mandatory": true}, +// {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, +// {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "Event-Timestamp", "mandatory": true}, +// {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*ccr_usage", "mandatory": true}, +// {"tag": "SubscriberID", "field_id": "SubscriberId", "type": "*composed", "value": "Subscription-Id>Subscription-Id-Data", "mandatory": true}, +// ], +// "cca_fields":[ // fields returned in CCA +// {"tag": "GrantedUnits", "field_id": "Granted-Service-Unit>CC-Time", "type": "*handler", "handler_id": "*cca_usage", "mandatory": true}, +// ], +// }, +// ], +// }, -//"historys": { -// "enabled": false, // starts History service: . -// "history_dir": "/var/log/cgrates/history", // location on disk where to store history files. -// "save_interval": "1s", // interval to save changed cache into .git archive -//}, +// "historys": { +// "enabled": false, // starts History service: . +// "history_dir": "/var/log/cgrates/history", // location on disk where to store history files. +// "save_interval": "1s", // interval to save changed cache into .git archive +// }, -//"pubsubs": { -// "enabled": false, // starts PubSub service: . -//}, +// "pubsubs": { +// "enabled": false, // starts PubSub service: . +// }, -//"aliases": { -// "enabled": false, // starts Aliases service: . -//}, +// "aliases": { +// "enabled": false, // starts Aliases service: . +// }, -//"users": { -// "enabled": false, // starts User service: . -// "indexes": [], // user profile field indexes -//}, +// "users": { +// "enabled": false, // starts User service: . +// "indexes": [], // user profile field indexes +// }, -//"mailer": { -// "server": "localhost", // the server to use when sending emails out -// "auth_user": "cgrates", // authenticate to email server using this user -// "auth_passwd": "CGRateS.org", // authenticate to email server with this password -// "from_address": "cgr-mailer@localhost.localdomain" // from address used when sending emails out -//}, +// "mailer": { +// "server": "localhost", // the server to use when sending emails out +// "auth_user": "cgrates", // authenticate to email server using this user +// "auth_password": "CGRateS.org", // authenticate to email server with this password +// "from_address": "cgr-mailer@localhost.localdomain" // from address used when sending emails out +// }, -//"suretax": { -// "url": "", // API url -// "client_number": "", // client number, provided by SureTax -// "validation_key": "", // validation key provided by SureTax -// "business_unit": "", // client’s Business Unit -// "timezone": "Local", // convert the time of the events to this timezone before sending request out -// "include_local_cost": false, // sum local calculated cost with tax one in final cost -// "return_file_code": "0", // default or Quote purposes <0|Q> -// "response_group": "03", // determines how taxes are grouped for the response <03|13> -// "response_type": "D4", // determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response -// "regulatory_code": "03", // provider type -// "client_tracking": "CgrId", // template extracting client information out of StoredCdr; <$RSRFields> -// "customer_number": "Subject", // template extracting customer number out of StoredCdr; <$RSRFields> -// "orig_number": "Subject", // template extracting origination number out of StoredCdr; <$RSRFields> -// "term_number": "Destination", // template extracting termination number out of StoredCdr; <$RSRFields> -// "bill_to_number": "", // template extracting billed to number out of StoredCdr; <$RSRFields> -// "zipcode": "", // template extracting billing zip code out of StoredCdr; <$RSRFields> -// "plus4": "", // template extracting billing zip code extension out of StoredCdr; <$RSRFields> -// "p2pzipcode": "", // template extracting secondary zip code out of StoredCdr; <$RSRFields> -// "p2pplus4": "", // template extracting secondary zip code extension out of StoredCdr; <$RSRFields> -// "units": "^1", // template extracting number of “lines” or unique charges contained within the revenue out of StoredCdr; <$RSRFields> -// "unit_type": "^00", // template extracting number of unique access lines out of StoredCdr; <$RSRFields> -// "tax_included": "^0", // template extracting tax included in revenue out of StoredCdr; <$RSRFields> -// "tax_situs_rule": "^04", // template extracting tax situs rule out of StoredCdr; <$RSRFields> -// "trans_type_code": "^010101", // template extracting transaction type indicator out of StoredCdr; <$RSRFields> -// "sales_type_code": "^R", // template extracting sales type code out of StoredCdr; <$RSRFields> -// "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> -//}, +// "suretax": { +// "url": "", // API url +// "client_number": "", // client number, provided by SureTax +// "validation_key": "", // validation key provided by SureTax +// "business_unit": "", // client’s Business Unit +// "timezone": "Local", // convert the time of the events to this timezone before sending request out +// "include_local_cost": false, // sum local calculated cost with tax one in final cost +// "return_file_code": "0", // default or Quote purposes <0|Q> +// "response_group": "03", // determines how taxes are grouped for the response <03|13> +// "response_type": "D4", // determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response +// "regulatory_code": "03", // provider type +// "client_tracking": "CGRID", // template extracting client information out of StoredCdr; <$RSRFields> +// "customer_number": "Subject", // template extracting customer number out of StoredCdr; <$RSRFields> +// "orig_number": "Subject", // template extracting origination number out of StoredCdr; <$RSRFields> +// "term_number": "Destination", // template extracting termination number out of StoredCdr; <$RSRFields> +// "bill_to_number": "", // template extracting billed to number out of StoredCdr; <$RSRFields> +// "zipcode": "", // template extracting billing zip code out of StoredCdr; <$RSRFields> +// "plus4": "", // template extracting billing zip code extension out of StoredCdr; <$RSRFields> +// "p2pzipcode": "", // template extracting secondary zip code out of StoredCdr; <$RSRFields> +// "p2pplus4": "", // template extracting secondary zip code extension out of StoredCdr; <$RSRFields> +// "units": "^1", // template extracting number of “lines” or unique charges contained within the revenue out of StoredCdr; <$RSRFields> +// "unit_type": "^00", // template extracting number of unique access lines out of StoredCdr; <$RSRFields> +// "tax_included": "^0", // template extracting tax included in revenue out of StoredCdr; <$RSRFields> +// "tax_situs_rule": "^04", // template extracting tax situs rule out of StoredCdr; <$RSRFields> +// "trans_type_code": "^010101", // template extracting transaction type indicator out of StoredCdr; <$RSRFields> +// "sales_type_code": "^R", // template extracting sales type code out of StoredCdr; <$RSRFields> +// "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> +// }, -} +} \ No newline at end of file From 0cdc9861634e4f2117865a9f2cd21e0c2817bdfe Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 22 Apr 2016 17:00:05 +0200 Subject: [PATCH 180/227] Fix build on hapool --- apier/v1/apier.go | 2 +- cmd/cgr-engine/cgr-engine.go | 200 +++++++++++++++++------------ cmd/cgr-engine/rater.go | 74 ++++++----- cmd/cgr-engine/registration.go | 8 +- engine/cdr_test.go | 2 +- engine/cdrs.go | 2 +- engine/libengine.go | 18 ++- sessionmanager/fssessionmanager.go | 6 +- sessionmanager/kamailiosm.go | 4 +- 9 files changed, 183 insertions(+), 133 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 94434b5bb..e6e62711f 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -817,7 +817,7 @@ func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.Cach } cs.CdrStats = len(queueIds) } - if self.Config.RaterUserServer == utils.INTERNAL { + if self.Users != nil { var ups engine.UserProfiles if err := self.Users.Call("UsersV1.GetUsers", &engine.UserProfile{}, &ups); err != nil { return utils.NewErrServerError(err) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index e5fd78bcd..00b983106 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -104,25 +104,16 @@ func startCdrcs(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConn // Fires up a cdrc instance func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConnection, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, closeChan chan struct{}, exitChan chan bool) { - var cdrsConn rpcclient.RpcClientConnection var cdrcCfg *config.CdrcConfig for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break } - if cdrcCfg.Cdrs == utils.INTERNAL { - cdrsChan := <-internalCdrSChan // This will signal that the cdrs part is populated in internalRaterChan - internalCdrSChan <- cdrsChan // Put it back for other components - resp := <-internalRaterChan - cdrsConn = resp - internalRaterChan <- resp - } else { - conn, err := rpcclient.NewRpcClient("tcp", cdrcCfg.Cdrs, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to CDRS via RPC: %v", err)) - exitChan <- true - return - } - cdrsConn = conn + cdrsConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cdrcCfg.CdrsConns, internalCdrSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to CDRS via RPC: %s", err.Error())) + exitChan <- true + return } cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan, cfg.DefaultTimezone) if err != nil { @@ -139,18 +130,18 @@ func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConne func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SM-Generic service.") ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmGenericConfig.RaterConns, internalRaterChan) + cfg.SmGenericConfig.RALsConns, internalRaterChan, cfg.InternalTtl) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } var cdrsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.SmGenericConfig.RaterConns, cfg.SmGenericConfig.CdrsConns) { + if reflect.DeepEqual(cfg.SmGenericConfig.RALsConns, cfg.SmGenericConfig.CDRsConns) { cdrsConn = ralConn } else { cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmGenericConfig.CdrsConns, internalCDRSChan) + cfg.SmGenericConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true @@ -179,18 +170,23 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal func startDiameterAgent(internalSMGChan, internalPubSubSChan chan rpcclient.RpcClientConnection, exitChan chan bool) { utils.Logger.Info("Starting CGRateS DiameterAgent service.") smgConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.DiameterAgentCfg().SMGenericConns, internalSMGChan) + cfg.DiameterAgentCfg().SMGenericConns, internalSMGChan, cfg.InternalTtl) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to SMG: %s", err.Error())) exitChan <- true return } - pubsubConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.DiameterAgentCfg().PubSubConns, internalPubSubSChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubS: %s", err.Error())) - exitChan <- true - return + var pubsubConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.DiameterAgentCfg().SMGenericConns, cfg.DiameterAgentCfg().PubSubConns) { + pubsubConn = smgConn + } else { + pubsubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.DiameterAgentCfg().PubSubConns, internalPubSubSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubS: %s", err.Error())) + exitChan <- true + return + } } da, err := agents.NewDiameterAgent(cfg, smgConn, pubsubConn) if err != nil { @@ -205,68 +201,83 @@ func startDiameterAgent(internalSMGChan, internalPubSubSChan chan rpcclient.RpcC } func startSmFreeSWITCH(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { - utils.Logger.Info("Starting CGRateS SM-FreeSWITCH service.") + utils.Logger.Info("Starting CGRateS SMFreeSWITCH service.") ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmFsConfig.RaterConns, internalRaterChan) + cfg.SmFsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } - cdrsConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmFsConfig.CdrsConns, internalRaterChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + var cdrsConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.SmFsConfig.RALsConns, cfg.SmFsConfig.CDRsConns) { + cdrsConn = ralConn + } else { + cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmFsConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + exitChan <- true + return + } } sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, ralConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) if err = sm.Connect(); err != nil { - utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) + utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) } exitChan <- true } func startSmKamailio(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { - utils.Logger.Info("Starting CGRateS SM-Kamailio service.") + utils.Logger.Info("Starting CGRateS SMKamailio service.") ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmKamConfig.RaterConns, internalRaterChan) + cfg.SmKamConfig.RALsConns, internalRaterChan, cfg.InternalTtl) if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } - cdrsConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmKamConfig.CdrsConns, internalRaterChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + var cdrsConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.SmKamConfig.RALsConns, cfg.SmKamConfig.CDRsConns) { + cdrsConn = ralConn + } else { + cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmKamConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + exitChan <- true + return + } } sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, ralConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) if err = sm.Connect(); err != nil { - utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) + utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) } exitChan <- true } func startSmOpenSIPS(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { - utils.Logger.Info("Starting CGRateS SM-OpenSIPS service.") + utils.Logger.Info("Starting CGRateS SMOpenSIPS service.") ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmOsipsConfig.RaterConns, internalRaterChan) + cfg.SmOsipsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RALs: %s", err.Error())) exitChan <- true return } - cdrsConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmOsipsConfig.CdrsConns, internalRaterChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + var cdrsConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.SmOsipsConfig.RALsConns, cfg.SmOsipsConfig.CDRsConns) { + cdrsConn = ralConn + } else { + cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmOsipsConfig.CDRsConns, internalRaterChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to CDRs: %s", err.Error())) + exitChan <- true + return + } } sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, cfg.Reconnects, ralConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) @@ -283,45 +294,64 @@ func startCDRS(internalCdrSChan chan rpcclient.RpcClientConnection, logDb engine utils.Logger.Info("Starting CGRateS CDRS service.") // Conn pool towards RAL ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSRaterConns, internalRaterChan) + cfg.CDRSRaterConns, internalRaterChan, cfg.InternalTtl) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } // Pubsub connection init - pubSubConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSPubSubSConns, internalPubSubSChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) - exitChan <- true - return + var pubSubConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSPubSubSConns) { + pubSubConn = ralConn + } else { + pubSubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSPubSubSConns, internalPubSubSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) + exitChan <- true + return + } } // Users connection init - usersConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSUserSConns, internalUserSChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) - exitChan <- true - return + var usersConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSUserSConns) { + pubSubConn = ralConn + } else { + usersConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSUserSConns, internalUserSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) + exitChan <- true + return + } } // Aliases connection init - aliasesConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSAliaseSConns, internalAliaseSChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) - exitChan <- true - return + var aliasesConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSAliaseSConns) { + pubSubConn = ralConn + } else { + aliasesConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSAliaseSConns, internalAliaseSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) + exitChan <- true + return + } } // Stats connection init - statsConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSStatSConns, internalCdrStatSChan) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) - exitChan <- true - return + var statsConn *rpcclient.RpcClientPool + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSStatSConns) { + pubSubConn = ralConn + } else { + statsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSStatSConns, internalCdrStatSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) + exitChan <- true + return + } } - cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, ralConn, pubSubConn, usersConn, aliasesConn, statsConn) cdrServer.SetTimeToLive(cfg.ResponseCacheTTL, nil) utils.Logger.Info("Registering CDRS HTTP Handlers.") @@ -459,7 +489,7 @@ func main() { } config.SetCgrConfig(cfg) // Share the config object if *raterEnabled { - cfg.RaterEnabled = *raterEnabled + cfg.RALsEnabled = *raterEnabled } if *schedEnabled { cfg.SchedulerEnabled = *schedEnabled @@ -472,7 +502,7 @@ func main() { var logDb engine.LogStorage var loadDb engine.LoadStorage var cdrDb engine.CdrStorage - if cfg.RaterEnabled || cfg.SchedulerEnabled || cfg.CDRStatsEnabled { // Only connect to dataDb if necessary + if cfg.RALsEnabled || cfg.SchedulerEnabled || cfg.CDRStatsEnabled { // Only connect to dataDb if necessary ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort, cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding) if err != nil { // Cannot configure getter database, show stopper @@ -482,7 +512,7 @@ func main() { defer ratingDb.Close() engine.SetRatingStorage(ratingDb) } - if cfg.RaterEnabled || cfg.CDRStatsEnabled || cfg.PubSubServerEnabled || cfg.AliasesServerEnabled || cfg.UserServerEnabled { + if cfg.RALsEnabled || cfg.CDRStatsEnabled || cfg.PubSubServerEnabled || cfg.AliasesServerEnabled || cfg.UserServerEnabled { accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort, cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding) if err != nil { // Cannot configure getter database, show stopper @@ -492,7 +522,7 @@ func main() { defer accountDb.Close() engine.SetAccountingStorage(accountDb) } - if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary + if cfg.RALsEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { // Cannot configure logger database, show stopper @@ -536,7 +566,7 @@ func main() { } // Start rater service - if cfg.RaterEnabled { + if cfg.RALsEnabled { go startRater(internalRaterChan, cacheDoneChan, internalBalancerChan, internalSchedulerChan, internalCdrStatSChan, internalHistorySChan, internalPubSubSChan, internalUserSChan, internalAliaseSChan, server, ratingDb, accountDb, loadDb, cdrDb, logDb, &stopHandled, exitChan) } diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index e56892ee4..b1f21d4b4 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -87,12 +87,12 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC // Connection to balancer var bal *balancer2go.Balancer - if cfg.RaterBalancer != "" { + if cfg.RALsBalancer != "" { balTaskChan := make(chan struct{}) waitTasks = append(waitTasks, balTaskChan) go func() { defer close(balTaskChan) - if cfg.RaterBalancer == utils.INTERNAL { + if cfg.RALsBalancer == utils.INTERNAL { select { case bal = <-internalBalancerChan: internalBalancerChan <- bal // Put it back if someone else is interested about @@ -108,39 +108,37 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - - // Connection to CDRStats - var cdrStats rpcclient.RpcClientConnection - if cfg.RaterCdrStats != "" { + // Connections to CDRStats + var cdrStats *rpcclient.RpcClientPool + if len(cfg.RALsCDRStatSConns) != 0 { cdrstatTaskChan := make(chan struct{}) waitTasks = append(waitTasks, cdrstatTaskChan) go func() { defer close(cdrstatTaskChan) - if cfg.RaterCdrStats == utils.INTERNAL { - select { - case cdrStats = <-internalCdrStatSChan: - internalCdrStatSChan <- cdrStats - case <-time.After(cfg.InternalTtl): - utils.Logger.Crit(": Internal cdrstats connection timeout.") - exitChan <- true - return - } - } else if cdrStats, err = rpcclient.NewRpcClient("tcp", cfg.RaterCdrStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to cdrstats, error: %s", err.Error())) + cdrStats, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSRaterConns, internalCdrStatSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to CDRStatS, error: %s", err.Error())) exitChan <- true return } }() } - // Connection to HistoryS - if cfg.RaterHistoryServer != "" { + // Connection to HistoryS, + // FixMe via multiple connections + var ralsHistoryServer string + for _, connCfg := range cfg.RALsHistorySConns { + ralsHistoryServer = connCfg.Address + break + } + if ralsHistoryServer != "" { histTaskChan := make(chan struct{}) waitTasks = append(waitTasks, histTaskChan) go func() { defer close(histTaskChan) var scribeServer rpcclient.RpcClientConnection - if cfg.RaterHistoryServer == utils.INTERNAL { + if ralsHistoryServer == utils.INTERNAL { select { case scribeServer = <-internalHistorySChan: internalHistorySChan <- scribeServer @@ -149,7 +147,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC exitChan <- true return } - } else if scribeServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterHistoryServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { + } else if scribeServer, err = rpcclient.NewRpcClient("tcp", ralsHistoryServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect historys, error: %s", err.Error())) exitChan <- true return @@ -157,15 +155,19 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC engine.SetHistoryScribe(scribeServer) }() } - // Connection to pubsubs - if cfg.RaterPubSubServer != "" { + var ralsPubSubServer string + for _, connCfg := range cfg.RALsPubSubSConns { + ralsPubSubServer = connCfg.Address + break + } + if ralsPubSubServer != "" { pubsubTaskChan := make(chan struct{}) waitTasks = append(waitTasks, pubsubTaskChan) go func() { defer close(pubsubTaskChan) var pubSubServer rpcclient.RpcClientConnection - if cfg.RaterPubSubServer == utils.INTERNAL { + if ralsPubSubServer == utils.INTERNAL { select { case pubSubServer = <-internalPubSubSChan: internalPubSubSChan <- pubSubServer @@ -174,7 +176,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC exitChan <- true return } - } else if pubSubServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterPubSubServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { + } else if pubSubServer, err = rpcclient.NewRpcClient("tcp", ralsPubSubServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to pubsubs: %s", err.Error())) exitChan <- true return @@ -184,13 +186,18 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } // Connection to AliasService - if cfg.RaterAliasesServer != "" { + var ralsAliasServer string + for _, connCfg := range cfg.RALsAliasSConns { + ralsAliasServer = connCfg.Address + break + } + if ralsAliasServer != "" { aliasesTaskChan := make(chan struct{}) waitTasks = append(waitTasks, aliasesTaskChan) go func() { defer close(aliasesTaskChan) var aliasesServer rpcclient.RpcClientConnection - if cfg.RaterAliasesServer == utils.INTERNAL { + if ralsAliasServer == utils.INTERNAL { select { case aliasesServer = <-internalAliaseSChan: internalAliaseSChan <- aliasesServer @@ -199,7 +206,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC exitChan <- true return } - } else if aliasesServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterAliasesServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { + } else if aliasesServer, err = rpcclient.NewRpcClient("tcp", ralsAliasServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to aliases, error: %s", err.Error())) exitChan <- true return @@ -209,13 +216,18 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } // Connection to UserService + var ralsUserServer string + for _, connCfg := range cfg.RALsUserSConns { + ralsUserServer = connCfg.Address + break + } var userServer rpcclient.RpcClientConnection - if cfg.RaterUserServer != "" { + if ralsUserServer != "" { usersTaskChan := make(chan struct{}) waitTasks = append(waitTasks, usersTaskChan) go func() { defer close(usersTaskChan) - if cfg.RaterUserServer == utils.INTERNAL { + if ralsUserServer == utils.INTERNAL { select { case userServer = <-internalUserSChan: internalUserSChan <- userServer @@ -224,7 +236,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC exitChan <- true return } - } else if userServer, err = rpcclient.NewRpcClient("tcp", cfg.RaterUserServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { + } else if userServer, err = rpcclient.NewRpcClient("tcp", ralsUserServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect users, error: %s", err.Error())) exitChan <- true return diff --git a/cmd/cgr-engine/registration.go b/cmd/cgr-engine/registration.go index a25514911..de7a724f1 100644 --- a/cmd/cgr-engine/registration.go +++ b/cmd/cgr-engine/registration.go @@ -83,14 +83,14 @@ func stopRaterSignalHandler(internalCdrStatSChan chan rpcclient.RpcClientConnect Connects to the balancer and calls unregister RPC method. */ func unregisterFromBalancer(exitChan chan bool) { - client, err := rpc.Dial("tcp", cfg.RaterBalancer) + client, err := rpc.Dial("tcp", cfg.RALsBalancer) if err != nil { utils.Logger.Crit("Cannot contact the balancer!") exitChan <- true return } var reply int - utils.Logger.Info(fmt.Sprintf("Unregistering from balancer %s", cfg.RaterBalancer)) + utils.Logger.Info(fmt.Sprintf("Unregistering from balancer %s", cfg.RALsBalancer)) client.Call("Responder.UnRegisterRater", cfg.RPCGOBListen, &reply) if err := client.Close(); err != nil { utils.Logger.Crit("Could not close balancer unregistration!") @@ -102,14 +102,14 @@ func unregisterFromBalancer(exitChan chan bool) { Connects to the balancer and rehisters the engine to the server. */ func registerToBalancer(exitChan chan bool) { - client, err := rpc.Dial("tcp", cfg.RaterBalancer) + client, err := rpc.Dial("tcp", cfg.RALsBalancer) if err != nil { utils.Logger.Crit(fmt.Sprintf("Cannot contact the balancer: %v", err)) exitChan <- true return } var reply int - utils.Logger.Info(fmt.Sprintf("Registering to balancer %s", cfg.RaterBalancer)) + utils.Logger.Info(fmt.Sprintf("Registering to balancer %s", cfg.RALsBalancer)) client.Call("Responder.RegisterRater", cfg.RPCGOBListen, &reply) if err := client.Close(); err != nil { utils.Logger.Crit("Could not close balancer registration!") diff --git a/engine/cdr_test.go b/engine/cdr_test.go index 013e0c8dc..af2529958 100644 --- a/engine/cdr_test.go +++ b/engine/cdr_test.go @@ -566,7 +566,7 @@ func TestUsageReqAsCD(t *testing.T) { Account: "1001", Subject: "1001", Destination: "1002", SetupTime: "2013-11-07T08:42:20Z", AnswerTime: "2013-11-07T08:42:26Z", Usage: "0.00000001", } - eCD := &CallDescriptor{CgrId: "9473e7b2e075d168b9da10ae957ee68fe5a217e4", TOR: req.ToR, Direction: req.Direction, Tenant: req.Tenant, Category: req.Category, Account: req.Account, Subject: req.Subject, Destination: req.Destination, TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), TimeEnd: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).Add(time.Duration(10))} + eCD := &CallDescriptor{CgrID: "9473e7b2e075d168b9da10ae957ee68fe5a217e4", TOR: req.ToR, Direction: req.Direction, Tenant: req.Tenant, Category: req.Category, Account: req.Account, Subject: req.Subject, Destination: req.Destination, TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), TimeEnd: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).Add(time.Duration(10))} if cd, err := req.AsCallDescriptor(""); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCD, cd) { diff --git a/engine/cdrs.go b/engine/cdrs.go index 4431b9137..c31badcd3 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -486,7 +486,7 @@ func (self *CdrServer) replicateCdr(cdr *CDR) error { self.cgrCfg.HttpFailedDir, rplCfg.FallbackFileName()) _, err := utils.HttpPoster( - rplCfg.Server, self.cgrCfg.HttpSkipTlsVerify, body, + rplCfg.Address, self.cgrCfg.HttpSkipTlsVerify, body, content, rplCfg.Attempts, fallbackPath) if err != nil { utils.Logger.Err(fmt.Sprintf( diff --git a/engine/libengine.go b/engine/libengine.go index e694f599c..d541de76d 100644 --- a/engine/libengine.go +++ b/engine/libengine.go @@ -19,23 +19,31 @@ along with this program. If not, see package engine import ( + "errors" + "time" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" "github.com/cgrates/rpcclient" ) func NewRPCPool(dispatchStrategy string, connAttempts, reconnects int, codec string, - rpcConnCfgs []*config.HaPoolConfig, internalConnChan chan rpcclient.RpcClientConnection) (*rpcclient.RpcClientPool, error) { + rpcConnCfgs []*config.HaPoolConfig, internalConnChan chan rpcclient.RpcClientConnection, ttl time.Duration) (*rpcclient.RpcClientPool, error) { var rpcClient *rpcclient.RpcClient var err error rpcPool := rpcclient.NewRpcClientPool(dispatchStrategy) for _, rpcConnCfg := range rpcConnCfgs { - if rpcConnCfg.Server == utils.INTERNAL { - internalConn := <-internalConnChan - internalConnChan <- internalConn + if rpcConnCfg.Address == utils.MetaInternal { + var internalConn rpcclient.RpcClientConnection + select { + case internalConn := <-internalConnChan: + internalConnChan <- internalConn + case <-time.After(ttl): + return nil, errors.New("TTL triggered") + } rpcClient, err = rpcclient.NewRpcClient("", "", 0, 0, rpcclient.INTERNAL_RPC, internalConn) } else { - rpcClient, err = rpcclient.NewRpcClient("tcp", rpcConnCfg.Server, connAttempts, reconnects, codec, nil) + rpcClient, err = rpcclient.NewRpcClient("tcp", rpcConnCfg.Address, connAttempts, reconnects, codec, nil) } if err != nil { break diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 7b64860be..f18ee52ec 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -231,9 +231,9 @@ func (sm *FSSessionManager) onChannelHangupComplete(ev engine.Event) { func (sm *FSSessionManager) Connect() error { eventFilters := map[string]string{"Call-Direction": "inbound"} errChan := make(chan error) - for _, connCfg := range sm.cfg.Connections { + for _, connCfg := range sm.cfg.EventSocketConns { connId := utils.GenUUID() - fSock, err := fsock.NewFSock(connCfg.Server, connCfg.Password, connCfg.Reconnects, sm.createHandlers(), eventFilters, utils.Logger.(*syslog.Writer), connId) + fSock, err := fsock.NewFSock(connCfg.Address, connCfg.Password, connCfg.Reconnects, sm.createHandlers(), eventFilters, utils.Logger.(*syslog.Writer), connId) if err != nil { return err } else if !fSock.Connected() { @@ -246,7 +246,7 @@ func (sm *FSSessionManager) Connect() error { errChan <- err } }() - if fsSenderPool, err := fsock.NewFSockPool(5, connCfg.Server, connCfg.Password, 1, sm.cfg.MaxWaitConnection, + if fsSenderPool, err := fsock.NewFSockPool(5, connCfg.Address, connCfg.Password, 1, sm.cfg.MaxWaitConnection, make(map[string][]func(string, string)), make(map[string]string), utils.Logger.(*syslog.Writer), connId); err != nil { return fmt.Errorf("Cannot connect FreeSWITCH senders pool, error: %s", err.Error()) } else if fsSenderPool == nil { diff --git a/sessionmanager/kamailiosm.go b/sessionmanager/kamailiosm.go index b56d0c96d..af7c5fb53 100644 --- a/sessionmanager/kamailiosm.go +++ b/sessionmanager/kamailiosm.go @@ -170,9 +170,9 @@ func (self *KamailioSessionManager) Connect() error { regexp.MustCompile("CGR_CALL_END"): []func([]byte, string){self.onCallEnd}, } errChan := make(chan error) - for _, connCfg := range self.cfg.Connections { + for _, connCfg := range self.cfg.EvapiConns { connId := utils.GenUUID() - if self.conns[connId], err = kamevapi.NewKamEvapi(connCfg.EvapiAddr, connId, connCfg.Reconnects, eventHandlers, utils.Logger.(*syslog.Writer)); err != nil { + if self.conns[connId], err = kamevapi.NewKamEvapi(connCfg.Address, connId, connCfg.Reconnects, eventHandlers, utils.Logger.(*syslog.Writer)); err != nil { return err } go func() { // Start reading in own goroutine, return on error From 1125c2425ce470eb1134a7ccb939e87b1e11c844 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 23 Apr 2016 15:00:03 +0300 Subject: [PATCH 181/227] startd db structures version management --- engine/storage_interface.go | 6 ++ engine/storage_map.go | 71 +++++++++++++ engine/storage_mongo_datadb.go | 52 ++++++++++ engine/storage_redis.go | 53 ++++++++++ engine/storage_sql.go | 8 ++ engine/version.go | 176 +++++++++++++++++++++++++++++++++ utils/consts.go | 3 + 7 files changed, 369 insertions(+) create mode 100644 engine/version.go diff --git a/engine/storage_interface.go b/engine/storage_interface.go index c2123970e..35b1236cc 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -68,6 +68,8 @@ type RatingStorage interface { GetAllActionPlans() (map[string]*ActionPlan, error) PushTask(*Task) error PopTask() (*Task, error) + GetRatingStructuresVersion() (*RatingStructuresVersion, error) + SetRatingStructuresVersion(*RatingStructuresVersion) error } type AccountingStorage interface { @@ -92,6 +94,8 @@ type AccountingStorage interface { RemoveAlias(string) error GetLoadHistory(int, bool) ([]*LoadInstance, error) AddLoadHistory(*LoadInstance, int) error + GetAccountingStructuresVersion() (*AccountingStructuresVersion, error) + SetAccountingStructuresVersion(*AccountingStructuresVersion) error } type CdrStorage interface { @@ -100,6 +104,8 @@ type CdrStorage interface { SetSMCost(smc *SMCost) error GetSMCosts(cgrid, runid, originHost, originIDPrfx string) ([]*SMCost, error) GetCDRs(*utils.CDRsFilter, bool) ([]*CDR, int64, error) + GetCdrStructuresVersion() (*CdrStructuresVersion, error) + SetCdrStructuresVersion(*CdrStructuresVersion) error } type LogStorage interface { diff --git a/engine/storage_map.go b/engine/storage_map.go index f7e21306a..72e436497 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -924,3 +924,74 @@ func (ms *MapStorage) LogActionTiming(source string, at *ActionTiming, as Action ms.dict[utils.LOG_ACTION_TIMMING_PREFIX+source+"_"+time.Now().Format(time.RFC3339Nano)] = []byte(fmt.Sprintf("%s*%s", string(mat), string(mas))) return } + +func (ms *MapStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() + var result []byte + result, err = ms.ms.Marshal(v) + if err != nil { + return + } + ms.dict[utils.RATING_VERSION_PREFIX+"version"] = result + return +} + +func (ms *MapStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + rsv = &RatingStructuresVersion{} + if values, ok := ms.dict[utils.RATING_VERSION_PREFIX+"version"]; ok { + err = ms.ms.Unmarshal(values, &rsv) + } else { + return nil, utils.ErrNotFound + } + return +} + +func (ms *MapStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() + var result []byte + result, err = ms.ms.Marshal(v) + if err != nil { + return + } + ms.dict[utils.ACCOUNTING_VERSION_PREFIX+"version"] = result + return +} + +func (ms *MapStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + asv = &AccountingStructuresVersion{} + if values, ok := ms.dict[utils.ACCOUNTING_VERSION_PREFIX+"version"]; ok { + err = ms.ms.Unmarshal(values, &asv) + } else { + return nil, utils.ErrNotFound + } + return +} +func (ms *MapStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() + var result []byte + result, err = ms.ms.Marshal(v) + if err != nil { + return + } + ms.dict[utils.CDR_VERSION_PREFIX+"version"] = result + return +} + +func (ms *MapStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { + ms.mu.RLock() + defer ms.mu.RUnlock() + csv = &CdrStructuresVersion{} + if values, ok := ms.dict[utils.CDR_VERSION_PREFIX+"version"]; ok { + err = ms.ms.Unmarshal(values, &csv) + } else { + return nil, utils.ErrNotFound + } + return +} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 51abc1989..2201588e9 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -53,6 +53,7 @@ const ( colLogAtr = "action_trigger_logs" colLogApl = "action_plan_logs" colLogErr = "error_logs" + colVer = "versions" ) var ( @@ -1363,3 +1364,54 @@ func (ms *MongoStorage) GetAllCdrStats() (css []*CdrStats, err error) { err = iter.Close() return } + +func (ms *MongoStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { + _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.RATING_VERSION_PREFIX + "version"}, &struct { + Key string + Value *RatingStructuresVersion + }{utils.RATING_VERSION_PREFIX + "version", v}) + return +} + +func (ms *MongoStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { + rsv = new(RatingStructuresVersion) + err = ms.db.C(colVer).Find(bson.M{"key": utils.RATING_VERSION_PREFIX + "version"}).One(rsv) + if err == mgo.ErrNotFound { + rsv = nil + } + return +} + +func (ms *MongoStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { + _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.ACCOUNTING_VERSION_PREFIX + "version"}, &struct { + Key string + Value *AccountingStructuresVersion + }{utils.ACCOUNTING_VERSION_PREFIX + "version", v}) + return +} + +func (ms *MongoStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { + asv = new(AccountingStructuresVersion) + err = ms.db.C(colVer).Find(bson.M{"key": utils.ACCOUNTING_VERSION_PREFIX + "version"}).One(asv) + if err == mgo.ErrNotFound { + asv = nil + } + return +} + +func (ms *MongoStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { + _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.CDR_VERSION_PREFIX + "version"}, &struct { + Key string + Value *CdrStructuresVersion + }{utils.CDR_VERSION_PREFIX + "version", v}) + return +} + +func (ms *MongoStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { + csv = new(CdrStructuresVersion) + err = ms.db.C(colVer).Find(bson.M{"key": utils.CDR_VERSION_PREFIX + "version"}).One(csv) + if err == mgo.ErrNotFound { + csv = nil + } + return +} diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 8d8b1660a..5c69570df 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1093,3 +1093,56 @@ func (rs *RedisStorage) LogActionTiming(source string, at *ActionTiming, as Acti } return rs.db.Cmd("SET", utils.LOG_ACTION_TIMMING_PREFIX+source+"_"+time.Now().Format(time.RFC3339Nano), []byte(fmt.Sprintf("%v*%v", string(mat), string(mas)))).Err } + +func (rs *RedisStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { + var result []byte + result, err = rs.ms.Marshal(v) + if err != nil { + return + } + return rs.db.Cmd("SET", utils.RATING_VERSION_PREFIX+"version", result).Err +} + +func (rs *RedisStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { + var values []byte + rsv = &RatingStructuresVersion{} + if values, err = rs.db.Cmd("GET", utils.RATING_VERSION_PREFIX+"version").Bytes(); err == nil { + err = rs.ms.Unmarshal(values, &rsv) + } + return +} + +func (rs *RedisStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { + var result []byte + result, err = rs.ms.Marshal(v) + if err != nil { + return + } + return rs.db.Cmd("SET", utils.ACCOUNTING_VERSION_PREFIX+"version", result).Err +} + +func (rs *RedisStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { + var values []byte + asv = &AccountingStructuresVersion{} + if values, err = rs.db.Cmd("GET", utils.ACCOUNTING_VERSION_PREFIX+"version").Bytes(); err == nil { + err = rs.ms.Unmarshal(values, &asv) + } + return +} +func (rs *RedisStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { + var result []byte + result, err = rs.ms.Marshal(v) + if err != nil { + return + } + return rs.db.Cmd("SET", utils.CDR_VERSION_PREFIX+"version", result).Err +} + +func (rs *RedisStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { + var values []byte + csv = &CdrStructuresVersion{} + if values, err = rs.db.Cmd("GET", utils.CDR_VERSION_PREFIX+"version").Bytes(); err == nil { + err = rs.ms.Unmarshal(values, &csv) + } + return +} diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 5c8ab5528..7eeb8f166 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -1343,3 +1343,11 @@ func (self *SQLStorage) GetTpAliases(filter *TpAlias) ([]TpAlias, error) { return tpAliases, nil } + +func (self *SQLStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { + return utils.ErrNotImplemented +} + +func (self *SQLStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { + return nil, utils.ErrNotImplemented +} diff --git a/engine/version.go b/engine/version.go new file mode 100644 index 000000000..9dc14c93c --- /dev/null +++ b/engine/version.go @@ -0,0 +1,176 @@ +package engine + +import ( + "fmt" + + "github.com/cgrates/cgrates/utils" +) + +func init() { + // get current db version + dbRsv, err := ratingStorage.GetRatingStructuresVersion() + if err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) + return + } + // comparing versions + if currentRsv.CompareAndMigrate(dbRsv) { + // write the new values + if err := ratingStorage.SetRatingStructuresVersion(currentRsv); err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + } + } + dbAsv, err := accountingStorage.GetAccountingStructuresVersion() + if err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) + return + } + // comparing versions + if currentAsv.CompareAndMigrate(dbAsv) { + // write the new values + if err := accountingStorage.SetAccountingStructuresVersion(currentAsv); err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + } + } + dbCsv, err := cdrStorage.GetCdrStructuresVersion() + if err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) + return + } + // comparing versions + if currentCsv.CompareAndMigrate(dbCsv) { + // write the new values + if err := cdrStorage.SetCdrStructuresVersion(currentCsv); err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + } + } +} + +var ( + currentRsv = &RatingStructuresVersion{ + Destinations: "1", + RatingPlans: "1", + RatingProfiles: "1", + Lcrs: "1", + DerivedChargers: "1", + Actions: "1", + ActionPlans: "1", + ActionTriggers: "1", + SharedGroups: "1", + } + + currentAsv = &AccountingStructuresVersion{ + Accounts: "1", + CdrStats: "1", + Users: "1", + Alias: "1", + PubSubs: "1", + LoadHistory: "1", + } + + currentCsv = &CdrStructuresVersion{ + Cdrs: "1", + SMCosts: "1", + } +) + +type RatingStructuresVersion struct { + Destinations string + RatingPlans string + RatingProfiles string + Lcrs string + DerivedChargers string + Actions string + ActionPlans string + ActionTriggers string + SharedGroups string +} + +func (rsv *RatingStructuresVersion) CompareAndMigrate(dbRsv *RatingStructuresVersion) bool { + migrationPerformed := false + if rsv.Destinations != dbRsv.Destinations { + migrationPerformed = true + + } + if rsv.RatingPlans != dbRsv.RatingPlans { + migrationPerformed = true + + } + if rsv.RatingProfiles != dbRsv.RatingPlans { + migrationPerformed = true + + } + if rsv.Lcrs != dbRsv.Lcrs { + migrationPerformed = true + + } + if rsv.DerivedChargers != dbRsv.DerivedChargers { + migrationPerformed = true + + } + if rsv.Actions != dbRsv.Actions { + migrationPerformed = true + + } + if rsv.ActionPlans != dbRsv.ActionPlans { + migrationPerformed = true + + } + if rsv.ActionTriggers != dbRsv.ActionTriggers { + migrationPerformed = true + + } + if rsv.SharedGroups != dbRsv.SharedGroups { + migrationPerformed = true + + } + return migrationPerformed +} + +type AccountingStructuresVersion struct { + Accounts string + CdrStats string + Users string + Alias string + PubSubs string + LoadHistory string +} + +func (asv *AccountingStructuresVersion) CompareAndMigrate(dbAsv *AccountingStructuresVersion) bool { + migrationPerformed := false + if asv.Accounts != dbAsv.Accounts { + migrationPerformed = true + } + if asv.CdrStats != dbAsv.CdrStats { + migrationPerformed = true + } + if asv.Users != dbAsv.Users { + migrationPerformed = true + } + if asv.Alias != dbAsv.Alias { + migrationPerformed = true + } + if asv.PubSubs != dbAsv.PubSubs { + migrationPerformed = true + } + if asv.LoadHistory != dbAsv.LoadHistory { + migrationPerformed = true + } + return migrationPerformed +} + +type CdrStructuresVersion struct { + Cdrs string + SMCosts string +} + +func (csv *CdrStructuresVersion) CompareAndMigrate(dbCsv *CdrStructuresVersion) bool { + migrationPerformed := false + if csv.Cdrs != dbCsv.Cdrs { + migrationPerformed = true + } + if csv.SMCosts != dbCsv.SMCosts { + migrationPerformed = true + } + return migrationPerformed +} diff --git a/utils/consts.go b/utils/consts.go index b83e10808..cc39ffc95 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -197,6 +197,9 @@ const ( LOG_CALL_COST_PREFIX = "cco_" LOG_ACTION_TIMMING_PREFIX = "ltm_" LOG_ACTION_TRIGGER_PREFIX = "ltr_" + RATING_VERSION_PREFIX = "rve_" + ACCOUNTING_VERSION_PREFIX = "ave_" + CDR_VERSION_PREFIX = "cve_" LOG_ERR = "ler_" LOG_CDR = "cdr_" LOG_MEDIATED_CDR = "mcd_" From 51aff554e644f57e82f84120e41db9b7628b33c4 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 23 Apr 2016 16:20:31 +0200 Subject: [PATCH 182/227] Pass connection pool to engine services --- cmd/cgr-engine/rater.go | 105 ++++++++++------------------------------ 1 file changed, 25 insertions(+), 80 deletions(-) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index b1f21d4b4..4f8e17be6 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -126,122 +126,67 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } // Connection to HistoryS, - // FixMe via multiple connections - var ralsHistoryServer string - for _, connCfg := range cfg.RALsHistorySConns { - ralsHistoryServer = connCfg.Address - break - } - if ralsHistoryServer != "" { + if len(cfg.RALsHistorySConns) != 0 { histTaskChan := make(chan struct{}) waitTasks = append(waitTasks, histTaskChan) go func() { defer close(histTaskChan) - var scribeServer rpcclient.RpcClientConnection - if ralsHistoryServer == utils.INTERNAL { - select { - case scribeServer = <-internalHistorySChan: - internalHistorySChan <- scribeServer - case <-time.After(cfg.InternalTtl): - utils.Logger.Crit(": Internal historys connection timeout.") - exitChan <- true - return - } - } else if scribeServer, err = rpcclient.NewRpcClient("tcp", ralsHistoryServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect historys, error: %s", err.Error())) + if historySConns, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.RALsHistorySConns, internalHistorySChan, cfg.InternalTtl); err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect HistoryS, error: %s", err.Error())) exitChan <- true return + } else { + engine.SetHistoryScribe(historySConns) } - engine.SetHistoryScribe(scribeServer) }() } // Connection to pubsubs - var ralsPubSubServer string - for _, connCfg := range cfg.RALsPubSubSConns { - ralsPubSubServer = connCfg.Address - break - } - if ralsPubSubServer != "" { + if len(cfg.RALsPubSubSConns) != 0 { pubsubTaskChan := make(chan struct{}) waitTasks = append(waitTasks, pubsubTaskChan) go func() { defer close(pubsubTaskChan) - var pubSubServer rpcclient.RpcClientConnection - if ralsPubSubServer == utils.INTERNAL { - select { - case pubSubServer = <-internalPubSubSChan: - internalPubSubSChan <- pubSubServer - case <-time.After(cfg.InternalTtl): - utils.Logger.Crit(": Internal pubsub connection timeout.") - exitChan <- true - return - } - } else if pubSubServer, err = rpcclient.NewRpcClient("tcp", ralsPubSubServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to pubsubs: %s", err.Error())) + if pubSubSConns, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.RALsPubSubSConns, internalPubSubSChan, cfg.InternalTtl); err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubS: %s", err.Error())) exitChan <- true return + } else { + engine.SetPubSub(pubSubSConns) } - engine.SetPubSub(pubSubServer) }() } - // Connection to AliasService - var ralsAliasServer string - for _, connCfg := range cfg.RALsAliasSConns { - ralsAliasServer = connCfg.Address - break - } - if ralsAliasServer != "" { + if len(cfg.RALsAliasSConns) != 0 { aliasesTaskChan := make(chan struct{}) waitTasks = append(waitTasks, aliasesTaskChan) go func() { defer close(aliasesTaskChan) - var aliasesServer rpcclient.RpcClientConnection - if ralsAliasServer == utils.INTERNAL { - select { - case aliasesServer = <-internalAliaseSChan: - internalAliaseSChan <- aliasesServer - case <-time.After(cfg.InternalTtl): - utils.Logger.Crit(": Internal aliases connection timeout.") - exitChan <- true - return - } - } else if aliasesServer, err = rpcclient.NewRpcClient("tcp", ralsAliasServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to aliases, error: %s", err.Error())) + if aliaseSCons, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.RALsAliasSConns, internalAliaseSChan, cfg.InternalTtl); err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS, error: %s", err.Error())) exitChan <- true return + } else { + engine.SetAliasService(aliaseSCons) } - engine.SetAliasService(aliasesServer) }() } - // Connection to UserService - var ralsUserServer string - for _, connCfg := range cfg.RALsUserSConns { - ralsUserServer = connCfg.Address - break - } - var userServer rpcclient.RpcClientConnection - if ralsUserServer != "" { + var usersConns rpcclient.RpcClientConnection + if len(cfg.RALsUserSConns) != 0 { usersTaskChan := make(chan struct{}) waitTasks = append(waitTasks, usersTaskChan) go func() { defer close(usersTaskChan) - if ralsUserServer == utils.INTERNAL { - select { - case userServer = <-internalUserSChan: - internalUserSChan <- userServer - case <-time.After(cfg.InternalTtl): - utils.Logger.Crit(": Internal users connection timeout.") - exitChan <- true - return - } - } else if userServer, err = rpcclient.NewRpcClient("tcp", ralsUserServer, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, nil); err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect users, error: %s", err.Error())) + if usersConns, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.RALsAliasSConns, internalAliaseSChan, cfg.InternalTtl); err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect UserS, error: %s", err.Error())) exitChan <- true return } - engine.SetUserService(userServer) + engine.SetUserService(usersConns) }() } @@ -253,7 +198,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats} responder.SetTimeToLive(cfg.ResponseCacheTTL, nil) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, - Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer} + Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: usersConns} apierRpcV2 := &v2.ApierV2{ ApierV1: *apierRpcV1} // internalSchedulerChan shared here From eb3bf6e205bfb337eec8ffae10cae29c256645c7 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 23 Apr 2016 18:25:32 +0200 Subject: [PATCH 183/227] Test fixes --- cdrc/cdrc_local_test.go | 12 ++++++++---- cmd/cgr-engine/rater.go | 2 -- sessionmanager/session_test.go | 27 ++++++++++++++------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index 1fa731eae..be4668599 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -166,8 +166,10 @@ func TestCsvLclProcessCdrDir(t *testing.T) { for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break } - if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network - cdrcCfg.Cdrs = "127.0.0.1:2013" + for _, cdrsConn := range cdrcCfg.CdrsConns { + if cdrsConn.Address == utils.INTERNAL { + cdrsConn.Address = "127.0.0.1:2013" + } } if err := startEngine(); err != nil { t.Fatal(err.Error()) @@ -202,8 +204,10 @@ func TestCsvLclProcessCdr3Dir(t *testing.T) { if !*testLocal { return } - if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network - cdrcCfg.Cdrs = "127.0.0.1:2013" + for _, cdrsConn := range cdrcCfg.CdrsConns { + if cdrsConn.Address == utils.INTERNAL { + cdrsConn.Address = "127.0.0.1:2013" + } } if err := startEngine(); err != nil { t.Fatal(err.Error()) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 4f8e17be6..1647ee62a 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -189,12 +189,10 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC engine.SetUserService(usersConns) }() } - // Wait for all connections to complete before going further for _, chn := range waitTasks { <-chn } - responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats} responder.SetTimeToLive(cfg.ResponseCacheTTL, nil) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, diff --git a/sessionmanager/session_test.go b/sessionmanager/session_test.go index 247e12121..f23b7d37f 100644 --- a/sessionmanager/session_test.go +++ b/sessionmanager/session_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) //"github.com/cgrates/cgrates/config" @@ -89,26 +90,26 @@ func (mc *MockRpcClient) Call(methodName string, arg interface{}, reply interfac } return nil } -func (mc *MockConnector) GetCost(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) Debit(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) MaxDebit(*engine.CallDescriptor, *engine.CallCost) error { return nil } -func (mc *MockConnector) RefundIncrements(cd *engine.CallDescriptor, reply *float64) error { +func (mc *MockRpcClient) GetCost(*engine.CallDescriptor, *engine.CallCost) error { return nil } +func (mc *MockRpcClient) Debit(*engine.CallDescriptor, *engine.CallCost) error { return nil } +func (mc *MockRpcClient) MaxDebit(*engine.CallDescriptor, *engine.CallCost) error { return nil } +func (mc *MockRpcClient) RefundIncrements(cd *engine.CallDescriptor, reply *float64) error { mc.refundCd = cd return nil } -func (mc *MockConnector) RefundRounding(cd *engine.CallDescriptor, reply *float64) error { +func (mc *MockRpcClient) RefundRounding(cd *engine.CallDescriptor, reply *float64) error { return nil } -func (mc *MockConnector) GetMaxSessionTime(*engine.CallDescriptor, *float64) error { return nil } -func (mc *MockConnector) GetDerivedChargers(*utils.AttrDerivedChargers, *utils.DerivedChargers) error { +func (mc *MockRpcClient) GetMaxSessionTime(*engine.CallDescriptor, *float64) error { return nil } +func (mc *MockRpcClient) GetDerivedChargers(*utils.AttrDerivedChargers, *utils.DerivedChargers) error { return nil } -func (mc *MockConnector) GetDerivedMaxSessionTime(*engine.CDR, *float64) error { return nil } -func (mc *MockConnector) GetSessionRuns(*engine.CDR, *[]*engine.SessionRun) error { return nil } -func (mc *MockConnector) ProcessCdr(*engine.CDR, *string) error { return nil } -func (mc *MockConnector) StoreSMCost(engine.AttrCDRSStoreSMCost, *string) error { return nil } -func (mc *MockConnector) GetLCR(*engine.AttrGetLcr, *engine.LCRCost) error { return nil } -func (mc *MockConnector) GetTimeout(int, *time.Duration) error { return nil } +func (mc *MockRpcClient) GetDerivedMaxSessionTime(*engine.CDR, *float64) error { return nil } +func (mc *MockRpcClient) GetSessionRuns(*engine.CDR, *[]*engine.SessionRun) error { return nil } +func (mc *MockRpcClient) ProcessCdr(*engine.CDR, *string) error { return nil } +func (mc *MockRpcClient) StoreSMCost(engine.AttrCDRSStoreSMCost, *string) error { return nil } +func (mc *MockRpcClient) GetLCR(*engine.AttrGetLcr, *engine.LCRCost) error { return nil } +func (mc *MockRpcClient) GetTimeout(int, *time.Duration) error { return nil } func TestSessionRefund(t *testing.T) { mc := &MockRpcClient{} From c913eca95dd49c5d69c5a9ef75e5aa7ab942ef12 Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 24 Apr 2016 11:18:23 +0200 Subject: [PATCH 184/227] Updating sample configs with new format --- config/config_defaults.go | 32 +- data/conf/samples/actions/cgradmin.json | 14 +- data/conf/samples/apier/apier.json | 10 +- data/conf/samples/cdrcflatstore/cgrates.json | 6 +- data/conf/samples/cdrcfwv/cgrates.json | 6 +- .../cdrsreplicationmaster.json | 2 +- .../cdrsreplicationslave.json | 2 +- data/conf/samples/cdrstats/cdrstats.json | 17 +- data/conf/samples/cdrsv2mongo/cdrsv2psql.json | 6 +- .../conf/samples/cdrsv2mysql/cdrsv2mysql.json | 6 +- data/conf/samples/cdrsv2psql/cdrsv2psql.json | 6 +- data/conf/samples/cgradmin/cgradmin.json | 20 +- data/conf/samples/dmtagent/cgrates.json | 41 +-- data/conf/samples/fscsv/cgrates.json | 13 +- data/conf/samples/multifiles/b/b.json | 2 +- .../multiplecdrc/multiplecdrc_fwexport.json | 7 +- data/conf/samples/multiral1/cgrates.json | 2 +- data/conf/samples/multiral2/cgrates.json | 2 +- data/conf/samples/osips_cdrs_cdrstats.cfg | 61 ---- data/conf/samples/smfs/smfs.json | 16 +- data/conf/samples/smg/cgrates.json | 5 +- data/conf/samples/smgeneric/cgrates.json | 284 +----------------- data/conf/samples/tutlocal/cgrates.json | 19 +- 23 files changed, 141 insertions(+), 438 deletions(-) delete mode 100644 data/conf/samples/osips_cdrs_cdrstats.cfg diff --git a/config/config_defaults.go b/config/config_defaults.go index f5ada7074..bce7f6467 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -135,7 +135,7 @@ const CGRATES_CFG_JSON = ` "field_separator": ",", "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) - "mms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from MMS unit to call duration in some billing systems) + "mms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from MMS unit to call duration in some billing systems) "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding @@ -225,12 +225,12 @@ const CGRATES_CFG_JSON = ` "sm_freeswitch": { "enabled": false, // starts SessionManager service: - "rals_conns": [ - {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> - ], + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> + ], "cdrs_conns": [ - {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> - ], + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> + ], "create_cdr": false, // create CDR out of events and sends them to CDRS component "extra_fields": [], // extra fields to store in auth/CDRs when creating them "debit_interval": "10s", // interval to perform debits on. @@ -251,12 +251,12 @@ const CGRATES_CFG_JSON = ` "sm_kamailio": { "enabled": false, // starts SessionManager service: - "rals_conns": [ - {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> - ], + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> + ], "cdrs_conns": [ - {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> - ], + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> + ], "create_cdr": false, // create CDR out of events and sends them to CDRS component "debit_interval": "10s", // interval to perform debits on. "min_call_duration": "0s", // only authorize calls with allowed duration higher than this @@ -270,12 +270,12 @@ const CGRATES_CFG_JSON = ` "sm_opensips": { "enabled": false, // starts SessionManager service: "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS - "rals_conns": [ - {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> - ], + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> + ], "cdrs_conns": [ - {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> - ], + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> + ], "reconnects": 5, // number of reconnects if connection is lost "create_cdr": false, // create CDR out of events and sends it to CDRS component "debit_interval": "10s", // interval to perform debits on. diff --git a/data/conf/samples/actions/cgradmin.json b/data/conf/samples/actions/cgradmin.json index 4ea1f0eef..b9d6b75e2 100644 --- a/data/conf/samples/actions/cgradmin.json +++ b/data/conf/samples/actions/cgradmin.json @@ -10,11 +10,17 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "pubsubs": "internal", - "users": "internal", - "aliases": "internal", + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { diff --git a/data/conf/samples/apier/apier.json b/data/conf/samples/apier/apier.json index e7385e90e..30a533b19 100644 --- a/data/conf/samples/apier/apier.json +++ b/data/conf/samples/apier/apier.json @@ -10,9 +10,11 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "aliases": "internal" + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { @@ -21,8 +23,8 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> ], }, diff --git a/data/conf/samples/cdrcflatstore/cgrates.json b/data/conf/samples/cdrcflatstore/cgrates.json index 454a5db8a..5c25da8c5 100644 --- a/data/conf/samples/cdrcflatstore/cgrates.json +++ b/data/conf/samples/cdrcflatstore/cgrates.json @@ -7,7 +7,7 @@ // This is what you get when you load CGRateS with an empty configuration file. -"rater": { +"rals": { "enabled": true, // enable Rater service: }, @@ -25,7 +25,9 @@ "cdrc": { "FLATSTORE": { "enabled": true, // enable CDR client functionality - "cdrs": "internal", // address where to reach CDR server. + "cdrs_conns": [ + {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> + ], "cdr_format": "opensips_flatstore", // CDR file format "field_separator": "|", // separator used in case of csv files "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify diff --git a/data/conf/samples/cdrcfwv/cgrates.json b/data/conf/samples/cdrcfwv/cgrates.json index c9b30158e..2feb548e9 100644 --- a/data/conf/samples/cdrcfwv/cgrates.json +++ b/data/conf/samples/cdrcfwv/cgrates.json @@ -7,7 +7,7 @@ // This is what you get when you load CGRateS with an empty configuration file. -"rater": { +"rals": { "enabled": true, // enable Rater service: }, @@ -26,7 +26,9 @@ "FWV1": { "enabled": true, // enable CDR client functionality "dry_run": true, - "cdrs": "internal", // address where to reach CDR server. + "cdrs_conns": [ + {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> + ], "cdr_format": "fwv", // CDR file format "cdr_in_dir": "/tmp/cgr_fwv/cdrc/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgr_fwv/cdrc/out", // absolute path towards the directory where processed CDRs will be moved diff --git a/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json b/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json index 20b8605d6..2777272fc 100644 --- a/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json +++ b/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json @@ -4,7 +4,7 @@ // Used in apier_local_tests // Starts rater, cdrs and mediator connecting over internal channel -"rater": { +"rals": { "enabled": true, // enable Rater service: }, diff --git a/data/conf/samples/cdrsreplicationslave/cdrsreplicationslave.json b/data/conf/samples/cdrsreplicationslave/cdrsreplicationslave.json index 2d06fb5a8..a9d04ca13 100644 --- a/data/conf/samples/cdrsreplicationslave/cdrsreplicationslave.json +++ b/data/conf/samples/cdrsreplicationslave/cdrsreplicationslave.json @@ -10,7 +10,7 @@ "http": "127.0.0.1:12080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, diff --git a/data/conf/samples/cdrstats/cdrstats.json b/data/conf/samples/cdrstats/cdrstats.json index 8c91dbc9d..5e47affa5 100644 --- a/data/conf/samples/cdrstats/cdrstats.json +++ b/data/conf/samples/cdrstats/cdrstats.json @@ -10,17 +10,22 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "cdrstats": "internal", + "cdrstats_conns": [ + {"address": "*internal"} + ], // address where to reach the cdrstats service, empty to disable stats functionality: <""|*internal|x.y.z.y:1234> }, + "cdrs": { "enabled": true, // start the CDR Server service: "store_cdrs": false, // store cdrs in storDb - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - ], - "cdrstats": "internal", // address where to reach the cdrstats service. Empty to disable stats gathering out of mediated CDRs <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> + ], + "cdrstats_conns": [ + {"address": "*internal"} + ], // address where to reach the cdrstats service. Empty to disable stats gathering out of mediated CDRs <""|internal|x.y.z.y:1234> }, "cdrstats": { diff --git a/data/conf/samples/cdrsv2mongo/cdrsv2psql.json b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json index d3590440f..8b7600efb 100644 --- a/data/conf/samples/cdrsv2mongo/cdrsv2psql.json +++ b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json @@ -10,14 +10,14 @@ }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, "cdrs": { "enabled": true, // start the CDR Server service: - "rater": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> ], }, diff --git a/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json b/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json index e1f724ad2..4f424a0f8 100644 --- a/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json +++ b/data/conf/samples/cdrsv2mysql/cdrsv2mysql.json @@ -4,14 +4,14 @@ // Used in apier_local_tests // Starts rater, cdrs and mediator connecting over internal channel -"rater": { +"rals": { "enabled": true, // enable Rater service: }, "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> ], }, diff --git a/data/conf/samples/cdrsv2psql/cdrsv2psql.json b/data/conf/samples/cdrsv2psql/cdrsv2psql.json index fe166b1d3..0a715bc07 100644 --- a/data/conf/samples/cdrsv2psql/cdrsv2psql.json +++ b/data/conf/samples/cdrsv2psql/cdrsv2psql.json @@ -10,14 +10,14 @@ }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "rals_conns": [ + {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> ], }, diff --git a/data/conf/samples/cgradmin/cgradmin.json b/data/conf/samples/cgradmin/cgradmin.json index 9dd30abcf..431ad027d 100644 --- a/data/conf/samples/cgradmin/cgradmin.json +++ b/data/conf/samples/cgradmin/cgradmin.json @@ -14,28 +14,34 @@ "db_type": "mongo", // stor database type to use: "db_host": "127.0.0.1", // the host to connect to "db_port": 27017, // the port to reach the stordb - "db_name": "tpdb", + "db_name": "tpdb", }, "data_db": { // database used to store offline tariff plans and CDRs "db_type": "mongo", // stor database type to use: "db_host": "127.0.0.1", // the host to connect to "db_port": 27017, // the port to reach the stordb - "db_name": "datadb", + "db_name": "datadb", }, "stor_db": { // database used to store offline tariff plans and CDRs "db_type": "mongo", // stor database type to use: "db_host": "127.0.0.1", // the host to connect to "db_port": 27017, // the port to reach the stordb - "db_name": "stordb", + "db_name": "stordb", }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "pubsubs": "internal", - "users": "internal", - "aliases": "internal", + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { diff --git a/data/conf/samples/dmtagent/cgrates.json b/data/conf/samples/dmtagent/cgrates.json index 90a610dae..9599f11d2 100644 --- a/data/conf/samples/dmtagent/cgrates.json +++ b/data/conf/samples/dmtagent/cgrates.json @@ -10,28 +10,35 @@ "http": ":2080", // HTTP listening address }, -"rater": { - "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal", +"rals": { + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { - "enabled": true, // start Scheduler service: + "enabled": true, }, "cdrs": { - "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} ], - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, "cdrstats": { - "enabled": true, // starts the cdrstats service: + "enabled": true, }, "pubsubs": { @@ -49,17 +56,13 @@ "sm_generic": { "enabled": true, - "rater_conns": [ - {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> - ], - "rater_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing - ], }, "diameter_agent": { "enabled": true, - "pubsubs": "internal", + "pubsubs_conns": [ + {"address": "*internal"} + ], }, } diff --git a/data/conf/samples/fscsv/cgrates.json b/data/conf/samples/fscsv/cgrates.json index 8ec9bb202..233fb6f1f 100644 --- a/data/conf/samples/fscsv/cgrates.json +++ b/data/conf/samples/fscsv/cgrates.json @@ -6,9 +6,11 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} + ], }, "scheduler": { @@ -17,15 +19,14 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} ], - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, "cdrstats": { "enabled": true, // starts the cdrstats service: - "save_interval": "5s" + "save_interval": "5s" }, } diff --git a/data/conf/samples/multifiles/b/b.json b/data/conf/samples/multifiles/b/b.json index 2187cae01..84e72f697 100644 --- a/data/conf/samples/multifiles/b/b.json +++ b/data/conf/samples/multifiles/b/b.json @@ -4,7 +4,7 @@ // Should be the second file loaded "general": { - "default_reqtype": "*pseudoprepaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> + "default_request_type": "*pseudoprepaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> }, "cdre": { diff --git a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json index d6285e750..1d47e5365 100644 --- a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json +++ b/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json @@ -4,7 +4,7 @@ // Used in mediator_local_test // Starts rater, cdrs and mediator connecting over internal channel -"rater": { +"rals": { "enabled": true, // enable Rater service: }, @@ -14,7 +14,6 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "mediator": "internal", // address where to reach the Mediator. Empty for disabling mediation. <""|internal> }, "cdrc": { @@ -67,10 +66,6 @@ } }, -"mediator": { - "enabled": true, // starts Mediator service: . -}, - "cdre": { "CDRE-FW1": { "cdr_format": "fwv", diff --git a/data/conf/samples/multiral1/cgrates.json b/data/conf/samples/multiral1/cgrates.json index 310b3861d..db96253dc 100644 --- a/data/conf/samples/multiral1/cgrates.json +++ b/data/conf/samples/multiral1/cgrates.json @@ -10,7 +10,7 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, diff --git a/data/conf/samples/multiral2/cgrates.json b/data/conf/samples/multiral2/cgrates.json index 8f9270ae6..7bc2d4e8f 100644 --- a/data/conf/samples/multiral2/cgrates.json +++ b/data/conf/samples/multiral2/cgrates.json @@ -10,7 +10,7 @@ "http": ":12080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, diff --git a/data/conf/samples/osips_cdrs_cdrstats.cfg b/data/conf/samples/osips_cdrs_cdrstats.cfg deleted file mode 100644 index fbdeb4dba..000000000 --- a/data/conf/samples/osips_cdrs_cdrstats.cfg +++ /dev/null @@ -1,61 +0,0 @@ -# Real-time Charging System for Telecom & ISP environments -# Copyright (C) ITsysCOM GmbH -# -# This file contains the default configuration hardcoded into CGRateS. -# This is what you get when you load CGRateS with an empty configuration file. - -[global] -rpc_json_listen = :2012 # RPC JSON listening address - -[rater] -enabled = true # Enable RaterCDRSExportPath service: . - -[scheduler] -enabled = true # Starts Scheduler service: . - -[cdrs] -enabled = true # Start the CDR Server service: . -mediator = internal # Address where to reach the Mediator. Empty for disabling mediation. <""|internal> -# cdrstats = # Address where to reach the cdrstats service: - -[mediator] -enabled = true # Starts Mediator service: . -# rater = internal # Address where to reach the Rater: -# cdrstats = internal # Address where to reach the cdrstats service: - -[cdrstats] -enabled = true # Starts the cdrstats service: -#queue_length = 50 # Number of items in the stats buffer -time_window = 1h # Will only keep the CDRs who's call setup time is not older than time.Now()-TimeWindow -save_interval = 5s -# metrics = ASR, ACD, ACC # Stat metric ids to build -# setup_interval = # Filter on CDR SetupTime -# tors = # Filter on CDR TOR fields -# cdr_hosts= # Filter on CDR CdrHost fields -# cdr_sources = # Filter on CDR CdrSource fields -# req_types = # Filter on CDR ReqType fields -# directions = # Filter on CDR Direction fields -# tenants = # Filter on CDR Tenant fields -# categories = # Filter on CDR Category fields -# accounts = # Filter on CDR Account fields -# subjects = # Filter on CDR Subject fields -# destination_prefixes = # Filter on CDR Destination prefixes -# usage_interval = # Filter on CDR Usage -# mediation_run_ids = # Filter on CDR MediationRunId fields -# rated_accounts = # Filter on CDR RatedAccount fields -# rated_subjects = # Filter on CDR RatedSubject fields -# cost_intervals = # Filter on CDR Cost - -[session_manager] -enabled = true # Starts SessionManager service: -switch_type = opensips # Defines the type of switch behind: - -[opensips] -listen_udp = :2020 # Address where to listen for datagram events coming from OpenSIPS -mi_addr = 172.16.254.77:8020 # Adress where to reach OpenSIPS mi_datagram module - -[mailer] -# server = localhost # The server to use when sending emails out -# auth_user = cgrates # Authenticate to email server using this user -# auth_passwd = CGRateS.org # Authenticate to email server with this password -# from_address = cgr-mailer@localhost.localdomain # From address used when sending emails out diff --git a/data/conf/samples/smfs/smfs.json b/data/conf/samples/smfs/smfs.json index 5176b1802..094e7eecf 100644 --- a/data/conf/samples/smfs/smfs.json +++ b/data/conf/samples/smfs/smfs.json @@ -8,13 +8,17 @@ "sm_freeswitch": { - "enabled": true, // starts SessionManager service: - "rater": "127.0.0.1:2013", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "127.0.0.1:2013", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> - "debit_interval": "5s", // interval to perform debits on. + "enabled": true, + "rals_conns": [ + {"address": "127.0.0.1:2013"} + ], + "cdrs_conns": [ + {"address": "127.0.0.1:2013"} + ], + "debit_interval": "5s", "channel_sync_interval": "10s", - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 15} + "event_socket_conns":[ + {"address": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} ], }, diff --git a/data/conf/samples/smg/cgrates.json b/data/conf/samples/smg/cgrates.json index 82ef9fa86..9f343a378 100644 --- a/data/conf/samples/smg/cgrates.json +++ b/data/conf/samples/smg/cgrates.json @@ -10,7 +10,7 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: }, @@ -20,13 +20,10 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> }, "sm_generic": { "enabled": true, - "rater": "internal", - "cdrs": "internal", "session_ttl": "50ms", }, diff --git a/data/conf/samples/smgeneric/cgrates.json b/data/conf/samples/smgeneric/cgrates.json index e54c93b48..02d95e649 100644 --- a/data/conf/samples/smgeneric/cgrates.json +++ b/data/conf/samples/smgeneric/cgrates.json @@ -6,77 +6,18 @@ // This file contains the default configuration hardcoded into CGRateS. // This is what you get when you load CGRateS with an empty configuration file. -//"general": { -// "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate -// "rounding_decimals": 5, // system level precision for floats -// "dbdata_encoding": "msgpack", // encoding used to store object data in strings: -// "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans -// "http_failed_dir": "/var/log/cgrates/http_failed", // directory path where we store failed http requests -// "default_reqtype": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> -// "default_category": "call", // default Type of Record to consider when missing from requests -// "default_tenant": "cgrates.org", // default Tenant to consider when missing from requests -// "default_subject": "cgrates", // default rating Subject to consider when missing from requests -// "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> -// "connect_attempts": 3, // initial server connect attempts -// "reconnects": -1, // number of retries in case of connection lost -// "response_cache_ttl": "3s", // the life span of a cached response -// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up -//}, - -//"listen": { -// "rpc_json": "127.0.0.1:2012", // RPC JSON listening address -// "rpc_gob": "127.0.0.1:2013", // RPC GOB listening address -// "http": "127.0.0.1:2080", // HTTP listening address -//}, - - -//"tariffplan_db": { // database used to store active tariff plan configuration -// "db_type": "redis", // tariffplan_db type: -// "db_host": "127.0.0.1", // tariffplan_db host address -// "db_port": 6379, // port to reach the tariffplan_db -// "db_name": "10", // tariffplan_db name to connect to -// "db_user": "", // sername to use when connecting to tariffplan_db -// "db_passwd": "", // password to use when connecting to tariffplan_db -//}, - - -//"data_db": { // database used to store runtime data (eg: accounts, cdr stats) -// "db_type": "redis", // data_db type: -// "db_host": "127.0.0.1", // data_db host address -// "db_port": 6379, // data_db port to reach the database -// "db_name": "11", // data_db database name to connect to -// "db_user": "", // username to use when connecting to data_db -// "db_passwd": "", // password to use when connecting to data_db -// "load_history_size": 10, // Number of records in the load history -//}, - - -//"stor_db": { // database used to store offline tariff plans and CDRs -// "db_type": "mysql", // stor database type to use: -// "db_host": "127.0.0.1", // the host to connect to -// "db_port": 3306, // the port to reach the stordb -// "db_name": "cgrates", // stor database name -// "db_user": "cgrates", // username to use when connecting to stordb -// "db_passwd": "CGRateS.org", // password to use when connecting to stordb -// "max_open_conns": 100, // maximum database connections opened -// "max_idle_conns": 10, // maximum database connections idle -//}, - - -//"balancer": { -// "enabled": false, // start Balancer service: -//}, - - -"rater": { +"rals": { "enabled": true, // enable Rater service: -// "balancer": "", // register to balancer as worker: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234> -// "historys": "", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> -// "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, @@ -87,186 +28,19 @@ "cdrs": { "enabled": true, // start the CDR Server service: -// "extra_fields": [], // extra fields to store in CDRs for non-generic CDRs -// "store_cdrs": true, // store cdrs in storDb - // "rater_conns": [ - //{"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - //], -// "pubsubs": "", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> -// "users": "", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> -// "aliases": "", // address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234> -// "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> -// "cdr_replication":[], // replicate the raw CDR to a number of servers }, "cdrstats": { "enabled": true, // starts the cdrstats service: -// "save_interval": "1m", // interval to save changed stats into dataDb storage }, -//"cdre": { -// "*default": { -// "cdr_format": "csv", // exported CDRs format -// "field_separator": ",", -// "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) -// "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) -// "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) -// "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT -// "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding -// "cost_shift_digits": 0, // shift digits in the cost on export (eg: convert from EUR to cents) -// "mask_destination_id": "MASKED_DESTINATIONS", // destination id containing called addresses to be masked on export -// "mask_length": 0, // length of the destination suffix to be masked -// "export_dir": "/var/log/cgrates/cdre", // path where the exported CDRs will be placed -// "header_fields": [], // template of the exported header fields -// "content_fields": [ // template of the exported content fields -// {"tag": "CgrId", "cdr_field_id": "CgrId", "type": "cdrfield", "value": "CgrId"}, -// {"tag":"RunId", "cdr_field_id": "MediationRunId", "type": "cdrfield", "value": "MediationRunId"}, -// {"tag":"Tor", "cdr_field_id": "TOR", "type": "cdrfield", "value": "TOR"}, -// {"tag":"AccId", "cdr_field_id": "AccId", "type": "cdrfield", "value": "AccId"}, -// {"tag":"ReqType", "cdr_field_id": "ReqType", "type": "cdrfield", "value": "ReqType"}, -// {"tag":"Direction", "cdr_field_id": "Direction", "type": "cdrfield", "value": "Direction"}, -// {"tag":"Tenant", "cdr_field_id": "Tenant", "type": "cdrfield", "value": "Tenant"}, -// {"tag":"Category", "cdr_field_id": "Category", "type": "cdrfield", "value": "Category"}, -// {"tag":"Account", "cdr_field_id": "Account", "type": "cdrfield", "value": "Account"}, -// {"tag":"Subject", "cdr_field_id": "Subject", "type": "cdrfield", "value": "Subject"}, -// {"tag":"Destination", "cdr_field_id": "Destination", "type": "cdrfield", "value": "Destination"}, -// {"tag":"SetupTime", "cdr_field_id": "SetupTime", "type": "cdrfield", "value": "SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"tag":"AnswerTime", "cdr_field_id": "AnswerTime", "type": "cdrfield", "value": "AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"tag":"Usage", "cdr_field_id": "Usage", "type": "cdrfield", "value": "Usage"}, -// {"tag":"Cost", "cdr_field_id": "Cost", "type": "cdrfield", "value": "Cost"}, -// ], -// "trailer_fields": [], // template of the exported trailer fields -// } -//}, - - -//"cdrc": { -// "*default": { -// "enabled": false, // enable CDR client functionality -// "dry_run": false, // do not send the CDRs to CDRS, just parse them -// "cdrs": "internal", // address where to reach CDR server. -// "cdr_format": "csv", // CDR file format -// "field_separator": ",", // separator used in case of csv files -// "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> -// "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify -// "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited -// "data_usage_multiply_factor": 1024, // conversion factor for data usage -// "cdr_in_dir": "/var/log/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored -// "cdr_out_dir": "/var/log/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved -// "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records -// "cdr_source_id": "freeswitch_csv", // free form field, tag identifying the source of the CDRs within CDRS database -// "cdr_filter": "", // filter CDR records to import -// "partial_record_cache": "10s", // duration to cache partial records when not pairing -// "header_fields": [], // template of the import header fields -// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value -// {"tag": "tor", "cdr_field_id": "TOR", "type": "cdrfield", "value": "2", "mandatory": true}, -// {"tag": "accid", "cdr_field_id": "AccId", "type": "cdrfield", "value": "3", "mandatory": true}, -// {"tag": "reqtype", "cdr_field_id": "ReqType", "type": "cdrfield", "value": "4", "mandatory": true}, -// {"tag": "direction", "cdr_field_id": "Direction", "type": "cdrfield", "value": "5", "mandatory": true}, -// {"tag": "tenant", "cdr_field_id": "Tenant", "type": "cdrfield", "value": "6", "mandatory": true}, -// {"tag": "category", "cdr_field_id": "Category", "type": "cdrfield", "value": "7", "mandatory": true}, -// {"tag": "account", "cdr_field_id": "Account", "type": "cdrfield", "value": "8", "mandatory": true}, -// {"tag": "subject", "cdr_field_id": "Subject", "type": "cdrfield", "value": "9", "mandatory": true}, -// {"tag": "destination", "cdr_field_id": "Destination", "type": "cdrfield", "value": "10", "mandatory": true}, -// {"tag": "setup_time", "cdr_field_id": "SetupTime", "type": "cdrfield", "value": "11", "mandatory": true}, -// {"tag": "answer_time", "cdr_field_id": "AnswerTime", "type": "cdrfield", "value": "12", "mandatory": true}, -// {"tag": "usage", "cdr_field_id": "Usage", "type": "cdrfield", "value": "13", "mandatory": true}, -// ], -// "trailer_fields": [], // template of the import trailer fields -// } -//}, - "sm_generic": { "enabled": true, // starts SessionManager service: -// "listen_bijson": "127.0.0.1:2014", // address where to listen for bidirectional JSON-RPC requests -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_conns": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last }, -//"sm_freeswitch": { -// "enabled": false, // starts SessionManager service: -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "extra_fields": [], // extra fields to store in auth/CDRs when creating them -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "min_dur_low_balance": "5s", // threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval) -// "low_balance_ann_file": "", // file to be played when low balance is reached for prepaid calls -// "empty_balance_context": "", // if defined, prepaid calls will be transfered to this context on empty balance -// "empty_balance_ann_file": "", // file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined) -// "subscribe_park": true, // subscribe via fsock to receive park events -// "channel_sync_interval": "5m", // sync channels with freeswitch regularly -// "connections":[ // instantiate connections to multiple FreeSWITCH servers -// {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} -// ], -//}, - - -//"sm_kamailio": { -// "enabled": false, // starts SessionManager service: -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "connections":[ // instantiate connections to multiple Kamailio servers -// {"evapi_addr": "127.0.0.1:8448", "reconnects": 5} -// ], -//}, - - -//"sm_opensips": { -// "enabled": false, // starts SessionManager service: -// "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "rater_conns": [ -// {"server": "internal"} // address where to reach the Rater <""|internal|127.0.0.1:2013> -// ], -// "cdrs_": [ -// {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing -// ], -// "reconnects": 5, // number of reconnects if connection is lost -// "create_cdr": false, // create CDR out of events and sends them to CDRS component -// "debit_interval": "10s", // interval to perform debits on. -// "min_call_duration": "0s", // only authorize calls with allowed duration higher than this -// "max_call_duration": "3h", // maximum call duration a prepaid call can last -// "events_subscribe_interval": "60s", // automatic events subscription to OpenSIPS, 0 to disable it -// "mi_addr": "127.0.0.1:8020", // address where to reach OpenSIPS MI to send session disconnects -//}, - - -//"historys": { -// "enabled": false, // starts History service: . -// "history_dir": "/var/log/cgrates/history", // location on disk where to store history files. -// "save_interval": "1s", // interval to save changed cache into .git archive -//}, - - -//"pubsubs": { -// "enabled": false, // starts PubSub service: . -//}, - - "aliases": { "enabled": true, // starts Aliases service: . }, @@ -274,45 +48,7 @@ "users": { "enabled": true, // starts User service: . -// "indexes": [], // user profile field indexes }, -//"mailer": { -// "server": "localhost", // the server to use when sending emails out -// "auth_user": "cgrates", // authenticate to email server using this user -// "auth_passwd": "CGRateS.org", // authenticate to email server with this password -// "from_address": "cgr-mailer@localhost.localdomain" // from address used when sending emails out -//}, - - -//"suretax": { -// "url": "", // API url -// "client_number": "", // client number, provided by SureTax -// "validation_key": "", // validation key provided by SureTax -// "business_unit": "", // client’s Business Unit -// "timezone": "Local", // convert the time of the events to this timezone before sending request out -// "include_local_cost": false, // sum local calculated cost with tax one in final cost -// "return_file_code": "0", // default or Quote purposes <0|Q> -// "response_group": "03", // determines how taxes are grouped for the response <03|13> -// "response_type": "D4", // determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response -// "regulatory_code": "03", // provider type -// "client_tracking": "CgrId", // template extracting client information out of StoredCdr; <$RSRFields> -// "customer_number": "Subject", // template extracting customer number out of StoredCdr; <$RSRFields> -// "orig_number": "Subject", // template extracting origination number out of StoredCdr; <$RSRFields> -// "term_number": "Destination", // template extracting termination number out of StoredCdr; <$RSRFields> -// "bill_to_number": "", // template extracting billed to number out of StoredCdr; <$RSRFields> -// "zipcode": "", // template extracting billing zip code out of StoredCdr; <$RSRFields> -// "plus4": "", // template extracting billing zip code extension out of StoredCdr; <$RSRFields> -// "p2pzipcode": "", // template extracting secondary zip code out of StoredCdr; <$RSRFields> -// "p2pplus4": "", // template extracting secondary zip code extension out of StoredCdr; <$RSRFields> -// "units": "^1", // template extracting number of “lines” or unique charges contained within the revenue out of StoredCdr; <$RSRFields> -// "unit_type": "^00", // template extracting number of unique access lines out of StoredCdr; <$RSRFields> -// "tax_included": "^0", // template extracting tax included in revenue out of StoredCdr; <$RSRFields> -// "tax_situs_rule": "^04", // template extracting tax situs rule out of StoredCdr; <$RSRFields> -// "trans_type_code": "^010101", // template extracting transaction type indicator out of StoredCdr; <$RSRFields> -// "sales_type_code": "^R", // template extracting sales type code out of StoredCdr; <$RSRFields> -// "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> -//}, - } diff --git a/data/conf/samples/tutlocal/cgrates.json b/data/conf/samples/tutlocal/cgrates.json index df77fa492..3e3682d80 100644 --- a/data/conf/samples/tutlocal/cgrates.json +++ b/data/conf/samples/tutlocal/cgrates.json @@ -10,11 +10,17 @@ "http": ":2080", // HTTP listening address }, -"rater": { +"rals": { "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], }, "scheduler": { @@ -23,10 +29,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server":"internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} ], - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, "cdrstats": { From 44f0553e92d63a770eb7ca159ca5294d9199d9ee Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 24 Apr 2016 12:26:40 +0200 Subject: [PATCH 185/227] Update tutorial configuration files with new structure --- .../cgrates/etc/cgrates/cgrates.json | 33 ++++-- .../kamevapi/cgrates/etc/cgrates/cgrates.json | 104 ++++++++++-------- .../cgrates/etc/cgrates/cgrates.json | 36 +++--- 3 files changed, 99 insertions(+), 74 deletions(-) diff --git a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json index b21f7fd59..1425caab7 100644 --- a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json @@ -7,13 +7,23 @@ // This is what you get when you load CGRateS with an empty configuration file. -"rater": { +"rals": { "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "historys": "internal", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal" + "cdrstats_conns": [ + {"address": "*internal"} + ], + "historys_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, @@ -24,8 +34,9 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} + ], }, @@ -99,12 +110,10 @@ "sm_freeswitch": { "enabled": true, // starts SessionManager service: - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> "debit_interval": "5s", // interval to perform debits on. "channel_sync_interval": "10s", - "connections":[ // instantiate connections to multiple FreeSWITCH servers - {"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 15} + "event_socket_conns":[ // instantiate connections to multiple FreeSWITCH servers + {"address": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5} ], }, diff --git a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json index 6d9890381..3a3764e9c 100644 --- a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json @@ -7,48 +7,59 @@ // This is what you get when you load CGRateS with an empty configuration file. -"rater": { - "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "historys": "internal", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal" +"rals": { + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], + "historys_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { - "enabled": true, // start Scheduler service: + "enabled": true, }, "cdrs": { - "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service. Empty to disable stats gathering <""|internal|x.y.z.y:1234> + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], }, "cdrstats": { - "enabled": true, // starts the cdrstats service: + "enabled": true, }, "cdre": { "*default": { - "cdr_format": "csv", // exported CDRs format + "cdr_format": "csv", "field_separator": ",", - "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) - "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) - "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENRIC unit to call duration in some billing systems) - "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT - "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding - "cost_shift_digits": 0, // shift digits in the cost on export (eg: convert from EUR to cents) - "mask_destination_id": "MASKED_DESTINATIONS", // destination id containing called addresses to be masked on export - "mask_length": 0, // length of the destination suffix to be masked - "export_dir": "/tmp/cgr_kamevapi/cgrates/cdre", // path where the exported CDRs will be placed - "header_fields": [], // template of the exported header fields - "content_fields": [ // template of the exported content fields + "data_usage_multiply_factor": 1, + "sms_usage_multiply_factor": 1, + "generic_usage_multiply_factor": 1, + "cost_multiply_factor": 1, + "cost_rounding_decimals": -1, + "cost_shift_digits": 0, + "mask_destination_id": "MASKED_DESTINATIONS", + "mask_length": 0, + "export_dir": "/tmp/cgr_kamevapi/cgrates/cdre", + "header_fields": [], + "content_fields": [ {"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"}, {"tag":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"}, {"tag":"Tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "tor"}, @@ -65,22 +76,22 @@ {"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, {"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, ], - "trailer_fields": [], // template of the exported trailer fields + "trailer_fields": [], }, "customer_tpl": { - "cdr_format": "csv", // exported CDRs format + "cdr_format": "csv", "field_separator": ";", - "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) - "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) - "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) - "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT - "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding - "cost_shift_digits": 0, // shift digits in the cost on export (eg: convert from EUR to cents) - "mask_destination_id": "MASKED_DESTINATIONS", // destination id containing called addresses to be masked on export - "mask_length": 0, // length of the destination suffix to be masked - "export_dir": "/tmp/cgr_kamevapi/cgrates/cdre", // path where the exported CDRs will be placed - "header_fields": [], // template of the exported header fields - "content_fields": [ // template of the exported content fields + "data_usage_multiply_factor": 1, + "sms_usage_multiply_factor": 1, + "generic_usage_multiply_factor": 1, + "cost_multiply_factor": 1, + "cost_rounding_decimals": -1, + "cost_shift_digits": 0, + "mask_destination_id": "MASKED_DESTINATIONS", + "mask_length": 0, + "export_dir": "/tmp/cgr_kamevapi/cgrates/cdre", + "header_fields": [], + "content_fields": [ {"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"}, {"tag":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"}, {"tag":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"}, @@ -92,37 +103,36 @@ {"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, {"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, ], - "trailer_fields": [], // template of the exported trailer fields + "trailer_fields": [], } }, "sm_kamailio": { - "enabled": true, // starts SessionManager service: - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> - "create_cdr": true, // create CDR out of events and sends them to CDRS component + "enabled": true, + "create_cdr": true, }, "historys": { - "enabled": true, // starts History service: . - "history_dir": "/tmp/cgr_kamevapi/cgrates/history", // location on disk where to store history files. + "enabled": true, + "history_dir": "/tmp/cgr_kamevapi/cgrates/history", }, "pubsubs": { - "enabled": true, // starts PubSub service: . + "enabled": true, }, "aliases": { - "enabled": true, // starts PubSub service: . + "enabled": true, }, "users": { - "enabled": true, // starts User service: . - "indexes": ["Uuid"], // user profile field indexes + "enabled": true, + "indexes": ["Uuid"], }, diff --git a/data/tutorials/osips_async/cgrates/etc/cgrates/cgrates.json b/data/tutorials/osips_async/cgrates/etc/cgrates/cgrates.json index 9a3488e09..225830f24 100644 --- a/data/tutorials/osips_async/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/osips_async/cgrates/etc/cgrates/cgrates.json @@ -7,25 +7,36 @@ // This is what you get when you load CGRateS with an empty configuration file. -"rater": { - "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "historys": "internal", // address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal", +"rals": { + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], + "historys_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { - "enabled": true, // start Scheduler service: + "enabled": true, }, "cdrs": { "enabled": true, // start the CDR Server service: - "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> + "cdrstats_conns": [ + {"address": "*internal"} + ], }, @@ -99,13 +110,8 @@ "sm_opensips": { "enabled": true, // starts SessionManager service: - "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS - "rater": "internal", // address where to reach the Rater <""|internal|127.0.0.1:2013> - "cdrs": "internal", // address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234> "create_cdr": true, // create CDR out of events and sends them to CDRS component "debit_interval": "2s", // interval to perform debits on. - "events_subscribe_interval": "60s", // automatic events subscription to OpenSIPS, 0 to disable it - "mi_addr": "127.0.0.1:8020", // address where to reach OpenSIPS MI to send session disconnects }, From bd619afdd04514a11aba8c70b8a59145f720b7f1 Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 24 Apr 2016 20:47:18 +0200 Subject: [PATCH 186/227] Various fixes --- apier/v1/apier.go | 2 +- cdrc/cdrc_local_test.go | 4 +- cmd/cgr-engine/cgr-engine.go | 101 ++++++++++---------- cmd/cgr-engine/rater.go | 10 +- data/conf/samples/apier/apier.json | 5 + data/conf/samples/cdrstats/cdrstats.json | 20 ++-- data/conf/samples/hapool/cgrrater1/cgr.json | 34 ++++--- data/conf/samples/hapool/cgrrater2/cgr.json | 34 ++++--- data/conf/samples/hapool/cgrsmg1/cgr.json | 14 +-- data/conf/samples/hapool/cgrsmg2/cgr.json | 14 +-- data/conf/samples/hapool/dagent/cgr.json | 4 +- engine/action.go | 2 +- engine/actions_test.go | 2 +- engine/cdrs.go | 15 +++ engine/libengine.go | 2 +- engine/responder.go | 3 +- utils/consts.go | 2 +- 17 files changed, 155 insertions(+), 113 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index e6e62711f..95fba7081 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -810,7 +810,7 @@ func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.Cach cs.DerivedChargers = cache2go.CountEntries(utils.DERIVEDCHARGERS_PREFIX) cs.LcrProfiles = cache2go.CountEntries(utils.LCR_PREFIX) cs.Aliases = cache2go.CountEntries(utils.ALIASES_PREFIX) - if self.CdrStatsSrv != nil && self.Config.CDRStatsEnabled { + if self.CdrStatsSrv != nil { var queueIds []string if err := self.CdrStatsSrv.Call("CDRStatsV1.GetQueueIds", 0, &queueIds); err != nil { return utils.NewErrServerError(err) diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index be4668599..3fd2f31f7 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -167,7 +167,7 @@ func TestCsvLclProcessCdrDir(t *testing.T) { break } for _, cdrsConn := range cdrcCfg.CdrsConns { - if cdrsConn.Address == utils.INTERNAL { + if cdrsConn.Address == utils.MetaInternal { cdrsConn.Address = "127.0.0.1:2013" } } @@ -205,7 +205,7 @@ func TestCsvLclProcessCdr3Dir(t *testing.T) { return } for _, cdrsConn := range cdrcCfg.CdrsConns { - if cdrsConn.Address == utils.INTERNAL { + if cdrsConn.Address == utils.MetaInternal { cdrsConn.Address = "127.0.0.1:2013" } } diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 00b983106..6c5f3eb04 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -292,64 +292,67 @@ func startCDRS(internalCdrSChan chan rpcclient.RpcClientConnection, logDb engine internalUserSChan chan rpcclient.RpcClientConnection, internalAliaseSChan chan rpcclient.RpcClientConnection, internalCdrStatSChan chan rpcclient.RpcClientConnection, server *utils.Server, exitChan chan bool) { utils.Logger.Info("Starting CGRateS CDRS service.") - // Conn pool towards RAL - ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSRaterConns, internalRaterChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return - } - // Pubsub connection init - var pubSubConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSPubSubSConns) { - pubSubConn = ralConn - } else { - pubSubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSPubSubSConns, internalPubSubSChan, cfg.InternalTtl) + var ralConn, pubSubConn, usersConn, aliasesConn, statsConn *rpcclient.RpcClientPool + if len(cfg.CDRSRaterConns) != 0 { // Conn pool towards RAL + ralConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSRaterConns, internalRaterChan, cfg.InternalTtl) if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } } - // Users connection init - var usersConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSUserSConns) { - pubSubConn = ralConn - } else { - usersConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSUserSConns, internalUserSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) - exitChan <- true - return + if len(cfg.CDRSPubSubSConns) != 0 { // Pubsub connection init + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSPubSubSConns) { + pubSubConn = ralConn + } else { + pubSubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSPubSubSConns, internalPubSubSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) + exitChan <- true + return + } } } - // Aliases connection init - var aliasesConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSAliaseSConns) { - pubSubConn = ralConn - } else { - aliasesConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSAliaseSConns, internalAliaseSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) - exitChan <- true - return + if len(cfg.CDRSUserSConns) != 0 { // Users connection init + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSUserSConns) { + usersConn = ralConn + } else { + usersConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSUserSConns, internalUserSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) + exitChan <- true + return + } } } - // Stats connection init - var statsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSStatSConns) { - pubSubConn = ralConn - } else { - statsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSStatSConns, internalCdrStatSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) - exitChan <- true - return + if len(cfg.CDRSAliaseSConns) != 0 { // Aliases connection init + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSAliaseSConns) { + aliasesConn = ralConn + } else { + aliasesConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSAliaseSConns, internalAliaseSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) + exitChan <- true + return + } + } + } + + if len(cfg.CDRSStatSConns) != 0 { // Stats connection init + if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSStatSConns) { + statsConn = ralConn + } else { + statsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSStatSConns, internalCdrStatSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) + exitChan <- true + return + } } } cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, ralConn, pubSubConn, usersConn, aliasesConn, statsConn) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 1647ee62a..9613256de 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -92,7 +92,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC waitTasks = append(waitTasks, balTaskChan) go func() { defer close(balTaskChan) - if cfg.RALsBalancer == utils.INTERNAL { + if cfg.RALsBalancer == utils.MetaInternal { select { case bal = <-internalBalancerChan: internalBalancerChan <- bal // Put it back if someone else is interested about @@ -193,12 +193,16 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC for _, chn := range waitTasks { <-chn } - responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats} + responder := &engine.Responder{Bal: bal, ExitChan: exitChan} responder.SetTimeToLive(cfg.ResponseCacheTTL, nil) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, - Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: usersConns} + Config: cfg, Responder: responder, Users: usersConns} apierRpcV2 := &v2.ApierV2{ ApierV1: *apierRpcV1} + if cdrStats != nil { // ToDo: Fix here properly the init of stats + responder.Stats = cdrStats + apierRpcV1.CdrStatsSrv = cdrStats + } // internalSchedulerChan shared here server.RpcRegister(responder) server.RpcRegister(apierRpcV1) diff --git a/data/conf/samples/apier/apier.json b/data/conf/samples/apier/apier.json index 30a533b19..8234d67b0 100644 --- a/data/conf/samples/apier/apier.json +++ b/data/conf/samples/apier/apier.json @@ -38,4 +38,9 @@ } }, +"cdrstats": { + "enabled": true, // starts the cdrstats service: + "save_interval": "0s", // interval to save changed stats into dataDb storage +}, + } diff --git a/data/conf/samples/cdrstats/cdrstats.json b/data/conf/samples/cdrstats/cdrstats.json index 5e47affa5..fc4d074b0 100644 --- a/data/conf/samples/cdrstats/cdrstats.json +++ b/data/conf/samples/cdrstats/cdrstats.json @@ -5,31 +5,31 @@ // Starts rater, cdrs and mediator connecting over internal channel "listen": { - "rpc_json": ":2012", // RPC JSON listening address - "rpc_gob": ":2013", // RPC GOB listening address - "http": ":2080", // HTTP listening address + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", }, "rals": { - "enabled": true, // enable Rater service: + "enabled": true, "cdrstats_conns": [ {"address": "*internal"} - ], // address where to reach the cdrstats service, empty to disable stats functionality: <""|*internal|x.y.z.y:1234> + ], }, "cdrs": { - "enabled": true, // start the CDR Server service: - "store_cdrs": false, // store cdrs in storDb + "enabled": true, + "store_cdrs": false, "rals_conns": [ - {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> + {"address": "*internal"} ], "cdrstats_conns": [ {"address": "*internal"} - ], // address where to reach the cdrstats service. Empty to disable stats gathering out of mediated CDRs <""|internal|x.y.z.y:1234> + ] }, "cdrstats": { - "enabled": true, // starts the cdrstats service: + "enabled": true, "save_interval": "1s", }, diff --git a/data/conf/samples/hapool/cgrrater1/cgr.json b/data/conf/samples/hapool/cgrrater1/cgr.json index cfde7c390..d33b19f7b 100644 --- a/data/conf/samples/hapool/cgrrater1/cgr.json +++ b/data/conf/samples/hapool/cgrrater1/cgr.json @@ -1,32 +1,40 @@ { "listen": { - "rpc_json": ":2014", // RPC JSON listening address - "rpc_gob": ":2015", // RPC GOB listening address - "http": ":2081", // HTTP listening address + "rpc_json": ":2014", + "rpc_gob": ":2015", + "http": ":2081", }, -"rater": { - "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal", +"rals": { + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { - "enabled": true, // start Scheduler service: + "enabled": true, }, "cdrstats": { - "enabled": true, // starts the cdrstats service: + "enabled": true, }, "pubsubs": { - "enabled": true, // starts PubSub service: . + "enabled": true, }, "aliases": { - "enabled": true, // starts Aliases service: . + "enabled": true, }, "users": { diff --git a/data/conf/samples/hapool/cgrrater2/cgr.json b/data/conf/samples/hapool/cgrrater2/cgr.json index dd64d4442..604f5b9ee 100644 --- a/data/conf/samples/hapool/cgrrater2/cgr.json +++ b/data/conf/samples/hapool/cgrrater2/cgr.json @@ -1,33 +1,41 @@ { "listen": { - "rpc_json": ":2016", // RPC JSON listening address - "rpc_gob": ":2017", // RPC GOB listening address - "http": ":2082", // HTTP listening address + "rpc_json": ":2016", + "rpc_gob": ":2017", + "http": ":2082", }, -"rater": { - "enabled": true, // enable Rater service: - "cdrstats": "internal", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> - "pubsubs": "internal", // address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234> - "users": "internal", // address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234> - "aliases": "internal", +"rals": { + "enabled": true, + "cdrstats_conns": [ + {"address": "*internal"} + ], + "pubsubs_conns": [ + {"address": "*internal"} + ], + "users_conns": [ + {"address": "*internal"} + ], + "aliases_conns": [ + {"address": "*internal"} + ], }, "scheduler": { - "enabled": true, // start Scheduler service: + "enabled": true, }, "cdrstats": { - "enabled": true, // starts the cdrstats service: + "enabled": true, }, "pubsubs": { - "enabled": true, // starts PubSub service: . + "enabled": true, }, "aliases": { - "enabled": true, // starts Aliases service: . + "enabled": true, }, "users": { diff --git a/data/conf/samples/hapool/cgrsmg1/cgr.json b/data/conf/samples/hapool/cgrsmg1/cgr.json index 9b8ec46b1..0473fb2bc 100644 --- a/data/conf/samples/hapool/cgrsmg1/cgr.json +++ b/data/conf/samples/hapool/cgrsmg1/cgr.json @@ -7,21 +7,21 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server": "127.0.0.1:2014"}, - {"server": "127.0.0.1:2016"} + "rals_conns": [ + {"address": "127.0.0.1:2014"}, + {"address": "127.0.0.1:2016"} ], "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, "sm_generic": { "enabled": true, - "rater_conns": [ - {"server": "127.0.0.1:2014"}, - {"server": "127.0.0.1:2016"} + "rals_conns": [ + {"address": "127.0.0.1:2014"}, + {"address": "127.0.0.1:2016"} ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing ], }, } diff --git a/data/conf/samples/hapool/cgrsmg2/cgr.json b/data/conf/samples/hapool/cgrsmg2/cgr.json index c61812057..645a8ccf3 100644 --- a/data/conf/samples/hapool/cgrsmg2/cgr.json +++ b/data/conf/samples/hapool/cgrsmg2/cgr.json @@ -7,21 +7,21 @@ "cdrs": { "enabled": true, // start the CDR Server service: - "rater_conns": [ - {"server": "127.0.0.1:2014"}, - {"server": "127.0.0.1:2016"} + "rals_conns": [ + {"address": "127.0.0.1:2014"}, + {"address": "127.0.0.1:2016"} ], "cdrstats": "", // address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234> }, "sm_generic": { "enabled": true, - "rater_conns": [ - {"server": "127.0.0.1:2014"}, - {"server": "127.0.0.1:2016"} + "rals_conns": [ + {"address": "127.0.0.1:2014"}, + {"address": "127.0.0.1:2016"} ], "cdrs_conns": [ - {"server": "internal"} // address where to reach CDR Server, empty to disable CDR capturing + {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing ], }, diff --git a/data/conf/samples/hapool/dagent/cgr.json b/data/conf/samples/hapool/dagent/cgr.json index 2f02d5669..bf06a164c 100644 --- a/data/conf/samples/hapool/dagent/cgr.json +++ b/data/conf/samples/hapool/dagent/cgr.json @@ -3,8 +3,8 @@ "enabled": true, "listen": "127.0.0.1:3868", "sm_generic_conns": [ - {"server": "127.0.0.1:2018"}, - {"server": "127.0.0.1:2020"} + {"address": "127.0.0.1:2018"}, + {"address": "127.0.0.1:2020"} ], }, } diff --git a/engine/action.go b/engine/action.go index 84669ff1f..358f87dc0 100644 --- a/engine/action.go +++ b/engine/action.go @@ -670,7 +670,7 @@ func cgrRPCAction(account *Account, sq *StatsQueueTriggered, a *Action, acs Acti return err } var client rpcclient.RpcClientConnection - if req.Address != utils.INTERNAL { + if req.Address != utils.MetaInternal { if client, err = rpcclient.NewRpcClient(req.Method, req.Address, req.Attempts, 0, req.Transport, nil); err != nil { return err } diff --git a/engine/actions_test.go b/engine/actions_test.go index f9297d25a..14c3dd104 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2207,7 +2207,7 @@ func TestCgrRpcAction(t *testing.T) { trpcp := &TestRPCParameters{} utils.RegisterRpcParams("", trpcp) a := &Action{ - ExtraParameters: `{"Address": "internal", + ExtraParameters: `{"Address": "*internal", "Transport": "*gob", "Method": "TestRPCParameters.Hopa", "Attempts":1, diff --git a/engine/cdrs.go b/engine/cdrs.go index c31badcd3..7c00dd2e5 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -71,6 +71,21 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater, pubsub, users, aliases, stats rpcclient.RpcClientConnection) (*CdrServer, error) { + if rater == nil || reflect.ValueOf(rater).IsNil() { // Work around so we store actual nil instead of nil interface value + rater = nil + } + if pubsub == nil || reflect.ValueOf(pubsub).IsNil() { + pubsub = nil + } + if users == nil || reflect.ValueOf(users).IsNil() { + users = nil + } + if aliases == nil || reflect.ValueOf(aliases).IsNil() { + aliases = nil + } + if stats == nil || reflect.ValueOf(stats).IsNil() { + stats = nil + } return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, client: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: Guardian}, nil } diff --git a/engine/libengine.go b/engine/libengine.go index d541de76d..b555f596a 100644 --- a/engine/libengine.go +++ b/engine/libengine.go @@ -36,7 +36,7 @@ func NewRPCPool(dispatchStrategy string, connAttempts, reconnects int, codec str if rpcConnCfg.Address == utils.MetaInternal { var internalConn rpcclient.RpcClientConnection select { - case internalConn := <-internalConnChan: + case internalConn = <-internalConnChan: internalConnChan <- internalConn case <-time.After(ttl): return nil, errors.New("TTL triggered") diff --git a/engine/responder.go b/engine/responder.go index f4a25c957..8c6009db6 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -93,7 +93,6 @@ func (rs *Responder) GetCost(arg *CallDescriptor, reply *CallCost) (err error) { }, arg, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { return err } - if rs.Bal != nil { r, e := rs.getCallCost(arg, "Responder.GetCost") *reply, err = *r, e @@ -497,7 +496,7 @@ func (rs *Responder) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *u } func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { - cacheKey := "GetLCR" + attrs.CgrID + cacheKey := utils.LCRCachePrefix + attrs.CgrID + attrs.RunID if attrs.CallDescriptor.Subject == "" { attrs.CallDescriptor.Subject = attrs.CallDescriptor.Account } diff --git a/utils/consts.go b/utils/consts.go index 4c81a53c8..15889b7d8 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -143,7 +143,6 @@ const ( FWV = "fwv" DRYRUN = "dry_run" META_COMBIMED = "*combimed" - INTERNAL = "internal" MetaInternal = "*internal" ZERO_RATING_SUBJECT_PREFIX = "*zero" OK = "OK" @@ -243,6 +242,7 @@ const ( REFUND_ROUND_CACHE_PREFIX = "REFUND_ROUND_" GET_SESS_RUNS_CACHE_PREFIX = "GET_SESS_RUNS_" LOG_CALL_COST_CACHE_PREFIX = "LOG_CALL_COSTS_" + LCRCachePrefix = "LCR_" ALIAS_CONTEXT_RATING = "*rating" NOT_AVAILABLE = "N/A" CALL = "call" From 56943624bb63d1abd596991f29c7d6fc42893c46 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 25 Apr 2016 15:35:08 +0300 Subject: [PATCH 187/227] fixed globar rounding digit count --- engine/calldesc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/calldesc.go b/engine/calldesc.go index b06a21ed5..e126df79e 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -74,7 +74,7 @@ var ( storageLogger LogStorage cdrStorage CdrStorage debitPeriod = 10 * time.Second - globalRoundingDecimals = 5 + globalRoundingDecimals = 6 historyScribe rpcclient.RpcClientConnection pubSubServer rpcclient.RpcClientConnection userService rpcclient.RpcClientConnection From e6ce7a3bed37a7407fed9f37d92bdcc83d901124 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 25 Apr 2016 18:45:20 +0300 Subject: [PATCH 188/227] only one version structure in the db --- engine/storage_interface.go | 8 +- engine/storage_map.go | 57 +--------- engine/storage_mongo_datadb.go | 50 ++------- engine/storage_redis.go | 45 +------- engine/storage_sql.go | 8 -- engine/version.go | 189 +++++++++++++-------------------- utils/consts.go | 4 +- 7 files changed, 94 insertions(+), 267 deletions(-) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 35b1236cc..96a4b32c6 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -68,8 +68,6 @@ type RatingStorage interface { GetAllActionPlans() (map[string]*ActionPlan, error) PushTask(*Task) error PopTask() (*Task, error) - GetRatingStructuresVersion() (*RatingStructuresVersion, error) - SetRatingStructuresVersion(*RatingStructuresVersion) error } type AccountingStorage interface { @@ -94,8 +92,8 @@ type AccountingStorage interface { RemoveAlias(string) error GetLoadHistory(int, bool) ([]*LoadInstance, error) AddLoadHistory(*LoadInstance, int) error - GetAccountingStructuresVersion() (*AccountingStructuresVersion, error) - SetAccountingStructuresVersion(*AccountingStructuresVersion) error + GetStructVersion() (*StructVersion, error) + SetStructVersion(*StructVersion) error } type CdrStorage interface { @@ -104,8 +102,6 @@ type CdrStorage interface { SetSMCost(smc *SMCost) error GetSMCosts(cgrid, runid, originHost, originIDPrfx string) ([]*SMCost, error) GetCDRs(*utils.CDRsFilter, bool) ([]*CDR, int64, error) - GetCdrStructuresVersion() (*CdrStructuresVersion, error) - SetCdrStructuresVersion(*CdrStructuresVersion) error } type LogStorage interface { diff --git a/engine/storage_map.go b/engine/storage_map.go index 72e436497..11afce2c7 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -925,7 +925,7 @@ func (ms *MapStorage) LogActionTiming(source string, at *ActionTiming, as Action return } -func (ms *MapStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { +func (ms *MapStorage) SetStructVersion(v *StructVersion) (err error) { ms.mu.Lock() defer ms.mu.Unlock() var result []byte @@ -933,65 +933,18 @@ func (ms *MapStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (er if err != nil { return } - ms.dict[utils.RATING_VERSION_PREFIX+"version"] = result + ms.dict[utils.VERSION_PREFIX+"struct"] = result return } -func (ms *MapStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { +func (ms *MapStorage) GetStructVersion() (rsv *StructVersion, err error) { ms.mu.RLock() defer ms.mu.RUnlock() - rsv = &RatingStructuresVersion{} - if values, ok := ms.dict[utils.RATING_VERSION_PREFIX+"version"]; ok { + rsv = &StructVersion{} + if values, ok := ms.dict[utils.VERSION_PREFIX+"struct"]; ok { err = ms.ms.Unmarshal(values, &rsv) } else { return nil, utils.ErrNotFound } return } - -func (ms *MapStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { - ms.mu.Lock() - defer ms.mu.Unlock() - var result []byte - result, err = ms.ms.Marshal(v) - if err != nil { - return - } - ms.dict[utils.ACCOUNTING_VERSION_PREFIX+"version"] = result - return -} - -func (ms *MapStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { - ms.mu.RLock() - defer ms.mu.RUnlock() - asv = &AccountingStructuresVersion{} - if values, ok := ms.dict[utils.ACCOUNTING_VERSION_PREFIX+"version"]; ok { - err = ms.ms.Unmarshal(values, &asv) - } else { - return nil, utils.ErrNotFound - } - return -} -func (ms *MapStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { - ms.mu.Lock() - defer ms.mu.Unlock() - var result []byte - result, err = ms.ms.Marshal(v) - if err != nil { - return - } - ms.dict[utils.CDR_VERSION_PREFIX+"version"] = result - return -} - -func (ms *MapStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { - ms.mu.RLock() - defer ms.mu.RUnlock() - csv = &CdrStructuresVersion{} - if values, ok := ms.dict[utils.CDR_VERSION_PREFIX+"version"]; ok { - err = ms.ms.Unmarshal(values, &csv) - } else { - return nil, utils.ErrNotFound - } - return -} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 2201588e9..36656fd70 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -53,7 +53,7 @@ const ( colLogAtr = "action_trigger_logs" colLogApl = "action_plan_logs" colLogErr = "error_logs" - colVer = "versions" + colVer = "version" ) var ( @@ -1365,53 +1365,19 @@ func (ms *MongoStorage) GetAllCdrStats() (css []*CdrStats, err error) { return } -func (ms *MongoStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { - _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.RATING_VERSION_PREFIX + "version"}, &struct { +func (ms *MongoStorage) SetStructVersion(v *StructVersion) (err error) { + _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.VERSION_PREFIX + "struct"}, &struct { Key string - Value *RatingStructuresVersion - }{utils.RATING_VERSION_PREFIX + "version", v}) + Value *StructVersion + }{utils.VERSION_PREFIX + "struct", v}) return } -func (ms *MongoStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { - rsv = new(RatingStructuresVersion) - err = ms.db.C(colVer).Find(bson.M{"key": utils.RATING_VERSION_PREFIX + "version"}).One(rsv) +func (ms *MongoStorage) GetStructVersion() (rsv *StructVersion, err error) { + rsv = new(StructVersion) + err = ms.db.C(colVer).Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(rsv) if err == mgo.ErrNotFound { rsv = nil } return } - -func (ms *MongoStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { - _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.ACCOUNTING_VERSION_PREFIX + "version"}, &struct { - Key string - Value *AccountingStructuresVersion - }{utils.ACCOUNTING_VERSION_PREFIX + "version", v}) - return -} - -func (ms *MongoStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { - asv = new(AccountingStructuresVersion) - err = ms.db.C(colVer).Find(bson.M{"key": utils.ACCOUNTING_VERSION_PREFIX + "version"}).One(asv) - if err == mgo.ErrNotFound { - asv = nil - } - return -} - -func (ms *MongoStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { - _, err = ms.db.C(colVer).Upsert(bson.M{"key": utils.CDR_VERSION_PREFIX + "version"}, &struct { - Key string - Value *CdrStructuresVersion - }{utils.CDR_VERSION_PREFIX + "version", v}) - return -} - -func (ms *MongoStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { - csv = new(CdrStructuresVersion) - err = ms.db.C(colVer).Find(bson.M{"key": utils.CDR_VERSION_PREFIX + "version"}).One(csv) - if err == mgo.ErrNotFound { - csv = nil - } - return -} diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 5c69570df..b4aef0fb7 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1094,55 +1094,20 @@ func (rs *RedisStorage) LogActionTiming(source string, at *ActionTiming, as Acti return rs.db.Cmd("SET", utils.LOG_ACTION_TIMMING_PREFIX+source+"_"+time.Now().Format(time.RFC3339Nano), []byte(fmt.Sprintf("%v*%v", string(mat), string(mas)))).Err } -func (rs *RedisStorage) SetRatingStructuresVersion(v *RatingStructuresVersion) (err error) { +func (rs *RedisStorage) SetStructVersion(v *StructVersion) (err error) { var result []byte result, err = rs.ms.Marshal(v) if err != nil { return } - return rs.db.Cmd("SET", utils.RATING_VERSION_PREFIX+"version", result).Err + return rs.db.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err } -func (rs *RedisStorage) GetRatingStructuresVersion() (rsv *RatingStructuresVersion, err error) { +func (rs *RedisStorage) GetStructVersion() (rsv *StructVersion, err error) { var values []byte - rsv = &RatingStructuresVersion{} - if values, err = rs.db.Cmd("GET", utils.RATING_VERSION_PREFIX+"version").Bytes(); err == nil { + rsv = &StructVersion{} + if values, err = rs.db.Cmd("GET", utils.VERSION_PREFIX+"struct").Bytes(); err == nil { err = rs.ms.Unmarshal(values, &rsv) } return } - -func (rs *RedisStorage) SetAccountingStructuresVersion(v *AccountingStructuresVersion) (err error) { - var result []byte - result, err = rs.ms.Marshal(v) - if err != nil { - return - } - return rs.db.Cmd("SET", utils.ACCOUNTING_VERSION_PREFIX+"version", result).Err -} - -func (rs *RedisStorage) GetAccountingStructuresVersion() (asv *AccountingStructuresVersion, err error) { - var values []byte - asv = &AccountingStructuresVersion{} - if values, err = rs.db.Cmd("GET", utils.ACCOUNTING_VERSION_PREFIX+"version").Bytes(); err == nil { - err = rs.ms.Unmarshal(values, &asv) - } - return -} -func (rs *RedisStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { - var result []byte - result, err = rs.ms.Marshal(v) - if err != nil { - return - } - return rs.db.Cmd("SET", utils.CDR_VERSION_PREFIX+"version", result).Err -} - -func (rs *RedisStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { - var values []byte - csv = &CdrStructuresVersion{} - if values, err = rs.db.Cmd("GET", utils.CDR_VERSION_PREFIX+"version").Bytes(); err == nil { - err = rs.ms.Unmarshal(values, &csv) - } - return -} diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 7eeb8f166..5c8ab5528 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -1343,11 +1343,3 @@ func (self *SQLStorage) GetTpAliases(filter *TpAlias) ([]TpAlias, error) { return tpAliases, nil } - -func (self *SQLStorage) SetCdrStructuresVersion(v *CdrStructuresVersion) (err error) { - return utils.ErrNotImplemented -} - -func (self *SQLStorage) GetCdrStructuresVersion() (csv *CdrStructuresVersion, err error) { - return nil, utils.ErrNotImplemented -} diff --git a/engine/version.go b/engine/version.go index 9dc14c93c..047036261 100644 --- a/engine/version.go +++ b/engine/version.go @@ -8,46 +8,22 @@ import ( func init() { // get current db version - dbRsv, err := ratingStorage.GetRatingStructuresVersion() + dbVersion, err := accountingStorage.GetStructVersion() if err != nil { utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) return } // comparing versions - if currentRsv.CompareAndMigrate(dbRsv) { + if currentVersion.CompareAndMigrate(dbVersion) { // write the new values - if err := ratingStorage.SetRatingStructuresVersion(currentRsv); err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) - } - } - dbAsv, err := accountingStorage.GetAccountingStructuresVersion() - if err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) - return - } - // comparing versions - if currentAsv.CompareAndMigrate(dbAsv) { - // write the new values - if err := accountingStorage.SetAccountingStructuresVersion(currentAsv); err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) - } - } - dbCsv, err := cdrStorage.GetCdrStructuresVersion() - if err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) - return - } - // comparing versions - if currentCsv.CompareAndMigrate(dbCsv) { - // write the new values - if err := cdrStorage.SetCdrStructuresVersion(currentCsv); err != nil { + if err := accountingStorage.SetStructVersion(currentVersion); err != nil { utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) } } } var ( - currentRsv = &RatingStructuresVersion{ + currentVersion = &StructVersion{ Destinations: "1", RatingPlans: "1", RatingProfiles: "1", @@ -57,24 +33,19 @@ var ( ActionPlans: "1", ActionTriggers: "1", SharedGroups: "1", - } - - currentAsv = &AccountingStructuresVersion{ - Accounts: "1", - CdrStats: "1", - Users: "1", - Alias: "1", - PubSubs: "1", - LoadHistory: "1", - } - - currentCsv = &CdrStructuresVersion{ - Cdrs: "1", - SMCosts: "1", + Accounts: "1", + CdrStats: "1", + Users: "1", + Alias: "1", + PubSubs: "1", + LoadHistory: "1", + Cdrs: "1", + SMCosts: "1", } ) -type RatingStructuresVersion struct { +type StructVersion struct { + // rating Destinations string RatingPlans string RatingProfiles string @@ -84,92 +55,78 @@ type RatingStructuresVersion struct { ActionPlans string ActionTriggers string SharedGroups string -} - -func (rsv *RatingStructuresVersion) CompareAndMigrate(dbRsv *RatingStructuresVersion) bool { - migrationPerformed := false - if rsv.Destinations != dbRsv.Destinations { - migrationPerformed = true - - } - if rsv.RatingPlans != dbRsv.RatingPlans { - migrationPerformed = true - - } - if rsv.RatingProfiles != dbRsv.RatingPlans { - migrationPerformed = true - - } - if rsv.Lcrs != dbRsv.Lcrs { - migrationPerformed = true - - } - if rsv.DerivedChargers != dbRsv.DerivedChargers { - migrationPerformed = true - - } - if rsv.Actions != dbRsv.Actions { - migrationPerformed = true - - } - if rsv.ActionPlans != dbRsv.ActionPlans { - migrationPerformed = true - - } - if rsv.ActionTriggers != dbRsv.ActionTriggers { - migrationPerformed = true - - } - if rsv.SharedGroups != dbRsv.SharedGroups { - migrationPerformed = true - - } - return migrationPerformed -} - -type AccountingStructuresVersion struct { + // accounting Accounts string CdrStats string Users string Alias string PubSubs string LoadHistory string -} - -func (asv *AccountingStructuresVersion) CompareAndMigrate(dbAsv *AccountingStructuresVersion) bool { - migrationPerformed := false - if asv.Accounts != dbAsv.Accounts { - migrationPerformed = true - } - if asv.CdrStats != dbAsv.CdrStats { - migrationPerformed = true - } - if asv.Users != dbAsv.Users { - migrationPerformed = true - } - if asv.Alias != dbAsv.Alias { - migrationPerformed = true - } - if asv.PubSubs != dbAsv.PubSubs { - migrationPerformed = true - } - if asv.LoadHistory != dbAsv.LoadHistory { - migrationPerformed = true - } - return migrationPerformed -} - -type CdrStructuresVersion struct { + // cdr Cdrs string SMCosts string } -func (csv *CdrStructuresVersion) CompareAndMigrate(dbCsv *CdrStructuresVersion) bool { +func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) bool { migrationPerformed := false - if csv.Cdrs != dbCsv.Cdrs { + if sv.Destinations != dbVer.Destinations { + migrationPerformed = true + + } + if sv.RatingPlans != dbVer.RatingPlans { + migrationPerformed = true + + } + if sv.RatingProfiles != dbVer.RatingPlans { + migrationPerformed = true + + } + if sv.Lcrs != dbVer.Lcrs { + migrationPerformed = true + + } + if sv.DerivedChargers != dbVer.DerivedChargers { + migrationPerformed = true + + } + if sv.Actions != dbVer.Actions { + migrationPerformed = true + + } + if sv.ActionPlans != dbVer.ActionPlans { + migrationPerformed = true + + } + if sv.ActionTriggers != dbVer.ActionTriggers { + migrationPerformed = true + + } + if sv.SharedGroups != dbVer.SharedGroups { + migrationPerformed = true + + } + if sv.Accounts != dbVer.Accounts { migrationPerformed = true } - if csv.SMCosts != dbCsv.SMCosts { + if sv.CdrStats != dbVer.CdrStats { + migrationPerformed = true + } + if sv.Users != dbVer.Users { + migrationPerformed = true + } + if sv.Alias != dbVer.Alias { + migrationPerformed = true + } + if sv.PubSubs != dbVer.PubSubs { + migrationPerformed = true + } + if sv.LoadHistory != dbVer.LoadHistory { + migrationPerformed = true + } + if sv.Cdrs != dbVer.Cdrs { + migrationPerformed = true + } + if sv.SMCosts != dbVer.SMCosts { migrationPerformed = true } return migrationPerformed diff --git a/utils/consts.go b/utils/consts.go index cc39ffc95..290824d53 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -197,9 +197,7 @@ const ( LOG_CALL_COST_PREFIX = "cco_" LOG_ACTION_TIMMING_PREFIX = "ltm_" LOG_ACTION_TRIGGER_PREFIX = "ltr_" - RATING_VERSION_PREFIX = "rve_" - ACCOUNTING_VERSION_PREFIX = "ave_" - CDR_VERSION_PREFIX = "cve_" + VERSION_PREFIX = "ver_" LOG_ERR = "ler_" LOG_CDR = "cdr_" LOG_MEDIATED_CDR = "mcd_" From f549f8f08bcf9d1039f378bc74f87f36a3843082 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 25 Apr 2016 19:31:34 +0200 Subject: [PATCH 189/227] Various test fixes, ApierV1 passing tests --- apier/v1/cdrstatsv1_local_test.go | 1 - apier/v2/apier.go | 9 +- cmd/cgr-engine/cgr-engine.go | 182 ++++++++++++++---------------- cmd/cgr-engine/rater.go | 34 +++--- engine/cdrs.go | 2 +- 5 files changed, 101 insertions(+), 127 deletions(-) diff --git a/apier/v1/cdrstatsv1_local_test.go b/apier/v1/cdrstatsv1_local_test.go index b9a2b7354..9d6b82130 100644 --- a/apier/v1/cdrstatsv1_local_test.go +++ b/apier/v1/cdrstatsv1_local_test.go @@ -147,7 +147,6 @@ func TestCDRStatsLclPostCdrs(t *testing.T) { } } time.Sleep(time.Duration(*waitRater) * time.Millisecond) - } func TestCDRStatsLclGetMetrics1(t *testing.T) { diff --git a/apier/v2/apier.go b/apier/v2/apier.go index c8464e8d2..9f5f4cc2a 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -220,14 +220,12 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc } aps, _ := loader.GetLoadedIds(utils.ACTION_PLAN_PREFIX) - utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.") + utils.Logger.Info("ApierV2.LoadTariffPlanFromFolder, reloading cache.") cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX) userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX) li := loader.GetLoadInstance() - - // release the tp data - loader.Init() + loader.Init() // release the tp data if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{ utils.DESTINATION_PREFIX: dstKeys, @@ -247,7 +245,7 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, return err } if len(aps) != 0 && self.Sched != nil { - utils.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading scheduler.") + utils.Logger.Info("ApierV2.LoadTariffPlanFromFolder, reloading scheduler.") self.Sched.Reload(true) } if len(cstKeys) != 0 && self.CdrStatsSrv != nil { @@ -256,7 +254,6 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, return err } } - if len(userKeys) != 0 && self.Users != nil { var r string if err := self.Users.Call("UsersV1.ReloadUsers", "", &r); err != nil { diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 6c5f3eb04..1ae8ab419 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -23,7 +23,6 @@ import ( "fmt" "log" "os" - "reflect" "runtime" "runtime/pprof" "strconv" @@ -128,30 +127,30 @@ func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConne } func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, server *utils.Server, exitChan chan bool) { - utils.Logger.Info("Starting CGRateS SM-Generic service.") - ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmGenericConfig.RALsConns, internalRaterChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + utils.Logger.Info("Starting CGRateS SMGeneric service.") + var ralsConns, cdrsConn *rpcclient.RpcClientPool + if len(cfg.SmGenericConfig.RALsConns) != 0 { + ralsConns, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmGenericConfig.RALsConns, internalRaterChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + exitChan <- true + return + } } - var cdrsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.SmGenericConfig.RALsConns, cfg.SmGenericConfig.CDRsConns) { - cdrsConn = ralConn - } else { + if len(cfg.SmGenericConfig.CDRsConns) != 0 { cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, cfg.SmGenericConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) exitChan <- true return } } smg_econns := sessionmanager.NewSMGExternalConnections() - sm := sessionmanager.NewSMGeneric(cfg, ralConn, cdrsConn, cfg.DefaultTimezone, smg_econns) + sm := sessionmanager.NewSMGeneric(cfg, ralsConns, cdrsConn, cfg.DefaultTimezone, smg_econns) if err = sm.Connect(); err != nil { - utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) + utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) } // Register RPC handler smgRpc := v1.NewSMGenericV1(sm) @@ -169,17 +168,17 @@ func startSmGeneric(internalSMGChan chan rpcclient.RpcClientConnection, internal func startDiameterAgent(internalSMGChan, internalPubSubSChan chan rpcclient.RpcClientConnection, exitChan chan bool) { utils.Logger.Info("Starting CGRateS DiameterAgent service.") - smgConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.DiameterAgentCfg().SMGenericConns, internalSMGChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to SMG: %s", err.Error())) - exitChan <- true - return + var smgConn, pubsubConn *rpcclient.RpcClientPool + if len(cfg.DiameterAgentCfg().SMGenericConns) != 0 { + smgConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.DiameterAgentCfg().SMGenericConns, internalSMGChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to SMG: %s", err.Error())) + exitChan <- true + return + } } - var pubsubConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.DiameterAgentCfg().SMGenericConns, cfg.DiameterAgentCfg().PubSubConns) { - pubsubConn = smgConn - } else { + if len(cfg.DiameterAgentCfg().PubSubConns) != 0 { pubsubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, cfg.DiameterAgentCfg().PubSubConns, internalPubSubSChan, cfg.InternalTtl) if err != nil { @@ -202,17 +201,17 @@ func startDiameterAgent(internalSMGChan, internalPubSubSChan chan rpcclient.RpcC func startSmFreeSWITCH(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SMFreeSWITCH service.") - ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmFsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + var ralsConn, cdrsConn *rpcclient.RpcClientPool + if len(cfg.SmFsConfig.RALsConns) != 0 { + ralsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmFsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + exitChan <- true + return + } } - var cdrsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.SmFsConfig.RALsConns, cfg.SmFsConfig.CDRsConns) { - cdrsConn = ralConn - } else { + if len(cfg.SmFsConfig.CDRsConns) != 0 { cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, cfg.SmFsConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) if err != nil { @@ -221,7 +220,7 @@ func startSmFreeSWITCH(internalRaterChan, internalCDRSChan chan rpcclient.RpcCli return } } - sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, ralConn, cdrsConn, cfg.DefaultTimezone) + sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, ralsConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) if err = sm.Connect(); err != nil { utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) @@ -231,17 +230,17 @@ func startSmFreeSWITCH(internalRaterChan, internalCDRSChan chan rpcclient.RpcCli func startSmKamailio(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SMKamailio service.") - ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmKamConfig.RALsConns, internalRaterChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) - exitChan <- true - return + var ralsConn, cdrsConn *rpcclient.RpcClientPool + if len(cfg.SmKamConfig.RALsConns) != 0 { + ralsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmKamConfig.RALsConns, internalRaterChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RAL: %s", err.Error())) + exitChan <- true + return + } } - var cdrsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.SmKamConfig.RALsConns, cfg.SmKamConfig.CDRsConns) { - cdrsConn = ralConn - } else { + if len(cfg.SmKamConfig.CDRsConns) != 0 { cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, cfg.SmKamConfig.CDRsConns, internalCDRSChan, cfg.InternalTtl) if err != nil { @@ -250,7 +249,7 @@ func startSmKamailio(internalRaterChan, internalCDRSChan chan rpcclient.RpcClien return } } - sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, ralConn, cdrsConn, cfg.DefaultTimezone) + sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, ralsConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) if err = sm.Connect(); err != nil { utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) @@ -260,17 +259,17 @@ func startSmKamailio(internalRaterChan, internalCDRSChan chan rpcclient.RpcClien func startSmOpenSIPS(internalRaterChan, internalCDRSChan chan rpcclient.RpcClientConnection, cdrDb engine.CdrStorage, exitChan chan bool) { utils.Logger.Info("Starting CGRateS SMOpenSIPS service.") - ralConn, err := engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.SmOsipsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to RALs: %s", err.Error())) - exitChan <- true - return + var ralsConn, cdrsConn *rpcclient.RpcClientPool + if len(cfg.SmOsipsConfig.RALsConns) != 0 { + ralsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.SmOsipsConfig.RALsConns, internalRaterChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to RALs: %s", err.Error())) + exitChan <- true + return + } } - var cdrsConn *rpcclient.RpcClientPool - if reflect.DeepEqual(cfg.SmOsipsConfig.RALsConns, cfg.SmOsipsConfig.CDRsConns) { - cdrsConn = ralConn - } else { + if len(cfg.SmOsipsConfig.CDRsConns) != 0 { cdrsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, cfg.SmOsipsConfig.CDRsConns, internalRaterChan, cfg.InternalTtl) if err != nil { @@ -279,7 +278,7 @@ func startSmOpenSIPS(internalRaterChan, internalCDRSChan chan rpcclient.RpcClien return } } - sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, cfg.Reconnects, ralConn, cdrsConn, cfg.DefaultTimezone) + sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, cfg.Reconnects, ralsConn, cdrsConn, cfg.DefaultTimezone) smRpc.SMs = append(smRpc.SMs, sm) if err := sm.Connect(); err != nil { utils.Logger.Err(fmt.Sprintf(" error: %s!", err)) @@ -303,56 +302,40 @@ func startCDRS(internalCdrSChan chan rpcclient.RpcClientConnection, logDb engine } } if len(cfg.CDRSPubSubSConns) != 0 { // Pubsub connection init - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSPubSubSConns) { - pubSubConn = ralConn - } else { - pubSubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSPubSubSConns, internalPubSubSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) - exitChan <- true - return - } + pubSubConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSPubSubSConns, internalPubSubSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to PubSubSystem: %s", err.Error())) + exitChan <- true + return } } if len(cfg.CDRSUserSConns) != 0 { // Users connection init - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSUserSConns) { - usersConn = ralConn - } else { - usersConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSUserSConns, internalUserSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) - exitChan <- true - return - } + usersConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSUserSConns, internalUserSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to UserS: %s", err.Error())) + exitChan <- true + return } } if len(cfg.CDRSAliaseSConns) != 0 { // Aliases connection init - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSAliaseSConns) { - aliasesConn = ralConn - } else { - aliasesConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSAliaseSConns, internalAliaseSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) - exitChan <- true - return - } + aliasesConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSAliaseSConns, internalAliaseSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to AliaseS: %s", err.Error())) + exitChan <- true + return } } if len(cfg.CDRSStatSConns) != 0 { // Stats connection init - if reflect.DeepEqual(cfg.CDRSRaterConns, cfg.CDRSStatSConns) { - statsConn = ralConn - } else { - statsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSStatSConns, internalCdrStatSChan, cfg.InternalTtl) - if err != nil { - utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) - exitChan <- true - return - } + statsConn, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, + cfg.CDRSStatSConns, internalCdrStatSChan, cfg.InternalTtl) + if err != nil { + utils.Logger.Crit(fmt.Sprintf(" Could not connect to StatS: %s", err.Error())) + exitChan <- true + return } } cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, ralConn, pubSubConn, usersConn, aliasesConn, statsConn) @@ -428,7 +411,7 @@ func startUsersServer(internalUserSChan chan rpcclient.RpcClientConnection, acco internalUserSChan <- userServer } -func startRpc(server *utils.Server, internalRaterChan chan rpcclient.RpcClientConnection, +func startRpc(server *utils.Server, internalRaterChan, internalCdrSChan, internalCdrStatSChan, internalHistorySChan, internalPubSubSChan, internalUserSChan, internalAliaseSChan chan rpcclient.RpcClientConnection) { select { // Any of the rpc methods will unlock listening to rpc requests @@ -450,7 +433,6 @@ func startRpc(server *utils.Server, internalRaterChan chan rpcclient.RpcClientCo go server.ServeJSON(cfg.RPCJSONListen) go server.ServeGOB(cfg.RPCGOBListen) go server.ServeHTTP(cfg.HTTPListen) - } func writePid() { diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 9613256de..a0d72d6a7 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -84,10 +84,8 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC }() } - - // Connection to balancer var bal *balancer2go.Balancer - if cfg.RALsBalancer != "" { + if cfg.RALsBalancer != "" { // Connection to balancer balTaskChan := make(chan struct{}) waitTasks = append(waitTasks, balTaskChan) go func() { @@ -108,15 +106,14 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - // Connections to CDRStats var cdrStats *rpcclient.RpcClientPool - if len(cfg.RALsCDRStatSConns) != 0 { + if len(cfg.RALsCDRStatSConns) != 0 { // Connections to CDRStats cdrstatTaskChan := make(chan struct{}) waitTasks = append(waitTasks, cdrstatTaskChan) go func() { defer close(cdrstatTaskChan) cdrStats, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.CDRSRaterConns, internalCdrStatSChan, cfg.InternalTtl) + cfg.RALsCDRStatSConns, internalCdrStatSChan, cfg.InternalTtl) if err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect to CDRStatS, error: %s", err.Error())) exitChan <- true @@ -124,9 +121,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - - // Connection to HistoryS, - if len(cfg.RALsHistorySConns) != 0 { + if len(cfg.RALsHistorySConns) != 0 { // Connection to HistoryS, histTaskChan := make(chan struct{}) waitTasks = append(waitTasks, histTaskChan) go func() { @@ -141,8 +136,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - // Connection to pubsubs - if len(cfg.RALsPubSubSConns) != 0 { + if len(cfg.RALsPubSubSConns) != 0 { // Connection to pubsubs pubsubTaskChan := make(chan struct{}) waitTasks = append(waitTasks, pubsubTaskChan) go func() { @@ -157,8 +151,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - // Connection to AliasService - if len(cfg.RALsAliasSConns) != 0 { + if len(cfg.RALsAliasSConns) != 0 { // Connection to AliasService aliasesTaskChan := make(chan struct{}) waitTasks = append(waitTasks, aliasesTaskChan) go func() { @@ -173,15 +166,14 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC } }() } - // Connection to UserService var usersConns rpcclient.RpcClientConnection - if len(cfg.RALsUserSConns) != 0 { + if len(cfg.RALsUserSConns) != 0 { // Connection to UserService usersTaskChan := make(chan struct{}) waitTasks = append(waitTasks, usersTaskChan) go func() { defer close(usersTaskChan) if usersConns, err = engine.NewRPCPool(rpcclient.POOL_FIRST, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB, - cfg.RALsAliasSConns, internalAliaseSChan, cfg.InternalTtl); err != nil { + cfg.RALsUserSConns, internalUserSChan, cfg.InternalTtl); err != nil { utils.Logger.Crit(fmt.Sprintf(" Could not connect UserS, error: %s", err.Error())) exitChan <- true return @@ -196,13 +188,17 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC responder := &engine.Responder{Bal: bal, ExitChan: exitChan} responder.SetTimeToLive(cfg.ResponseCacheTTL, nil) apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched, - Config: cfg, Responder: responder, Users: usersConns} - apierRpcV2 := &v2.ApierV2{ - ApierV1: *apierRpcV1} + Config: cfg, Responder: responder} if cdrStats != nil { // ToDo: Fix here properly the init of stats responder.Stats = cdrStats apierRpcV1.CdrStatsSrv = cdrStats } + if usersConns != nil { + apierRpcV1.Users = usersConns + } + apierRpcV2 := &v2.ApierV2{ + ApierV1: *apierRpcV1} + // internalSchedulerChan shared here server.RpcRegister(responder) server.RpcRegister(apierRpcV1) diff --git a/engine/cdrs.go b/engine/cdrs.go index 7c00dd2e5..d907d9f5d 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -71,7 +71,7 @@ func fsCdrHandler(w http.ResponseWriter, r *http.Request) { } func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater, pubsub, users, aliases, stats rpcclient.RpcClientConnection) (*CdrServer, error) { - if rater == nil || reflect.ValueOf(rater).IsNil() { // Work around so we store actual nil instead of nil interface value + if rater == nil || reflect.ValueOf(rater).IsNil() { // Work around so we store actual nil instead of nil interface value, faster to check here than in CdrServer code rater = nil } if pubsub == nil || reflect.ValueOf(pubsub).IsNil() { From 34e604900d4873bdad691208373fdb6ab15c371c Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 26 Apr 2016 12:04:07 +0200 Subject: [PATCH 190/227] More fixes --- apier/v1/cdrsv1.go | 4 ++-- .../cdrsreplicationmaster/cdrsreplicationmaster.json | 4 ++-- engine/cdrs.go | 9 ++++++--- sessionmanager/smg_session.go | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/apier/v1/cdrsv1.go b/apier/v1/cdrsv1.go index 10aa6b9df..09df4c5a6 100644 --- a/apier/v1/cdrsv1.go +++ b/apier/v1/cdrsv1.go @@ -59,8 +59,8 @@ func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { return nil } -func (self *CdrsV1) StoreSMCost(attr *engine.AttrCDRSStoreSMCost, reply *string) error { - if err := self.CdrSrv.StoreSMCost(attr.Cost, attr.CheckDuplicate); err != nil { +func (self *CdrsV1) StoreSMCost(attr engine.AttrCDRSStoreSMCost, reply *string) error { + if err := self.CdrSrv.StoreSMCost(attr, reply); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK diff --git a/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json b/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json index 2777272fc..030824c84 100644 --- a/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json +++ b/data/conf/samples/cdrsreplicationmaster/cdrsreplicationmaster.json @@ -12,8 +12,8 @@ "enabled": true, // start the CDR Server service: "store_cdrs": false, // store cdrs in storDb "cdr_replication":[ // replicate the rated CDR to a number of servers - {"transport": "*http_post", "server": "http://127.0.0.1:12080/cdr_http", "attempts": 1}, - //{"transport": "*http_post", "server": "http://127.0.0.1:8000/mycdr"}, + {"transport": "*http_post", "address": "http://127.0.0.1:12080/cdr_http", "attempts": 1}, + //{"transport": "*http_post", "address": "http://127.0.0.1:8000/mycdr"}, ], }, diff --git a/engine/cdrs.go b/engine/cdrs.go index d907d9f5d..dad06dd92 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -151,8 +151,7 @@ func (self *CdrServer) ProcessExternalCdr(eCDR *ExternalCDR) error { return self.processCdr(cdr) } -// RPC method, used to log callcosts to db -func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { +func (self *CdrServer) storeSMCost(smCost *SMCost, checkDuplicate bool) error { smCost.CostDetails.UpdateCost() // make sure the total cost reflect the increments smCost.CostDetails.UpdateRatedUsage() // make sure rated usage is updated lockKey := utils.CDRS_SOURCE + smCost.CGRID + smCost.RunID + smCost.OriginID // Will lock on this ID @@ -172,6 +171,11 @@ func (self *CdrServer) StoreSMCost(smCost *SMCost, checkDuplicate bool) error { return self.cdrDb.SetSMCost(smCost) } +// RPC method, differs from storeSMCost through it's signature +func (self *CdrServer) StoreSMCost(attr AttrCDRSStoreSMCost, reply *string) error { + return self.storeSMCost(attr.Cost, attr.CheckDuplicate) +} + // Called by rate/re-rate API func (self *CdrServer) RateCDRs(cdrFltr *utils.CDRsFilter, sendToStats bool) error { cdrs, _, err := self.cdrDb.GetCDRs(cdrFltr, false) @@ -532,7 +536,6 @@ func (cdrsrv *CdrServer) Call(serviceMethod string, args interface{}, reply inte // construct the params params := []reflect.Value{reflect.ValueOf(args), reflect.ValueOf(reply)} - ret := method.Call(params) if len(ret) != 1 { return utils.ErrServerError diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index af8f1a153..d1701a905 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -266,7 +266,7 @@ func (self *SMGSession) saveOperations(originID string) error { CostDetails: firstCC, } var reply string - if err := self.cdrsrv.Call("CdrServer.StoreSMCost", engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil { + if err := self.cdrsrv.Call("CdrsV1.StoreSMCost", engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil { if err == utils.ErrExists { self.refund(self.cd.GetDuration()) // Refund entire duration } else { From d4ac5548cb01852f911f14aaf1311a40de328e40 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 26 Apr 2016 13:42:34 +0200 Subject: [PATCH 191/227] Default request caching disabled --- cmd/cgr-engine/rater.go | 1 + config/config_defaults.go | 2 +- engine/responder.go | 1 + sessionmanager/smg_it_test.go | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 127fac4f3..0c3f90021 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -26,6 +26,7 @@ import ( "github.com/cgrates/cgrates/apier/v2" "github.com/cgrates/cgrates/balancer2go" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/history" "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/utils" "github.com/cgrates/rpcclient" diff --git a/config/config_defaults.go b/config/config_defaults.go index bce7f6467..a02d31c55 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -39,7 +39,7 @@ const CGRATES_CFG_JSON = ` "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> "connect_attempts": 3, // initial server connect attempts "reconnects": -1, // number of retries in case of connection lost - "response_cache_ttl": "3s", // the life span of a cached response + "response_cache_ttl": "0s", // the life span of a cached response "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up }, diff --git a/engine/responder.go b/engine/responder.go index 8c6009db6..f396ba032 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -194,6 +194,7 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) } func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err error) { + utils.Logger.Debug(fmt.Sprintf("Responder.RefundIncrements, arg: %+v", arg)) if item, err := rs.getCache().Get(utils.REFUND_INCR_CACHE_PREFIX + arg.CgrID + arg.RunID); err == nil && item != nil { *reply = *(item.Value.(*float64)) return item.Err diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 85796545b..07bcdffc2 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -195,7 +195,7 @@ func TestSMGVoiceVoiceRefund(t *testing.T) { t.Error(err) } if maxUsage != 90 { - t.Error("Bad max usage: ", maxUsage) + t.Error("Received: ", maxUsage) } var acnt *engine.Account attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001"} From 342439213be029b9c5d6f49af6f739237f0087a5 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 26 Apr 2016 16:18:32 +0200 Subject: [PATCH 192/227] Default configuration updated --- agents/hapool_it_test.go | 2 ++ config/config_json_test.go | 2 +- data/conf/cgrates/cgrates.json | 36 +++++++++++++++++----------------- engine/responder.go | 1 - 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/agents/hapool_it_test.go b/agents/hapool_it_test.go index 81908d728..aa4fac2e5 100644 --- a/agents/hapool_it_test.go +++ b/agents/hapool_it_test.go @@ -18,6 +18,7 @@ along with this program. If not, see package agents +/* import ( "os/exec" "path" @@ -120,3 +121,4 @@ func TestHaPoolCdrs(t *testing.T) { func TestHaPoolStopEngine(t *testing.T) { TestDmtAgentStopEngine(t) } +*/ diff --git a/config/config_json_test.go b/config/config_json_test.go index c6de54cdd..0a21b7ee9 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -50,7 +50,7 @@ func TestDfGeneralJsonCfg(t *testing.T) { Default_timezone: utils.StringPointer("Local"), Connect_attempts: utils.IntPointer(3), Reconnects: utils.IntPointer(-1), - Response_cache_ttl: utils.StringPointer("3s"), + Response_cache_ttl: utils.StringPointer("0s"), Internal_ttl: utils.StringPointer("2m")} if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil { t.Error(err) diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index b585790bc..b8fd14d00 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -18,7 +18,7 @@ // "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> // "connect_attempts": 3, // initial server connect attempts // "reconnects": -1, // number of retries in case of connection lost -// "response_cache_ttl": "3s", // the life span of a cached response +// "response_cache_ttl": "0s", // the life span of a cached response // "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up // }, @@ -93,7 +93,7 @@ // "store_cdrs": true, // store cdrs in storDb // "rals_conns": [ // {"address": "*internal"} // address where to reach the Rater for cost calculation, empty to disable functionality: <""|*internal|x.y.z.y:1234> -// ], +// ], // "pubsubs_conns": [], // address where to reach the pubusb service, empty to disable pubsub functionality: <""|*internal|x.y.z.y:1234> // "users_conns": [], // address where to reach the user service, empty to disable user profile functionality: <""|*internal|x.y.z.y:1234> // "aliases_conns": [], // address where to reach the aliases service, empty to disable aliases functionality: <""|*internal|x.y.z.y:1234> @@ -114,7 +114,7 @@ // "field_separator": ",", // "data_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from KBytes to Bytes) // "sms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems) -// "mms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from MMS unit to call duration in some billing systems) +// "mms_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from MMS unit to call duration in some billing systems) // "generic_usage_multiply_factor": 1, // multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems) // "cost_multiply_factor": 1, // multiply cost before export, eg: add VAT // "cost_rounding_decimals": -1, // rounding decimals for Cost values. -1 to disable rounding @@ -204,12 +204,12 @@ // "sm_freeswitch": { // "enabled": false, // starts SessionManager service: -// "rals_conns": [ -// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> -// ], +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], // "cdrs_conns": [ -// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> -// ], +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "extra_fields": [], // extra fields to store in auth/CDRs when creating them // "debit_interval": "10s", // interval to perform debits on. @@ -230,12 +230,12 @@ // "sm_kamailio": { // "enabled": false, // starts SessionManager service: -// "rals_conns": [ -// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> -// ], +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], // "cdrs_conns": [ -// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> -// ], +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], // "create_cdr": false, // create CDR out of events and sends them to CDRS component // "debit_interval": "10s", // interval to perform debits on. // "min_call_duration": "0s", // only authorize calls with allowed duration higher than this @@ -249,12 +249,12 @@ // "sm_opensips": { // "enabled": false, // starts SessionManager service: // "listen_udp": "127.0.0.1:2020", // address where to listen for datagram events coming from OpenSIPS -// "rals_conns": [ -// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> -// ], +// "rals_conns": [ +// {"address": "*internal"} // address where to reach the Rater <""|*internal|127.0.0.1:2013> +// ], // "cdrs_conns": [ -// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> -// ], +// {"address": "*internal"} // address where to reach CDR Server, empty to disable CDR capturing <*internal|x.y.z.y:1234> +// ], // "reconnects": 5, // number of reconnects if connection is lost // "create_cdr": false, // create CDR out of events and sends it to CDRS component // "debit_interval": "10s", // interval to perform debits on. diff --git a/engine/responder.go b/engine/responder.go index f396ba032..8c6009db6 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -194,7 +194,6 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) } func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err error) { - utils.Logger.Debug(fmt.Sprintf("Responder.RefundIncrements, arg: %+v", arg)) if item, err := rs.getCache().Get(utils.REFUND_INCR_CACHE_PREFIX + arg.CgrID + arg.RunID); err == nil && item != nil { *reply = *(item.Value.(*float64)) return item.Err From e1d8d964028cbe5c47dd8bb68d79f7ef81005e44 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 26 Apr 2016 18:04:03 +0300 Subject: [PATCH 193/227] updated dependencies version --- glide.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/glide.lock b/glide.lock index 95ea5db91..69a1eef99 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 5c488630d1d32687b7a3c3b22c47ceaf7eb3cffb764799706728a6accbcd3ff5 -updated: 2016-04-11T11:24:07.55158903+03:00 +updated: 2016-04-26T18:00:43.651987521+03:00 imports: - name: github.com/cenkalti/hub version: b864404b5f990410d56858a1b0a6fac23a85443f @@ -12,7 +12,7 @@ imports: - name: github.com/cgrates/osipsdagram version: 3d6beed663452471dec3ca194137a30d379d9e8f - name: github.com/cgrates/rpcclient - version: 64c9ff2b700d5a97a969a59d648bd75d09274143 + version: 9a6185f8a2093ce10f1a08242b0d757f24795800 - name: github.com/DisposaBoy/JsonConfigReader version: 33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4 - name: github.com/fiorix/go-diameter @@ -40,7 +40,7 @@ imports: subpackages: - oid - name: github.com/mediocregopher/radix.v2 - version: 7bdaf7c45ec452ca691ab20535471e24460f0876 + version: 74e50e64194d2d2f4836212451c28b127f9d7fa1 subpackages: - pool - redis @@ -51,20 +51,20 @@ imports: subpackages: - codec - name: golang.org/x/net - version: e45385e9b226f570b1f086bf287b25d3d4117776 + version: b797637b7aeeed133049c7281bfa31dcc9ca42d6 subpackages: - websocket - context +- name: golang.org/x/sys + version: f64b50fbea64174967a8882830d621a18ee1548e + subpackages: + - unix - name: gopkg.in/fsnotify.v1 - version: 875cf421b32f8f1b31bd43776297876d01542279 + version: 30411dbcefb7a1da7e84f75530ad3abe4011b4f8 - name: gopkg.in/mgo.v2 version: b6e2fa371e64216a45e61072a96d4e3859f169da subpackages: - bson -- name: gopkg.in/tomb.v2 - version: 14b3d72120e8d10ea6e6b7f87f7175734b1faab8 -devImports: [] -======= - internal/sasl - internal/scram devImports: [] From b16cc28389cfccbdbbbedb3fef17dd13d4c863b0 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 21 Apr 2016 20:35:48 +0300 Subject: [PATCH 194/227] started value formula --- apier/v1/accounts.go | 10 ++-- apier/v1/apier.go | 8 +-- cmd/cgr-engine/cgr-engine.go | 1 + cmd/cgr-engine/rater.go | 3 +- cmd/cgr-loader/migrator_rc8.go | 2 +- cmd/cgr-loader/migrator_rc8int.go | 2 +- engine/account.go | 2 +- engine/account_test.go | 8 +-- engine/action.go | 81 +++++++++++-------------------- engine/actions_test.go | 66 ++++++++++++------------- engine/balance_filter.go | 59 +++++++++++++++++++--- engine/calldesc_test.go | 6 +-- engine/loader_csv_test.go | 10 ++-- engine/tp_reader.go | 12 ++--- utils/rpc_params.go | 6 +-- 15 files changed, 149 insertions(+), 127 deletions(-) diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index add9e9b97..39173d652 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -426,7 +426,7 @@ func (self *ApierV1) modifyBalance(aType string, attr *AttrAddBalance, reply *st Uuid: attr.BalanceUuid, ID: attr.BalanceId, Type: utils.StringPointer(attr.BalanceType), - Value: utils.Float64Pointer(attr.Value), + Value: &engine.ValueFormula{Static: attr.Value}, ExpirationDate: expTime, RatingSubject: attr.RatingSubject, Weight: attr.Weight, @@ -514,7 +514,6 @@ func (self *ApierV1) SetBalance(attr *AttrSetBalance, reply *string) error { Uuid: attr.BalanceUUID, ID: attr.BalanceID, Type: utils.StringPointer(attr.BalanceType), - Value: attr.Value, ExpirationDate: expTime, RatingSubject: attr.RatingSubject, Weight: attr.Weight, @@ -522,6 +521,9 @@ func (self *ApierV1) SetBalance(attr *AttrSetBalance, reply *string) error { Disabled: attr.Disabled, }, } + if attr.Value != nil { + a.Balance.Value = &engine.ValueFormula{Static: *attr.Value} + } if attr.Directions != nil { a.Balance.Directions = utils.StringMapPointer(utils.ParseStringMap(*attr.Directions)) } @@ -572,7 +574,6 @@ func (self *ApierV1) RemoveBalances(attr *AttrSetBalance, reply *string) error { Uuid: attr.BalanceUUID, ID: attr.BalanceID, Type: utils.StringPointer(attr.BalanceType), - Value: attr.Value, ExpirationDate: expTime, RatingSubject: attr.RatingSubject, Weight: attr.Weight, @@ -580,6 +581,9 @@ func (self *ApierV1) RemoveBalances(attr *AttrSetBalance, reply *string) error { Disabled: attr.Disabled, }, } + if attr.Value != nil { + a.Balance.Value = &engine.ValueFormula{Static: *attr.Value} + } if attr.Directions != nil { a.Balance.Directions = utils.StringMapPointer(utils.ParseStringMap(*attr.Directions)) } diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 95fba7081..24a5d3a49 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -504,10 +504,10 @@ func (self *ApierV1) SetActions(attrs utils.AttrSetActions, reply *string) error } storeActions := make(engine.Actions, len(attrs.Actions)) for idx, apiAct := range attrs.Actions { - var units *float64 + var vf *engine.ValueFormula if apiAct.Units != "" { - if x, err := strconv.ParseFloat(apiAct.Units, 64); err == nil { - units = &x + if x, err := engine.ParseBalanceFilterValue(apiAct.Units); err == nil { + vf = x } else { return err } @@ -533,7 +533,7 @@ func (self *ApierV1) SetActions(attrs utils.AttrSetActions, reply *string) error Uuid: utils.StringPointer(utils.GenUUID()), ID: utils.StringPointer(apiAct.BalanceId), Type: utils.StringPointer(apiAct.BalanceType), - Value: units, + Value: vf, Weight: weight, Directions: utils.StringMapPointer(utils.ParseStringMap(apiAct.Directions)), DestinationIDs: utils.StringMapPointer(utils.ParseStringMap(apiAct.DestinationIds)), diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 1ae8ab419..de9a51586 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -22,6 +22,7 @@ import ( "flag" "fmt" "log" + _ "net/http/pprof" "os" "runtime" "runtime/pprof" diff --git a/cmd/cgr-engine/rater.go b/cmd/cgr-engine/rater.go index 0c3f90021..8fe1ee27d 100644 --- a/cmd/cgr-engine/rater.go +++ b/cmd/cgr-engine/rater.go @@ -211,7 +211,6 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC utils.RegisterRpcParams("PubSubV1", &engine.PubSub{}) utils.RegisterRpcParams("AliasesV1", &engine.AliasHandler{}) utils.RegisterRpcParams("UsersV1", &engine.UserMap{}) - utils.RegisterRpcParams("UsersV1", &engine.UserMap{}) utils.RegisterRpcParams("", &v1.CdrsV1{}) utils.RegisterRpcParams("", &v2.CdrsV2{}) utils.RegisterRpcParams("", &v1.SessionManagerV1{}) @@ -219,6 +218,6 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC utils.RegisterRpcParams("", responder) utils.RegisterRpcParams("", apierRpcV1) utils.RegisterRpcParams("", apierRpcV2) - + utils.GetRpcParams("") internalRaterChan <- responder // Rater done } diff --git a/cmd/cgr-loader/migrator_rc8.go b/cmd/cgr-loader/migrator_rc8.go index 9d085d2f6..f28e04766 100644 --- a/cmd/cgr-loader/migrator_rc8.go +++ b/cmd/cgr-loader/migrator_rc8.go @@ -482,7 +482,7 @@ func (mig MigratorRC8) migrateActions() error { bf.Type = utils.StringPointer(oldAc.BalanceType) } if oldAc.Balance.Value != 0 { - bf.Value = utils.Float64Pointer(oldAc.Balance.Value) + bf.Value = &engine.ValueFormula{Static: oldAc.Balance.Value} } if oldAc.Balance.RatingSubject != "" { bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) diff --git a/cmd/cgr-loader/migrator_rc8int.go b/cmd/cgr-loader/migrator_rc8int.go index 41f55afb1..3d335c914 100644 --- a/cmd/cgr-loader/migrator_rc8int.go +++ b/cmd/cgr-loader/migrator_rc8int.go @@ -364,7 +364,7 @@ func (mig MigratorRC8) migrateActionsInt() error { bf.Type = utils.StringPointer(oldAc.BalanceType) } if oldAc.Balance.Value != 0 { - bf.Value = utils.Float64Pointer(oldAc.Balance.Value) + bf.Value = &engine.ValueFormula{Static: oldAc.Balance.Value} } if oldAc.Balance.RatingSubject != "" { bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) diff --git a/engine/account.go b/engine/account.go index b49ef4edd..1bd3ac42e 100644 --- a/engine/account.go +++ b/engine/account.go @@ -127,7 +127,7 @@ func (acc *Account) setBalanceAction(a *Action) error { if a.Balance.ID != nil && *a.Balance.ID == utils.META_DEFAULT { balance.ID = utils.META_DEFAULT if a.Balance.Value != nil { - balance.Value = *a.Balance.Value + balance.Value = a.Balance.GetValue() } } else { a.Balance.ModifyBalance(balance) diff --git a/engine/account_test.go b/engine/account_test.go index 90be903d2..42dd1cb4c 100644 --- a/engine/account_test.go +++ b/engine/account_test.go @@ -856,7 +856,7 @@ func TestAccountdebitBalanceExists(t *testing.T) { BalanceMap: map[string]Balances{utils.SMS: Balances{&Balance{Value: 14}}, utils.DATA: Balances{&Balance{Value: 1024}}, utils.VOICE: Balances{&Balance{Value: 15, Weight: 20, DestinationIDs: utils.StringMap{"NAT": true}, Directions: utils.NewStringMap(utils.OUT)}, &Balance{Weight: 10, DestinationIDs: utils.StringMap{"RET": true}}}}, } newMb := &BalanceFilter{ - Value: utils.Float64Pointer(-10), + Value: &ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), @@ -883,19 +883,19 @@ func TestAccountAddMinuteNil(t *testing.T) { func TestAccountAddMinutBucketEmpty(t *testing.T) { mb1 := &BalanceFilter{ - Value: utils.Float64Pointer(-10), + Value: &ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), } mb2 := &BalanceFilter{ - Value: utils.Float64Pointer(-10), + Value: &ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), } mb3 := &BalanceFilter{ - Value: utils.Float64Pointer(-10), + Value: &ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"OTHER": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), diff --git a/engine/action.go b/engine/action.go index 903ce7d90..5a3c8e754 100644 --- a/engine/action.go +++ b/engine/action.go @@ -93,59 +93,36 @@ func (a *Action) Clone() *Action { type actionTypeFunc func(*Account, *StatsQueueTriggered, *Action, Actions) error func getActionFunc(typ string) (actionTypeFunc, bool) { - switch typ { - case LOG: - return logAction, true - case CDRLOG: - return cdrLogAction, true - case RESET_TRIGGERS: - return resetTriggersAction, true - case SET_RECURRENT: - return setRecurrentAction, true - case UNSET_RECURRENT: - return unsetRecurrentAction, true - case ALLOW_NEGATIVE: - return allowNegativeAction, true - case DENY_NEGATIVE: - return denyNegativeAction, true - case RESET_ACCOUNT: - return resetAccountAction, true - case TOPUP_RESET: - return topupResetAction, true - case TOPUP: - return topupAction, true - case DEBIT_RESET: - return debitResetAction, true - case DEBIT: - return debitAction, true - case RESET_COUNTERS: - return resetCountersAction, true - case ENABLE_ACCOUNT: - return enableUserAction, true - case DISABLE_ACCOUNT: - return disableUserAction, true - //case ENABLE_DISABLE_BALANCE: - // return enableDisableBalanceAction, true - case CALL_URL: - return callUrl, true - case CALL_URL_ASYNC: - return callUrlAsync, true - case MAIL_ASYNC: - return mailAsync, true - case SET_DDESTINATIONS: - return setddestinations, true - case REMOVE_ACCOUNT: - return removeAccountAction, true - case REMOVE_BALANCE: - return removeBalanceAction, true - case SET_BALANCE: - return setBalanceAction, true - case TRANSFER_MONETARY_DEFAULT: - return transferMonetaryDefaultAction, true - case CGR_RPC: - return cgrRPCAction, true + actionFuncMap := map[string]actionTypeFunc{ + LOG: logAction, + CDRLOG: cdrLogAction, + RESET_TRIGGERS: resetTriggersAction, + SET_RECURRENT: setRecurrentAction, + UNSET_RECURRENT: unsetRecurrentAction, + ALLOW_NEGATIVE: allowNegativeAction, + DENY_NEGATIVE: denyNegativeAction, + RESET_ACCOUNT: resetAccountAction, + TOPUP_RESET: topupResetAction, + TOPUP: topupAction, + DEBIT_RESET: debitResetAction, + DEBIT: debitAction, + RESET_COUNTERS: resetCountersAction, + ENABLE_ACCOUNT: enableUserAction, + DISABLE_ACCOUNT: disableUserAction, + //case ENABLE_DISABLE_BALANCE: + // return enableDisableBalanceAction, true + CALL_URL: callUrl, + CALL_URL_ASYNC: callUrlAsync, + MAIL_ASYNC: mailAsync, + SET_DDESTINATIONS: setddestinations, + REMOVE_ACCOUNT: removeAccountAction, + REMOVE_BALANCE: removeBalanceAction, + SET_BALANCE: setBalanceAction, + TRANSFER_MONETARY_DEFAULT: transferMonetaryDefaultAction, + CGR_RPC: cgrRPCAction, } - return nil, false + f, exists := actionFuncMap[typ] + return f, exists } func logAction(ub *Account, sq *StatsQueueTriggered, a *Action, acs Actions) (err error) { diff --git a/engine/actions_test.go b/engine/actions_test.go index fb1481ef4..d37695055 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -413,7 +413,7 @@ func TestActionPlanLogFunction(t *testing.T) { ActionType: "*log", Balance: &BalanceFilter{ Type: utils.StringPointer("test"), - Value: utils.Float64Pointer(1.1), + Value: &ValueFormula{Static: 1.1}, }, } at := &ActionTiming{ @@ -430,7 +430,7 @@ func TestActionPlanFunctionNotAvailable(t *testing.T) { ActionType: "VALID_FUNCTION_TYPE", Balance: &BalanceFilter{ Type: utils.StringPointer("test"), - Value: utils.Float64Pointer(1.1), + Value: &ValueFormula{Static: 1.1}, }, } at := &ActionTiming{ @@ -659,7 +659,7 @@ func TestActionTriggerMatchAll(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), RatingSubject: utils.StringPointer("test1"), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: utils.Float64Pointer(2), + Value: &ValueFormula{Static: 2}, Weight: utils.Float64Pointer(1.0), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), SharedGroups: utils.StringMapPointer(utils.NewStringMap("test2")), @@ -669,7 +669,7 @@ func TestActionTriggerMatchAll(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), RatingSubject: utils.StringPointer("test1"), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: utils.Float64Pointer(2), + Value: &ValueFormula{Static: 2}, Weight: utils.Float64Pointer(1.0), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), SharedGroups: utils.StringMapPointer(utils.NewStringMap("test2")), @@ -799,7 +799,7 @@ func TestActionTopupResetCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 10 || @@ -818,7 +818,7 @@ func TestActionTopupValueFactor(t *testing.T) { a := &Action{ Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: utils.Float64Pointer(10), + Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), }, ExtraParameters: `{"*monetary":2.0}`, @@ -839,7 +839,7 @@ func TestActionTopupResetCreditId(t *testing.T) { }, }, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), ID: utils.StringPointer("TEST_B"), Value: utils.Float64Pointer(10), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), ID: utils.StringPointer("TEST_B"), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 110 || @@ -858,7 +858,7 @@ func TestActionTopupResetCreditNoId(t *testing.T) { }, }, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 20 || @@ -876,7 +876,7 @@ func TestActionTopupResetMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: utils.Float64Pointer(5), Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE].GetTotalValue() != 5 || @@ -895,7 +895,7 @@ func TestActionTopupCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 110 || @@ -913,7 +913,7 @@ func TestActionTopupMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: utils.Float64Pointer(5), Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE].GetTotalValue() != 15 || @@ -932,7 +932,7 @@ func TestActionDebitCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} debitAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 90 || @@ -950,7 +950,7 @@ func TestActionDebitMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: utils.Float64Pointer(5), Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} debitAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE][0].GetValue() != 5 || @@ -1119,7 +1119,7 @@ func TestActionPlanLogging(t *testing.T) { } func TestActionMakeNegative(t *testing.T) { - a := &Action{Balance: &BalanceFilter{Value: utils.Float64Pointer(10)}} + a := &Action{Balance: &BalanceFilter{Value: &ValueFormula{Static: 10}}} genericMakeNegative(a) if a.Balance.GetValue() > 0 { t.Error("Failed to make negative: ", a) @@ -1153,7 +1153,7 @@ func TestTopupAction(t *testing.T) { initialUb, _ := accountingStorage.GetAccount("vdf:minu") a := &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, } at := &ActionTiming{ @@ -1174,7 +1174,7 @@ func TestTopupActionLoaded(t *testing.T) { initialUb, _ := accountingStorage.GetAccount("vdf:minitsboy") a := &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, } at := &ActionTiming{ @@ -1201,7 +1201,7 @@ func TestActionCdrlogEmpty(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1223,11 +1223,11 @@ func TestActionCdrlogWithParams(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, &Action{ ActionType: DEBIT_RESET, - Balance: &BalanceFilter{Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1250,11 +1250,11 @@ func TestActionCdrLogParamsWithOverload(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, &Action{ ActionType: DEBIT_RESET, - Balance: &BalanceFilter{Value: utils.Float64Pointer(25), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1335,11 +1335,11 @@ func TestActionTransactionFuncType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, }, &Action{ ActionType: "VALID_FUNCTION_TYPE", - Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer("test")}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer("test")}, }, }, } @@ -1371,7 +1371,7 @@ func TestActionTransactionBalanceType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, }, &Action{ ActionType: TOPUP, @@ -1407,7 +1407,7 @@ func TestActionTransactionBalanceNotType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: utils.Float64Pointer(1.1), Type: utils.StringPointer(utils.VOICE)}, + Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.VOICE)}, }, &Action{ ActionType: TOPUP, @@ -1445,14 +1445,14 @@ func TestActionWithExpireWithoutExpire(t *testing.T) { ActionType: TOPUP, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.VOICE), - Value: utils.Float64Pointer(15), + Value: &ValueFormula{Static: 15}, }, }, &Action{ ActionType: TOPUP, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.VOICE), - Value: utils.Float64Pointer(30), + Value: &ValueFormula{Static: 30}, ExpirationDate: utils.TimePointer(time.Date(2025, time.November, 11, 22, 39, 0, 0, time.UTC)), }, }, @@ -1672,7 +1672,7 @@ func TestActionConditionalTopup(t *testing.T) { Filter: `{"Type":"*monetary","Value":1,"Weight":10}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: utils.Float64Pointer(11), + Value: &ValueFormula{Static: 11}, Weight: utils.Float64Pointer(30), }, } @@ -1736,7 +1736,7 @@ func TestActionConditionalTopupNoMatch(t *testing.T) { Filter: `{"Type":"*monetary","Value":2,"Weight":10}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: utils.Float64Pointer(11), + Value: &ValueFormula{Static: 11}, Weight: utils.Float64Pointer(30), }, } @@ -1800,7 +1800,7 @@ func TestActionConditionalTopupExistingBalance(t *testing.T) { Filter: `{"Type":"*voice","Value":{"*gte":100}}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: utils.Float64Pointer(11), + Value: &ValueFormula{Static: 11}, Weight: utils.Float64Pointer(10), }, } @@ -2021,7 +2021,7 @@ func TestActionSetBalance(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("m2"), Type: utils.StringPointer(utils.MONETARY), - Value: utils.Float64Pointer(11), + Value: &ValueFormula{Static: 11}, Weight: utils.Float64Pointer(10), }, } @@ -2122,7 +2122,7 @@ func TestActionCdrlogBalanceValue(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("*default"), Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), - Value: utils.Float64Pointer(1.1), + Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY), }, }, @@ -2132,7 +2132,7 @@ func TestActionCdrlogBalanceValue(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("*default"), Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), - Value: utils.Float64Pointer(2.1), + Value: &ValueFormula{Static: 2.1}, Type: utils.StringPointer(utils.MONETARY), }, }, diff --git a/engine/balance_filter.go b/engine/balance_filter.go index f9e408d39..bc98c2bae 100644 --- a/engine/balance_filter.go +++ b/engine/balance_filter.go @@ -1,7 +1,10 @@ package engine import ( + "encoding/json" + "errors" "reflect" + "strconv" "time" "github.com/cgrates/cgrates/utils" @@ -11,7 +14,7 @@ type BalanceFilter struct { Uuid *string ID *string Type *string - Value *float64 + Value *ValueFormula Directions *utils.StringMap ExpirationDate *time.Time Weight *float64 @@ -58,7 +61,7 @@ func (bf *BalanceFilter) Clone() *BalanceFilter { *result.ID = *bf.ID } if bf.Value != nil { - result.Value = new(float64) + result.Value = new(ValueFormula) *result.Value = *bf.Value } if bf.RatingSubject != nil { @@ -116,7 +119,7 @@ func (bf *BalanceFilter) LoadFromBalance(b *Balance) *BalanceFilter { bf.ID = &b.ID } if b.Value != 0 { - bf.Value = &b.Value + bf.Value.Static = b.Value } if !b.Directions.IsEmpty() { bf.Directions = &b.Directions @@ -173,14 +176,22 @@ func (bp *BalanceFilter) GetValue() float64 { if bp == nil || bp.Value == nil { return 0.0 } - return *bp.Value + if bp.Value.Method == "" { + return bp.Value.Static + } + // calculate using formula + formula, exists := valueFormulas[bp.Value.Method] + if !exists { + return 0.0 + } + return formula(bp.Value.Params) } func (bp *BalanceFilter) SetValue(v float64) { if bp.Value == nil { - bp.Value = new(float64) + bp.Value = new(ValueFormula) } - *bp.Value = v + bp.Value.Static = v } func (bp *BalanceFilter) GetUuid() string { @@ -292,7 +303,7 @@ func (bf *BalanceFilter) ModifyBalance(b *Balance) { b.Directions = *bf.Directions } if bf.Value != nil { - b.Value = *bf.Value + b.Value = bf.GetValue() } if bf.ExpirationDate != nil { b.ExpirationDate = *bf.ExpirationDate @@ -323,3 +334,37 @@ func (bf *BalanceFilter) ModifyBalance(b *Balance) { } b.SetDirty() // Mark the balance as dirty since we have modified and it should be checked by action triggers } + +//for computing a dynamic value for Value field +type ValueFormula struct { + Method string + Params map[string]interface{} + Static float64 +} + +func ParseBalanceFilterValue(val string) (*ValueFormula, error) { + u, err := strconv.ParseFloat(val, 64) + if err == nil { + return &ValueFormula{Static: u}, err + } + var vf ValueFormula + err = json.Unmarshal([]byte(val), &vf) + if err == nil { + return &vf, err + } + return nil, errors.New("Invalid value: " + val) +} + +type valueFormula func(map[string]interface{}) float64 + +const ( + PERIODIC = "*periodic" +) + +var valueFormulas = map[string]valueFormula{ + PERIODIC: periodicFormula, +} + +func periodicFormula(params map[string]interface{}) float64 { + return 0.0 +} diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index 607a97a23..d37b417b0 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -41,12 +41,12 @@ func init() { func populateDB() { ats := []*Action{ - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10)}}, - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), Value: utils.Float64Pointer(10), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT"))}}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}}}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), Value: &ValueFormula{Static: 10}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT"))}}, } ats1 := []*Action{ - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: utils.Float64Pointer(10)}, Weight: 10}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}}, Weight: 10}, &Action{ActionType: "*reset_account", Weight: 20}, } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index d26ebbcc6..851d474e9 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -840,7 +840,7 @@ func TestLoadActions(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), Uuid: as1[0].Balance.Uuid, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: utils.Float64Pointer(10), + Value: &ValueFormula{Static: 10}, Weight: utils.Float64Pointer(10), DestinationIDs: nil, TimingIDs: nil, @@ -860,7 +860,7 @@ func TestLoadActions(t *testing.T) { Type: utils.StringPointer(utils.VOICE), Uuid: as1[1].Balance.Uuid, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: utils.Float64Pointer(100), + Value: &ValueFormula{Static: 100}, Weight: utils.Float64Pointer(10), RatingSubject: utils.StringPointer("test"), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), @@ -873,7 +873,7 @@ func TestLoadActions(t *testing.T) { }, } if !reflect.DeepEqual(as1, expected) { - t.Errorf("Error loading action1: %+v", as1[0].Balance) + t.Errorf("Error loading action1: %s", utils.ToIJSON(as1)) } as2 := csvr.actions["SHARED"] expected = []*Action{ @@ -887,7 +887,7 @@ func TestLoadActions(t *testing.T) { Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), DestinationIDs: nil, Uuid: as2[0].Balance.Uuid, - Value: utils.Float64Pointer(100), + Value: &ValueFormula{Static: 100}, Weight: utils.Float64Pointer(10), SharedGroups: utils.StringMapPointer(utils.NewStringMap("SG1")), TimingIDs: nil, @@ -898,7 +898,7 @@ func TestLoadActions(t *testing.T) { }, } if !reflect.DeepEqual(as2, expected) { - t.Errorf("Error loading action: %+v", as2[0].Balance) + t.Errorf("Error loading action: %s", utils.ToIJSON(as2)) } as3 := csvr.actions["DEFEE"] expected = []*Action{ diff --git a/engine/tp_reader.go b/engine/tp_reader.go index e458e130e..6f85f3d89 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -531,11 +531,11 @@ func (tpr *TpReader) LoadActions() (err error) { } if tpact.Units != "" && tpact.Units != utils.ANY { - u, err := strconv.ParseFloat(tpact.Units, 64) + vf, err := ParseBalanceFilterValue(tpact.Units) if err != nil { return err } - acts[idx].Balance.Value = utils.Float64Pointer(u) + acts[idx].Balance.Value = vf } if tpact.BalanceWeight != "" && tpact.BalanceWeight != utils.ANY { @@ -1007,11 +1007,11 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *TpAccountAction) error } if tpact.Units != "" && tpact.Units != utils.ANY { - u, err := strconv.ParseFloat(tpact.Units, 64) + vf, err := ParseBalanceFilterValue(tpact.Units) if err != nil { return err } - acts[idx].Balance.Value = utils.Float64Pointer(u) + acts[idx].Balance.Value = vf } if tpact.BalanceWeight != "" && tpact.BalanceWeight != utils.ANY { @@ -1355,11 +1355,11 @@ func (tpr *TpReader) LoadCdrStatsFiltered(tag string, save bool) (err error) { } if tpact.Units != "" && tpact.Units != utils.ANY { - u, err := strconv.ParseFloat(tpact.Units, 64) + vf, err := ParseBalanceFilterValue(tpact.Units) if err != nil { return err } - acts[idx].Balance.Value = utils.Float64Pointer(u) + acts[idx].Balance.Value = vf } if tpact.BalanceWeight != "" && tpact.BalanceWeight != utils.ANY { diff --git a/utils/rpc_params.go b/utils/rpc_params.go index c48e84085..3e5f3f166 100644 --- a/utils/rpc_params.go +++ b/utils/rpc_params.go @@ -1,8 +1,6 @@ package utils -import ( - "reflect" -) +import "reflect" var rpcParamsMap map[string]*RpcParams @@ -39,12 +37,10 @@ func RegisterRpcParams(name string, obj interface{}) { OutParam: reflect.New(out).Interface(), } } - } } func GetRpcParams(method string) (*RpcParams, error) { - Logger.Info("REGISTERED: " + ToJSON(rpcParamsMap)) x, found := rpcParamsMap[method] if !found { return nil, ErrNotFound From 206d33242faa3682c9c2ef64eb786a8b49f09ede Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 22 Apr 2016 13:00:15 +0300 Subject: [PATCH 195/227] updated go version in docker --- data/docker/devel/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/docker/devel/Dockerfile b/data/docker/devel/Dockerfile index 5e4917784..5bf440c97 100644 --- a/data/docker/devel/Dockerfile +++ b/data/docker/devel/Dockerfile @@ -26,7 +26,7 @@ COPY mongod.conf /etc/mongod.conf RUN useradd -c CGRateS -d /var/run/cgrates -s /bin/false -r cgrates # install golang -RUN wget -qO- https://storage.googleapis.com/golang/go1.6.1.linux-amd64.tar.gz | tar xzf - -C /root/ +RUN wget -qO- https://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz | tar xzf - -C /root/ #install glide RUN GOROOT=/root/go GOPATH=/root/code /root/go/bin/go get github.com/Masterminds/glide From 768cdcc43a129d24685aa3ff577d386c7d0688c6 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 22 Apr 2016 15:07:05 +0300 Subject: [PATCH 196/227] value formula passing test --- apier/v1/accounts.go | 6 +-- apier/v1/apier.go | 4 +- cmd/cgr-loader/migrator_rc8.go | 2 +- cmd/cgr-loader/migrator_rc8int.go | 2 +- engine/account.go | 1 + engine/account_test.go | 8 +-- engine/action_plan.go | 1 - engine/actions_test.go | 82 +++++++++++++++++------------ engine/balance_filter.go | 45 ++-------------- engine/calldesc_test.go | 6 +-- engine/loader_csv_test.go | 12 +++-- engine/storage_test.go | 2 +- engine/tp_reader.go | 6 +-- utils/dateseries.go | 10 ++++ utils/dateseries_test.go | 27 ++++++++++ utils/value_formula.go | 85 +++++++++++++++++++++++++++++++ utils/value_formula_test.go | 39 ++++++++++++++ 17 files changed, 240 insertions(+), 98 deletions(-) create mode 100644 utils/value_formula.go create mode 100644 utils/value_formula_test.go diff --git a/apier/v1/accounts.go b/apier/v1/accounts.go index 39173d652..620b64c48 100644 --- a/apier/v1/accounts.go +++ b/apier/v1/accounts.go @@ -426,7 +426,7 @@ func (self *ApierV1) modifyBalance(aType string, attr *AttrAddBalance, reply *st Uuid: attr.BalanceUuid, ID: attr.BalanceId, Type: utils.StringPointer(attr.BalanceType), - Value: &engine.ValueFormula{Static: attr.Value}, + Value: &utils.ValueFormula{Static: attr.Value}, ExpirationDate: expTime, RatingSubject: attr.RatingSubject, Weight: attr.Weight, @@ -522,7 +522,7 @@ func (self *ApierV1) SetBalance(attr *AttrSetBalance, reply *string) error { }, } if attr.Value != nil { - a.Balance.Value = &engine.ValueFormula{Static: *attr.Value} + a.Balance.Value = &utils.ValueFormula{Static: *attr.Value} } if attr.Directions != nil { a.Balance.Directions = utils.StringMapPointer(utils.ParseStringMap(*attr.Directions)) @@ -582,7 +582,7 @@ func (self *ApierV1) RemoveBalances(attr *AttrSetBalance, reply *string) error { }, } if attr.Value != nil { - a.Balance.Value = &engine.ValueFormula{Static: *attr.Value} + a.Balance.Value = &utils.ValueFormula{Static: *attr.Value} } if attr.Directions != nil { a.Balance.Directions = utils.StringMapPointer(utils.ParseStringMap(*attr.Directions)) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 24a5d3a49..cf3f86389 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -504,9 +504,9 @@ func (self *ApierV1) SetActions(attrs utils.AttrSetActions, reply *string) error } storeActions := make(engine.Actions, len(attrs.Actions)) for idx, apiAct := range attrs.Actions { - var vf *engine.ValueFormula + var vf *utils.ValueFormula if apiAct.Units != "" { - if x, err := engine.ParseBalanceFilterValue(apiAct.Units); err == nil { + if x, err := utils.ParseBalanceFilterValue(apiAct.Units); err == nil { vf = x } else { return err diff --git a/cmd/cgr-loader/migrator_rc8.go b/cmd/cgr-loader/migrator_rc8.go index f28e04766..b5574a1d5 100644 --- a/cmd/cgr-loader/migrator_rc8.go +++ b/cmd/cgr-loader/migrator_rc8.go @@ -482,7 +482,7 @@ func (mig MigratorRC8) migrateActions() error { bf.Type = utils.StringPointer(oldAc.BalanceType) } if oldAc.Balance.Value != 0 { - bf.Value = &engine.ValueFormula{Static: oldAc.Balance.Value} + bf.Value = &utils.ValueFormula{Static: oldAc.Balance.Value} } if oldAc.Balance.RatingSubject != "" { bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) diff --git a/cmd/cgr-loader/migrator_rc8int.go b/cmd/cgr-loader/migrator_rc8int.go index 3d335c914..dd04b91ec 100644 --- a/cmd/cgr-loader/migrator_rc8int.go +++ b/cmd/cgr-loader/migrator_rc8int.go @@ -364,7 +364,7 @@ func (mig MigratorRC8) migrateActionsInt() error { bf.Type = utils.StringPointer(oldAc.BalanceType) } if oldAc.Balance.Value != 0 { - bf.Value = &engine.ValueFormula{Static: oldAc.Balance.Value} + bf.Value = &utils.ValueFormula{Static: oldAc.Balance.Value} } if oldAc.Balance.RatingSubject != "" { bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) diff --git a/engine/account.go b/engine/account.go index 1bd3ac42e..d70a974d3 100644 --- a/engine/account.go +++ b/engine/account.go @@ -170,6 +170,7 @@ func (ub *Account) debitBalanceAction(a *Action, reset bool) error { return errors.New("nil action") } bClone := a.Balance.CreateBalance() + //log.Print("Bclone: ", utils.ToJSON(a.Balance)) if bClone == nil { return errors.New("nil balance") } diff --git a/engine/account_test.go b/engine/account_test.go index 42dd1cb4c..7abd8d6c1 100644 --- a/engine/account_test.go +++ b/engine/account_test.go @@ -856,7 +856,7 @@ func TestAccountdebitBalanceExists(t *testing.T) { BalanceMap: map[string]Balances{utils.SMS: Balances{&Balance{Value: 14}}, utils.DATA: Balances{&Balance{Value: 1024}}, utils.VOICE: Balances{&Balance{Value: 15, Weight: 20, DestinationIDs: utils.StringMap{"NAT": true}, Directions: utils.NewStringMap(utils.OUT)}, &Balance{Weight: 10, DestinationIDs: utils.StringMap{"RET": true}}}}, } newMb := &BalanceFilter{ - Value: &ValueFormula{Static: -10}, + Value: &utils.ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), @@ -883,19 +883,19 @@ func TestAccountAddMinuteNil(t *testing.T) { func TestAccountAddMinutBucketEmpty(t *testing.T) { mb1 := &BalanceFilter{ - Value: &ValueFormula{Static: -10}, + Value: &utils.ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), } mb2 := &BalanceFilter{ - Value: &ValueFormula{Static: -10}, + Value: &utils.ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"NAT": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), } mb3 := &BalanceFilter{ - Value: &ValueFormula{Static: -10}, + Value: &utils.ValueFormula{Static: -10}, Type: utils.StringPointer(utils.VOICE), DestinationIDs: utils.StringMapPointer(utils.StringMap{"OTHER": true}), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), diff --git a/engine/action_plan.go b/engine/action_plan.go index 79095227c..563ea9413 100644 --- a/engine/action_plan.go +++ b/engine/action_plan.go @@ -291,7 +291,6 @@ func (at *ActionTiming) Execute() (err error) { transactionFailed := false removeAccountActionFound := false for _, a := range aac { - //log.Print("A: ", utils.ToJSON(a)) // check action filter if len(a.Filter) > 0 { matched, err := acc.matchActionFilter(a.Filter) diff --git a/engine/actions_test.go b/engine/actions_test.go index d37695055..ebfd7b86f 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -413,7 +413,7 @@ func TestActionPlanLogFunction(t *testing.T) { ActionType: "*log", Balance: &BalanceFilter{ Type: utils.StringPointer("test"), - Value: &ValueFormula{Static: 1.1}, + Value: &utils.ValueFormula{Static: 1.1}, }, } at := &ActionTiming{ @@ -430,7 +430,7 @@ func TestActionPlanFunctionNotAvailable(t *testing.T) { ActionType: "VALID_FUNCTION_TYPE", Balance: &BalanceFilter{ Type: utils.StringPointer("test"), - Value: &ValueFormula{Static: 1.1}, + Value: &utils.ValueFormula{Static: 1.1}, }, } at := &ActionTiming{ @@ -659,7 +659,7 @@ func TestActionTriggerMatchAll(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), RatingSubject: utils.StringPointer("test1"), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: &ValueFormula{Static: 2}, + Value: &utils.ValueFormula{Static: 2}, Weight: utils.Float64Pointer(1.0), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), SharedGroups: utils.StringMapPointer(utils.NewStringMap("test2")), @@ -669,7 +669,7 @@ func TestActionTriggerMatchAll(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), RatingSubject: utils.StringPointer("test1"), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: &ValueFormula{Static: 2}, + Value: &utils.ValueFormula{Static: 2}, Weight: utils.Float64Pointer(1.0), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), SharedGroups: utils.StringMapPointer(utils.NewStringMap("test2")), @@ -799,7 +799,7 @@ func TestActionTopupResetCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 10 || @@ -818,7 +818,7 @@ func TestActionTopupValueFactor(t *testing.T) { a := &Action{ Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: &ValueFormula{Static: 10}, + Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), }, ExtraParameters: `{"*monetary":2.0}`, @@ -839,7 +839,7 @@ func TestActionTopupResetCreditId(t *testing.T) { }, }, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), ID: utils.StringPointer("TEST_B"), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), ID: utils.StringPointer("TEST_B"), Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 110 || @@ -858,7 +858,7 @@ func TestActionTopupResetCreditNoId(t *testing.T) { }, }, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 20 || @@ -876,7 +876,7 @@ func TestActionTopupResetMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &utils.ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupResetAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE].GetTotalValue() != 5 || @@ -895,7 +895,7 @@ func TestActionTopupCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 110 || @@ -913,7 +913,7 @@ func TestActionTopupMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &utils.ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} topupAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE].GetTotalValue() != 15 || @@ -932,7 +932,7 @@ func TestActionDebitCredit(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1, Filter: &BalanceFilter{Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} debitAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.MONETARY].GetTotalValue() != 90 || @@ -950,7 +950,7 @@ func TestActionDebitMinutes(t *testing.T) { UnitCounters: UnitCounters{utils.MONETARY: []*UnitCounter{&UnitCounter{Counters: CounterFilters{&CounterFilter{Value: 1}}}}}, ActionTriggers: ActionTriggers{&ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}, &ActionTrigger{Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY)}, ThresholdValue: 2, ActionsID: "TEST_ACTIONS", Executed: true}}, } - a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} + a := &Action{Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Value: &utils.ValueFormula{Static: 5}, Weight: utils.Float64Pointer(20), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT))}} debitAction(ub, nil, a, nil) if ub.AllowNegative || ub.BalanceMap[utils.VOICE][0].GetValue() != 5 || @@ -1119,7 +1119,7 @@ func TestActionPlanLogging(t *testing.T) { } func TestActionMakeNegative(t *testing.T) { - a := &Action{Balance: &BalanceFilter{Value: &ValueFormula{Static: 10}}} + a := &Action{Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 10}}} genericMakeNegative(a) if a.Balance.GetValue() > 0 { t.Error("Failed to make negative: ", a) @@ -1153,7 +1153,7 @@ func TestTopupAction(t *testing.T) { initialUb, _ := accountingStorage.GetAccount("vdf:minu") a := &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, } at := &ActionTiming{ @@ -1174,7 +1174,7 @@ func TestTopupActionLoaded(t *testing.T) { initialUb, _ := accountingStorage.GetAccount("vdf:minitsboy") a := &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), Weight: utils.Float64Pointer(20)}, } at := &ActionTiming{ @@ -1201,7 +1201,7 @@ func TestActionCdrlogEmpty(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1223,11 +1223,11 @@ func TestActionCdrlogWithParams(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, &Action{ ActionType: DEBIT_RESET, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1250,11 +1250,11 @@ func TestActionCdrLogParamsWithOverload(t *testing.T) { err := cdrLogAction(acnt, nil, cdrlog, Actions{ &Action{ ActionType: DEBIT, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, &Action{ ActionType: DEBIT_RESET, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 25}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("RET")), Weight: utils.Float64Pointer(20)}, }, }) if err != nil { @@ -1335,11 +1335,11 @@ func TestActionTransactionFuncType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, }, &Action{ ActionType: "VALID_FUNCTION_TYPE", - Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer("test")}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 1.1}, Type: utils.StringPointer("test")}, }, }, } @@ -1371,7 +1371,7 @@ func TestActionTransactionBalanceType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY)}, }, &Action{ ActionType: TOPUP, @@ -1407,7 +1407,7 @@ func TestActionTransactionBalanceNotType(t *testing.T) { actions: []*Action{ &Action{ ActionType: TOPUP, - Balance: &BalanceFilter{Value: &ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.VOICE)}, + Balance: &BalanceFilter{Value: &utils.ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.VOICE)}, }, &Action{ ActionType: TOPUP, @@ -1445,14 +1445,14 @@ func TestActionWithExpireWithoutExpire(t *testing.T) { ActionType: TOPUP, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.VOICE), - Value: &ValueFormula{Static: 15}, + Value: &utils.ValueFormula{Static: 15}, }, }, &Action{ ActionType: TOPUP, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.VOICE), - Value: &ValueFormula{Static: 30}, + Value: &utils.ValueFormula{Static: 30}, ExpirationDate: utils.TimePointer(time.Date(2025, time.November, 11, 22, 39, 0, 0, time.UTC)), }, }, @@ -1672,7 +1672,7 @@ func TestActionConditionalTopup(t *testing.T) { Filter: `{"Type":"*monetary","Value":1,"Weight":10}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: &ValueFormula{Static: 11}, + Value: &utils.ValueFormula{Static: 11}, Weight: utils.Float64Pointer(30), }, } @@ -1736,7 +1736,7 @@ func TestActionConditionalTopupNoMatch(t *testing.T) { Filter: `{"Type":"*monetary","Value":2,"Weight":10}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: &ValueFormula{Static: 11}, + Value: &utils.ValueFormula{Static: 11}, Weight: utils.Float64Pointer(30), }, } @@ -1800,7 +1800,7 @@ func TestActionConditionalTopupExistingBalance(t *testing.T) { Filter: `{"Type":"*voice","Value":{"*gte":100}}`, Balance: &BalanceFilter{ Type: utils.StringPointer(utils.MONETARY), - Value: &ValueFormula{Static: 11}, + Value: &utils.ValueFormula{Static: 11}, Weight: utils.Float64Pointer(10), }, } @@ -2021,7 +2021,7 @@ func TestActionSetBalance(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("m2"), Type: utils.StringPointer(utils.MONETARY), - Value: &ValueFormula{Static: 11}, + Value: &utils.ValueFormula{Static: 11}, Weight: utils.Float64Pointer(10), }, } @@ -2122,7 +2122,7 @@ func TestActionCdrlogBalanceValue(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("*default"), Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), - Value: &ValueFormula{Static: 1.1}, + Value: &utils.ValueFormula{Static: 1.1}, Type: utils.StringPointer(utils.MONETARY), }, }, @@ -2132,7 +2132,7 @@ func TestActionCdrlogBalanceValue(t *testing.T) { Balance: &BalanceFilter{ ID: utils.StringPointer("*default"), Uuid: utils.StringPointer("25a02c82-f09f-4c6e-bacf-8ed4b076475a"), - Value: &ValueFormula{Static: 2.1}, + Value: &utils.ValueFormula{Static: 2.1}, Type: utils.StringPointer(utils.MONETARY), }, }, @@ -2222,6 +2222,22 @@ func TestCgrRpcAction(t *testing.T) { } } +func TestValueFormulaDebit(t *testing.T) { + if _, err := accountingStorage.GetAccount("cgrates.org:vf"); err != nil { + t.Errorf("account to be removed not found: %v", err) + } + + at := &ActionTiming{ + accountIDs: utils.StringMap{"cgrates.org:vf": true}, + ActionsID: "VF", + } + at.Execute() + afterUb, err := accountingStorage.GetAccount("cgrates.org:vf") + if err != nil || afterUb.BalanceMap[utils.MONETARY].GetTotalValue() != -0.333334 { + t.Error("error debiting account: ", err, utils.ToIJSON(afterUb)) + } +} + /**************** Benchmarks ********************************/ func BenchmarkUUID(b *testing.B) { diff --git a/engine/balance_filter.go b/engine/balance_filter.go index bc98c2bae..78ff64140 100644 --- a/engine/balance_filter.go +++ b/engine/balance_filter.go @@ -1,10 +1,7 @@ package engine import ( - "encoding/json" - "errors" "reflect" - "strconv" "time" "github.com/cgrates/cgrates/utils" @@ -14,7 +11,7 @@ type BalanceFilter struct { Uuid *string ID *string Type *string - Value *ValueFormula + Value *utils.ValueFormula Directions *utils.StringMap ExpirationDate *time.Time Weight *float64 @@ -61,7 +58,7 @@ func (bf *BalanceFilter) Clone() *BalanceFilter { *result.ID = *bf.ID } if bf.Value != nil { - result.Value = new(ValueFormula) + result.Value = new(utils.ValueFormula) *result.Value = *bf.Value } if bf.RatingSubject != nil { @@ -180,7 +177,7 @@ func (bp *BalanceFilter) GetValue() float64 { return bp.Value.Static } // calculate using formula - formula, exists := valueFormulas[bp.Value.Method] + formula, exists := utils.ValueFormulas[bp.Value.Method] if !exists { return 0.0 } @@ -189,7 +186,7 @@ func (bp *BalanceFilter) GetValue() float64 { func (bp *BalanceFilter) SetValue(v float64) { if bp.Value == nil { - bp.Value = new(ValueFormula) + bp.Value = new(utils.ValueFormula) } bp.Value.Static = v } @@ -334,37 +331,3 @@ func (bf *BalanceFilter) ModifyBalance(b *Balance) { } b.SetDirty() // Mark the balance as dirty since we have modified and it should be checked by action triggers } - -//for computing a dynamic value for Value field -type ValueFormula struct { - Method string - Params map[string]interface{} - Static float64 -} - -func ParseBalanceFilterValue(val string) (*ValueFormula, error) { - u, err := strconv.ParseFloat(val, 64) - if err == nil { - return &ValueFormula{Static: u}, err - } - var vf ValueFormula - err = json.Unmarshal([]byte(val), &vf) - if err == nil { - return &vf, err - } - return nil, errors.New("Invalid value: " + val) -} - -type valueFormula func(map[string]interface{}) float64 - -const ( - PERIODIC = "*periodic" -) - -var valueFormulas = map[string]valueFormula{ - PERIODIC: periodicFormula, -} - -func periodicFormula(params map[string]interface{}) float64 { - return 0.0 -} diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index d37b417b0..f2dbbdd08 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -41,12 +41,12 @@ func init() { func populateDB() { ats := []*Action{ - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}}}, - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), Value: &ValueFormula{Static: 10}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT"))}}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}}}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.VOICE), Weight: utils.Float64Pointer(20), Value: &utils.ValueFormula{Static: 10}, DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT"))}}, } ats1 := []*Action{ - &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &ValueFormula{Static: 10}}, Weight: 10}, + &Action{ActionType: "*topup", Balance: &BalanceFilter{Type: utils.StringPointer(utils.MONETARY), Value: &utils.ValueFormula{Static: 10}}, Weight: 10}, &Action{ActionType: "*reset_account", Weight: 20}, } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 851d474e9..e88b57218 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -180,6 +180,7 @@ BLOCK_EMPTY,*topup,,,bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 FILTER,*topup,,"{""*and"":[{""Value"":{""*lt"":0}},{""Id"":{""*eq"":""*default""}}]}",bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 EXP,*topup,,,,*voice,*out,,,,,*monthly,*any,300,10,false,false,10 NOEXP,*topup,,,,*voice,*out,,,,,*unlimited,*any,50,10,false,false,10 +VF,*debit,,,,*monetary,*out,,,,,*unlimited,*any,"{""Method"":""*periodic"",""Params"":{""Units"":10, ""Interval"":""month"", ""Increment"":""day""}}",10,false,false,10 ` actionPlans = ` MORE_MINUTES,MINI,ONE_TIME_RUN,10 @@ -223,6 +224,7 @@ cgrates.org,block,BLOCK_AT,,false,false cgrates.org,block_empty,BLOCK_EMPTY_AT,,false,false cgrates.org,expo,EXP_AT,,false,false cgrates.org,expnoexp,,,false,false +cgrates.org,vf,,,false,false ` derivedCharges = ` @@ -825,7 +827,7 @@ func TestLoadRatingProfiles(t *testing.T) { } func TestLoadActions(t *testing.T) { - if len(csvr.actions) != 14 { + if len(csvr.actions) != 15 { t.Error("Failed to load actions: ", len(csvr.actions)) } as1 := csvr.actions["MINI"] @@ -840,7 +842,7 @@ func TestLoadActions(t *testing.T) { Type: utils.StringPointer(utils.MONETARY), Uuid: as1[0].Balance.Uuid, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: &ValueFormula{Static: 10}, + Value: &utils.ValueFormula{Static: 10}, Weight: utils.Float64Pointer(10), DestinationIDs: nil, TimingIDs: nil, @@ -860,7 +862,7 @@ func TestLoadActions(t *testing.T) { Type: utils.StringPointer(utils.VOICE), Uuid: as1[1].Balance.Uuid, Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), - Value: &ValueFormula{Static: 100}, + Value: &utils.ValueFormula{Static: 100}, Weight: utils.Float64Pointer(10), RatingSubject: utils.StringPointer("test"), DestinationIDs: utils.StringMapPointer(utils.NewStringMap("NAT")), @@ -887,7 +889,7 @@ func TestLoadActions(t *testing.T) { Directions: utils.StringMapPointer(utils.NewStringMap(utils.OUT)), DestinationIDs: nil, Uuid: as2[0].Balance.Uuid, - Value: &ValueFormula{Static: 100}, + Value: &utils.ValueFormula{Static: 100}, Weight: utils.Float64Pointer(10), SharedGroups: utils.StringMapPointer(utils.NewStringMap("SG1")), TimingIDs: nil, @@ -1106,7 +1108,7 @@ func TestLoadActionTriggers(t *testing.T) { } func TestLoadAccountActions(t *testing.T) { - if len(csvr.accountActions) != 15 { + if len(csvr.accountActions) != 16 { t.Error("Failed to load account actions: ", len(csvr.accountActions)) } aa := csvr.accountActions["vdf:minitsboy"] diff --git a/engine/storage_test.go b/engine/storage_test.go index eb007243b..06c40e0d1 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -274,7 +274,7 @@ func TestDifferentUuid(t *testing.T) { func TestStorageTask(t *testing.T) { // clean previous unused tasks - for i := 0; i < 19; i++ { + for i := 0; i < 20; i++ { ratingStorage.PopTask() } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index 6f85f3d89..a2a71470b 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -531,7 +531,7 @@ func (tpr *TpReader) LoadActions() (err error) { } if tpact.Units != "" && tpact.Units != utils.ANY { - vf, err := ParseBalanceFilterValue(tpact.Units) + vf, err := utils.ParseBalanceFilterValue(tpact.Units) if err != nil { return err } @@ -1007,7 +1007,7 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *TpAccountAction) error } if tpact.Units != "" && tpact.Units != utils.ANY { - vf, err := ParseBalanceFilterValue(tpact.Units) + vf, err := utils.ParseBalanceFilterValue(tpact.Units) if err != nil { return err } @@ -1355,7 +1355,7 @@ func (tpr *TpReader) LoadCdrStatsFiltered(tag string, save bool) (err error) { } if tpact.Units != "" && tpact.Units != utils.ANY { - vf, err := ParseBalanceFilterValue(tpact.Units) + vf, err := utils.ParseBalanceFilterValue(tpact.Units) if err != nil { return err } diff --git a/utils/dateseries.go b/utils/dateseries.go index 130bac5f2..818efa414 100644 --- a/utils/dateseries.go +++ b/utils/dateseries.go @@ -277,3 +277,13 @@ func (wd WeekDays) Serialize(sep string) string { } return wdStr } + +func DaysInMonth(year int, month time.Month) float64 { + return float64(time.Date(year, month, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, -1).Day()) +} + +func DaysInYear(year int) float64 { + first := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC) + last := first.AddDate(1, 0, 0) + return float64(last.Sub(first).Hours() / 24) +} diff --git a/utils/dateseries_test.go b/utils/dateseries_test.go index 363aee178..ffd498827 100644 --- a/utils/dateseries_test.go +++ b/utils/dateseries_test.go @@ -161,3 +161,30 @@ func TestDateseriesMonthsIsCompleteYes(t *testing.T) { t.Error("Error months IsComplete: ", months) } } + +func TestDateseriesDaysInMonth(t *testing.T) { + if n := DaysInMonth(2016, 4); n != 30 { + t.Error("error calculating days: ", n) + } + if n := DaysInMonth(2016, 2); n != 29 { + t.Error("error calculating days: ", n) + } + if n := DaysInMonth(2016, 1); n != 31 { + t.Error("error calculating days: ", n) + } + if n := DaysInMonth(2016, 12); n != 31 { + t.Error("error calculating days: ", n) + } + if n := DaysInMonth(2015, 2); n != 28 { + t.Error("error calculating days: ", n) + } +} + +func TestDateseriesDaysInYear(t *testing.T) { + if n := DaysInYear(2016); n != 366 { + t.Error("error calculating days: ", n) + } + if n := DaysInYear(2015); n != 365 { + t.Error("error calculating days: ", n) + } +} diff --git a/utils/value_formula.go b/utils/value_formula.go new file mode 100644 index 000000000..2a554f39e --- /dev/null +++ b/utils/value_formula.go @@ -0,0 +1,85 @@ +package utils + +import ( + "encoding/json" + "errors" + "log" + "strconv" + "time" +) + +//for computing a dynamic value for Value field +type ValueFormula struct { + Method string + Params map[string]interface{} + Static float64 +} + +func ParseBalanceFilterValue(val string) (*ValueFormula, error) { + u, err := strconv.ParseFloat(val, 64) + if err == nil { + return &ValueFormula{Static: u}, err + } + var vf ValueFormula + if err := json.Unmarshal([]byte(val), &vf); err == nil { + return &vf, err + } + return nil, errors.New("Invalid value: " + val) +} + +type valueFormula func(map[string]interface{}) float64 + +const ( + PERIODIC = "*periodic" +) + +var ValueFormulas = map[string]valueFormula{ + PERIODIC: periodicFormula, +} + +func periodicFormula(params map[string]interface{}) float64 { + // check parameters + unitsInterface, unitsFound := params["Units"] + intervalInterface, intervalFound := params["Interval"] + incrementInterface, incrementFound := params["Increment"] + + if !unitsFound || !intervalFound || !incrementFound { + return 0.0 + } + units, ok := unitsInterface.(float64) + if !ok { + log.Print("units") + return 0.0 + } + var interval string + switch intr := intervalInterface.(type) { + case string: + interval = intr + case []byte: + interval = string(intr) + default: + return 0.0 + } + var increment string + switch incr := incrementInterface.(type) { + case string: + increment = incr + case []byte: + increment = string(incr) + default: + return 0.0 + } + now := time.Now() + if increment == "day" { + if interval == "week" { + return units / 7 + } + if interval == "month" { + return units / DaysInMonth(now.Year(), now.Month()) + } + if interval == "year" { + return units / DaysInYear(now.Year()) + } + } + return 0.0 +} diff --git a/utils/value_formula_test.go b/utils/value_formula_test.go new file mode 100644 index 000000000..bb140a15d --- /dev/null +++ b/utils/value_formula_test.go @@ -0,0 +1,39 @@ +package utils + +import ( + "encoding/json" + "testing" + "time" +) + +func TestValueFormulaDayWeek(t *testing.T) { + params := make(map[string]interface{}) + if err := json.Unmarshal([]byte(`{"Units":10, "Interval":"week", "Increment":"day"}`), ¶ms); err != nil { + t.Error("error unmarshalling params: ", err) + } + if x := periodicFormula(params); x != 10/7.0 { + t.Error("error caclulating value using formula: ", x) + } +} + +func TestValueFormulaDayMonth(t *testing.T) { + params := make(map[string]interface{}) + if err := json.Unmarshal([]byte(`{"Units":10, "Interval":"month", "Increment":"day"}`), ¶ms); err != nil { + t.Error("error unmarshalling params: ", err) + } + now := time.Now() + if x := periodicFormula(params); x != 10/DaysInMonth(now.Year(), now.Month()) { + t.Error("error caclulating value using formula: ", x) + } +} + +func TestValueFormulaDayYear(t *testing.T) { + params := make(map[string]interface{}) + if err := json.Unmarshal([]byte(`{"Units":10, "Interval":"year", "Increment":"day"}`), ¶ms); err != nil { + t.Error("error unmarshalling params: ", err) + } + now := time.Now() + if x := periodicFormula(params); x != 10/DaysInYear(now.Year()) { + t.Error("error caclulating value using formula: ", x) + } +} From 30f42c07f2e2b27b4b5944e06c04a4e61929e14d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 25 Apr 2016 22:26:20 +0300 Subject: [PATCH 197/227] renamed *periodic to *incremental --- utils/value_formula.go | 6 +++--- utils/value_formula_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/value_formula.go b/utils/value_formula.go index 2a554f39e..1ac113aa1 100644 --- a/utils/value_formula.go +++ b/utils/value_formula.go @@ -30,14 +30,14 @@ func ParseBalanceFilterValue(val string) (*ValueFormula, error) { type valueFormula func(map[string]interface{}) float64 const ( - PERIODIC = "*periodic" + INCREMENTAL = "*incremental" ) var ValueFormulas = map[string]valueFormula{ - PERIODIC: periodicFormula, + INCREMENTAL: incrementalFormula, } -func periodicFormula(params map[string]interface{}) float64 { +func incrementalFormula(params map[string]interface{}) float64 { // check parameters unitsInterface, unitsFound := params["Units"] intervalInterface, intervalFound := params["Interval"] diff --git a/utils/value_formula_test.go b/utils/value_formula_test.go index bb140a15d..1aab18106 100644 --- a/utils/value_formula_test.go +++ b/utils/value_formula_test.go @@ -11,7 +11,7 @@ func TestValueFormulaDayWeek(t *testing.T) { if err := json.Unmarshal([]byte(`{"Units":10, "Interval":"week", "Increment":"day"}`), ¶ms); err != nil { t.Error("error unmarshalling params: ", err) } - if x := periodicFormula(params); x != 10/7.0 { + if x := incrementalFormula(params); x != 10/7.0 { t.Error("error caclulating value using formula: ", x) } } @@ -22,7 +22,7 @@ func TestValueFormulaDayMonth(t *testing.T) { t.Error("error unmarshalling params: ", err) } now := time.Now() - if x := periodicFormula(params); x != 10/DaysInMonth(now.Year(), now.Month()) { + if x := incrementalFormula(params); x != 10/DaysInMonth(now.Year(), now.Month()) { t.Error("error caclulating value using formula: ", x) } } @@ -33,7 +33,7 @@ func TestValueFormulaDayYear(t *testing.T) { t.Error("error unmarshalling params: ", err) } now := time.Now() - if x := periodicFormula(params); x != 10/DaysInYear(now.Year()) { + if x := incrementalFormula(params); x != 10/DaysInYear(now.Year()) { t.Error("error caclulating value using formula: ", x) } } From 03190245d25b921f3dd0be9479649dd8311a10d4 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 25 Apr 2016 23:09:50 +0300 Subject: [PATCH 198/227] created new int migrator --- cmd/cgr-loader/cgr-loader.go | 5 ++ cmd/cgr-loader/migrator_rc8int2.go | 102 +++++++++++++++++++++++++++++ engine/loader_csv_test.go | 2 +- 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 cmd/cgr-loader/migrator_rc8int2.go diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 7f26375fb..fba746007 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -163,6 +163,11 @@ func main() { log.Print(err.Error()) } } + if strings.Contains(*migrateRC8, "int2") { + if err := migratorRC8rat.migrateActionsInt2(); err != nil { + log.Print(err.Error()) + } + } log.Print("Done!") return } diff --git a/cmd/cgr-loader/migrator_rc8int2.go b/cmd/cgr-loader/migrator_rc8int2.go new file mode 100644 index 000000000..46c7caeeb --- /dev/null +++ b/cmd/cgr-loader/migrator_rc8int2.go @@ -0,0 +1,102 @@ +package main + +import ( + "log" + "time" + + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +type BalanceFilter2 struct { + Uuid *string + ID *string + Type *string + Value *float64 + Directions *utils.StringMap + ExpirationDate *time.Time + Weight *float64 + DestinationIDs *utils.StringMap + RatingSubject *string + Categories *utils.StringMap + SharedGroups *utils.StringMap + TimingIDs *utils.StringMap + Timings []*engine.RITiming + Disabled *bool + Factor *engine.ValueFactor + Blocker *bool +} + +type Action2 struct { + Id string + ActionType string + ExtraParameters string + Filter string + ExpirationString string // must stay as string because it can have relative values like 1month + Weight float64 + Balance *BalanceFilter2 +} + +type Actions2 []*Action2 + +func (mig MigratorRC8) migrateActionsInt2() error { + keys, err := mig.db.Cmd("KEYS", utils.ACTION_PREFIX+"*").List() + if err != nil { + return err + } + newAcsMap := make(map[string]engine.Actions, len(keys)) + for _, key := range keys { + log.Printf("Migrating action: %s...", key) + var oldAcs Actions2 + var values []byte + if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { + if err := mig.ms.Unmarshal(values, &oldAcs); err != nil { + return err + } + } + newAcs := make(engine.Actions, len(oldAcs)) + for index, oldAc := range oldAcs { + a := &engine.Action{ + Id: oldAc.Id, + ActionType: oldAc.ActionType, + ExtraParameters: oldAc.ExtraParameters, + ExpirationString: oldAc.ExpirationString, + Filter: oldAc.Filter, + Weight: oldAc.Weight, + Balance: &engine.BalanceFilter{ + Uuid: oldAc.Balance.Uuid, + ID: oldAc.Balance.ID, + Type: oldAc.Balance.Type, + Directions: oldAc.Balance.Directions, + ExpirationDate: oldAc.Balance.ExpirationDate, + Weight: oldAc.Balance.Weight, + DestinationIDs: oldAc.Balance.DestinationIDs, + RatingSubject: oldAc.Balance.RatingSubject, + Categories: oldAc.Balance.Categories, + SharedGroups: oldAc.Balance.SharedGroups, + TimingIDs: oldAc.Balance.TimingIDs, + Timings: oldAc.Balance.Timings, + Disabled: oldAc.Balance.Disabled, + Factor: oldAc.Balance.Factor, + Blocker: oldAc.Balance.Blocker, + }, + } + if oldAc.Balance.Value != nil { + a.Balance.Value = &utils.ValueFormula{Static: *oldAc.Balance.Value} + } + newAcs[index] = a + } + newAcsMap[key] = newAcs + } + // write data back + for key, acs := range newAcsMap { + result, err := mig.ms.Marshal(&acs) + if err != nil { + return err + } + if err = mig.db.Cmd("SET", key, result).Err; err != nil { + return err + } + } + return nil +} diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index e88b57218..f2fb1f898 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -180,7 +180,7 @@ BLOCK_EMPTY,*topup,,,bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 FILTER,*topup,,"{""*and"":[{""Value"":{""*lt"":0}},{""Id"":{""*eq"":""*default""}}]}",bfree,*monetary,*out,,,,,*unlimited,,20,10,false,false,10 EXP,*topup,,,,*voice,*out,,,,,*monthly,*any,300,10,false,false,10 NOEXP,*topup,,,,*voice,*out,,,,,*unlimited,*any,50,10,false,false,10 -VF,*debit,,,,*monetary,*out,,,,,*unlimited,*any,"{""Method"":""*periodic"",""Params"":{""Units"":10, ""Interval"":""month"", ""Increment"":""day""}}",10,false,false,10 +VF,*debit,,,,*monetary,*out,,,,,*unlimited,*any,"{""Method"":""*incremental"",""Params"":{""Units"":10, ""Interval"":""month"", ""Increment"":""day""}}",10,false,false,10 ` actionPlans = ` MORE_MINUTES,MINI,ONE_TIME_RUN,10 From bdb1e134d714ac2cd1959c85edab0c03b30d6917 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 25 Apr 2016 23:30:41 +0300 Subject: [PATCH 199/227] use fv instead of int2 --- cmd/cgr-loader/cgr-loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index fba746007..d03b0476f 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -163,7 +163,7 @@ func main() { log.Print(err.Error()) } } - if strings.Contains(*migrateRC8, "int2") { + if strings.Contains(*migrateRC8, "vf") { if err := migratorRC8rat.migrateActionsInt2(); err != nil { log.Print(err.Error()) } From 0c9ab06a658a6e3374212d7cbae8ca378740740f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 26 Apr 2016 20:23:17 +0300 Subject: [PATCH 200/227] migration for value formula in balance filter --- cmd/cgr-engine/cgr-engine.go | 4 + cmd/cgr-loader/cgr-loader.go | 3 + cmd/cgr-loader/migrator_rc8.go | 8 ++ engine/storage_mongo_datadb.go | 2 +- engine/version.go | 156 ++++++++++++++++++++++++--------- 5 files changed, 133 insertions(+), 40 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index de9a51586..c5bc24dbb 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -507,6 +507,10 @@ func main() { } defer accountDb.Close() engine.SetAccountingStorage(accountDb) + if err := engine.CheckVersion(); err != nil { + fmt.Println(err.Error()) + return + } } if cfg.RALsEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index d03b0476f..993761a7e 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -167,6 +167,9 @@ func main() { if err := migratorRC8rat.migrateActionsInt2(); err != nil { log.Print(err.Error()) } + if err := migratorRC8acc.writeVersion(); err != nil { + log.Print(err.Error()) + } } log.Print("Done!") return diff --git a/cmd/cgr-loader/migrator_rc8.go b/cmd/cgr-loader/migrator_rc8.go index b5574a1d5..15579db98 100644 --- a/cmd/cgr-loader/migrator_rc8.go +++ b/cmd/cgr-loader/migrator_rc8.go @@ -669,3 +669,11 @@ func (mig MigratorRC8) migrateSharedGroups() error { } return nil } + +func (mig MigratorRC8) writeVersion() error { + result, err := mig.ms.Marshal(engine.CurrentVersion) + if err != nil { + return err + } + return mig.db.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err +} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 68ba06277..c297d3565 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -53,7 +53,7 @@ const ( colLogAtr = "action_trigger_logs" colLogApl = "action_plan_logs" colLogErr = "error_logs" - colVer = "version" + colVer = "versions" ) var ( diff --git a/engine/version.go b/engine/version.go index 047036261..08849db5c 100644 --- a/engine/version.go +++ b/engine/version.go @@ -1,29 +1,41 @@ package engine import ( + "errors" "fmt" "github.com/cgrates/cgrates/utils" ) -func init() { +func CheckVersion() error { // get current db version dbVersion, err := accountingStorage.GetStructVersion() if err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not retrive current version from db: %v", err)) - return - } - // comparing versions - if currentVersion.CompareAndMigrate(dbVersion) { - // write the new values - if err := accountingStorage.SetStructVersion(currentVersion); err != nil { - utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + if lhList, err := accountingStorage.GetLoadHistory(1, true); err != nil || len(lhList) == 0 { + // no data, write version + if err := accountingStorage.SetStructVersion(CurrentVersion); err != nil { + utils.Logger.Warning(fmt.Sprintf("Could not write current version to db: %v", err)) + } + } else { + // has data but no version => run migration + msg := "Could not detect data structures version: run appropriate migration" + utils.Logger.Crit(msg) + return errors.New(msg) + } + } else { + // comparing versions + if len(CurrentVersion.CompareAndMigrate(dbVersion)) > 0 { + // write the new values + msg := "Migration needed: please backup cgr data and run cgr-cloader -migrate" + utils.Logger.Crit(msg) + return errors.New(msg) } } + return nil } var ( - currentVersion = &StructVersion{ + CurrentVersion = &StructVersion{ Destinations: "1", RatingPlans: "1", RatingProfiles: "1", @@ -67,67 +79,133 @@ type StructVersion struct { SMCosts string } -func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) bool { - migrationPerformed := false +type MigrationInfo struct { + Prefix string + DbVersion string + CurrentVersion string +} + +func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) []*MigrationInfo { + var migrationInfoList []*MigrationInfo if sv.Destinations != dbVer.Destinations { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.DESTINATION_PREFIX, + DbVersion: dbVer.Destinations, + CurrentVersion: CurrentVersion.Destinations, + }) } if sv.RatingPlans != dbVer.RatingPlans { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.RATING_PLAN_PREFIX, + DbVersion: dbVer.RatingPlans, + CurrentVersion: CurrentVersion.RatingPlans, + }) } - if sv.RatingProfiles != dbVer.RatingPlans { - migrationPerformed = true - + if sv.RatingProfiles != dbVer.RatingProfiles { + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.RATING_PROFILE_PREFIX, + DbVersion: dbVer.RatingProfiles, + CurrentVersion: CurrentVersion.RatingProfiles, + }) } if sv.Lcrs != dbVer.Lcrs { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.LCR_PREFIX, + DbVersion: dbVer.Lcrs, + CurrentVersion: CurrentVersion.Lcrs, + }) } if sv.DerivedChargers != dbVer.DerivedChargers { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.DERIVEDCHARGERS_PREFIX, + DbVersion: dbVer.DerivedChargers, + CurrentVersion: CurrentVersion.DerivedChargers, + }) } if sv.Actions != dbVer.Actions { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.ACTION_PREFIX, + DbVersion: dbVer.Actions, + CurrentVersion: CurrentVersion.Actions, + }) } if sv.ActionPlans != dbVer.ActionPlans { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.ACTION_PLAN_PREFIX, + DbVersion: dbVer.ActionPlans, + CurrentVersion: CurrentVersion.ActionPlans, + }) } if sv.ActionTriggers != dbVer.ActionTriggers { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.ACTION_TRIGGER_PREFIX, + DbVersion: dbVer.ActionTriggers, + CurrentVersion: CurrentVersion.ActionTriggers, + }) } if sv.SharedGroups != dbVer.SharedGroups { - migrationPerformed = true - + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.SHARED_GROUP_PREFIX, + DbVersion: dbVer.SharedGroups, + CurrentVersion: CurrentVersion.SharedGroups, + }) } if sv.Accounts != dbVer.Accounts { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.ACCOUNT_PREFIX, + DbVersion: dbVer.Accounts, + CurrentVersion: CurrentVersion.Accounts, + }) } if sv.CdrStats != dbVer.CdrStats { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.CDR_STATS_PREFIX, + DbVersion: dbVer.CdrStats, + CurrentVersion: CurrentVersion.CdrStats, + }) } if sv.Users != dbVer.Users { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.USERS_PREFIX, + DbVersion: dbVer.Users, + CurrentVersion: CurrentVersion.Users, + }) } if sv.Alias != dbVer.Alias { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.ALIASES_PREFIX, + DbVersion: dbVer.Alias, + CurrentVersion: CurrentVersion.Alias, + }) } if sv.PubSubs != dbVer.PubSubs { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.PUBSUB_SUBSCRIBERS_PREFIX, + DbVersion: dbVer.PubSubs, + CurrentVersion: CurrentVersion.PubSubs, + }) } if sv.LoadHistory != dbVer.LoadHistory { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.LOADINST_KEY, + DbVersion: dbVer.LoadHistory, + CurrentVersion: CurrentVersion.LoadHistory, + }) } if sv.Cdrs != dbVer.Cdrs { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.CDRS_SOURCE, + DbVersion: dbVer.RatingPlans, + CurrentVersion: CurrentVersion.RatingPlans, + }) } if sv.SMCosts != dbVer.SMCosts { - migrationPerformed = true + migrationInfoList = append(migrationInfoList, &MigrationInfo{ + Prefix: utils.SMG, + DbVersion: dbVer.SMCosts, + CurrentVersion: CurrentVersion.SMCosts, + }) } - return migrationPerformed + return migrationInfoList } From 6bdb091a92cfc8b60f40bd2e5dc6f0ad86d11129 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 27 Apr 2016 15:10:36 +0300 Subject: [PATCH 201/227] extend support for *unlimited in expiration time --- utils/coreutils.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/coreutils.go b/utils/coreutils.go index 461fb2d98..abdbc1fa7 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -127,8 +127,9 @@ func Round(x float64, prec int, method string) float64 { } func ParseTimeDetectLayout(tmStr string, timezone string) (time.Time, error) { + tmStr = strings.TrimSpace(tmStr) var nilTime time.Time - if len(tmStr) == 0 { + if len(tmStr) == 0 || tmStr == UNLIMITED { return nilTime, nil } loc, err := time.LoadLocation(timezone) @@ -179,7 +180,7 @@ func ParseTimeDetectLayout(tmStr string, timezone string) (time.Time, error) { func ParseDate(date string) (expDate time.Time, err error) { date = strings.TrimSpace(date) switch { - case date == "*unlimited" || date == "": + case date == UNLIMITED || date == "": // leave it at zero case strings.HasPrefix(date, "+"): d, err := time.ParseDuration(date[1:]) From 90381b7366153a9035e042813a76e48b5a6721bf Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 27 Apr 2016 16:08:40 +0300 Subject: [PATCH 202/227] extended units column in tp tables --- data/storage/mysql/create_tariffplan_tables.sql | 2 +- data/storage/postgres/create_tariffplan_tables.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index b6240cdc5..52bbf074b 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -152,7 +152,7 @@ CREATE TABLE `tp_actions` ( `balance_tag` varchar(64) NOT NULL, `balance_type` varchar(24) NOT NULL, `directions` varchar(8) NOT NULL, - `units` varchar(24) NOT NULL, + `units` varchar(256) NOT NULL, `expiry_time` varchar(24) NOT NULL, `timing_tags` varchar(128) NOT NULL, `destination_tags` varchar(64) NOT NULL, diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index b5d8193c8..e2bc89a57 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -147,7 +147,7 @@ CREATE TABLE tp_actions ( balance_tag VARCHAR(64) NOT NULL, balance_type VARCHAR(24) NOT NULL, directions VARCHAR(8) NOT NULL, - units VARCHAR(10) NOT NULL, + units VARCHAR(256) NOT NULL, expiry_time VARCHAR(24) NOT NULL, timing_tags VARCHAR(128) NOT NULL, destination_tags VARCHAR(64) NOT NULL, From a205b630962c2a7548ae109e1fbd674750b5603d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 27 Apr 2016 18:41:11 +0300 Subject: [PATCH 203/227] first version of mongo migrator --- cmd/cgr-loader/cgr-loader.go | 167 +++++++++++++++++-------------- cmd/cgr-loader/migrator_mongo.go | 97 ++++++++++++++++++ engine/storage_mongo_datadb.go | 9 +- 3 files changed, 196 insertions(+), 77 deletions(-) create mode 100644 cmd/cgr-loader/migrator_mongo.go diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 993761a7e..2805795d0 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -92,85 +92,102 @@ func main() { var rater, cdrstats, users *rpc.Client var loader engine.LoadReader if *migrateRC8 != "" { - var db_nb int - db_nb, err = strconv.Atoi(*datadb_name) - if err != nil { - log.Print("Redis db name must be an integer!") - return - } - host := *datadb_host - if *datadb_port != "" { - host += ":" + *datadb_port - } - migratorRC8acc, err := NewMigratorRC8(host, db_nb, *datadb_pass, *dbdata_encoding) - if err != nil { - log.Print(err.Error()) - return - } - if strings.Contains(*migrateRC8, "acc") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8acc.migrateAccounts(); err != nil { + if *datadb_type == "redis" && *tpdb_type == "redis" { + var db_nb int + db_nb, err = strconv.Atoi(*datadb_name) + if err != nil { + log.Print("Redis db name must be an integer!") + return + } + host := *datadb_host + if *datadb_port != "" { + host += ":" + *datadb_port + } + migratorRC8acc, err := NewMigratorRC8(host, db_nb, *datadb_pass, *dbdata_encoding) + if err != nil { log.Print(err.Error()) + return + } + if strings.Contains(*migrateRC8, "acc") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8acc.migrateAccounts(); err != nil { + log.Print(err.Error()) + } + } + + db_nb, err = strconv.Atoi(*tpdb_name) + if err != nil { + log.Print("Redis db name must be an integer!") + return + } + host = *tpdb_host + if *tpdb_port != "" { + host += ":" + *tpdb_port + } + migratorRC8rat, err := NewMigratorRC8(host, db_nb, *tpdb_pass, *dbdata_encoding) + if err != nil { + log.Print(err.Error()) + return + } + if strings.Contains(*migrateRC8, "atr") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8rat.migrateActionTriggers(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "act") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8rat.migrateActions(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "dcs") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8rat.migrateDerivedChargers(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "apl") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8rat.migrateActionPlans(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "shg") || strings.Contains(*migrateRC8, "*all") { + if err := migratorRC8rat.migrateSharedGroups(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "int") { + if err := migratorRC8acc.migrateAccountsInt(); err != nil { + log.Print(err.Error()) + } + if err := migratorRC8rat.migrateActionTriggersInt(); err != nil { + log.Print(err.Error()) + } + if err := migratorRC8rat.migrateActionsInt(); err != nil { + log.Print(err.Error()) + } + } + if strings.Contains(*migrateRC8, "vf") { + if err := migratorRC8rat.migrateActionsInt2(); err != nil { + log.Print(err.Error()) + } + if err := migratorRC8acc.writeVersion(); err != nil { + log.Print(err.Error()) + } + } + } else if *datadb_type == "mongo" && *tpdb_type == "mongo" { + mongoMigrator, err := NewMongoMigrator(*datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass) + if err != nil { + log.Print(err.Error()) + return + } + if strings.Contains(*migrateRC8, "vf") { + if err := mongoMigrator.migrateActions(); err != nil { + log.Print(err.Error()) + } + if err := mongoMigrator.writeVersion(); err != nil { + log.Print(err.Error()) + } } } - db_nb, err = strconv.Atoi(*tpdb_name) - if err != nil { - log.Print("Redis db name must be an integer!") - return - } - host = *tpdb_host - if *tpdb_port != "" { - host += ":" + *tpdb_port - } - migratorRC8rat, err := NewMigratorRC8(host, db_nb, *tpdb_pass, *dbdata_encoding) - if err != nil { - log.Print(err.Error()) - return - } - if strings.Contains(*migrateRC8, "atr") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8rat.migrateActionTriggers(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "act") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8rat.migrateActions(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "dcs") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8rat.migrateDerivedChargers(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "apl") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8rat.migrateActionPlans(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "shg") || strings.Contains(*migrateRC8, "*all") { - if err := migratorRC8rat.migrateSharedGroups(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "int") { - if err := migratorRC8acc.migrateAccountsInt(); err != nil { - log.Print(err.Error()) - } - if err := migratorRC8rat.migrateActionTriggersInt(); err != nil { - log.Print(err.Error()) - } - if err := migratorRC8rat.migrateActionsInt(); err != nil { - log.Print(err.Error()) - } - } - if strings.Contains(*migrateRC8, "vf") { - if err := migratorRC8rat.migrateActionsInt2(); err != nil { - log.Print(err.Error()) - } - if err := migratorRC8acc.writeVersion(); err != nil { - log.Print(err.Error()) - } - } log.Print("Done!") return } diff --git a/cmd/cgr-loader/migrator_mongo.go b/cmd/cgr-loader/migrator_mongo.go new file mode 100644 index 000000000..9436dfd25 --- /dev/null +++ b/cmd/cgr-loader/migrator_mongo.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "log" + + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +type MongoMigrator struct { + session *mgo.Session + db *mgo.Database +} + +func NewMongoMigrator(host, port, db, user, pass string) (*MongoMigrator, error) { + address := fmt.Sprintf("%s:%s", host, port) + if user != "" && pass != "" { + address = fmt.Sprintf("%s:%s@%s", user, pass, address) + } + session, err := mgo.Dial(address) + if err != nil { + return nil, err + } + ndb := session.DB(db) + return &MongoMigrator{session: session, db: ndb}, nil +} + +func (mig MongoMigrator) migrateActions() error { + newAcsMap := make(map[string]engine.Actions) + iter := mig.db.C("actions").Find(nil).Iter() + var oldAcs struct { + Key string + Value Actions2 + } + for iter.Next(&oldAcs) { + log.Printf("Migrating action: %s...", oldAcs.Key) + newAcs := make(engine.Actions, len(oldAcs.Value)) + for index, oldAc := range oldAcs.Value { + a := &engine.Action{ + Id: oldAc.Id, + ActionType: oldAc.ActionType, + ExtraParameters: oldAc.ExtraParameters, + ExpirationString: oldAc.ExpirationString, + Filter: oldAc.Filter, + Weight: oldAc.Weight, + Balance: &engine.BalanceFilter{ + Uuid: oldAc.Balance.Uuid, + ID: oldAc.Balance.ID, + Type: oldAc.Balance.Type, + Directions: oldAc.Balance.Directions, + ExpirationDate: oldAc.Balance.ExpirationDate, + Weight: oldAc.Balance.Weight, + DestinationIDs: oldAc.Balance.DestinationIDs, + RatingSubject: oldAc.Balance.RatingSubject, + Categories: oldAc.Balance.Categories, + SharedGroups: oldAc.Balance.SharedGroups, + TimingIDs: oldAc.Balance.TimingIDs, + Timings: oldAc.Balance.Timings, + Disabled: oldAc.Balance.Disabled, + Factor: oldAc.Balance.Factor, + Blocker: oldAc.Balance.Blocker, + }, + } + if oldAc.Balance.Value != nil { + a.Balance.Value = &utils.ValueFormula{Static: *oldAc.Balance.Value} + } + newAcs[index] = a + } + newAcsMap[oldAcs.Key] = newAcs + } + if err := iter.Close(); err != nil { + return err + } + + // write data back + for key, acs := range newAcsMap { + if _, err := mig.db.C("actions").Upsert(bson.M{"key": key}, &struct { + Key string + Value engine.Actions + }{Key: key, Value: acs}); err != nil { + return err + } + } + return nil +} + +func (mig MongoMigrator) writeVersion() error { + _, err := mig.db.C("versions").Upsert(bson.M{"key": utils.VERSION_PREFIX + "struct"}, &struct { + Key string + Value *engine.StructVersion + }{utils.VERSION_PREFIX + "struct", engine.CurrentVersion}) + return err +} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index c297d3565..ec754c2a4 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1374,10 +1374,15 @@ func (ms *MongoStorage) SetStructVersion(v *StructVersion) (err error) { } func (ms *MongoStorage) GetStructVersion() (rsv *StructVersion, err error) { - rsv = new(StructVersion) - err = ms.db.C(colVer).Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(rsv) + var result struct { + Key string + Value StructVersion + } + + err = ms.db.C(colVer).Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(result) if err == mgo.ErrNotFound { rsv = nil } + rsv = &result.Value return } From 47f98b0729845af0d724f3e0502b0f12f9223f0e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 27 Apr 2016 19:04:19 +0300 Subject: [PATCH 204/227] working migrator version --- cmd/cgr-loader/cgr-loader.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 2805795d0..3ac60d57c 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -173,16 +173,21 @@ func main() { } } } else if *datadb_type == "mongo" && *tpdb_type == "mongo" { - mongoMigrator, err := NewMongoMigrator(*datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass) + mongoMigratorAcc, err := NewMongoMigrator(*datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass) + if err != nil { + log.Print(err.Error()) + return + } + mongoMigratorRat, err := NewMongoMigrator(*tpdb_host, *tpdb_port, *tpdb_name, *tpdb_user, *tpdb_pass) if err != nil { log.Print(err.Error()) return } if strings.Contains(*migrateRC8, "vf") { - if err := mongoMigrator.migrateActions(); err != nil { + if err := mongoMigratorRat.migrateActions(); err != nil { log.Print(err.Error()) } - if err := mongoMigrator.writeVersion(); err != nil { + if err := mongoMigratorAcc.writeVersion(); err != nil { log.Print(err.Error()) } } From b664fc25f46ea17621dc4e29d80afbc596cebd42 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Wed, 27 Apr 2016 19:18:40 +0300 Subject: [PATCH 205/227] mongo version fix --- engine/storage_mongo_datadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index ec754c2a4..0e2e59113 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -1379,7 +1379,7 @@ func (ms *MongoStorage) GetStructVersion() (rsv *StructVersion, err error) { Value StructVersion } - err = ms.db.C(colVer).Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(result) + err = ms.db.C(colVer).Find(bson.M{"key": utils.VERSION_PREFIX + "struct"}).One(&result) if err == mgo.ErrNotFound { rsv = nil } From f819912a0f13402320dd5a83bfc0c60efd33c269 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 28 Apr 2016 15:40:14 +0200 Subject: [PATCH 206/227] CDRC configuration change to increase flexibility, list now --- cdrc/cdrc.go | 4 +-- cdrc/cdrc_local_test.go | 2 +- cdrc/csv.go | 22 ++++++------- cdrc/csv_test.go | 22 ++++++------- cdrc/flatstore_local_test.go | 6 +++- cdrc/fwv.go | 32 +++++++++---------- cdrc/fwv_local_test.go | 11 +++++-- cdrc/fwv_test.go | 4 +-- cmd/cgr-engine/cgr-engine.go | 17 +++++----- config/cdrcconfig.go | 5 +++ config/cfg_data.json | 10 +++--- config/cfg_data2.json | 18 +++-------- config/config.go | 30 ++++++++--------- config/config_defaults.go | 9 +++--- config/config_json.go | 4 +-- config/config_json_test.go | 18 ++++++----- config/configcdrc_test.go | 32 +++++++++++-------- config/libconfig_json.go | 1 + data/conf/samples/cdrcflatstore/cgrates.json | 7 ++-- data/conf/samples/cdrcfwv/cgrates.json | 7 ++-- .../conf/samples/fscsv/freeswitch_csvcdr.json | 7 ++-- .../multiplecdrc/multiplecdrc_fwexport.json | 13 +++++--- 22 files changed, 151 insertions(+), 130 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index a60c073e8..2b35dd828 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -55,7 +55,7 @@ Common parameters within configs processed: Parameters specific per config instance: * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields */ -func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs rpcclient.RpcClientConnection, closeChan chan struct{}, dfltTimezone string) (*Cdrc, error) { +func NewCdrc(cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, cdrs rpcclient.RpcClientConnection, closeChan chan struct{}, dfltTimezone string) (*Cdrc, error) { var cdrcCfg *config.CdrcConfig for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one break @@ -83,7 +83,7 @@ func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs type Cdrc struct { httpSkipTlsCheck bool - cdrcCfgs map[string]*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name) + cdrcCfgs []*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name) dfltCdrcCfg *config.CdrcConfig timezone string cdrs rpcclient.RpcClientConnection diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index 3fd2f31f7..44832531f 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -49,7 +49,7 @@ README: var cfgPath string var cfg *config.CGRConfig -var cdrcCfgs map[string]*config.CdrcConfig +var cdrcCfgs []*config.CdrcConfig var cdrcCfg *config.CdrcConfig var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args diff --git a/cdrc/csv.go b/cdrc/csv.go index 740ff14ac..00e9bd823 100644 --- a/cdrc/csv.go +++ b/cdrc/csv.go @@ -181,7 +181,7 @@ func (self *PartialRecordsCache) UncachePartial(fileName string, pr *PartialFlat } func NewCsvRecordsProcessor(csvReader *csv.Reader, timezone, fileName string, - dfltCdrcCfg *config.CdrcConfig, cdrcCfgs map[string]*config.CdrcConfig, + dfltCdrcCfg *config.CdrcConfig, cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, partialRecordsCache *PartialRecordsCache) *CsvRecordsProcessor { return &CsvRecordsProcessor{csvReader: csvReader, timezone: timezone, fileName: fileName, dfltCdrcCfg: dfltCdrcCfg, cdrcCfgs: cdrcCfgs, @@ -194,7 +194,7 @@ type CsvRecordsProcessor struct { timezone string // Timezone for CDRs which are not clearly specifying it fileName string dfltCdrcCfg *config.CdrcConfig - cdrcCfgs map[string]*config.CdrcConfig + cdrcCfgs []*config.CdrcConfig processedRecordsNr int64 // Number of content records in file httpSkipTlsCheck bool partialRecordsCache *PartialRecordsCache // Shared by cdrc so we can cache for all files in a folder @@ -247,8 +247,8 @@ func (self *CsvRecordsProcessor) processPartialRecord(record []string) ([]string // Takes the record from a slice and turns it into StoredCdrs, posting them to the cdrServer func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.CDR, error) { - recordCdrs := make([]*engine.CDR, 0) // More CDRs based on the number of filters and field templates - for cdrcId, cdrcCfg := range self.cdrcCfgs { // cdrFields coming from more templates will produce individual storCdr records + recordCdrs := make([]*engine.CDR, 0) // More CDRs based on the number of filters and field templates + for _, cdrcCfg := range self.cdrcCfgs { // cdrFields coming from more templates will produce individual storCdr records // Make sure filters are matching filterBreak := false for _, rsrFilter := range cdrcCfg.CdrFilter { @@ -265,12 +265,12 @@ func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.CDR, if filterBreak { // Stop importing cdrc fields profile due to non matching filter continue } - if storedCdr, err := self.recordToStoredCdr(record, cdrcId); err != nil { + if storedCdr, err := self.recordToStoredCdr(record, cdrcCfg); err != nil { return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error()) } else { recordCdrs = append(recordCdrs, storedCdr) } - if !self.cdrcCfgs[cdrcId].ContinueOnSuccess { + if !cdrcCfg.ContinueOnSuccess { break } } @@ -278,11 +278,11 @@ func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.CDR, } // Takes the record out of csv and turns it into storedCdr which can be processed by CDRS -func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcId string) (*engine.CDR, error) { - storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: self.cdrcCfgs[cdrcId].CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} +func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *config.CdrcConfig) (*engine.CDR, error) { + storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var err error var lazyHttpFields []*config.CfgCdrField - for _, cdrFldCfg := range self.cdrcCfgs[cdrcId].ContentFields { + for _, cdrFldCfg := range cdrcCfg.ContentFields { if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.dfltCdrcCfg.CdrFormat) { // Hardcode some values in case of flatstore switch cdrFldCfg.FieldId { case utils.ACCID: @@ -315,8 +315,8 @@ func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcId strin } } storedCdr.CGRID = utils.Sha1(storedCdr.OriginID, storedCdr.SetupTime.UTC().String()) - if storedCdr.ToR == utils.DATA && self.cdrcCfgs[cdrcId].DataUsageMultiplyFactor != 0 { - storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * self.cdrcCfgs[cdrcId].DataUsageMultiplyFactor) + if storedCdr.ToR == utils.DATA && cdrcCfg.DataUsageMultiplyFactor != 0 { + storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * cdrcCfg.DataUsageMultiplyFactor) } for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields var outValByte []byte diff --git a/cdrc/csv_test.go b/cdrc/csv_test.go index 066f564db..24458cbd0 100644 --- a/cdrc/csv_test.go +++ b/cdrc/csv_test.go @@ -30,21 +30,21 @@ import ( func TestCsvRecordForkCdr(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] + cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][0] cdrcConfig.CdrSourceId = "TEST_CDRC" cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "SupplierTest", Type: utils.META_COMPOSED, FieldId: utils.SUPPLIER, Value: []*utils.RSRField{&utils.RSRField{Id: "14"}}}) cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "DisconnectCauseTest", Type: utils.META_COMPOSED, FieldId: utils.DISCONNECT_CAUSE, Value: []*utils.RSRField{&utils.RSRField{Id: "16"}}}) // - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: map[string]*config.CdrcConfig{"*default": cdrcConfig}} + csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcConfig{cdrcConfig}} cdrRow := []string{"firstField", "secondField"} - _, err := csvProcessor.recordToStoredCdr(cdrRow, "*default") + _, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig) if err == nil { t.Error("Failed to corectly detect missing fields from record") } cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", utils.META_PREPAID, "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963", "2013-02-03 19:50:00", "2013-02-03 19:54:00", "62", "supplier1", "172.16.1.1", "NORMAL_DISCONNECT"} - rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, "*default") + rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig) if err != nil { t.Error("Failed to parse CDR in rated cdr", err) } @@ -76,14 +76,14 @@ func TestCsvRecordForkCdr(t *testing.T) { func TestCsvDataMultiplyFactor(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] + cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][0] cdrcConfig.CdrSourceId = "TEST_CDRC" cdrcConfig.ContentFields = []*config.CfgCdrField{&config.CfgCdrField{Tag: "TORField", Type: utils.META_COMPOSED, FieldId: utils.TOR, Value: []*utils.RSRField{&utils.RSRField{Id: "0"}}}, &config.CfgCdrField{Tag: "UsageField", Type: utils.META_COMPOSED, FieldId: utils.USAGE, Value: []*utils.RSRField{&utils.RSRField{Id: "1"}}}} - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: map[string]*config.CdrcConfig{"*default": cdrcConfig}} - csvProcessor.cdrcCfgs["*default"].DataUsageMultiplyFactor = 0 + csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcConfig{cdrcConfig}} + csvProcessor.cdrcCfgs[0].DataUsageMultiplyFactor = 0 cdrRow := []string{"*data", "1"} - rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, "*default") + rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig) if err != nil { t.Error("Failed to parse CDR in rated cdr", err) } @@ -100,7 +100,7 @@ func TestCsvDataMultiplyFactor(t *testing.T) { if !reflect.DeepEqual(expectedCdr, rtCdr) { t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) } - csvProcessor.cdrcCfgs["*default"].DataUsageMultiplyFactor = 1024 + csvProcessor.cdrcCfgs[0].DataUsageMultiplyFactor = 1024 expectedCdr = &engine.CDR{ CGRID: utils.Sha1("", sTime.String()), ToR: cdrRow[0], @@ -110,7 +110,7 @@ func TestCsvDataMultiplyFactor(t *testing.T) { ExtraFields: map[string]string{}, Cost: -1, } - if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, "*default"); !reflect.DeepEqual(expectedCdr, rtCdr) { + if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig); !reflect.DeepEqual(expectedCdr, rtCdr) { t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) } cdrRow = []string{"*voice", "1"} @@ -123,7 +123,7 @@ func TestCsvDataMultiplyFactor(t *testing.T) { ExtraFields: map[string]string{}, Cost: -1, } - if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, "*default"); !reflect.DeepEqual(expectedCdr, rtCdr) { + if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig); !reflect.DeepEqual(expectedCdr, rtCdr) { t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) } } diff --git a/cdrc/flatstore_local_test.go b/cdrc/flatstore_local_test.go index 44c3dcdc7..15653d577 100644 --- a/cdrc/flatstore_local_test.go +++ b/cdrc/flatstore_local_test.go @@ -85,7 +85,11 @@ func TestFlatstoreLclCreateCdrFiles(t *testing.T) { if flatstoreCfg == nil { t.Fatal("Empty default cdrc configuration") } - flatstoreCdrcCfg = flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"]["FLATSTORE"] + for _, cdrcCfg := range flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"] { + if cdrcCfg.ID == "FLATSTORE" { + flatstoreCdrcCfg = cdrcCfg + } + } if err := os.RemoveAll(flatstoreCdrcCfg.CdrInDir); err != nil { t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrInDir, err) } diff --git a/cdrc/fwv.go b/cdrc/fwv.go index 7940d268d..d672b58eb 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -49,14 +49,14 @@ func fwvValue(cdrLine string, indexStart, width int, padding string) string { return rawVal } -func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, cdrcCfgs map[string]*config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool, timezone string) *FwvRecordsProcessor { +func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcConfig, cdrcCfgs []*config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool, timezone string) *FwvRecordsProcessor { return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone} } type FwvRecordsProcessor struct { file *os.File dfltCfg *config.CdrcConfig // General parameters - cdrcCfgs map[string]*config.CdrcConfig + cdrcCfgs []*config.CdrcConfig httpClient *http.Client httpSkipTlsCheck bool timezone string @@ -125,11 +125,11 @@ func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { } self.processedRecordsNr += 1 record := string(buf) - for cfgKey, cdrcCfg := range self.cdrcCfgs { - if passes := self.recordPassesCfgFilter(record, cfgKey); !passes { + for _, cdrcCfg := range self.cdrcCfgs { + if passes := self.recordPassesCfgFilter(record, cdrcCfg); !passes { continue } - if storedCdr, err := self.recordToStoredCdr(record, cfgKey); err != nil { + if storedCdr, err := self.recordToStoredCdr(record, cdrcCfg, cdrcCfg.ID); err != nil { return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error()) } else { recordCdrs = append(recordCdrs, storedCdr) @@ -141,9 +141,9 @@ func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { return recordCdrs, nil } -func (self *FwvRecordsProcessor) recordPassesCfgFilter(record, configKey string) bool { +func (self *FwvRecordsProcessor) recordPassesCfgFilter(record string, cdrcCfg *config.CdrcConfig) bool { filterPasses := true - for _, rsrFilter := range self.cdrcCfgs[configKey].CdrFilter { + for _, rsrFilter := range cdrcCfg.CdrFilter { if rsrFilter == nil { // Nil filter does not need to match anything continue } @@ -158,8 +158,8 @@ func (self *FwvRecordsProcessor) recordPassesCfgFilter(record, configKey string) return filterPasses } -// Converts a record (header or normal) to StoredCdr -func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.CDR, error) { +// Converts a record (header or normal) to CDR +func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *config.CdrcConfig, cfgKey string) (*engine.CDR, error) { var err error var lazyHttpFields []*config.CfgCdrField var cfgFields []*config.CfgCdrField @@ -171,13 +171,13 @@ func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) storedCdr = &engine.CDR{OriginHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1} } if cfgKey == "*header" { - cfgFields = self.dfltCfg.HeaderFields - storedCdr.Source = self.dfltCfg.CdrSourceId - duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor + cfgFields = cdrcCfg.HeaderFields + storedCdr.Source = cdrcCfg.CdrSourceId + duMultiplyFactor = cdrcCfg.DataUsageMultiplyFactor } else { - cfgFields = self.cdrcCfgs[cfgKey].ContentFields - storedCdr.Source = self.cdrcCfgs[cfgKey].CdrSourceId - duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor + cfgFields = cdrcCfg.ContentFields + storedCdr.Source = cdrcCfg.CdrSourceId + duMultiplyFactor = cdrcCfg.DataUsageMultiplyFactor } for _, cdrFldCfg := range cfgFields { var fieldVal string @@ -244,7 +244,7 @@ func (self *FwvRecordsProcessor) processHeader() error { return fmt.Errorf("In header, line len: %d, have read: %d", self.lineLen, nRead) } var err error - if self.headerCdr, err = self.recordToStoredCdr(string(buf), "*header"); err != nil { + if self.headerCdr, err = self.recordToStoredCdr(string(buf), self.dfltCfg, "*header"); err != nil { return err } return nil diff --git a/cdrc/fwv_local_test.go b/cdrc/fwv_local_test.go index 7f4b96fb4..53730ac12 100644 --- a/cdrc/fwv_local_test.go +++ b/cdrc/fwv_local_test.go @@ -19,8 +19,6 @@ along with this program. If not, see package cdrc import ( - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" "io/ioutil" "net/rpc" "net/rpc/jsonrpc" @@ -28,6 +26,9 @@ import ( "path" "testing" "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" ) var fwvCfgPath string @@ -91,7 +92,11 @@ func TestFwvLclCreateCdrFiles(t *testing.T) { if fwvCfg == nil { t.Fatal("Empty default cdrc configuration") } - fwvCdrcCfg = fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"]["FWV1"] + for _, cdrcCfg := range fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"] { + if cdrcCfg.ID == "FWV1" { + fwvCdrcCfg = cdrcCfg + } + } if err := os.RemoveAll(fwvCdrcCfg.CdrInDir); err != nil { t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrInDir, err) } diff --git a/cdrc/fwv_test.go b/cdrc/fwv_test.go index a5a1175cd..7623ec124 100644 --- a/cdrc/fwv_test.go +++ b/cdrc/fwv_test.go @@ -45,11 +45,11 @@ func TestFwvValue(t *testing.T) { func TestFwvRecordPassesCfgFilter(t *testing.T) { //record, configKey string) bool { cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters + cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][0] // We don't really care that is for .csv since all we want to test are the filters cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(^+49123123120)`, utils.INFIELD_SEP) fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]} cdrLine := "CDR0000010 0 20120708181506000123451234 0040123123120 004 000018009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009" - if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter { + if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, cdrcConfig); !passesFilter { t.Error("Not passes filter") } } diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 1ae8ab419..653382ced 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -86,22 +86,23 @@ func startCdrcs(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConn } // Start CDRCs for _, cdrcCfgs := range cfg.CdrcProfiles { - var cdrcCfg *config.CdrcConfig - for _, cdrcCfg = range cdrcCfgs { // Take a random config out since they should be the same - break + var enabledCfgs []*config.CdrcConfig + for _, cdrcCfg := range cdrcCfgs { // Take a random config out since they should be the same + if cdrcCfg.Enabled { + enabledCfgs = append(enabledCfgs, cdrcCfg) + } } - if cdrcCfg.Enabled == false { - continue // Ignore not enabled + + if len(enabledCfgs) != 0 { + go startCdrc(internalCdrSChan, internalRaterChan, cdrcCfgs, cfg.HttpSkipTlsVerify, cdrcChildrenChan, exitChan) } - go startCdrc(internalCdrSChan, internalRaterChan, cdrcCfgs, cfg.HttpSkipTlsVerify, cdrcChildrenChan, exitChan) } cdrcInitialized = true // Initialized - } } // Fires up a cdrc instance -func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConnection, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, +func startCdrc(internalCdrSChan, internalRaterChan chan rpcclient.RpcClientConnection, cdrcCfgs []*config.CdrcConfig, httpSkipTlsCheck bool, closeChan chan struct{}, exitChan chan bool) { var cdrcCfg *config.CdrcConfig for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one diff --git a/config/cdrcconfig.go b/config/cdrcconfig.go index abac4b187..c4b6c9bdd 100644 --- a/config/cdrcconfig.go +++ b/config/cdrcconfig.go @@ -25,6 +25,7 @@ import ( ) type CdrcConfig struct { + ID string // free-form text identifying this CDRC instance Enabled bool // Enable/Disable the profile DryRun bool // Do not post CDRs to the server CdrsConns []*HaPoolConfig // The address where CDRs can be reached @@ -51,6 +52,9 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { return nil } var err error + if jsnCfg.Id != nil { + self.ID = *jsnCfg.Id + } if jsnCfg.Enabled != nil { self.Enabled = *jsnCfg.Enabled } @@ -129,6 +133,7 @@ func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error { // Clone itself into a new CdrcConfig func (self *CdrcConfig) Clone() *CdrcConfig { clnCdrc := new(CdrcConfig) + clnCdrc.ID = self.ID clnCdrc.Enabled = self.Enabled clnCdrc.CdrsConns = make([]*HaPoolConfig, len(self.CdrsConns)) for idx, cdrConn := range self.CdrsConns { diff --git a/config/cfg_data.json b/config/cfg_data.json index 000854998..e01b9dee0 100644 --- a/config/cfg_data.json +++ b/config/cfg_data.json @@ -18,14 +18,16 @@ "enabled": true, // enable Rater service: }, -"cdrc": { - "CDRC-CSV1": { +"cdrc": [ + { + "id": "CDRC-CSV1", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database }, - "CDRC-CSV2": { + { + "id": "CDRC-CSV2", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved @@ -38,7 +40,7 @@ {"field_id": "Usage", "value": "~9:s/^(\\d+)$/${1}s/"}, ], }, -}, +], "sm_freeswitch": { "enabled": true, // starts SessionManager service: diff --git a/config/cfg_data2.json b/config/cfg_data2.json index dc001f760..5115f5364 100644 --- a/config/cfg_data2.json +++ b/config/cfg_data2.json @@ -1,24 +1,14 @@ { -"cdrc": { - "CDRC-CSV2": { - "enabled": true, // enable CDR client functionality - "cdr_in_dir": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_dir": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "data_usage_multiply_factor": 0.000976563, - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"field_id": "ToR", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"}, - {"field_id": "AnswerTime", "value": "2"}, - ], - }, - "CDRC-CSV3": { +"cdrc": [ + { + "id": "CDRC-CSV3", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database }, -}, +], "sm_freeswitch": { "enabled": true, // starts SessionManager service: diff --git a/config/config.go b/config/config.go index b605cfe35..f9682ae58 100644 --- a/config/config.go +++ b/config/config.go @@ -85,7 +85,7 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { return nil, err } cfg.dfltCdreProfile = cfg.CdreProfiles[utils.META_DEFAULT].Clone() // So default will stay unique, will have nil pointer in case of no defaults loaded which is an extra check - cfg.dfltCdrcProfile = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT].Clone() + cfg.dfltCdrcProfile = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"][0].Clone() dfltFsConnConfig = cfg.SmFsConfig.EventSocketConns[0] // We leave it crashing here on purpose if no Connection defaults defined dfltKamConnConfig = cfg.SmKamConfig.EvapiConns[0] if err := cfg.checkConfigSanity(); err != nil { @@ -231,7 +231,7 @@ type CGRConfig struct { CDRStatsEnabled bool // Enable CDR Stats service CDRStatsSaveInterval time.Duration // Save interval duration CdreProfiles map[string]*CdreConfig - CdrcProfiles map[string]map[string]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath]map[instanceName]{Configs} + CdrcProfiles map[string][]*CdrcConfig // Number of CDRC instances running imports, format map[dirPath][]{Configs} SmGenericConfig *SmGenericConfig SmFsConfig *SmFsConfig // SMFreeSWITCH configuration SmKamConfig *SmKamConfig // SM-Kamailio Configuration @@ -319,12 +319,12 @@ func (self *CGRConfig) checkConfigSanity() error { } // CDRC sanity checks for _, cdrcCfgs := range self.CdrcProfiles { - for instID, cdrcInst := range cdrcCfgs { + for _, cdrcInst := range cdrcCfgs { if !cdrcInst.Enabled { continue } if len(cdrcInst.CdrsConns) == 0 { - return fmt.Errorf(" Instance: %s, CdrC enabled but no CDRS defined!", instID) + return fmt.Errorf(" Instance: %s, CdrC enabled but no CDRS defined!", cdrcInst.ID) } for _, conn := range cdrcInst.CdrsConns { if conn.Address == utils.MetaInternal && !self.CDRSEnabled { @@ -844,28 +844,26 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { } } } - if jsnCdrcCfg != nil { if self.CdrcProfiles == nil { - self.CdrcProfiles = make(map[string]map[string]*CdrcConfig) + self.CdrcProfiles = make(map[string][]*CdrcConfig) } - for profileName, jsnCrc1Cfg := range jsnCdrcCfg { + for _, jsnCrc1Cfg := range jsnCdrcCfg { if _, hasDir := self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir]; !hasDir { - self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir] = make(map[string]*CdrcConfig) + self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir] = make([]*CdrcConfig, 0) } - if _, hasProfile := self.CdrcProfiles[profileName]; !hasProfile { - if profileName == utils.META_DEFAULT { - self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir][profileName] = new(CdrcConfig) - } else { - self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir][profileName] = self.dfltCdrcProfile.Clone() // Clone default so we do not inherit pointers - } + var cdrcInstCfg *CdrcConfig + if *jsnCrc1Cfg.Id == utils.META_DEFAULT { + cdrcInstCfg = new(CdrcConfig) + } else { + cdrcInstCfg = self.dfltCdrcProfile.Clone() // Clone default so we do not inherit pointers } - if err = self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir][profileName].loadFromJsonCfg(jsnCrc1Cfg); err != nil { + if err := cdrcInstCfg.loadFromJsonCfg(jsnCrc1Cfg); err != nil { return err } + self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir] = append(self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir], cdrcInstCfg) } } - if jsnSmGenericCfg != nil { if err := self.SmGenericConfig.loadFromJsonCfg(jsnSmGenericCfg); err != nil { return err diff --git a/config/config_defaults.go b/config/config_defaults.go index a02d31c55..c10d0d06c 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -166,8 +166,9 @@ const CGRATES_CFG_JSON = ` }, -"cdrc": { - "*default": { +"cdrc": [ + { + "id": "*default", // identifier of the CDRC runner "enabled": false, // enable CDR client functionality "dry_run": false, // do not send the CDRs to CDRS, just parse them "cdrs_conns": [ @@ -202,8 +203,8 @@ const CGRATES_CFG_JSON = ` {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, ], "trailer_fields": [], // template of the import trailer fields - } -}, + }, +], "sm_generic": { "enabled": false, // starts SessionManager service: diff --git a/config/config_json.go b/config/config_json.go index 72b12a949..0336f5c71 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -188,12 +188,12 @@ func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) { return cfg, nil } -func (self CgrJsonCfg) CdrcJsonCfg() (map[string]*CdrcJsonCfg, error) { +func (self CgrJsonCfg) CdrcJsonCfg() ([]*CdrcJsonCfg, error) { rawCfg, hasKey := self[CDRC_JSN] if !hasKey { return nil, nil } - cfg := make(map[string]*CdrcJsonCfg) + cfg := make([]*CdrcJsonCfg, 0) if err := json.Unmarshal(*rawCfg, &cfg); err != nil { return nil, err } diff --git a/config/config_json_test.go b/config/config_json_test.go index 0a21b7ee9..520298908 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -302,8 +302,9 @@ func TestDfCdrcJsonCfg(t *testing.T) { &CdrFieldJsonCfg{Tag: utils.StringPointer("Usage"), Field_id: utils.StringPointer(utils.USAGE), Type: utils.StringPointer(utils.META_COMPOSED), Value: utils.StringPointer("13"), Mandatory: utils.BoolPointer(true)}, } - eCfg := map[string]*CdrcJsonCfg{ - "*default": &CdrcJsonCfg{ + eCfg := []*CdrcJsonCfg{ + &CdrcJsonCfg{ + Id: utils.StringPointer(utils.META_DEFAULT), Enabled: utils.BoolPointer(false), Dry_run: utils.BoolPointer(false), Cdrs_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{ @@ -330,7 +331,7 @@ func TestDfCdrcJsonCfg(t *testing.T) { if cfg, err := dfCgrJsonCfg.CdrcJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { - t.Error("Received: ", cfg["*default"]) + t.Errorf("Expecting: \n%s\n, received: \n%s\n: ", utils.ToIJSON(eCfg), utils.ToIJSON(cfg)) } } @@ -633,14 +634,16 @@ func TestNewCgrJsonCfgFromFile(t *testing.T) { &CdrFieldJsonCfg{Field_id: utils.StringPointer(utils.ANSWER_TIME), Value: utils.StringPointer("1")}, &CdrFieldJsonCfg{Field_id: utils.StringPointer(utils.USAGE), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)}, } - eCfgCdrc := map[string]*CdrcJsonCfg{ - "CDRC-CSV1": &CdrcJsonCfg{ + eCfgCdrc := []*CdrcJsonCfg{ + &CdrcJsonCfg{ + Id: utils.StringPointer("CDRC-CSV1"), Enabled: utils.BoolPointer(true), Cdr_in_dir: utils.StringPointer("/tmp/cgrates/cdrc1/in"), Cdr_out_dir: utils.StringPointer("/tmp/cgrates/cdrc1/out"), Cdr_source_id: utils.StringPointer("csv1"), }, - "CDRC-CSV2": &CdrcJsonCfg{ + &CdrcJsonCfg{ + Id: utils.StringPointer("CDRC-CSV2"), Enabled: utils.BoolPointer(true), Data_usage_multiply_factor: utils.Float64Pointer(0.000976563), Run_delay: utils.IntPointer(1), @@ -653,8 +656,7 @@ func TestNewCgrJsonCfgFromFile(t *testing.T) { if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfgCdrc, cfg) { - key := "CDRC-CSV2" - t.Errorf("Expecting:\n %+v\n received:\n %+v\n", utils.ToIJSON(eCfgCdrc[key]), utils.ToIJSON(cfg[key])) + t.Errorf("Expecting:\n %+v\n received:\n %+v\n", utils.ToIJSON(eCfgCdrc), utils.ToIJSON(cfg)) } eCfgSmFs := &SmFsJsonCfg{ Enabled: utils.BoolPointer(true), diff --git a/config/configcdrc_test.go b/config/configcdrc_test.go index 497f4bfd4..51d88f60c 100644 --- a/config/configcdrc_test.go +++ b/config/configcdrc_test.go @@ -32,10 +32,11 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { t.Error(err) } eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles = make(map[string]map[string]*CdrcConfig) + eCgrCfg.CdrcProfiles = make(map[string][]*CdrcConfig) // Default instance first - eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = map[string]*CdrcConfig{ - "*default": &CdrcConfig{ + eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = []*CdrcConfig{ + &CdrcConfig{ + ID: utils.META_DEFAULT, Enabled: false, CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", @@ -79,8 +80,9 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { TrailerFields: make([]*CfgCdrField, 0), }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = map[string]*CdrcConfig{ - "CDRC-CSV1": &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = []*CdrcConfig{ + &CdrcConfig{ + ID: "CDRC-CSV1", Enabled: true, CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", @@ -122,14 +124,15 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { TrailerFields: make([]*CfgCdrField, 0), }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = map[string]*CdrcConfig{ - "CDRC-CSV2": &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = []*CdrcConfig{ + &CdrcConfig{ + ID: "CDRC-CSV2", Enabled: true, CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", FieldSeparator: ',', DataUsageMultiplyFactor: 0.000976563, - RunDelay: 0, + RunDelay: 1000000000, MaxOpenFiles: 1024, CdrInDir: "/tmp/cgrates/cdrc2/in", CdrOutDir: "/tmp/cgrates/cdrc2/out", @@ -137,16 +140,19 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { CdrFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), HeaderFields: make([]*CfgCdrField, 0), ContentFields: []*CfgCdrField{ - &CfgCdrField{Tag: "", Type: "", FieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("~7:s/^(voice|data|sms|generic)$/*$1/", utils.INFIELD_SEP), + &CfgCdrField{FieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("~7:s/^(voice|data|sms|mms|generic)$/*$1/", utils.INFIELD_SEP), FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false}, - &CfgCdrField{Tag: "", Type: "", FieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP), + &CfgCdrField{Tag: "", Type: "", FieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false}, + &CfgCdrField{FieldId: utils.USAGE, Value: utils.ParseRSRFieldsMustCompile("~9:s/^(\\d+)$/${1}s/", utils.INFIELD_SEP), FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false}, }, TrailerFields: make([]*CfgCdrField, 0), }, } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = map[string]*CdrcConfig{ - "CDRC-CSV3": &CdrcConfig{ + eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = []*CdrcConfig{ + &CdrcConfig{ + ID: "CDRC-CSV3", Enabled: true, CdrsConns: []*HaPoolConfig{&HaPoolConfig{Address: utils.MetaInternal}}, CdrFormat: "csv", @@ -189,6 +195,6 @@ func TestLoadCdrcConfigMultipleFiles(t *testing.T) { }, } if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) { - t.Errorf("Expected: %+v, received: %+v", eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) + t.Errorf("Expected: \n%s\n, received: \n%s\n", utils.ToJSON(eCgrCfg.CdrcProfiles), utils.ToJSON(cgrCfg.CdrcProfiles)) } } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 8f8ea6c17..d05c7200e 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -143,6 +143,7 @@ type CdreJsonCfg struct { // Cdrc config section type CdrcJsonCfg struct { + Id *string Enabled *bool Dry_run *bool Cdrs_conns *[]*HaPoolJsonCfg diff --git a/data/conf/samples/cdrcflatstore/cgrates.json b/data/conf/samples/cdrcflatstore/cgrates.json index 5c25da8c5..5b637fa20 100644 --- a/data/conf/samples/cdrcflatstore/cgrates.json +++ b/data/conf/samples/cdrcflatstore/cgrates.json @@ -22,8 +22,9 @@ }, -"cdrc": { - "FLATSTORE": { +"cdrc": [ + { + "id": "FLATSTORE", "enabled": true, // enable CDR client functionality "cdrs_conns": [ {"address": "*internal"} // address where to reach CDR server. <*internal|x.y.z.y:1234> @@ -56,6 +57,6 @@ {"tag": "DialogId", "cdr_field_id": "DialogId", "type": "cdrfield", "value": "11"}, ], }, -}, +], } diff --git a/data/conf/samples/cdrcfwv/cgrates.json b/data/conf/samples/cdrcfwv/cgrates.json index 2feb548e9..6a3a3f3ab 100644 --- a/data/conf/samples/cdrcfwv/cgrates.json +++ b/data/conf/samples/cdrcfwv/cgrates.json @@ -22,8 +22,9 @@ }, -"cdrc": { - "FWV1": { +"cdrc": [ + { + "id": "FWV1", "enabled": true, // enable CDR client functionality "dry_run": true, "cdrs_conns": [ @@ -62,6 +63,6 @@ {"tag": "TotalDuration", "type": "metatag", "metatag_id":"total_duration", "value": "150", "width": 12}, ], }, -}, +], } diff --git a/data/conf/samples/fscsv/freeswitch_csvcdr.json b/data/conf/samples/fscsv/freeswitch_csvcdr.json index b20f3c3f4..27c9932b9 100644 --- a/data/conf/samples/fscsv/freeswitch_csvcdr.json +++ b/data/conf/samples/fscsv/freeswitch_csvcdr.json @@ -1,8 +1,9 @@ { // Contains CDRC template for FreeSWITCH CDR -"cdrc": { - "CDRC-CSV2": { +"cdrc": [ + { + "id": "CDRC-CSV2", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc_fs/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc_fs/out", // absolute path towards the directory where processed CDRs will be moved @@ -22,6 +23,6 @@ {"tag": "usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "~8:s/^(\\d+)$/${1}s/", "mandatory": true}, ], }, -}, +], } diff --git a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json index 1d47e5365..a9b18ad40 100644 --- a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json +++ b/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json @@ -16,14 +16,16 @@ "enabled": true, // start the CDR Server service: }, -"cdrc": { - "CDRC-CSV1": { +"cdrc": [ + { + "id": "CDRC-CSV1", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database }, - "CDRC-CSV2": { + { + "id": "CDRC-CSV2", "enabled": true, // enable CDR client functionality "cdr_in_dir": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored "cdr_out_dir": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved @@ -43,7 +45,8 @@ {"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"}, ], }, - "CDRC-CSV3": { + { + "id": "CDRC-CSV3", "enabled": true, // enable CDR client functionality "field_separator": ";", // separator used in case of csv files "cdr_in_dir": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored @@ -64,7 +67,7 @@ {"cdr_field_id": "usage", "value": "~6:s/^(\\d+)$/${1}s/"}, ], } -}, +], "cdre": { "CDRE-FW1": { From 03ca558963ae6ac75d5a69bcce1d9a48c2d81225 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 13:24:17 +0300 Subject: [PATCH 207/227] fix for SetActions API --- apier/v1/apier.go | 2 +- data/tariffplans/testtp/AccountActions.csv | 3 +- general_tests/tp_it_test.go | 52 ++++++++++++++++++++++ utils/apitpdata.go | 1 + 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index cf3f86389..28aceee33 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -530,7 +530,7 @@ func (self *ApierV1) SetActions(attrs utils.AttrSetActions, reply *string) error ExtraParameters: apiAct.ExtraParameters, Filter: apiAct.Filter, Balance: &engine.BalanceFilter{ // TODO: update this part - Uuid: utils.StringPointer(utils.GenUUID()), + Uuid: utils.StringPointer(apiAct.BalanceUuid), ID: utils.StringPointer(apiAct.BalanceId), Type: utils.StringPointer(apiAct.BalanceType), Value: vf, diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index 850d03f77..9fa3f8977 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -9,4 +9,5 @@ cgrates.org,1010,TEST_DATA_r,,true, cgrates.org,1011,TEST_VOICE,,, cgrates.org,1012,PREPAID_10,,, cgrates.org,1013,TEST_NEG,,, -cgrates.org,1014,TEST_RPC,,, \ No newline at end of file +cgrates.org,1014,TEST_RPC,,, +cgrates.org,1015,,,, diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 8165ed19c..bb7560921 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -239,3 +239,55 @@ func TestTpExecuteActionCgrRpc(t *testing.T) { t.Error("Got error on ApierV2.GetAccount: ", err.Error()) } } + +func TestTpCreateExecuteActionMatch(t *testing.T) { + if !*testIntegration { + return + } + var reply string + if err := tpRPC.Call("ApierV2.SetActions", utils.AttrSetActions{ + ActionsId: "PAYMENT_2056bd2fe137082970f97102b64e42fd", + Actions: []*utils.TPAction{ + &utils.TPAction{ + BalanceType: "*monetary", + Directions: "*out", + Identifier: "*topup", + RatingSubject: "", + Units: "10.500000", + Weight: 10, + }, + }, + }, &reply); err != nil { + t.Error("Got error on ApierV2.SetActions: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ApierV2.SetActions got reply: %s", reply) + } + if err := tpRPC.Call("ApierV2.ExecuteAction", utils.AttrExecuteAction{ + Tenant: "cgrates.org", + Account: "1015", + ActionsId: "PAYMENT_2056bd2fe137082970f97102b64e42fd", + }, &reply); err != nil { + t.Error("Got error on ApierV2.ExecuteAction: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ExecuteAction got reply: %s", reply) + } + if err := tpRPC.Call("ApierV2.ExecuteAction", utils.AttrExecuteAction{ + Tenant: "cgrates.org", + Account: "1015", + ActionsId: "PAYMENT_2056bd2fe137082970f97102b64e42fd", + }, &reply); err != nil { + t.Error("Got error on ApierV2.ExecuteAction: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ExecuteAction got reply: %s", reply) + } + var acnt engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1015"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error("Got error on ApierV2.GetAccount: ", err.Error()) + } + if len(acnt.BalanceMap) != 1 || + len(acnt.BalanceMap[utils.MONETARY]) != 1 || + acnt.BalanceMap[utils.MONETARY].GetTotalValue() != 21 { + t.Error("error matching previous created balance: ", utils.ToIJSON(acnt.BalanceMap)) + } +} diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 62f38434b..12bfc9173 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -275,6 +275,7 @@ type TPActions struct { type TPAction struct { Identifier string // Identifier mapped in the code BalanceId string // Balance identification string (account scope) + BalanceUuid string // Balance identification string (global scope) BalanceType string // Type of balance the action will operate on Directions string // Balance direction Units string // Number of units to add/deduct From 1822e895a45f338a9de4f825fd102a1e6bd832ea Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 15:22:06 +0300 Subject: [PATCH 208/227] more data usage tests --- sessionmanager/data_it_test.go | 342 +++++++++++++++++++++++++++++++++ 1 file changed, 342 insertions(+) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index dd21c6c8c..d8422d118 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -510,3 +510,345 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } } + +func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} + eAcntVal := 49997767680.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "1048576", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 1054720 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 49997767680.000000 // refunded + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } +} + +func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { + if !*testIntegration { + return + } + var acnt *engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1010"} + eAcntVal := 49997767680.000000 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv := SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.ANSWER_TIME: "2016-01-05 18:31:05", + utils.USAGE: "1048576", + } + var maxUsage float64 + if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 1054720 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.LastUsed: "0", + } + var rpl string + if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { + t.Error(err) + } + eAcntVal = 49997767680.000000 // refunded + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } +} From ee5d5b0bedb0998ef5fa635206e5f4a5b40d07a7 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 16:38:50 +0300 Subject: [PATCH 209/227] value formula string method --- apier/v1/apier.go | 5 +++-- utils/value_formula.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 28aceee33..f65b3df94 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -562,7 +562,8 @@ func (self *ApierV1) GetActions(actsId string, reply *[]*utils.TPAction) error { return utils.NewErrServerError(err) } for _, engAct := range engActs { - act := &utils.TPAction{Identifier: engAct.ActionType, + act := &utils.TPAction{ + Identifier: engAct.ActionType, ExpiryTime: engAct.ExpirationString, ExtraParameters: engAct.ExtraParameters, Filter: engAct.Filter, @@ -571,7 +572,7 @@ func (self *ApierV1) GetActions(actsId string, reply *[]*utils.TPAction) error { bf := engAct.Balance if bf != nil { act.BalanceType = bf.GetType() - act.Units = strconv.FormatFloat(bf.GetValue(), 'f', -1, 64) + act.Units = bf.Value.String() act.Directions = bf.GetDirections().String() act.DestinationIds = bf.GetDestinationIDs().String() act.RatingSubject = bf.GetRatingSubject() diff --git a/utils/value_formula.go b/utils/value_formula.go index 1ac113aa1..1e6221087 100644 --- a/utils/value_formula.go +++ b/utils/value_formula.go @@ -37,6 +37,10 @@ var ValueFormulas = map[string]valueFormula{ INCREMENTAL: incrementalFormula, } +func (vf *ValueFormula) String() string { + return ToJSON(vf) +} + func incrementalFormula(params map[string]interface{}) float64 { // check parameters unitsInterface, unitsFound := params["Units"] From dd48ee768778a449ffe9f993fd2ffb62a19e6237 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 17:35:09 +0300 Subject: [PATCH 210/227] data usage fix --- sessionmanager/data_it_test.go | 2 +- sessionmanager/smg_session.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index d8422d118..aecd35abd 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -845,7 +845,7 @@ func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { if err = smgRPC.Call("SMGenericV1.SessionEnd", smgEv, &rpl); err != nil || rpl != utils.OK { t.Error(err) } - eAcntVal = 49997767680.000000 // refunded + eAcntVal = 49997757440.000000 // 10240 (from the start) if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { t.Error(err) } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index d1701a905..79160139c 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -90,13 +90,15 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time. // total usage correction self.totalUsage -= self.lastUsage self.totalUsage += *lastUsed - //utils.Logger.Debug(fmt.Sprintf("Correction: %f", self.totalUsage.Seconds())) + //utils.Logger.Debug(fmt.Sprintf("TotalUsage Correction: %f", self.totalUsage.Seconds())) } } // apply correction from previous run if self.extraDuration < dur { dur -= self.extraDuration } else { + self.lastUsage = requestedDuration + self.totalUsage += self.lastUsage ccDuration := self.extraDuration // fake ccDuration self.extraDuration -= dur return ccDuration, nil From ff0c94c18a7416bba147a18f37c46142b494bc96 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 18:36:09 +0300 Subject: [PATCH 211/227] tests for total usage --- sessionmanager/data_it_test.go | 309 ++++++++++++++++++++++----------- sessionmanager/smg_session.go | 1 + 2 files changed, 205 insertions(+), 105 deletions(-) diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index aecd35abd..726e6d5d4 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -212,6 +212,12 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + aSessions := make([]*ActiveSession, 0) + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -238,6 +244,11 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1068576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -264,7 +275,11 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1088576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -291,6 +306,11 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1108576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -317,7 +337,11 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1128576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -341,6 +365,11 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 0 { + t.Errorf("wrong active sessions: %+v", aSessions) + } } func TestSMGDataDerivedChargingNoCredit(t *testing.T) { @@ -474,6 +503,12 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + aSessions := make([]*ActiveSession, 0) + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", @@ -509,9 +544,14 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 0 { + t.Errorf("wrong active sessions: %+v", aSessions) + } } -func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { +func TestSMGDataMultipleDataNoUsage(t *testing.T) { if !*testIntegration { return } @@ -551,31 +591,11 @@ func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - smgEv = SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.DATA, - utils.ACCID: "12349", - utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1010", - utils.SUBJECT: "1010", - utils.DESTINATION: "222", - utils.CATEGORY: "data", - utils.TENANT: "cgrates.org", - utils.REQTYPE: utils.META_PREPAID, - utils.USAGE: "1048576", - utils.LastUsed: "0", - } - if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + aSessions := make([]*ActiveSession, 0) + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { t.Error(err) - } - if maxUsage != 1.048576e+06 { - t.Error("Bad max usage: ", maxUsage) - } - eAcntVal = 49996712960.000000 // 0 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", @@ -603,32 +623,10 @@ func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - - smgEv = SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.DATA, - utils.ACCID: "12349", - utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1010", - utils.SUBJECT: "1010", - utils.DESTINATION: "222", - utils.CATEGORY: "data", - utils.TENANT: "cgrates.org", - utils.REQTYPE: utils.META_PREPAID, - utils.USAGE: "1048576", - utils.LastUsed: "0", - } - if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { t.Error(err) - } - if maxUsage != 1.048576e+06 { - t.Error("Bad max usage: ", maxUsage) - } - eAcntVal = 49996712960.000000 // 0 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", @@ -656,7 +654,73 @@ func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "0", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -680,9 +744,14 @@ func TestSMGDataLastUsedMultipleDataNoUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 0 { + t.Errorf("wrong active sessions: %+v", aSessions) + } } -func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { +func TestSMGDataMultipleDataConstantUsage(t *testing.T) { if !*testIntegration { return } @@ -722,57 +791,11 @@ func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - smgEv = SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.DATA, - utils.ACCID: "12349", - utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1010", - utils.SUBJECT: "1010", - utils.DESTINATION: "222", - utils.CATEGORY: "data", - utils.TENANT: "cgrates.org", - utils.REQTYPE: utils.META_PREPAID, - utils.USAGE: "1048576", - utils.LastUsed: "600", - } - if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + aSessions := make([]*ActiveSession, 0) + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { t.Error(err) - } - if maxUsage != 1.048576e+06 { - t.Error("Bad max usage: ", maxUsage) - } - eAcntVal = 49996712960.000000 // 0 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) - } - smgEv = SMGenericEvent{ - utils.EVENT_NAME: "TEST_EVENT", - utils.TOR: utils.DATA, - utils.ACCID: "12349", - utils.DIRECTION: utils.OUT, - utils.ACCOUNT: "1010", - utils.SUBJECT: "1010", - utils.DESTINATION: "222", - utils.CATEGORY: "data", - utils.TENANT: "cgrates.org", - utils.REQTYPE: utils.META_PREPAID, - utils.USAGE: "1048576", - utils.LastUsed: "600", - } - if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { - t.Error(err) - } - if maxUsage != 1.048576e+06 { - t.Error("Bad max usage: ", maxUsage) - } - eAcntVal = 49996712960.000000 // 0 - if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { - t.Error(err) - } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { - t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1048576 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) } smgEv = SMGenericEvent{ @@ -801,6 +824,11 @@ func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1049176 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -827,7 +855,73 @@ func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } - + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1049776 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1050376 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } + smgEv = SMGenericEvent{ + utils.EVENT_NAME: "TEST_EVENT", + utils.TOR: utils.DATA, + utils.ACCID: "12349", + utils.DIRECTION: utils.OUT, + utils.ACCOUNT: "1010", + utils.SUBJECT: "1010", + utils.DESTINATION: "222", + utils.CATEGORY: "data", + utils.TENANT: "cgrates.org", + utils.REQTYPE: utils.META_PREPAID, + utils.USAGE: "1048576", + utils.LastUsed: "600", + } + if err := smgRPC.Call("SMGenericV1.SessionUpdate", smgEv, &maxUsage); err != nil { + t.Error(err) + } + if maxUsage != 1.048576e+06 { + t.Error("Bad max usage: ", maxUsage) + } + eAcntVal = 49996712960.000000 // 0 + if err := smgRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { + t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) + } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 1 || aSessions[0].Usage.Seconds() != 1050976 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, @@ -851,4 +945,9 @@ func TestSMGDataLastUsedMultipleDataConstantUsage(t *testing.T) { } else if acnt.BalanceMap[utils.DATA].GetTotalValue() != eAcntVal { t.Errorf("Expected: %f, received: %f", eAcntVal, acnt.BalanceMap[utils.DATA].GetTotalValue()) } + if err := smgRPC.Call("SMGenericV1.ActiveSessions", utils.AttrSMGGetActiveSessions{}, &aSessions); err != nil { + t.Error(err) + } else if len(aSessions) != 0 { + t.Errorf("wrong active sessions: %f", aSessions[0].Usage.Seconds()) + } } diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index 79160139c..df622953b 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -113,6 +113,7 @@ func (self *SMGSession) debit(dur time.Duration, lastUsed *time.Duration) (time. self.cd.DurationIndex += dur cc := &engine.CallCost{} if err := self.rater.Call("Responder.MaxDebit", self.cd, cc); err != nil { + self.lastUsage = 0 self.lastDebit = 0 return 0, err } From d8e789c1a06d12ce659d2bdc16f97bba83507a30 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 18:50:12 +0300 Subject: [PATCH 212/227] fix GetActions test --- apier/v1/apier_local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 124b6a984..edfd7bd57 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -974,7 +974,7 @@ func TestApierGetActions(t *testing.T) { return } expectActs := []*utils.TPAction{ - &utils.TPAction{Identifier: engine.TOPUP_RESET, BalanceType: utils.MONETARY, Directions: utils.OUT, Units: "75", BalanceWeight: "0", BalanceBlocker: "false", BalanceDisabled: "false", ExpiryTime: engine.UNLIMITED, Weight: 20.0}} + &utils.TPAction{Identifier: engine.TOPUP_RESET, BalanceType: utils.MONETARY, Directions: utils.OUT, Units: "{\"Method\":\"\",\"Params\":null,\"Static\":75}", BalanceWeight: "0", BalanceBlocker: "false", BalanceDisabled: "false", ExpiryTime: engine.UNLIMITED, Weight: 20.0}} var reply []*utils.TPAction if err := rater.Call("ApierV1.GetActions", "ACTS_1", &reply); err != nil { From ac5fddd8ca83d80e0a0e1b2b8a5fdd8437e1e39c Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 21:08:10 +0300 Subject: [PATCH 213/227] improved GetActions in ApierV2 (params changes!) --- apier/v1/apier.go | 2 +- apier/v2/accounts.go | 2 +- apier/v2/apier.go | 55 +++++++++++++++++++++++++++++ console/actions.go | 11 +++--- data/tariffplans/testtp/Actions.csv | 2 ++ engine/storage_interface.go | 1 + engine/tp_reader.go | 6 ++-- 7 files changed, 70 insertions(+), 9 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index f65b3df94..6d489468d 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -572,7 +572,7 @@ func (self *ApierV1) GetActions(actsId string, reply *[]*utils.TPAction) error { bf := engAct.Balance if bf != nil { act.BalanceType = bf.GetType() - act.Units = bf.Value.String() + act.Units = strconv.FormatFloat(bf.GetValue(), 'f', -1, 64) act.Directions = bf.GetDirections().String() act.DestinationIds = bf.GetDestinationIDs().String() act.RatingSubject = bf.GetRatingSubject() diff --git a/apier/v2/accounts.go b/apier/v2/accounts.go index 08f6832dd..83d3d56f2 100644 --- a/apier/v2/accounts.go +++ b/apier/v2/accounts.go @@ -33,7 +33,7 @@ func (self *ApierV2) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Ac var accountKeys []string var err error if len(attr.AccountIds) == 0 { - if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Tenant), true); err != nil { + if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX+attr.Tenant, true); err != nil { return err } } else { diff --git a/apier/v2/apier.go b/apier/v2/apier.go index 9f5f4cc2a..4bf68bd4f 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -21,6 +21,7 @@ package v2 import ( "errors" "fmt" + "math" "os" "path" "strings" @@ -263,3 +264,57 @@ func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, *reply = *li return nil } + +type AttrGetActions struct { + ActionIDs []string + Offset int // Set the item offset + Limit int // Limit number of items retrieved +} + +// Retrieves actions attached to specific ActionsId within cache +func (self *ApierV2) GetActions(attr AttrGetActions, reply *map[string]engine.Actions) error { + var actionKeys []string + var err error + if len(attr.ActionIDs) == 0 { + if actionKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACTION_PREFIX, false); err != nil { + return err + } + } else { + for _, accID := range attr.ActionIDs { + if len(accID) == 0 { // Source of error returned from redis (key not found) + continue + } + actionKeys = append(actionKeys, utils.ACCOUNT_PREFIX+accID) + } + } + if len(actionKeys) == 0 { + return nil + } + if attr.Offset > len(actionKeys) { + attr.Offset = len(actionKeys) + } + if attr.Offset < 0 { + attr.Offset = 0 + } + var limitedActions []string + if attr.Limit != 0 { + max := math.Min(float64(attr.Offset+attr.Limit), float64(len(actionKeys))) + limitedActions = actionKeys[attr.Offset:int(max)] + } else { + limitedActions = actionKeys[attr.Offset:] + } + retActions := make(map[string]engine.Actions) + for _, accKey := range limitedActions { + key := accKey[len(utils.ACTION_PREFIX):] + acts, err := self.RatingDb.GetActions(key, false) + if err != nil { + return utils.NewErrServerError(err) + } + if len(acts) > 0 { + retActions[key] = acts + + } + } + *reply = retActions + return nil +} diff --git a/console/actions.go b/console/actions.go index 25e9af0e5..1d2504029 100644 --- a/console/actions.go +++ b/console/actions.go @@ -18,7 +18,10 @@ along with this program. If not, see package console -import "github.com/cgrates/cgrates/utils" +import ( + "github.com/cgrates/cgrates/apier/v2" + "github.com/cgrates/cgrates/engine" +) func init() { c := &CmdGetActions{ @@ -33,7 +36,7 @@ func init() { type CmdGetActions struct { name string rpcMethod string - rpcParams *StringWrapper + rpcParams *v2.AttrGetActions *CommandExecuter } @@ -47,7 +50,7 @@ func (self *CmdGetActions) RpcMethod() string { func (self *CmdGetActions) RpcParams(reset bool) interface{} { if reset || self.rpcParams == nil { - self.rpcParams = &StringWrapper{} + self.rpcParams = &v2.AttrGetActions{} } return self.rpcParams } @@ -57,6 +60,6 @@ func (self *CmdGetActions) PostprocessRpcParams() error { } func (self *CmdGetActions) RpcResult() interface{} { - a := make([]*utils.TPAction, 0) + a := make(map[string]engine.Actions, 0) return &a } diff --git a/data/tariffplans/testtp/Actions.csv b/data/tariffplans/testtp/Actions.csv index 680866202..20d61a27d 100644 --- a/data/tariffplans/testtp/Actions.csv +++ b/data/tariffplans/testtp/Actions.csv @@ -10,3 +10,5 @@ TOPUP_DATA_r,*topup,,,,*data,*out,,DATA_DEST,datar,,*unlimited,,50000000000,10,f TOPUP_VOICE,*topup,,,,*voice,*out,,GERMANY_MOBILE,,,*unlimited,,50000,10,false,false,10 TOPUP_NEG,*topup,,,,*voice,*out,,GERMANY;!GERMANY_MOBILE,*zero1m,,*unlimited,,100,10,false,false,10 RPC,*cgr_rpc,"{""Address"": ""localhost:2013"",""Transport"":""*gob"",""Method"":""ApierV2.SetAccount"",""Attempts"":1,""Async"" :false,""Params"":{""Account"":""rpc"",""Tenant"":""cgrates.org""}}",,,,,,,,,,,,,,, +DID,*debit,,,,*monetary,*out,,*any,,,*unlimited,*any,"{""Method"":""*incremental"",""Params"":{""Units"":1, ""Interval"":""month"",""Increment"":""day""}}",10.0,,,10.0 +DID,*cdrlog,"{""action"":""^DID"",""prev_balance"":""BalanceValue""}",,,*monetary,*out,,*any,,,*unlimited,,,10.0,,,10.0 diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 96a4b32c6..5fd6657db 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -204,6 +204,7 @@ func NewCodecMsgpackMarshaler() *CodecMsgpackMarshaler { cmm := &CodecMsgpackMarshaler{new(codec.MsgpackHandle)} mh := cmm.mh mh.MapType = reflect.TypeOf(map[string]interface{}(nil)) + mh.RawToString = true return cmm } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index a2a71470b..4eaf2e81f 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -514,7 +514,7 @@ func (tpr *TpReader) LoadActions() (err error) { } } acts[idx] = &Action{ - Id: tag + strconv.Itoa(idx), + Id: tag, ActionType: tpact.Identifier, //BalanceType: tpact.BalanceType, Weight: tpact.Weight, @@ -990,7 +990,7 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *TpAccountAction) error } } acts[idx] = &Action{ - Id: tag + strconv.Itoa(idx), + Id: tag, ActionType: tpact.Identifier, //BalanceType: tpact.BalanceType, Weight: tpact.Weight, @@ -1338,7 +1338,7 @@ func (tpr *TpReader) LoadCdrStatsFiltered(tag string, save bool) (err error) { } } acts[idx] = &Action{ - Id: tag + strconv.Itoa(idx), + Id: tag, ActionType: tpact.Identifier, //BalanceType: tpact.BalanceType, Weight: tpact.Weight, From 114447d6d3a13c15398c82c4c7ea941924a408ae Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 21:22:03 +0300 Subject: [PATCH 214/227] test fixes --- apier/v1/apier_local_test.go | 4 ++-- engine/loader_csv_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index edfd7bd57..0e1e7f09e 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -974,7 +974,7 @@ func TestApierGetActions(t *testing.T) { return } expectActs := []*utils.TPAction{ - &utils.TPAction{Identifier: engine.TOPUP_RESET, BalanceType: utils.MONETARY, Directions: utils.OUT, Units: "{\"Method\":\"\",\"Params\":null,\"Static\":75}", BalanceWeight: "0", BalanceBlocker: "false", BalanceDisabled: "false", ExpiryTime: engine.UNLIMITED, Weight: 20.0}} + &utils.TPAction{Identifier: engine.TOPUP_RESET, BalanceType: utils.MONETARY, Directions: utils.OUT, Units: "75", BalanceWeight: "0", BalanceBlocker: "false", BalanceDisabled: "false", ExpiryTime: engine.UNLIMITED, Weight: 20.0}} var reply []*utils.TPAction if err := rater.Call("ApierV1.GetActions", "ACTS_1", &reply); err != nil { @@ -1297,7 +1297,7 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { if rcvStats.Destinations != 5 || rcvStats.RatingPlans != 5 || rcvStats.RatingProfiles != 5 || - rcvStats.Actions != 10 || + rcvStats.Actions != 11 || rcvStats.DerivedChargers != 3 { t.Errorf("Calling ApierV1.GetCacheStats received: %+v", rcvStats) } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index f2fb1f898..fbcf9e8f2 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -833,7 +833,7 @@ func TestLoadActions(t *testing.T) { as1 := csvr.actions["MINI"] expected := []*Action{ &Action{ - Id: "MINI0", + Id: "MINI", ActionType: TOPUP_RESET, ExpirationString: UNLIMITED, ExtraParameters: "", @@ -853,7 +853,7 @@ func TestLoadActions(t *testing.T) { }, }, &Action{ - Id: "MINI1", + Id: "MINI", ActionType: TOPUP, ExpirationString: UNLIMITED, ExtraParameters: "", @@ -880,7 +880,7 @@ func TestLoadActions(t *testing.T) { as2 := csvr.actions["SHARED"] expected = []*Action{ &Action{ - Id: "SHARED0", + Id: "SHARED", ActionType: TOPUP, ExpirationString: UNLIMITED, Weight: 10, @@ -905,7 +905,7 @@ func TestLoadActions(t *testing.T) { as3 := csvr.actions["DEFEE"] expected = []*Action{ &Action{ - Id: "DEFEE0", + Id: "DEFEE", ActionType: CDRLOG, ExtraParameters: `{"Category":"^ddi","MediationRunId":"^did_run"}`, Weight: 10, From 0777c299b3c87bea75c131657345c8c0378ed949 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 29 Apr 2016 22:26:05 +0300 Subject: [PATCH 215/227] started RemActions API --- apier/v1/apier.go | 45 ++++++++++++++++++++++++++++++++++ engine/storage_interface.go | 1 + engine/storage_map.go | 7 ++++++ engine/storage_mongo_datadb.go | 4 +++ engine/storage_redis.go | 5 ++++ 5 files changed, 62 insertions(+) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 6d489468d..1ca28e9a0 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1064,3 +1064,48 @@ func (self *ApierV1) GetLoadHistory(attrs utils.Paginator, reply *[]*engine.Load *reply = loadHist[offset:nrItems] return nil } + +type AttrRemActions struct { + ActionIDs []string +} + +func (self *ApierV1) RemActions(attr AttrRemActions, reply *string) error { + if attr.ActionIDs == nil { + return nil + } + stringMap := utils.NewStringMap(attr.ActionIDs...) + keys, err := self.RatingDb.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX, true) + if err != nil { + return err + } + for _, key := range keys { + getAttrs, err := self.RatingDb.GetActionTriggers(key[len(utils.ACTION_TRIGGER_PREFIX):]) + if err != nil { + return err + } + for _, atr := range getAttrs { + if _, found := stringMap[atr.ActionsID]; found { + // found action trigger referencing action; abort + return fmt.Errorf("action %s refenced by action trigger %s", atr.ActionsID, atr.ID) + } + } + } + allAplsMap, err := self.RatingDb.GetAllActionPlans() + if err != nil { + return err + } + for _, apl := range allAplsMap { + for _, aID := range attr.ActionIDs { + if _, found := apl.AccountIDs[aID]; found { + return fmt.Errorf("action %s refenced by action plan %s", aID, apl.Id) + } + } + + } + for _, aID := range attr.ActionIDs { + if err := self.RatingDb.RemoveActions(aID); err != nil { + return err + } + } + return nil +} diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 5fd6657db..327be197f 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -59,6 +59,7 @@ type RatingStorage interface { SetDerivedChargers(string, *utils.DerivedChargers) error GetActions(string, bool) (Actions, error) SetActions(string, Actions) error + RemoveActions(string) error GetSharedGroup(string, bool) (*SharedGroup, error) SetSharedGroup(*SharedGroup) error GetActionTriggers(string) (ActionTriggers, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index afab1a06d..f9a747a5b 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -469,6 +469,13 @@ func (ms *MapStorage) SetActions(key string, as Actions) (err error) { return } +func (ms *MapStorage) RemoveActions(key string) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() + delete(ms.dict, utils.ACTION_PREFIX+key) + return +} + func (ms *MapStorage) GetSharedGroup(key string, skipCache bool) (sg *SharedGroup, err error) { ms.mu.RLock() defer ms.mu.RUnlock() diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 0e2e59113..70fc91dc4 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -916,6 +916,10 @@ func (ms *MongoStorage) SetActions(key string, as Actions) error { return err } +func (ms *MongoStorage) RemoveActions(key string) error { + return ms.db.C(colAct).Remove(bson.M{"key": key}) +} + func (ms *MongoStorage) GetSharedGroup(key string, skipCache bool) (sg *SharedGroup, err error) { if !skipCache { if x, err := cache2go.Get(utils.SHARED_GROUP_PREFIX + key); err == nil { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 896181922..be591ff48 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -590,6 +590,11 @@ func (rs *RedisStorage) SetActions(key string, as Actions) (err error) { return } +func (rs *RedisStorage) RemoveActions(key string) (err error) { + err = rs.db.Cmd("DEL", utils.ACTION_PREFIX+key).Err + return +} + func (rs *RedisStorage) GetSharedGroup(key string, skipCache bool) (sg *SharedGroup, err error) { key = utils.SHARED_GROUP_PREFIX + key if !skipCache { From ce04b07a43c065d834079644b3c2bba723a5635d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 30 Apr 2016 13:50:09 +0300 Subject: [PATCH 216/227] console command and test for RemActions --- apier/v1/apier.go | 29 +++++++++++---- apier/v2/apier.go | 1 + console/actions_remove.go | 62 ++++++++++++++++++++++++++++++++ general_tests/tp_it_test.go | 71 +++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 console/actions_remove.go diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 1ca28e9a0..21d64d422 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -523,7 +523,7 @@ func (self *ApierV1) SetActions(attrs utils.AttrSetActions, reply *string) error } a := &engine.Action{ - Id: utils.GenUUID(), + Id: attrs.ActionsId, ActionType: apiAct.Identifier, Weight: apiAct.Weight, ExpirationString: apiAct.ExpiryTime, @@ -1071,41 +1071,56 @@ type AttrRemActions struct { func (self *ApierV1) RemActions(attr AttrRemActions, reply *string) error { if attr.ActionIDs == nil { - return nil + err := utils.ErrNotFound + *reply = err.Error() + return err } stringMap := utils.NewStringMap(attr.ActionIDs...) keys, err := self.RatingDb.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX, true) if err != nil { + *reply = err.Error() return err } for _, key := range keys { getAttrs, err := self.RatingDb.GetActionTriggers(key[len(utils.ACTION_TRIGGER_PREFIX):]) if err != nil { + *reply = err.Error() return err } for _, atr := range getAttrs { if _, found := stringMap[atr.ActionsID]; found { // found action trigger referencing action; abort - return fmt.Errorf("action %s refenced by action trigger %s", atr.ActionsID, atr.ID) + err := fmt.Errorf("action %s refenced by action trigger %s", atr.ActionsID, atr.ID) + *reply = err.Error() + return err } } } allAplsMap, err := self.RatingDb.GetAllActionPlans() - if err != nil { + if err != nil && err != utils.ErrNotFound { + *reply = err.Error() return err } for _, apl := range allAplsMap { - for _, aID := range attr.ActionIDs { - if _, found := apl.AccountIDs[aID]; found { - return fmt.Errorf("action %s refenced by action plan %s", aID, apl.Id) + for _, atm := range apl.ActionTimings { + if _, found := stringMap[atm.ActionsID]; found { + err := fmt.Errorf("action %s refenced by action plan %s", atm.ActionsID, apl.Id) + *reply = err.Error() + return err } } } for _, aID := range attr.ActionIDs { if err := self.RatingDb.RemoveActions(aID); err != nil { + *reply = err.Error() return err } } + if err := self.RatingDb.CacheRatingPrefixes(utils.ACTION_PREFIX); err != nil { + *reply = err.Error() + return err + } + *reply = utils.OK return nil } diff --git a/apier/v2/apier.go b/apier/v2/apier.go index 4bf68bd4f..e337400b0 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -315,6 +315,7 @@ func (self *ApierV2) GetActions(attr AttrGetActions, reply *map[string]engine.Ac } } + *reply = retActions return nil } diff --git a/console/actions_remove.go b/console/actions_remove.go new file mode 100644 index 000000000..c7d5f54db --- /dev/null +++ b/console/actions_remove.go @@ -0,0 +1,62 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2015 ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package console + +import "github.com/cgrates/cgrates/apier/v1" + +func init() { + c := &CmdRemoveActions{ + name: "actions_remove", + rpcMethod: "ApierV1.RemActions", + } + commands[c.Name()] = c + c.CommandExecuter = &CommandExecuter{c} +} + +// Commander implementation +type CmdRemoveActions struct { + name string + rpcMethod string + rpcParams *v1.AttrRemActions + *CommandExecuter +} + +func (self *CmdRemoveActions) Name() string { + return self.name +} + +func (self *CmdRemoveActions) RpcMethod() string { + return self.rpcMethod +} + +func (self *CmdRemoveActions) RpcParams(reset bool) interface{} { + if reset || self.rpcParams == nil { + self.rpcParams = &v1.AttrRemActions{} + } + return self.rpcParams +} + +func (self *CmdRemoveActions) PostprocessRpcParams() error { + return nil +} + +func (self *CmdRemoveActions) RpcResult() interface{} { + var s string + return &s +} diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index bb7560921..723ebda93 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cgrates/cgrates/apier/v1" + "github.com/cgrates/cgrates/apier/v2" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" @@ -291,3 +292,73 @@ func TestTpCreateExecuteActionMatch(t *testing.T) { t.Error("error matching previous created balance: ", utils.ToIJSON(acnt.BalanceMap)) } } + +func TestTpSetRemActions(t *testing.T) { + if !*testIntegration { + return + } + var reply string + if err := tpRPC.Call("ApierV2.SetActions", utils.AttrSetActions{ + ActionsId: "TO_BE_DELETED", + Actions: []*utils.TPAction{ + &utils.TPAction{ + BalanceType: "*monetary", + Directions: "*out", + Identifier: "*topup", + RatingSubject: "", + Units: "10.500000", + Weight: 10, + }, + }, + }, &reply); err != nil { + t.Error("Got error on ApierV2.SetActions: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ApierV2.SetActions got reply: %s", reply) + } + actionsMap := make(map[string]engine.Actions) + if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ + ActionIDs: []string{"PAYMENT_2056bd2fe137082970f97102b64e42fd"}, + }, &actionsMap); err != nil { + t.Error("Got error on ApierV2.GetActions: ", err.Error()) + } else if len(actionsMap) != 1 { + t.Errorf("Calling ApierV2.GetActions got reply: %s", utils.ToIJSON(actionsMap)) + } + if err := tpRPC.Call("ApierV2.RemActions", v1.AttrRemActions{ + ActionIDs: []string{"PAYMENT_2056bd2fe137082970f97102b64e42fd"}, + }, &reply); err != nil { + t.Error("Got error on ApierV2.RemActions: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ApierV2.RemActions got reply: %s", reply) + } + if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ + ActionIDs: []string{"PAYMENT_2056bd2fe137082970f97102b64e42fd"}, + }, &actionsMap); err == nil { + t.Error("no error on ApierV2.GetActions: ", err) + } +} + +func TestTpRemActionsRefenced(t *testing.T) { + actionsMap := make(map[string]engine.Actions) + if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ + ActionIDs: []string{"TOPUP_VOICE"}, + }, &actionsMap); err != nil { + t.Error("Got error on ApierV2.GetActions: ", err.Error()) + } else if len(actionsMap) != 1 { + t.Errorf("Calling ApierV2.GetActions got reply: %s", utils.ToIJSON(actionsMap)) + } + var reply string + if err := tpRPC.Call("ApierV2.RemActions", v1.AttrRemActions{ + ActionIDs: []string{"TOPUP_VOICE"}, + }, &reply); err == nil { + t.Error("No error on ApierV2.RemActions: ", err.Error()) + } else if reply == utils.OK { + t.Errorf("Calling ApierV2.RemActions got reply: %s", reply) + } + if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ + ActionIDs: []string{"TOPUP_VOICE"}, + }, &actionsMap); err != nil { + t.Error("Got error on ApierV2.GetActions: ", err.Error()) + } else if len(actionsMap) != 1 { + t.Errorf("Calling ApierV2.GetActions got reply: %s", utils.ToIJSON(actionsMap)) + } +} From e96f5e34b0866bdb4b3b33d365aefc1f79f80058 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Sat, 30 Apr 2016 14:03:49 +0300 Subject: [PATCH 217/227] fixed test and removed pprof --- cmd/cgr-engine/cgr-engine.go | 2 +- general_tests/tp_it_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index c342e56ba..ddf5b3b14 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -22,7 +22,7 @@ import ( "flag" "fmt" "log" - _ "net/http/pprof" + // _ "net/http/pprof" "os" "runtime" "runtime/pprof" diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 723ebda93..a917c9961 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -338,6 +338,9 @@ func TestTpSetRemActions(t *testing.T) { } func TestTpRemActionsRefenced(t *testing.T) { + if !*testIntegration { + return + } actionsMap := make(map[string]engine.Actions) if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ ActionIDs: []string{"TOPUP_VOICE"}, From 4fb9a641eec956c7c1e4387f1a423ba2ec2767a2 Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 1 May 2016 17:06:25 +0200 Subject: [PATCH 218/227] Refactored CDRs internal functions, better CDRC .csv testing --- apier/v1/cdrstatsv1_local_test.go | 26 ++- cdrc/cdrc_local_test.go | 223 ------------------------- cdrc/csv_it_test.go | 185 ++++++++++++++++++++ config/config.go | 2 +- data/conf/cgrates/cgrates.json | 10 +- data/conf/samples/cdrccsv/cgrates.json | 67 ++++++++ data/tariffplans/cdrstats/CdrStats.csv | 4 +- engine/cdrs.go | 176 +++++++++---------- engine/stats.go | 1 + 9 files changed, 364 insertions(+), 330 deletions(-) delete mode 100644 cdrc/cdrc_local_test.go create mode 100644 cdrc/csv_it_test.go create mode 100644 data/conf/samples/cdrccsv/cgrates.json diff --git a/apier/v1/cdrstatsv1_local_test.go b/apier/v1/cdrstatsv1_local_test.go index 9d6b82130..814b36d00 100644 --- a/apier/v1/cdrstatsv1_local_test.go +++ b/apier/v1/cdrstatsv1_local_test.go @@ -104,6 +104,13 @@ func TestCDRStatsLclGetQueueIds2(t *testing.T) { } else if len(eQueueIds) != len(queueIds) { t.Errorf("Expecting: %v, received: %v", eQueueIds, queueIds) } + var rcvMetrics map[string]float64 + expectedMetrics := map[string]float64{"ASR": -1, "ACD": -1} + if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics); err != nil { + t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error()) + } else if !reflect.DeepEqual(expectedMetrics, rcvMetrics) { + t.Errorf("Expecting: %v, received: %v", expectedMetrics, rcvMetrics) + } } func TestCDRStatsLclPostCdrs(t *testing.T) { @@ -112,28 +119,28 @@ func TestCDRStatsLclPostCdrs(t *testing.T) { } httpClient := new(http.Client) storedCdrs := []*engine.CDR{ - &engine.CDR{CGRID: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", + &engine.CDR{CGRID: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsafa", OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(), RunID: utils.DEFAULT_RUNID, Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, }, - &engine.CDR{CGRID: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", + &engine.CDR{CGRID: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsafb", OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(), RunID: utils.DEFAULT_RUNID, Usage: time.Duration(5) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, }, - &engine.CDR{CGRID: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", + &engine.CDR{CGRID: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsafc", OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(), RunID: utils.DEFAULT_RUNID, Usage: time.Duration(30) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, }, - &engine.CDR{CGRID: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", + &engine.CDR{CGRID: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsafd", OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Time{}, @@ -146,7 +153,7 @@ func TestCDRStatsLclPostCdrs(t *testing.T) { t.Error(err.Error()) } } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) + time.Sleep(time.Duration(1000) * time.Millisecond) } func TestCDRStatsLclGetMetrics1(t *testing.T) { @@ -204,3 +211,12 @@ func TestCDRStatsLclResetMetrics(t *testing.T) { t.Errorf("Expecting: %v, received: %v", expectedMetrics2, rcvMetrics2) } } + +func TestCDRStatsLclKillEngine(t *testing.T) { + if !*testLocal { + return + } + if err := engine.KillEngine(*waitRater); err != nil { + t.Error(err) + } +} diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go deleted file mode 100644 index 44832531f..000000000 --- a/cdrc/cdrc_local_test.go +++ /dev/null @@ -1,223 +0,0 @@ -/* -Rating system designed to be used in VoIP Carriers World -Copyright (C) 2012-2015 ITsysCOM - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "errors" - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -/* -README: - - Enable local tests by passing '-local' to the go test command - It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data or passed via command arguments. - Prior running the tests, create database and users by running: - mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql - What these tests do: - * Flush tables in storDb. - * Start engine with default configuration and give it some time to listen (here caching can slow down). - * -*/ - -var cfgPath string -var cfg *config.CGRConfig -var cdrcCfgs []*config.CdrcConfig -var cdrcCfg *config.CdrcConfig - -var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args -var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") -var storDbType = flag.String("stordb_type", "mysql", "The type of the storDb database ") -var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for rater to start and cache") - -var fileContent1 = `accid11,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -accid12,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -dummy_data -accid13,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -` - -var fileContent2 = `accid21,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -accid22,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -#accid1,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1 -accid23,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1` - -var fileContent3 = `accid31;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1 -accid32;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1 -#accid1;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1 -accid33;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1` - -func startEngine() error { - enginePath, err := exec.LookPath("cgr-engine") - if err != nil { - return errors.New("Cannot find cgr-engine executable") - } - stopEngine() - engine := exec.Command(enginePath, "-config", cfgPath) - if err := engine.Start(); err != nil { - return fmt.Errorf("Cannot start cgr-engine: %s", err.Error()) - } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up - return nil -} - -func stopEngine() error { - exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it - return nil -} - -// Need it here and not in init since Travis has no possibility to load local file -func TestCsvLclLoadConfigt(*testing.T) { - if !*testLocal { - return - } - cfgPath = path.Join(*dataDir, "conf", "samples", "apier") - cfg, _ = config.NewCGRConfigFromFolder(cfgPath) - if len(cfg.CdrcProfiles) > 0 { - cdrcCfgs = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] - } -} - -func TestCsvLclEmptyTables(t *testing.T) { - if !*testLocal { - return - } - if *storDbType != utils.MYSQL { - t.Fatal("Unsupported storDbType") - } - mysql, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) - if err != nil { - t.Fatal("Error on opening database connection: ", err) - } - for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} { - if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil { - t.Fatal("Error on mysql creation: ", err.Error()) - return // No point in going further - } - } - if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", utils.TBL_CDRS)); err != nil { - t.Fatal(err.Error()) - } -} - -// Creates cdr files and starts the engine -func TestCsvLclCreateCdrFiles(t *testing.T) { - if !*testLocal { - return - } - if cdrcCfgs == nil { - t.Fatal("Empty default cdrc configuration") - } - for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one - break - } - if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil { - t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err) - } - if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err) - } - if err := os.RemoveAll(cdrcCfg.CdrOutDir); err != nil { - t.Fatal("Error removing folder: ", cdrcCfg.CdrOutDir, err) - } - if err := os.MkdirAll(cdrcCfg.CdrOutDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cdrcCfg.CdrOutDir, err) - } - if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil { - t.Fatal(err.Error) - } - if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil { - t.Fatal(err.Error) - } - -} - -func TestCsvLclProcessCdrDir(t *testing.T) { - if !*testLocal { - return - } - var cdrcCfg *config.CdrcConfig - for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one - break - } - for _, cdrsConn := range cdrcCfg.CdrsConns { - if cdrsConn.Address == utils.MetaInternal { - cdrsConn.Address = "127.0.0.1:2013" - } - } - if err := startEngine(); err != nil { - t.Fatal(err.Error()) - } - cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}), "") - if err != nil { - t.Fatal(err.Error()) - } - if err := cdrc.processCdrDir(); err != nil { - t.Error(err) - } - stopEngine() -} - -// Creates cdr files and starts the engine -func TestCsvLclCreateCdr3File(t *testing.T) { - if !*testLocal { - return - } - if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil { - t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err) - } - if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil { - t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err) - } - if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil { - t.Fatal(err.Error) - } -} - -func TestCsvLclProcessCdr3Dir(t *testing.T) { - if !*testLocal { - return - } - for _, cdrsConn := range cdrcCfg.CdrsConns { - if cdrsConn.Address == utils.MetaInternal { - cdrsConn.Address = "127.0.0.1:2013" - } - } - if err := startEngine(); err != nil { - t.Fatal(err.Error()) - } - cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}), "") - if err != nil { - t.Fatal(err.Error()) - } - if err := cdrc.processCdrDir(); err != nil { - t.Error(err) - } - stopEngine() -} diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go new file mode 100644 index 000000000..ea2491e05 --- /dev/null +++ b/cdrc/csv_it_test.go @@ -0,0 +1,185 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2012-2015 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package cdrc + +import ( + "flag" + "io/ioutil" + "net/rpc" + "net/rpc/jsonrpc" + "os" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" +) + +/* +README: + + Enable local tests by passing '-local' to the go test command + It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data or passed via command arguments. + Prior running the tests, create database and users by running: + mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql + What these tests do: + * Flush tables in storDb. + * Start engine with default configuration and give it some time to listen (here caching can slow down). + * +*/ + +var csvCfgPath string +var csvCfg *config.CGRConfig +var cdrcCfgs []*config.CdrcConfig +var cdrcCfg *config.CdrcConfig +var cdrcRpc *rpc.Client + +var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args +var testIT = flag.Bool("integration", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args +var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") +var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for rater to start and cache") + +var fileContent1 = `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,*rated,*out,cgrates.org,call,1001,1001,+4986517174963,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10s,1.0100,val_extra3,"",val_extra1 +dbafe9c8614c785a65aabd116dd3959c3c56f7f7,default,*voice,dsafdsag,*rated,*out,cgrates.org,call,1001,1001,+4986517174964,2013-11-07 09:42:25 +0000 UTC,2013-11-07 09:42:26 +0000 UTC,20s,1.0100,val_extra3,"",val_extra1 +` + +var fileContent2 = `accid21;*prepaid;itsyscom.com;1001;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 +accid22;*postpaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 +#accid1;*pseudoprepaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;12;val_extra3;"";val_extra1 +accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;"";val_extra1` + +func TestCsvITInitConfig(t *testing.T) { + if !*testIT { + return + } + var err error + csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") + if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil { + t.Fatal("Got config error: ", err.Error()) + } +} + +// InitDb so we can rely on count +func TestCsvITInitCdrDb(t *testing.T) { + if !*testIT { + return + } + if err := engine.InitStorDb(csvCfg); err != nil { + t.Fatal(err) + } +} + +/* +func TestCsvITCreateCdrDirs(t *testing.T) { + if !*testIT { + return + } + for _, cdrcProfiles := range csvCfg.CdrcProfiles { + for _, cdrcInst := range cdrcProfiles { + for _, dir := range []string{cdrcInst.CdrInDir, cdrcInst.CdrOutDir} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal("Error creating folder: ", dir, err) + } + } + } + } +} + + +func TestCsvITStartEngine(t *testing.T) { + if !*testIT { + return + } + if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} +*/ + +// Connect rpc client to rater +func TestCsvITRpcConn(t *testing.T) { + if !*testIT { + return + } + var err error + cdrcRpc, err = jsonrpc.Dial("tcp", csvCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal("Could not connect to rater: ", err.Error()) + } +} + +// The default scenario, out of cdrc defined in .cfg file +func TestCsvITHandleCdr1File(t *testing.T) { + if !*testIT { + return + } + fileName := "file1.csv" + tmpFilePath := path.Join("/tmp", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1), 0644); err != nil { + t.Fatal(err.Error) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctests/csvit1/in", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +// Scenario out of first .xml config +func TestCsvITHandleCdr2File(t *testing.T) { + if !*testIT { + return + } + fileName := "file2.csv" + tmpFilePath := path.Join("/tmp", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent2), 0644); err != nil { + t.Fatal(err.Error) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctests/csvit2/in", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func TestCsvITProcessedFiles(t *testing.T) { + if !*testIT { + return + } + time.Sleep(time.Duration(2**waitRater) * time.Millisecond) + if outContent1, err := ioutil.ReadFile("/tmp/cdrctests/csvit1/out/file1.csv"); err != nil { + t.Error(err) + } else if fileContent1 != string(outContent1) { + t.Errorf("Expecting: %q, received: %q", fileContent1, string(outContent1)) + } + if outContent2, err := ioutil.ReadFile("/tmp/cdrctests/csvit2/out/file2.csv"); err != nil { + t.Error(err) + } else if fileContent2 != string(outContent2) { + t.Errorf("Expecting: %q, received: %q", fileContent1, string(outContent2)) + } +} + +func TestCsvITKillEngine(t *testing.T) { + if !*testIT { + return + } + if err := engine.KillEngine(*waitRater); err != nil { + t.Error(err) + } +} diff --git a/config/config.go b/config/config.go index f9682ae58..016554415 100644 --- a/config/config.go +++ b/config/config.go @@ -853,7 +853,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { self.CdrcProfiles[*jsnCrc1Cfg.Cdr_in_dir] = make([]*CdrcConfig, 0) } var cdrcInstCfg *CdrcConfig - if *jsnCrc1Cfg.Id == utils.META_DEFAULT { + if *jsnCrc1Cfg.Id == utils.META_DEFAULT && self.dfltCdrcProfile == nil { cdrcInstCfg = new(CdrcConfig) } else { cdrcInstCfg = self.dfltCdrcProfile.Clone() // Clone default so we do not inherit pointers diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index b8fd14d00..b08f4d751 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -145,8 +145,9 @@ // }, -// "cdrc": { -// "*default": { +// "cdrc": [ +// { +// "id": "*default", // identifier of the CDRC runner // "enabled": false, // enable CDR client functionality // "dry_run": false, // do not send the CDRs to CDRS, just parse them // "cdrs_conns": [ @@ -181,8 +182,9 @@ // {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, // ], // "trailer_fields": [], // template of the import trailer fields -// } -// }, +// }, +// ], + // "sm_generic": { // "enabled": false, // starts SessionManager service: diff --git a/data/conf/samples/cdrccsv/cgrates.json b/data/conf/samples/cdrccsv/cgrates.json new file mode 100644 index 000000000..91c0ca30a --- /dev/null +++ b/data/conf/samples/cdrccsv/cgrates.json @@ -0,0 +1,67 @@ +{ + +// Real-time Charging System for Telecom & ISP environments +// Copyright (C) ITsysCOM GmbH +// +// This file contains the default configuration hardcoded into CGRateS. +// This is what you get when you load CGRateS with an empty configuration file. + + + "cdrs": { + "enabled": true, + "rals_conns": [], // no rating support, just *raw CDR testing +}, + + + + "cdrc": [ + { + "id": "*default", + "enabled": true, + "cdr_in_dir": "/tmp/cdrctests/csvit1/in", + "cdr_out_dir": "/tmp/cdrctests/csvit1/out", + "cdr_source_id": "csvit1", +// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value +// {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "2", "mandatory": true}, +// {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "3", "mandatory": true}, +// {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "4", "mandatory": true}, +// {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "5", "mandatory": true}, +// {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "6", "mandatory": true}, +// {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "7", "mandatory": true}, +// {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "8", "mandatory": true}, +// {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "9", "mandatory": true}, +// {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "10", "mandatory": true}, +// {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "11", "mandatory": true}, +// {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "12", "mandatory": true}, +// {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, +// ], + }, + { + "id": "*CSVit2", // identifier of the CDRC runner + "enabled": true, // enable CDR client functionality + "field_separator": ";", + "cdr_in_dir": "/tmp/cdrctests/csvit2/in", // absolute path towards the directory where the CDRs are stored + "cdr_out_dir": "/tmp/cdrctests/csvit2/out", // absolute path towards the directory where processed CDRs will be moved + "cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "^*voice", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "^*out", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "2", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "^call", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "3", "mandatory": true}, + {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "3", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "5", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "5", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "6", "mandatory": true}, + {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "6", "mandatory": true}, + {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "6", "mandatory": true}, + {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "6", "mandatory": true}, + ], + }, +], + + +} \ No newline at end of file diff --git a/data/tariffplans/cdrstats/CdrStats.csv b/data/tariffplans/cdrstats/CdrStats.csv index bf60b63b4..e52e8c15c 100644 --- a/data/tariffplans/cdrstats/CdrStats.csv +++ b/data/tariffplans/cdrstats/CdrStats.csv @@ -1,6 +1,6 @@ #Id[0],QueueLength[1],TimeWindow[2],SaveInerval[3],Metric[4],SetupInterval[5],TOR[6],CdrHost[7],CdrSource[8],ReqType[9],Direction[10],Tenant[11],Category[12],Account[13],Subject[14],DestinationPrefix[15],PddInterval[16],UsageInterval[17],Supplier[18],DisconnectCause[19],MediationRunIds[20],RatedAccount[21],RatedSubject[22],CostInterval[23],Triggers[24] -CDRST3,5,60m,,ASR,2014-07-29T15:00:00Z;2014-07-29T16:00:00Z,*voice,87.139.12.167,FS_JSON,rated,*out,cgrates.org,call,dan,dan,+49,,5m;10m,,,default,rif,rif,0;2,CDRST3_WARN_ASR +CDRST3,5,60m,,ASR,2014-07-29T15:00:00Z;2014-07-29T16:00:00Z,*voice,87.139.12.167,FS_JSON,rated,*out,cgrates.org,call,dan,dan,+49,,5m;10m,,,*default,rif,rif,0;2,CDRST3_WARN_ASR CDRST3,,,,ACD,,,,,,,,,,,,,,,,,,,,CDRST3_WARN_ACD CDRST3,,,,ACC,,,,,,,,,,,,,,,,,,,,CDRST3_WARN_ACC -CDRST4,10,0,,ASR,,,,,,,cgrates.org,call,,,,,,,,,,,,CDRST4_WARN_ASR +CDRST4,10,0,,ASR,,,,,,,cgrates.org,call,,,,,,,,*default,,,,CDRST4_WARN_ASR CDRST4,,,,ACD,,,,,,,,,,,,,,,,,,,,CDRST4_WARN_ACD diff --git a/engine/cdrs.go b/engine/cdrs.go index dad06dd92..a8bfbd3c3 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -86,13 +86,13 @@ func NewCdrServer(cgrCfg *config.CGRConfig, cdrDb CdrStorage, rater, pubsub, use if stats == nil || reflect.ValueOf(stats).IsNil() { stats = nil } - return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, client: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: Guardian}, nil + return &CdrServer{cgrCfg: cgrCfg, cdrDb: cdrDb, rals: rater, pubsub: pubsub, users: users, aliases: aliases, stats: stats, guard: Guardian}, nil } type CdrServer struct { cgrCfg *config.CGRConfig cdrDb CdrStorage - client rpcclient.RpcClientConnection + rals rpcclient.RpcClientConnection pubsub rpcclient.RpcClientConnection users rpcclient.RpcClientConnection aliases rpcclient.RpcClientConnection @@ -183,23 +183,7 @@ func (self *CdrServer) RateCDRs(cdrFltr *utils.CDRsFilter, sendToStats bool) err return err } for _, cdr := range cdrs { - // replace user profile fields - if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { - return err - } - // replace aliases for cases they were loaded after CDR received - if err := LoadAlias(&AttrMatchingAlias{ - Destination: cdr.Destination, - Direction: cdr.Direction, - Tenant: cdr.Tenant, - Category: cdr.Category, - Account: cdr.Account, - Subject: cdr.Subject, - Context: utils.ALIAS_CONTEXT_RATING, - }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { - return err - } - if err := self.rateStoreStatsReplicate(cdr, sendToStats); err != nil { + if err := self.deriveRateStoreStatsReplicate(cdr, self.cgrCfg.CDRSStoreCdrs, sendToStats, len(self.cgrCfg.CDRSCdrReplication) != 0); err != nil { utils.Logger.Err(fmt.Sprintf(" Processing CDR %+v, got error: %s", cdr, err.Error())) } } @@ -223,7 +207,7 @@ func (self *CdrServer) processCdr(cdr *CDR) (err error) { if cdr.Subject == "" { // Use account information as rating subject if missing cdr.Subject = cdr.Account } - if !cdr.Rated { + if !cdr.Rated { // Enforce the RunID if CDR is not rated cdr.RunID = utils.MetaRaw } if self.cgrCfg.CDRSStoreCdrs { // Store RawCDRs, this we do sync so we can reply with the status @@ -231,24 +215,92 @@ func (self *CdrServer) processCdr(cdr *CDR) (err error) { cdr.CostDetails.UpdateCost() cdr.CostDetails.UpdateRatedUsage() } - if err := self.cdrDb.SetCDR(cdr, false); err != nil { // Only original CDR stored in primary table, no derived + if err := self.cdrDb.SetCDR(cdr, false); err != nil { utils.Logger.Err(fmt.Sprintf(" Storing primary CDR %+v, got error: %s", cdr, err.Error())) return err // Error is propagated back and we don't continue processing the CDR if we cannot store it } } - go self.deriveRateStoreStatsReplicate(cdr) + // Attach raw CDR to stats + if self.stats != nil { // Send raw CDR to stats + var out int + go self.stats.Call("CDRStatsV1.AppendCDR", cdr, &out) + } + if len(self.cgrCfg.CDRSCdrReplication) != 0 { // Replicate raw CDR + go self.replicateCdr(cdr) + } + + if self.rals != nil && !cdr.Rated { // CDRs not rated will be processed by Rating + go self.deriveRateStoreStatsReplicate(cdr, self.cgrCfg.CDRSStoreCdrs, self.stats != nil, len(self.cgrCfg.CDRSCdrReplication) != 0) + } return nil } // Returns error if not able to properly store the CDR, mediation is async since we can always recover offline -func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR) error { +func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR, store, stats, replicate bool) error { cdrRuns, err := self.deriveCdrs(cdr) if err != nil { return err } + var ratedCDRs []*CDR // Gather all CDRs received from rating subsystem for _, cdrRun := range cdrRuns { - if err := self.rateStoreStatsReplicate(cdrRun, true); err != nil { - return err + utils.Logger.Debug(fmt.Sprintf("Processing CDR run: %+v", cdrRun)) + if err := LoadUserProfile(cdrRun, utils.EXTRA_FIELDS); err != nil { + utils.Logger.Err(fmt.Sprintf(" UserS handling for CDR %+v, got error: %s", cdrRun, err.Error())) + continue + } + if err := LoadAlias(&AttrMatchingAlias{ + Destination: cdrRun.Destination, + Direction: cdrRun.Direction, + Tenant: cdrRun.Tenant, + Category: cdrRun.Category, + Account: cdrRun.Account, + Subject: cdrRun.Subject, + Context: utils.ALIAS_CONTEXT_RATING, + }, cdrRun, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + utils.Logger.Err(fmt.Sprintf(" Aliasing CDR %+v, got error: %s", cdrRun, err.Error())) + continue + } + rcvRatedCDRs, err := self.rateCDR(cdrRun) + if err != nil { + cdrRun.Cost = -1.0 // If there was an error, mark the CDR + cdrRun.ExtraInfo = err.Error() + rcvRatedCDRs = []*CDR{cdrRun} + } + ratedCDRs = append(ratedCDRs, rcvRatedCDRs...) + } + // Request should be processed by SureTax + for _, ratedCDR := range ratedCDRs { + if ratedCDR.RunID == utils.META_SURETAX { + if err := SureTaxProcessCdr(ratedCDR); err != nil { + ratedCDR.Cost = -1.0 + ratedCDR.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo + } + } + } + // Store rated CDRs + if store { + for _, ratedCDR := range ratedCDRs { + if ratedCDR.CostDetails != nil { + ratedCDR.CostDetails.UpdateCost() + ratedCDR.CostDetails.UpdateRatedUsage() + } + if err := self.cdrDb.SetCDR(ratedCDR, true); err != nil { + utils.Logger.Err(fmt.Sprintf(" Storing rated CDR %+v, got error: %s", ratedCDR, err.Error())) + } + } + } + // Attach CDR to stats + if stats { // Send CDR to stats + for _, ratedCDR := range ratedCDRs { + var out int + if err := self.stats.Call("CDRStatsV1.AppendCDR", ratedCDR, &out); err != nil { + utils.Logger.Err(fmt.Sprintf(" Could not send CDR to stats: %s", err.Error())) + } + } + } + if replicate { + for _, ratedCDR := range ratedCDRs { + self.replicateCdr(ratedCDR) } } return nil @@ -259,6 +311,7 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { if cdr.RunID != utils.MetaRaw { // Only derive *raw CDRs return cdrRuns, nil } + cdr.RunID = utils.META_DEFAULT // Rewrite *raw with *default since we have it as first run if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { return nil, err } @@ -276,7 +329,7 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { attrsDC := &utils.AttrDerivedChargers{Tenant: cdr.Tenant, Category: cdr.Category, Direction: cdr.Direction, Account: cdr.Account, Subject: cdr.Subject, Destination: cdr.Destination} var dcs utils.DerivedChargers - if err := self.client.Call("Responder.GetDerivedChargers", attrsDC, &dcs); err != nil { + if err := self.rals.Call("Responder.GetDerivedChargers", attrsDC, &dcs); err != nil { utils.Logger.Err(fmt.Sprintf("Could not get derived charging for cgrid %s, error: %s", cdr.CGRID, err.Error())) return nil, err } @@ -321,73 +374,6 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { return cdrRuns, nil } -func (self *CdrServer) rateStoreStatsReplicate(cdr *CDR, sendToStats bool) error { - if cdr.RunID == utils.MetaRaw { // Overwrite *raw with *default for rating - cdr.RunID = utils.META_DEFAULT - } - if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { - return err - } - if err := LoadAlias(&AttrMatchingAlias{ - Destination: cdr.Destination, - Direction: cdr.Direction, - Tenant: cdr.Tenant, - Category: cdr.Category, - Account: cdr.Account, - Subject: cdr.Subject, - Context: utils.ALIAS_CONTEXT_RATING, - }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { - return err - } - // Rate CDR, can receive multiple due to SMCosts for OriginIDPrefix - var ratedCDRs []*CDR - var err error - if cdr.Rated { - ratedCDRs = []*CDR{cdr} - } else if self.client != nil && !cdr.Rated { - if ratedCDRs, err = self.rateCDR(cdr); err != nil { - cdr.Cost = -1.0 // If there was an error, mark the CDR - cdr.ExtraInfo = err.Error() - ratedCDRs = []*CDR{cdr} - } - } - for _, ratedCDR := range ratedCDRs { - if ratedCDR.RunID == utils.META_SURETAX { // Request should be processed by SureTax - if err := SureTaxProcessCdr(ratedCDR); err != nil { - ratedCDR.Cost = -1.0 - ratedCDR.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo - } - } - } - if self.cgrCfg.CDRSStoreCdrs { // Store CDRs - for _, ratedCDR := range ratedCDRs { - // Store RatedCDR - if ratedCDR.CostDetails != nil { - ratedCDR.CostDetails.UpdateCost() - ratedCDR.CostDetails.UpdateRatedUsage() - } - if err := self.cdrDb.SetCDR(ratedCDR, true); err != nil { - utils.Logger.Err(fmt.Sprintf(" Storing rated CDR %+v, got error: %s", ratedCDR, err.Error())) - } - } - } - // Attach CDR to stats - if self.stats != nil && sendToStats { // Send CDR to stats - for _, ratedCDR := range ratedCDRs { - var out int - if err := self.stats.Call("CDRStatsV1.AppendCDR", ratedCDR, &out); err != nil { - utils.Logger.Err(fmt.Sprintf(" Could not send CDR to stats: %s", err.Error())) - } - } - } - if len(self.cgrCfg.CDRSCdrReplication) != 0 { - for _, ratedCDR := range ratedCDRs { - self.replicateCdr(ratedCDR) - } - } - return nil -} - // rateCDR will populate cost field // Returns more than one rated CDR in case of SMCost retrieved based on prefix func (self *CdrServer) rateCDR(cdr *CDR) ([]*CDR, error) { @@ -462,9 +448,9 @@ func (self *CdrServer) getCostFromRater(cdr *CDR) (*CallCost, error) { PerformRounding: true, } if utils.IsSliceMember([]string{utils.META_PSEUDOPREPAID, utils.META_POSTPAID, utils.META_PREPAID, utils.PSEUDOPREPAID, utils.POSTPAID, utils.PREPAID}, cdr.RequestType) { // Prepaid - Cost can be recalculated in case of missing records from SM - err = self.client.Call("Responder.Debit", cd, cc) + err = self.rals.Call("Responder.Debit", cd, cc) } else { - err = self.client.Call("Responder.GetCost", cd, cc) + err = self.rals.Call("Responder.GetCost", cd, cc) } if err != nil { return cc, err diff --git a/engine/stats.go b/engine/stats.go index 571615ec6..b84fda549 100644 --- a/engine/stats.go +++ b/engine/stats.go @@ -289,6 +289,7 @@ func (s *Stats) setupQueueSaver(sq *StatsQueue) { func (s *Stats) AppendCDR(cdr *CDR, out *int) error { s.mux.RLock() defer s.mux.RUnlock() + utils.Logger.Debug(fmt.Sprintf("Stats.AppendCDR: %+v", cdr)) for _, sq := range s.queues { sq.AppendCDR(cdr) } From cd5993467de05fe592f95fb57a72810373a04ff8 Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 1 May 2016 18:42:08 +0200 Subject: [PATCH 219/227] Diameter fix for config loading in case of pubsubs --- config/config_json_test.go | 2 +- config/daconfig.go | 6 +++--- config/libconfig_json.go | 2 +- engine/cdrs.go | 1 - engine/stats.go | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/config/config_json_test.go b/config/config_json_test.go index 520298908..c879439cf 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -460,7 +460,7 @@ func TestDiameterAgentJsonCfg(t *testing.T) { &HaPoolJsonCfg{ Address: utils.StringPointer(utils.MetaInternal), }}, - Pubsub_conns: nil, + Pubsubs_conns: &[]*HaPoolJsonCfg{}, Create_cdr: utils.BoolPointer(true), Debit_interval: utils.StringPointer("5m"), Timezone: utils.StringPointer(""), diff --git a/config/daconfig.go b/config/daconfig.go index ee7adad15..d309a2290 100644 --- a/config/daconfig.go +++ b/config/daconfig.go @@ -61,9 +61,9 @@ func (self *DiameterAgentCfg) loadFromJsonCfg(jsnCfg *DiameterAgentJsonCfg) erro self.SMGenericConns[idx].loadFromJsonCfg(jsnHaCfg) } } - if jsnCfg.Pubsub_conns != nil { - self.PubSubConns = make([]*HaPoolConfig, len(*jsnCfg.Pubsub_conns)) - for idx, jsnHaCfg := range *jsnCfg.Pubsub_conns { + if jsnCfg.Pubsubs_conns != nil { + self.PubSubConns = make([]*HaPoolConfig, len(*jsnCfg.Pubsubs_conns)) + for idx, jsnHaCfg := range *jsnCfg.Pubsubs_conns { self.PubSubConns[idx] = NewDfltHaPoolConfig() self.PubSubConns[idx].loadFromJsonCfg(jsnHaCfg) } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index d05c7200e..eb2bf969b 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -255,7 +255,7 @@ type DiameterAgentJsonCfg struct { Listen *string // address where to listen for diameter requests Dictionaries_dir *string // path towards additional dictionaries Sm_generic_conns *[]*HaPoolJsonCfg // Connections towards generic SM - Pubsub_conns *[]*HaPoolJsonCfg // connection towards pubsubs + Pubsubs_conns *[]*HaPoolJsonCfg // connection towards pubsubs Create_cdr *bool Debit_interval *string Timezone *string // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> diff --git a/engine/cdrs.go b/engine/cdrs.go index a8bfbd3c3..285666ae7 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -243,7 +243,6 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR, store, stats, rep } var ratedCDRs []*CDR // Gather all CDRs received from rating subsystem for _, cdrRun := range cdrRuns { - utils.Logger.Debug(fmt.Sprintf("Processing CDR run: %+v", cdrRun)) if err := LoadUserProfile(cdrRun, utils.EXTRA_FIELDS); err != nil { utils.Logger.Err(fmt.Sprintf(" UserS handling for CDR %+v, got error: %s", cdrRun, err.Error())) continue diff --git a/engine/stats.go b/engine/stats.go index b84fda549..571615ec6 100644 --- a/engine/stats.go +++ b/engine/stats.go @@ -289,7 +289,6 @@ func (s *Stats) setupQueueSaver(sq *StatsQueue) { func (s *Stats) AppendCDR(cdr *CDR, out *int) error { s.mux.RLock() defer s.mux.RUnlock() - utils.Logger.Debug(fmt.Sprintf("Stats.AppendCDR: %+v", cdr)) for _, sq := range s.queues { sq.AppendCDR(cdr) } From e537659f0eec83abd9f4e24564d37e09f0dbc7bc Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 1 May 2016 19:30:14 +0200 Subject: [PATCH 220/227] Check CDRs in CDR .csv test --- cdrc/csv_it_test.go | 17 ++++++++++++++--- data/conf/samples/cdrccsv/cgrates.json | 4 ++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go index ea2491e05..f43cef288 100644 --- a/cdrc/csv_it_test.go +++ b/cdrc/csv_it_test.go @@ -30,6 +30,7 @@ import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) /* @@ -86,7 +87,6 @@ func TestCsvITInitCdrDb(t *testing.T) { } } -/* func TestCsvITCreateCdrDirs(t *testing.T) { if !*testIT { return @@ -105,7 +105,6 @@ func TestCsvITCreateCdrDirs(t *testing.T) { } } - func TestCsvITStartEngine(t *testing.T) { if !*testIT { return @@ -114,7 +113,6 @@ func TestCsvITStartEngine(t *testing.T) { t.Fatal(err) } } -*/ // Connect rpc client to rater func TestCsvITRpcConn(t *testing.T) { @@ -175,6 +173,19 @@ func TestCsvITProcessedFiles(t *testing.T) { } } +func TestCsvITAnalyseCDRs(t *testing.T) { + if !*testIT { + return + } + var reply []*engine.ExternalCDR + req := utils.RPCCDRsFilter{} + if err := cdrcRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 6 { // 1 injected, 1 rated, 1 *raw and it's pair in *default run + t.Error("Unexpected number of CDRs returned: ", len(reply)) + } +} + func TestCsvITKillEngine(t *testing.T) { if !*testIT { return diff --git a/data/conf/samples/cdrccsv/cgrates.json b/data/conf/samples/cdrccsv/cgrates.json index 91c0ca30a..7793ec97b 100644 --- a/data/conf/samples/cdrccsv/cgrates.json +++ b/data/conf/samples/cdrccsv/cgrates.json @@ -7,6 +7,10 @@ // This is what you get when you load CGRateS with an empty configuration file. + "rals": { + "enabled": true // so we can query CDRs + }, + "cdrs": { "enabled": true, "rals_conns": [], // no rating support, just *raw CDR testing From f562b617ccc7a5c6394d508a8f1383aa1559d854 Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 1 May 2016 19:37:41 +0200 Subject: [PATCH 221/227] CDRC .csv test making sure conversion was done for the data via rsr --- cdrc/csv_it_test.go | 9 +++++++-- data/conf/samples/cdrccsv/cgrates.json | 16 +--------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go index f43cef288..b2cf53b2d 100644 --- a/cdrc/csv_it_test.go +++ b/cdrc/csv_it_test.go @@ -178,12 +178,17 @@ func TestCsvITAnalyseCDRs(t *testing.T) { return } var reply []*engine.ExternalCDR - req := utils.RPCCDRsFilter{} - if err := cdrcRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + if err := cdrcRpc.Call("ApierV2.GetCdrs", utils.RPCCDRsFilter{}, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if len(reply) != 6 { // 1 injected, 1 rated, 1 *raw and it's pair in *default run t.Error("Unexpected number of CDRs returned: ", len(reply)) } + if err := cdrcRpc.Call("ApierV2.GetCdrs", utils.RPCCDRsFilter{DestinationPrefixes: []string{"08651"}}, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 0 { // Original 08651 was converted + t.Error("Unexpected number of CDRs returned: ", len(reply)) + } + } func TestCsvITKillEngine(t *testing.T) { diff --git a/data/conf/samples/cdrccsv/cgrates.json b/data/conf/samples/cdrccsv/cgrates.json index 7793ec97b..06768551f 100644 --- a/data/conf/samples/cdrccsv/cgrates.json +++ b/data/conf/samples/cdrccsv/cgrates.json @@ -13,7 +13,7 @@ "cdrs": { "enabled": true, - "rals_conns": [], // no rating support, just *raw CDR testing + "rals_conns": [], // no rating support, just *raw CDR testing }, @@ -25,20 +25,6 @@ "cdr_in_dir": "/tmp/cdrctests/csvit1/in", "cdr_out_dir": "/tmp/cdrctests/csvit1/out", "cdr_source_id": "csvit1", -// "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value -// {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "2", "mandatory": true}, -// {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "3", "mandatory": true}, -// {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "4", "mandatory": true}, -// {"tag": "Direction", "field_id": "Direction", "type": "*composed", "value": "5", "mandatory": true}, -// {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "6", "mandatory": true}, -// {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "7", "mandatory": true}, -// {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "8", "mandatory": true}, -// {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "9", "mandatory": true}, -// {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "10", "mandatory": true}, -// {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "11", "mandatory": true}, -// {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "12", "mandatory": true}, -// {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "13", "mandatory": true}, -// ], }, { "id": "*CSVit2", // identifier of the CDRC runner From 27d5a7ba2fe5e32a76babd20ba5af93961bbc223 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 2 May 2016 17:23:51 +0200 Subject: [PATCH 222/227] CDRs refactoring, adding versioning for internal RPC methods --- apier/v1/apier.go | 1 + apier/v1/apier_local_test.go | 2 +- apier/v1/cdrs.go | 9 +++ apier/v1/cdrsv1.go | 8 +- config/config.go | 5 ++ engine/cdrs.go | 113 ++++++++++++++++++--------- general_tests/tutorial_local_test.go | 24 +++--- sessionmanager/smgeneric.go | 6 +- utils/apitpdata.go | 7 ++ 9 files changed, 116 insertions(+), 59 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 21d64d422..146065699 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -51,6 +51,7 @@ type ApierV1 struct { Responder *engine.Responder CdrStatsSrv rpcclient.RpcClientConnection Users rpcclient.RpcClientConnection + CDRs rpcclient.RpcClientConnection // FixMe: populate it from cgr-engine } func (self *ApierV1) GetDestination(dstId string, reply *engine.Destination) error { diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 0e1e7f09e..7bc966260 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -1274,7 +1274,7 @@ func TestApierLoadTariffPlanFromFolder(t *testing.T) { } else if reply != "OK" { t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply) } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) + time.Sleep(time.Duration(2**waitRater) * time.Millisecond) } func TestApierResetDataAfterLoadFromFolder(t *testing.T) { diff --git a/apier/v1/cdrs.go b/apier/v1/cdrs.go index 4e87a2120..a4ce0b5af 100644 --- a/apier/v1/cdrs.go +++ b/apier/v1/cdrs.go @@ -19,6 +19,7 @@ along with this program. If not, see package v1 import ( + "errors" "fmt" "github.com/cgrates/cgrates/engine" @@ -85,3 +86,11 @@ func (apier *ApierV1) RemoveCDRs(attrs utils.RPCCDRsFilter, reply *string) error *reply = "OK" return nil } + +// New way of (re-)rating CDRs +func (apier *ApierV1) RateCDRs(attrs utils.AttrRateCDRs, reply *string) error { + if apier.CDRs == nil { + return errors.New("CDRS_NOT_ENABLED") + } + return apier.CDRs.Call("CDRsV1.RateCDRs", attrs, reply) +} diff --git a/apier/v1/cdrsv1.go b/apier/v1/cdrsv1.go index 09df4c5a6..c6b93d504 100644 --- a/apier/v1/cdrsv1.go +++ b/apier/v1/cdrsv1.go @@ -46,7 +46,7 @@ func (self *CdrsV1) ProcessExternalCdr(cdr *engine.ExternalCDR, reply *string) e return nil } -// Remotely start mediation with specific runid, runs asynchronously, it's status will be displayed in syslog +// Remotely (re)rating, deprecated func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { cdrsFltr, err := attrs.AsCDRsFilter(self.CdrSrv.Timezone()) if err != nil { @@ -60,9 +60,5 @@ func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error { } func (self *CdrsV1) StoreSMCost(attr engine.AttrCDRSStoreSMCost, reply *string) error { - if err := self.CdrSrv.StoreSMCost(attr, reply); err != nil { - return utils.NewErrServerError(err) - } - *reply = utils.OK - return nil + return self.CdrSrv.V1StoreSMCost(attr, reply) } diff --git a/config/config.go b/config/config.go index 016554415..736923a6c 100644 --- a/config/config.go +++ b/config/config.go @@ -429,6 +429,11 @@ func (self *CGRConfig) checkConfigSanity() error { return errors.New("SMGeneric not enabled but referenced by DiameterAgent component") } } + for _, daPubSubSConn := range self.diameterAgentCfg.PubSubConns { + if daPubSubSConn.Address == utils.MetaInternal && !self.PubSubServerEnabled { + return errors.New("PubSubS not enabled but requested by DiameterAgent component.") + } + } } return nil } diff --git a/engine/cdrs.go b/engine/cdrs.go index 285666ae7..56b643bac 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -122,27 +122,12 @@ func (self *CdrServer) RegisterHandlersToServer(server *utils.Server) { server.RegisterHttpFunc("/freeswitch_json", fsCdrHandler) } -func (self *CdrServer) ProcessCdr(cdr *CDR, reply *string) error { - cacheKey := "ProcessCdr" + cdr.CGRID - if item, err := self.getCache().Get(cacheKey); err == nil && item != nil { - *reply = item.Value.(string) - return item.Err - } - if err := self.LocalProcessCdr(cdr); err != nil { - self.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) - return utils.NewErrServerError(err) - } - self.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) - *reply = utils.OK - return nil -} - -// RPC method, used to internally process CDR +// Used to internally process CDR func (self *CdrServer) LocalProcessCdr(cdr *CDR) error { return self.processCdr(cdr) } -// RPC method, used to process external CDRs +// Used to process external CDRs func (self *CdrServer) ProcessExternalCdr(eCDR *ExternalCDR) error { cdr, err := NewCDRFromExternalCDR(eCDR, self.cgrCfg.DefaultTimezone) if err != nil { @@ -171,25 +156,6 @@ func (self *CdrServer) storeSMCost(smCost *SMCost, checkDuplicate bool) error { return self.cdrDb.SetSMCost(smCost) } -// RPC method, differs from storeSMCost through it's signature -func (self *CdrServer) StoreSMCost(attr AttrCDRSStoreSMCost, reply *string) error { - return self.storeSMCost(attr.Cost, attr.CheckDuplicate) -} - -// Called by rate/re-rate API -func (self *CdrServer) RateCDRs(cdrFltr *utils.CDRsFilter, sendToStats bool) error { - cdrs, _, err := self.cdrDb.GetCDRs(cdrFltr, false) - if err != nil { - return err - } - for _, cdr := range cdrs { - if err := self.deriveRateStoreStatsReplicate(cdr, self.cgrCfg.CDRSStoreCdrs, sendToStats, len(self.cgrCfg.CDRSCdrReplication) != 0); err != nil { - utils.Logger.Err(fmt.Sprintf(" Processing CDR %+v, got error: %s", cdr, err.Error())) - } - } - return nil -} - // Returns error if not able to properly store the CDR, mediation is async since we can always recover offline func (self *CdrServer) processCdr(cdr *CDR) (err error) { if cdr.Direction == "" { @@ -508,13 +474,86 @@ func (self *CdrServer) replicateCdr(cdr *CDR) error { return nil } +// Called by rate/re-rate API, FixMe: deprecate it once new APIer structure is operational +func (self *CdrServer) RateCDRs(cdrFltr *utils.CDRsFilter, sendToStats bool) error { + cdrs, _, err := self.cdrDb.GetCDRs(cdrFltr, false) + if err != nil { + return err + } + for _, cdr := range cdrs { + if err := self.deriveRateStoreStatsReplicate(cdr, self.cgrCfg.CDRSStoreCdrs, sendToStats, len(self.cgrCfg.CDRSCdrReplication) != 0); err != nil { + utils.Logger.Err(fmt.Sprintf(" Processing CDR %+v, got error: %s", cdr, err.Error())) + } + } + return nil +} + +func (self *CdrServer) V1ProcessCDR(cdr *CDR, reply *string) error { + cacheKey := "ProcessCdr" + cdr.CGRID + if item, err := self.getCache().Get(cacheKey); err == nil && item != nil { + *reply = item.Value.(string) + return item.Err + } + if err := self.LocalProcessCdr(cdr); err != nil { + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) + return utils.NewErrServerError(err) + } + self.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: utils.OK}) + *reply = utils.OK + return nil +} + +// Alias, deprecated after removing CdrServerV1.ProcessCdr +func (self *CdrServer) V1ProcessCdr(cdr *CDR, reply *string) error { + return self.V1ProcessCDR(cdr, reply) +} + +// RPC method, differs from storeSMCost through it's signature +func (self *CdrServer) V1StoreSMCost(attr AttrCDRSStoreSMCost, reply *string) error { + if err := self.storeSMCost(attr.Cost, attr.CheckDuplicate); err != nil { + return utils.NewErrServerError(err) + } + *reply = utils.OK + return nil +} + +// Called by rate/re-rate API, RPC method +func (self *CdrServer) V1RateCDRs(attrs utils.AttrRateCDRs, reply *string) error { + cdrFltr, err := attrs.RPCCDRsFilter.AsCDRsFilter(self.cgrCfg.DefaultTimezone) + if err != nil { + return utils.NewErrServerError(err) + } + cdrs, _, err := self.cdrDb.GetCDRs(cdrFltr, false) + if err != nil { + return err + } + storeCDRs := self.cgrCfg.CDRSStoreCdrs + if attrs.StoreCDRs != nil { + storeCDRs = *attrs.StoreCDRs + } + sendToStats := self.stats != nil + if attrs.SendToStatS != nil { + sendToStats = *attrs.SendToStatS + } + replicate := len(self.cgrCfg.CDRSCdrReplication) != 0 + if attrs.ReplicateCDRs != nil { + replicate = *attrs.ReplicateCDRs + } + for _, cdr := range cdrs { + if err := self.deriveRateStoreStatsReplicate(cdr, storeCDRs, sendToStats, replicate); err != nil { + utils.Logger.Err(fmt.Sprintf(" Processing CDR %+v, got error: %s", cdr, err.Error())) + } + } + return nil +} + func (cdrsrv *CdrServer) Call(serviceMethod string, args interface{}, reply interface{}) error { parts := strings.Split(serviceMethod, ".") if len(parts) != 2 { return utils.ErrNotImplemented } // get method - method := reflect.ValueOf(cdrsrv).MethodByName(parts[1]) + method := reflect.ValueOf(cdrsrv).MethodByName(parts[0][len(parts[0])-2:] + parts[1]) // Inherit the version in the method if !method.IsValid() { return utils.ErrNotImplemented } diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 17d986ea9..d57ce2fdd 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -545,7 +545,7 @@ func TestTutLocalProcessExternalCdr(t *testing.T) { Usage: "1", PDD: "7.0", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } var reply string - if err := tutLocalRpc.Call("CdrsV2.ProcessExternalCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessExternalCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -565,7 +565,7 @@ func TestTutLocalProcessExternalCdrUP(t *testing.T) { ExtraFields: map[string]string{"Cli": "+4986517174964", "fieldextr2": "valextr2", "SysUserName": utils.USERS}, } var reply string - if err := tutLocalRpc.Call("CdrsV2.ProcessExternalCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessExternalCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -645,7 +645,7 @@ func TestTutLocalCostErrors(t *testing.T) { Usage: "1", PDD: "7.0", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } var reply string - if err := tutLocalRpc.Call("CdrsV2.ProcessExternalCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessExternalCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -671,7 +671,7 @@ func TestTutLocalCostErrors(t *testing.T) { SetupTime: "2014-08-04T13:00:00Z", AnswerTime: "2014-08-04T13:00:07Z", Usage: "1", PDD: "7.0", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } - if err := tutLocalRpc.Call("CdrsV2.ProcessExternalCdr", cdr2, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessExternalCdr", cdr2, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -696,7 +696,7 @@ func TestTutLocalCostErrors(t *testing.T) { SetupTime: "2014-08-04T13:00:00Z", AnswerTime: "2014-08-04T13:00:07Z", Usage: "1", PDD: "7.0", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } - if err := tutLocalRpc.Call("CdrsV2.ProcessExternalCdr", cdr3, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessExternalCdr", cdr3, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -886,7 +886,7 @@ func TestTutLocalLcrQos(t *testing.T) { ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}} var reply string for _, cdr := range []*engine.CDR{testCdr1, testCdr2} { - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -916,7 +916,7 @@ func TestTutLocalLcrQos(t *testing.T) { Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1004", SetupTime: time.Date(2014, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2014, 12, 7, 8, 42, 26, 0, time.UTC), Usage: time.Duration(180) * time.Second, Supplier: "suppl2"} - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", testCdr3, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", testCdr3, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -983,7 +983,7 @@ func TestTutLocalLcrQosThreshold(t *testing.T) { SetupTime: time.Date(2014, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2014, 12, 7, 8, 42, 26, 0, time.UTC), Usage: time.Duration(60) * time.Second, Supplier: "suppl2"} var reply string - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", testCdr4, &reply); err != nil { // Should drop ACD under the 2m required by threshold, removing suppl2 from lcr + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", testCdr4, &reply); err != nil { // Should drop ACD under the 2m required by threshold, removing suppl2 from lcr t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -1047,7 +1047,7 @@ func TestTutLocalLcrQosThreshold(t *testing.T) { Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1004", SetupTime: time.Date(2014, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2014, 12, 7, 8, 42, 26, 0, time.UTC), Usage: time.Duration(1) * time.Second, Supplier: "suppl2"} - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", testCdr5, &reply); err != nil { // Should drop ACD under the 1m required by threshold, removing suppl2 from lcr + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", testCdr5, &reply); err != nil { // Should drop ACD under the 1m required by threshold, removing suppl2 from lcr t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -1325,12 +1325,12 @@ func TestTutLocalPrepaidCDRWithSMCost(t *testing.T) { }, } var reply string - if err := tutLocalRpc.Call("CdrsV2.StoreSMCost", &engine.AttrCDRSStoreSMCost{Cost: smCost}, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.StoreSMCost", &engine.AttrCDRSStoreSMCost{Cost: smCost}, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) } - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) @@ -1363,7 +1363,7 @@ func TestTutLocalPrepaidCDRWithoutSMCost(t *testing.T) { Usage: time.Duration(90) * time.Second, Supplier: "suppl1", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}} var reply string - if err := tutLocalRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", cdr, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 99a45bd8f..7fc17f5dc 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -91,7 +91,7 @@ func (self *SMGeneric) ttlTerminate(s *SMGSession, tmtr *smgSessionTerminator) { cdr := s.eventStart.AsStoredCdr(self.cgrCfg, self.timezone) cdr.Usage = s.TotalUsage() var reply string - self.cdrsrv.Call("CdrServer.ProcessCdr", cdr, &reply) + self.cdrsrv.Call("CdrsV1.ProcessCdr", cdr, &reply) } func (self *SMGeneric) indexSession(uuid string, s *SMGSession) { @@ -474,7 +474,7 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu OriginID: gev.GetUUID(), CostDetails: cc, } - if err := self.cdrsrv.Call("CdrServer.StoreSMCost", engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil && err != utils.ErrExists { + if err := self.cdrsrv.Call("CdrsV1.StoreSMCost", engine.AttrCDRSStoreSMCost{Cost: smCost, CheckDuplicate: true}, &reply); err != nil && err != utils.ErrExists { withErrors = true utils.Logger.Err(fmt.Sprintf(" Could not save CC: %+v, RunID: %s error: %s", cc, sR.DerivedCharger.RunID, err.Error())) } @@ -487,7 +487,7 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu func (self *SMGeneric) ProcessCdr(gev SMGenericEvent) error { var reply string - if err := self.cdrsrv.Call("CdrServer.ProcessCdr", gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { + if err := self.cdrsrv.Call("CdrsV1.ProcessCdr", gev.AsStoredCdr(self.cgrCfg, self.timezone), &reply); err != nil { return err } return nil diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 12bfc9173..80b428e9a 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1171,3 +1171,10 @@ type AttrSMGGetActiveSessions struct { Destination *string Supplier *string } + +type AttrRateCDRs struct { + RPCCDRsFilter + StoreCDRs *bool + SendToStatS *bool // Set to true if the CDRs should be sent to stats server + ReplicateCDRs *bool // Replicate results +} From e6169fd2bde7486d1b7721b80f75b06bef54ecfc Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 2 May 2016 19:18:31 +0200 Subject: [PATCH 223/227] CDRS - properly fork CDR in case of derived charging --- apier/v1/cdrstatsv1_local_test.go | 2 +- engine/cdrs.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apier/v1/cdrstatsv1_local_test.go b/apier/v1/cdrstatsv1_local_test.go index 814b36d00..e1cd0493b 100644 --- a/apier/v1/cdrstatsv1_local_test.go +++ b/apier/v1/cdrstatsv1_local_test.go @@ -153,7 +153,7 @@ func TestCDRStatsLclPostCdrs(t *testing.T) { t.Error(err.Error()) } } - time.Sleep(time.Duration(1000) * time.Millisecond) + time.Sleep(time.Duration(*waitRater) * time.Millisecond) } func TestCDRStatsLclGetMetrics1(t *testing.T) { diff --git a/engine/cdrs.go b/engine/cdrs.go index 56b643bac..293069549 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -272,11 +272,12 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(cdr *CDR, store, stats, rep } func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { - cdrRuns := []*CDR{cdr} + dfltCDRRun := cdr.Clone() + cdrRuns := []*CDR{dfltCDRRun} if cdr.RunID != utils.MetaRaw { // Only derive *raw CDRs return cdrRuns, nil } - cdr.RunID = utils.META_DEFAULT // Rewrite *raw with *default since we have it as first run + dfltCDRRun.RunID = utils.META_DEFAULT // Rewrite *raw with *default since we have it as first run if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { return nil, err } From ddc8636fa60d4ed5cd88d1db2634ac9e98284062 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 3 May 2016 10:31:37 +0300 Subject: [PATCH 224/227] no more dependecy check on RemActions --- apier/v1/apier.go | 55 +++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 146065699..b72335757 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1076,42 +1076,45 @@ func (self *ApierV1) RemActions(attr AttrRemActions, reply *string) error { *reply = err.Error() return err } - stringMap := utils.NewStringMap(attr.ActionIDs...) - keys, err := self.RatingDb.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX, true) - if err != nil { - *reply = err.Error() - return err - } - for _, key := range keys { - getAttrs, err := self.RatingDb.GetActionTriggers(key[len(utils.ACTION_TRIGGER_PREFIX):]) + // The check could lead to very long execution time. So we decided to leave it at the user's risck.' + /* + stringMap := utils.NewStringMap(attr.ActionIDs...) + keys, err := self.RatingDb.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX, true) if err != nil { *reply = err.Error() return err } - for _, atr := range getAttrs { - if _, found := stringMap[atr.ActionsID]; found { - // found action trigger referencing action; abort - err := fmt.Errorf("action %s refenced by action trigger %s", atr.ActionsID, atr.ID) + for _, key := range keys { + getAttrs, err := self.RatingDb.GetActionTriggers(key[len(utils.ACTION_TRIGGER_PREFIX):]) + if err != nil { *reply = err.Error() return err } - } - } - allAplsMap, err := self.RatingDb.GetAllActionPlans() - if err != nil && err != utils.ErrNotFound { - *reply = err.Error() - return err - } - for _, apl := range allAplsMap { - for _, atm := range apl.ActionTimings { - if _, found := stringMap[atm.ActionsID]; found { - err := fmt.Errorf("action %s refenced by action plan %s", atm.ActionsID, apl.Id) - *reply = err.Error() - return err + for _, atr := range getAttrs { + if _, found := stringMap[atr.ActionsID]; found { + // found action trigger referencing action; abort + err := fmt.Errorf("action %s refenced by action trigger %s", atr.ActionsID, atr.ID) + *reply = err.Error() + return err + } } } + allAplsMap, err := self.RatingDb.GetAllActionPlans() + if err != nil && err != utils.ErrNotFound { + *reply = err.Error() + return err + } + for _, apl := range allAplsMap { + for _, atm := range apl.ActionTimings { + if _, found := stringMap[atm.ActionsID]; found { + err := fmt.Errorf("action %s refenced by action plan %s", atm.ActionsID, apl.Id) + *reply = err.Error() + return err + } + } - } + } + */ for _, aID := range attr.ActionIDs { if err := self.RatingDb.RemoveActions(aID); err != nil { *reply = err.Error() From 83b6c9be653fd5926fedd847b3f0ed4527ae6815 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 3 May 2016 10:31:54 +0300 Subject: [PATCH 225/227] install ngrep on devel docker --- data/docker/devel/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/docker/devel/Dockerfile b/data/docker/devel/Dockerfile index 5bf440c97..dea7243d8 100644 --- a/data/docker/devel/Dockerfile +++ b/data/docker/devel/Dockerfile @@ -17,7 +17,7 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 RUN echo 'deb http://repo.mongodb.org/apt/debian wheezy/mongodb-org/3.2 main' | tee '/etc/apt/sources.list.d/mongodb-org-3.2.list' # install dependencies -RUN apt-get -y update && apt-get -y install git bzr mercurial redis-server mysql-server python-pycurl python-mysqldb postgresql postgresql-client sudo wget freeswitch-meta-vanilla vim zsh mongodb-org tmux rsyslog +RUN apt-get -y update && apt-get -y install git bzr mercurial redis-server mysql-server python-pycurl python-mysqldb postgresql postgresql-client sudo wget freeswitch-meta-vanilla vim zsh mongodb-org tmux rsyslog ngrep # add mongo conf COPY mongod.conf /etc/mongod.conf From 3db05157acf74470ac0342defe32ca83ec6c873a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 3 May 2016 15:17:40 +0300 Subject: [PATCH 226/227] tests passing on hapool with response_ttl > 0 --- cache2go/response_cache.go | 3 + data/conf/samples/smg/cgrates.json | 4 ++ data/tariffplans/testtp/AccountActions.csv | 2 +- data/tariffplans/testtp/ActionPlans.csv | 3 +- engine/actions_test.go | 4 +- engine/calldesc.go | 2 + engine/responder.go | 62 ++++++++++++++------ engine/responder_test.go | 6 +- general_tests/tp_it_test.go | 17 +++--- sessionmanager/data_it_test.go | 68 ++++++++++++---------- sessionmanager/smg_it_test.go | 24 ++++---- sessionmanager/smg_session.go | 6 +- sessionmanager/smgeneric.go | 14 ++--- utils/consts.go | 1 + 14 files changed, 129 insertions(+), 87 deletions(-) diff --git a/cache2go/response_cache.go b/cache2go/response_cache.go index eba9dcf2e..c7631cc57 100644 --- a/cache2go/response_cache.go +++ b/cache2go/response_cache.go @@ -32,6 +32,7 @@ func NewResponseCache(ttl time.Duration) *ResponseCache { } func (rc *ResponseCache) Cache(key string, item *CacheItem) { + //utils.Logger.Info("key: " + key) if rc.ttl == 0 { return } @@ -58,6 +59,7 @@ func (rc *ResponseCache) Get(key string) (*CacheItem, error) { item, ok := rc.cache[key] rc.mu.RUnlock() if ok { + //utils.Logger.Info(",,,,,,,,,,,,,,,,,,,,,Found key: " + key) return item, nil } rc.wait(key) // wait for other goroutine processsing this key @@ -67,6 +69,7 @@ func (rc *ResponseCache) Get(key string) (*CacheItem, error) { if !ok { return nil, ErrNotFound } + //utils.Logger.Info("............................Found key: " + key) return item, nil } diff --git a/data/conf/samples/smg/cgrates.json b/data/conf/samples/smg/cgrates.json index 9f343a378..d49f06d2b 100644 --- a/data/conf/samples/smg/cgrates.json +++ b/data/conf/samples/smg/cgrates.json @@ -4,6 +4,10 @@ // Used for cgradmin // Starts rater, scheduler +"general": { + "response_cache_ttl": "1s", +}, + "listen": { "rpc_json": ":2012", // RPC JSON listening address "rpc_gob": ":2013", // RPC GOB listening address diff --git a/data/tariffplans/testtp/AccountActions.csv b/data/tariffplans/testtp/AccountActions.csv index 9fa3f8977..3adcdcb1f 100644 --- a/data/tariffplans/testtp/AccountActions.csv +++ b/data/tariffplans/testtp/AccountActions.csv @@ -10,4 +10,4 @@ cgrates.org,1011,TEST_VOICE,,, cgrates.org,1012,PREPAID_10,,, cgrates.org,1013,TEST_NEG,,, cgrates.org,1014,TEST_RPC,,, -cgrates.org,1015,,,, +cgrates.org,1015,TEST_DID,,, diff --git a/data/tariffplans/testtp/ActionPlans.csv b/data/tariffplans/testtp/ActionPlans.csv index 897278d0e..636888218 100644 --- a/data/tariffplans/testtp/ActionPlans.csv +++ b/data/tariffplans/testtp/ActionPlans.csv @@ -5,4 +5,5 @@ TEST_EXE,TOPUP_EXE,ALWAYS,10 TEST_DATA_r,TOPUP_DATA_r,ASAP,10 TEST_VOICE,TOPUP_VOICE,ASAP,10 TEST_NEG,TOPUP_NEG,ASAP,10 -TEST_RPC,RPC,ALWAYS,10 \ No newline at end of file +TEST_RPC,RPC,ALWAYS,10 +TEST_DID,DID,ALWAYS,10 diff --git a/engine/actions_test.go b/engine/actions_test.go index ebfd7b86f..26c358c8d 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -2233,7 +2233,9 @@ func TestValueFormulaDebit(t *testing.T) { } at.Execute() afterUb, err := accountingStorage.GetAccount("cgrates.org:vf") - if err != nil || afterUb.BalanceMap[utils.MONETARY].GetTotalValue() != -0.333334 { + // not an exact value, depends of month + v := afterUb.BalanceMap[utils.MONETARY].GetTotalValue() + if err != nil || v > -0.30 || v < -0.35 { t.Error("error debiting account: ", err, utils.ToIJSON(afterUb)) } } diff --git a/engine/calldesc.go b/engine/calldesc.go index e126df79e..d62a0005e 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -933,6 +933,8 @@ func (cd *CallDescriptor) Clone() *CallDescriptor { ForceDuration: cd.ForceDuration, PerformRounding: cd.PerformRounding, DryRun: cd.DryRun, + CgrID: cd.CgrID, + RunID: cd.RunID, } } diff --git a/engine/responder.go b/engine/responder.go index 8c6009db6..751d3299c 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -147,7 +147,8 @@ func (rs *Responder) Debit(arg *CallDescriptor, reply *CallCost) (err error) { } func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) { - if item, err := rs.getCache().Get(utils.MAX_DEBIT_CACHE_PREFIX + arg.CgrID + arg.RunID); err == nil && item != nil { + cacheKey := utils.MAX_DEBIT_CACHE_PREFIX + arg.CgrID + arg.RunID + arg.DurationIndex.String() + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = *(item.Value.(*CallCost)) return item.Err } @@ -178,7 +179,7 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) } else { r, e := arg.MaxDebit() if e != nil { - rs.getCache().Cache(utils.MAX_DEBIT_CACHE_PREFIX+arg.CgrID+arg.RunID, &cache2go.CacheItem{ + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ Err: e, }) return e @@ -186,7 +187,7 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) *reply = *r } } - rs.getCache().Cache(utils.MAX_DEBIT_CACHE_PREFIX+arg.CgrID+arg.RunID, &cache2go.CacheItem{ + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ Value: reply, Err: err, }) @@ -194,7 +195,8 @@ func (rs *Responder) MaxDebit(arg *CallDescriptor, reply *CallCost) (err error) } func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err error) { - if item, err := rs.getCache().Get(utils.REFUND_INCR_CACHE_PREFIX + arg.CgrID + arg.RunID); err == nil && item != nil { + cacheKey := utils.REFUND_INCR_CACHE_PREFIX + arg.CgrID + arg.RunID + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = *(item.Value.(*float64)) return item.Err } @@ -216,6 +218,9 @@ func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err Subject: arg.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, arg, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ + Err: err, + }) return err } @@ -224,7 +229,7 @@ func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err } else { err = arg.RefundIncrements() } - rs.getCache().Cache(utils.REFUND_INCR_CACHE_PREFIX+arg.CgrID+arg.RunID, &cache2go.CacheItem{ + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ Value: reply, Err: err, }) @@ -232,7 +237,8 @@ func (rs *Responder) RefundIncrements(arg *CallDescriptor, reply *float64) (err } func (rs *Responder) RefundRounding(arg *CallDescriptor, reply *float64) (err error) { - if item, err := rs.getCache().Get(utils.REFUND_ROUND_CACHE_PREFIX + arg.CgrID + arg.RunID); err == nil && item != nil { + cacheKey := utils.REFUND_ROUND_CACHE_PREFIX + arg.CgrID + arg.RunID + arg.DurationIndex.String() + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { *reply = *(item.Value.(*float64)) return item.Err } @@ -254,6 +260,9 @@ func (rs *Responder) RefundRounding(arg *CallDescriptor, reply *float64) (err er Subject: arg.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, arg, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ + Err: err, + }) return err } @@ -262,7 +271,7 @@ func (rs *Responder) RefundRounding(arg *CallDescriptor, reply *float64) (err er } else { err = arg.RefundRounding() } - rs.getCache().Cache(utils.REFUND_ROUND_CACHE_PREFIX+arg.CgrID+arg.RunID, &cache2go.CacheItem{ + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ Value: reply, Err: err, }) @@ -305,11 +314,17 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { if rs.Bal != nil { return errors.New("unsupported method on the balancer") } + cacheKey := utils.GET_DERIV_MAX_SESS_TIME + ev.CGRID + ev.RunID + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *reply = *(item.Value.(*float64)) + return item.Err + } if ev.Subject == "" { ev.Subject = ev.Account } // replace user profile fields if err := LoadUserProfile(ev, utils.EXTRA_FIELDS); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } // replace aliases @@ -323,6 +338,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { Subject: ev.Subject, Context: utils.ALIAS_CONTEXT_RATING, }, ev, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } @@ -331,6 +347,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { Account: ev.GetAccount(utils.META_DEFAULT), Subject: ev.GetSubject(utils.META_DEFAULT)} dcs := &utils.DerivedChargers{} if err := rs.GetDerivedChargers(attrsDC, dcs); err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } dcs, _ = dcs.AppendDefaultRun() @@ -351,10 +368,12 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { } startTime, err := ev.GetSetupTime(utils.META_DEFAULT, rs.Timezone) if err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } usage, err := ev.GetDuration(utils.META_DEFAULT) if err != nil { + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if usage == 0 { @@ -362,7 +381,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { } cd := &CallDescriptor{ CgrID: ev.GetCgrId(rs.Timezone), - RunID: ev.RunID, + RunID: dc.RunID, TOR: ev.ToR, Direction: ev.GetDirection(dc.DirectionField), Tenant: ev.GetTenant(dc.TenantField), @@ -377,6 +396,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { err = rs.GetMaxSessionTime(cd, &remainingDuration) if err != nil { *reply = 0 + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return err } if utils.IsSliceMember([]string{utils.META_POSTPAID, utils.POSTPAID}, ev.GetReqType(dc.RequestTypeField)) { @@ -390,6 +410,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { maxCallDuration = remainingDuration } } + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: maxCallDuration}) *reply = maxCallDuration return nil } @@ -399,6 +420,11 @@ func (rs *Responder) GetSessionRuns(ev *CDR, sRuns *[]*SessionRun) error { if rs.Bal != nil { return errors.New("Unsupported method on the balancer") } + cacheKey := utils.GET_SESS_RUNS_CACHE_PREFIX + ev.CGRID + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *sRuns = *(item.Value.(*[]*SessionRun)) + return item.Err + } if ev.Subject == "" { ev.Subject = ev.Account } @@ -427,7 +453,7 @@ func (rs *Responder) GetSessionRuns(ev *CDR, sRuns *[]*SessionRun) error { //utils.Logger.Info(fmt.Sprintf("Derived chargers for: %+v", attrsDC)) dcs := &utils.DerivedChargers{} if err := rs.GetDerivedChargers(attrsDC, dcs); err != nil { - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CGRID, &cache2go.CacheItem{ + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{ Err: err, }) return err @@ -441,22 +467,18 @@ func (rs *Responder) GetSessionRuns(ev *CDR, sRuns *[]*SessionRun) error { } startTime, err := ev.GetAnswerTime(dc.AnswerTimeField, rs.Timezone) if err != nil { - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CGRID, &cache2go.CacheItem{ - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return errors.New("Error parsing answer event start time") } endTime, err := ev.GetEndTime("", rs.Timezone) if err != nil { - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CGRID, &cache2go.CacheItem{ - Err: err, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Err: err}) return errors.New("Error parsing answer event end time") } extraFields := ev.GetExtraFields() cd := &CallDescriptor{ CgrID: ev.GetCgrId(rs.Timezone), - RunID: ev.RunID, + RunID: dc.RunID, TOR: ev.ToR, Direction: ev.GetDirection(dc.DirectionField), Tenant: ev.GetTenant(dc.TenantField), @@ -477,9 +499,7 @@ func (rs *Responder) GetSessionRuns(ev *CDR, sRuns *[]*SessionRun) error { } //utils.Logger.Info(fmt.Sprintf("RUNS: %v", len(sesRuns))) *sRuns = sesRuns - rs.getCache().Cache(utils.GET_SESS_RUNS_CACHE_PREFIX+ev.CGRID, &cache2go.CacheItem{ - Value: sRuns, - }) + rs.getCache().Cache(cacheKey, &cache2go.CacheItem{Value: sRuns}) return nil } @@ -497,6 +517,10 @@ func (rs *Responder) GetDerivedChargers(attrs *utils.AttrDerivedChargers, dcs *u func (rs *Responder) GetLCR(attrs *AttrGetLcr, reply *LCRCost) error { cacheKey := utils.LCRCachePrefix + attrs.CgrID + attrs.RunID + if item, err := rs.getCache().Get(cacheKey); err == nil && item != nil { + *reply = *(item.Value.(*LCRCost)) + return item.Err + } if attrs.CallDescriptor.Subject == "" { attrs.CallDescriptor.Subject = attrs.CallDescriptor.Account } diff --git a/engine/responder_test.go b/engine/responder_test.go index ed3f002f2..9651382ec 100644 --- a/engine/responder_test.go +++ b/engine/responder_test.go @@ -153,10 +153,10 @@ func TestResponderGetSessionRuns(t *testing.T) { sesRuns := make([]*SessionRun, 0) eSRuns := []*SessionRun{ &SessionRun{DerivedCharger: extra1DC, - CallDescriptor: &CallDescriptor{CgrID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), RunID: "*default", Direction: "*out", Category: "0", + CallDescriptor: &CallDescriptor{CgrID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), RunID: "extra1", Direction: "*out", Category: "0", Tenant: "vdf", Subject: "rif", Account: "minitsboy", Destination: "0256", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), TimeEnd: time.Date(2013, 11, 7, 8, 42, 36, 0, time.UTC), TOR: utils.VOICE, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: extra2DC, - CallDescriptor: &CallDescriptor{CgrID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), RunID: "*default", Direction: "*out", Category: "call", + CallDescriptor: &CallDescriptor{CgrID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), RunID: "extra2", Direction: "*out", Category: "call", Tenant: "vdf", Subject: "ivo", Account: "ivo", Destination: "1002", TimeStart: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), TimeEnd: time.Date(2013, 11, 7, 8, 42, 36, 0, time.UTC), TOR: utils.VOICE, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}}, &SessionRun{DerivedCharger: dfDC, CallDescriptor: &CallDescriptor{CgrID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), RunID: "*default", Direction: "*out", Category: "call", @@ -165,7 +165,7 @@ func TestResponderGetSessionRuns(t *testing.T) { t.Error(err) } else if !reflect.DeepEqual(eSRuns, sesRuns) { for _, sr := range sesRuns { - t.Logf("sr cd: %+v", sr.CallDescriptor) + t.Logf("sr cd: %s", utils.ToIJSON(sr.CallDescriptor)) } t.Errorf("Expecting: %+v, received: %+v", eSRuns, sesRuns) } diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index a917c9961..8fb9ca7b2 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -341,6 +341,9 @@ func TestTpRemActionsRefenced(t *testing.T) { if !*testIntegration { return } + + // no more reference check for sake of speed! + actionsMap := make(map[string]engine.Actions) if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ ActionIDs: []string{"TOPUP_VOICE"}, @@ -352,16 +355,14 @@ func TestTpRemActionsRefenced(t *testing.T) { var reply string if err := tpRPC.Call("ApierV2.RemActions", v1.AttrRemActions{ ActionIDs: []string{"TOPUP_VOICE"}, - }, &reply); err == nil { - t.Error("No error on ApierV2.RemActions: ", err.Error()) - } else if reply == utils.OK { + }, &reply); err != nil { + t.Error("Error on ApierV2.RemActions: ", err.Error()) + } else if reply != utils.OK { t.Errorf("Calling ApierV2.RemActions got reply: %s", reply) } if err := tpRPC.Call("ApierV2.GetActions", v2.AttrGetActions{ - ActionIDs: []string{"TOPUP_VOICE"}, - }, &actionsMap); err != nil { - t.Error("Got error on ApierV2.GetActions: ", err.Error()) - } else if len(actionsMap) != 1 { - t.Errorf("Calling ApierV2.GetActions got reply: %s", utils.ToIJSON(actionsMap)) + ActionIDs: []string{"PAYMENT_2056bd2fe137082970f97102b64e42fd"}, + }, &actionsMap); err == nil { + t.Error("no error on ApierV2.GetActions: ", err) } } diff --git a/sessionmanager/data_it_test.go b/sessionmanager/data_it_test.go index 726e6d5d4..ee73c7f38 100644 --- a/sessionmanager/data_it_test.go +++ b/sessionmanager/data_it_test.go @@ -96,7 +96,7 @@ func TestSMGDataLastUsedData(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123491", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -104,7 +104,7 @@ func TestSMGDataLastUsedData(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:59", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -124,7 +124,7 @@ func TestSMGDataLastUsedData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123491", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -132,6 +132,8 @@ func TestSMGDataLastUsedData(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:59", + utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", utils.LastUsed: "20000", } @@ -150,7 +152,7 @@ func TestSMGDataLastUsedData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123491", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -158,6 +160,8 @@ func TestSMGDataLastUsedData(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, + utils.SETUP_TIME: "2016-01-05 18:30:59", + utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.LastUsed: "0", } var rpl string @@ -187,7 +191,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -195,7 +199,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:50", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -221,7 +225,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -252,7 +256,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -283,7 +287,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -314,7 +318,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -345,7 +349,7 @@ func TestSMGDataLastUsedMultipleData(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123492", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -387,7 +391,7 @@ func TestSMGDataDerivedChargingNoCredit(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234967", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1011", utils.SUBJECT: "1011", @@ -403,6 +407,8 @@ func TestSMGDataDerivedChargingNoCredit(t *testing.T) { if err := smgRPC.Call("SMGenericV1.SessionStart", smgEv, &maxUsage); err != nil { t.Error(err) } + // the second derived charging run has no credit + if maxUsage != 0 { t.Error("Bad max usage: ", maxUsage) } @@ -429,7 +435,7 @@ func TestSMGDataTTLExpired(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123494", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -437,7 +443,7 @@ func TestSMGDataTTLExpired(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:52", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -478,7 +484,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123495", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -486,7 +492,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:53", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -513,7 +519,7 @@ func TestSMGDataTTLExpiredMultiUpdates(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123495", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -566,7 +572,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -574,7 +580,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:54", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -600,7 +606,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -631,7 +637,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -662,7 +668,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -693,7 +699,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -724,7 +730,7 @@ func TestSMGDataMultipleDataNoUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123496", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -766,7 +772,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -774,7 +780,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { utils.CATEGORY: "data", utils.TENANT: "cgrates.org", utils.REQTYPE: utils.META_PREPAID, - utils.SETUP_TIME: "2016-01-05 18:30:49", + utils.SETUP_TIME: "2016-01-05 18:30:55", utils.ANSWER_TIME: "2016-01-05 18:31:05", utils.USAGE: "1048576", } @@ -801,7 +807,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -832,7 +838,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -863,7 +869,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -894,7 +900,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", @@ -925,7 +931,7 @@ func TestSMGDataMultipleDataConstantUsage(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.DATA, - utils.ACCID: "12349", + utils.ACCID: "123497", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1010", utils.SUBJECT: "1010", diff --git a/sessionmanager/smg_it_test.go b/sessionmanager/smg_it_test.go index 07bcdffc2..b14e4adf0 100644 --- a/sessionmanager/smg_it_test.go +++ b/sessionmanager/smg_it_test.go @@ -117,7 +117,7 @@ func TestSMGVoiceMonetaryRefund(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123451", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -147,7 +147,7 @@ func TestSMGVoiceMonetaryRefund(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123451", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -178,7 +178,7 @@ func TestSMGVoiceVoiceRefund(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123452", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -208,7 +208,7 @@ func TestSMGVoiceVoiceRefund(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123452", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -246,7 +246,7 @@ func TestSMGVoiceMixedRefund(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123453", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -278,7 +278,7 @@ func TestSMGVoiceMixedRefund(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12345", + utils.ACCID: "123453", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -438,7 +438,7 @@ func TestSMGVoiceLastUsedEnd(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234911", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -466,7 +466,7 @@ func TestSMGVoiceLastUsedEnd(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234911", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -492,7 +492,7 @@ func TestSMGVoiceLastUsedEnd(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234911", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -529,7 +529,7 @@ func TestSMGVoiceLastUsedNotFixed(t *testing.T) { smgEv := SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234922", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -557,7 +557,7 @@ func TestSMGVoiceLastUsedNotFixed(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234922", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", @@ -583,7 +583,7 @@ func TestSMGVoiceLastUsedNotFixed(t *testing.T) { smgEv = SMGenericEvent{ utils.EVENT_NAME: "TEST_EVENT", utils.TOR: utils.VOICE, - utils.ACCID: "12349", + utils.ACCID: "1234922", utils.DIRECTION: utils.OUT, utils.ACCOUNT: "1001", utils.SUBJECT: "1001", diff --git a/sessionmanager/smg_session.go b/sessionmanager/smg_session.go index df622953b..bb7e73f01 100644 --- a/sessionmanager/smg_session.go +++ b/sessionmanager/smg_session.go @@ -192,8 +192,10 @@ func (self *SMGSession) refund(refundDuration time.Duration) error { if len(refundIncrements) > 0 { cd := firstCC.CreateCallDescriptor() cd.Increments = refundIncrements + cd.CgrID = self.cd.CgrID + cd.RunID = self.cd.RunID cd.Increments.Compress() - utils.Logger.Info(fmt.Sprintf("Refunding duration %v with cd: %s", initialRefundDuration, utils.ToJSON(cd))) + utils.Logger.Info(fmt.Sprintf("Refunding %s duration %v with incerements: %s", cd.CgrID, initialRefundDuration, utils.ToJSON(cd.Increments))) var response float64 err := self.rater.Call("Responder.RefundIncrements", cd, &response) if err != nil { @@ -253,6 +255,8 @@ func (self *SMGSession) saveOperations(originID string) error { roundIncrements := firstCC.GetRoundIncrements() if len(roundIncrements) != 0 { cd := firstCC.CreateCallDescriptor() + cd.CgrID = self.cd.CgrID + cd.RunID = self.cd.RunID cd.Increments = roundIncrements var response float64 if err := self.rater.Call("Responder.RefundRounding", cd, &response); err != nil { diff --git a/sessionmanager/smgeneric.go b/sessionmanager/smgeneric.go index 7fc17f5dc..b81c79d95 100644 --- a/sessionmanager/smgeneric.go +++ b/sessionmanager/smgeneric.go @@ -423,16 +423,10 @@ func (self *SMGeneric) ChargeEvent(gev SMGenericEvent, clnt *rpc2.Client) (maxDu } // refund cc if len(refundIncrements) > 0 { - cd := &engine.CallDescriptor{ - Direction: cc.Direction, - Tenant: cc.Tenant, - Category: cc.Category, - Subject: cc.Subject, - Account: cc.Account, - Destination: cc.Destination, - TOR: cc.TOR, - Increments: refundIncrements, - } + cd := cc.CreateCallDescriptor() + cd.Increments = refundIncrements + cd.CgrID = sR.CallDescriptor.CgrID + cd.RunID = sR.CallDescriptor.RunID cd.Increments.Compress() utils.Logger.Info(fmt.Sprintf("Refunding session run callcost: %s", utils.ToJSON(cd))) var response float64 diff --git a/utils/consts.go b/utils/consts.go index b5a720173..d5f800d04 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -242,6 +242,7 @@ const ( REFUND_INCR_CACHE_PREFIX = "REFUND_INCR_" REFUND_ROUND_CACHE_PREFIX = "REFUND_ROUND_" GET_SESS_RUNS_CACHE_PREFIX = "GET_SESS_RUNS_" + GET_DERIV_MAX_SESS_TIME = "GET_DERIV_MAX_SESS_TIME_" LOG_CALL_COST_CACHE_PREFIX = "LOG_CALL_COSTS_" LCRCachePrefix = "LCR_" ALIAS_CONTEXT_RATING = "*rating" From 2ee445abcfdf5453e4a6f5288a3395db2d3f2886 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 3 May 2016 15:54:15 +0300 Subject: [PATCH 227/227] ResetAccountActionTriggers has Executed attribute --- apier/v1/triggers.go | 16 +++++++++++++--- general_tests/tp_it_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/apier/v1/triggers.go b/apier/v1/triggers.go index b6c6a3f5f..93d9e9ed1 100644 --- a/apier/v1/triggers.go +++ b/apier/v1/triggers.go @@ -129,7 +129,15 @@ func (self *ApierV1) RemoveAccountActionTriggers(attr AttrRemoveAccountActionTri return nil } -func (self *ApierV1) ResetAccountActionTriggers(attr AttrRemoveAccountActionTriggers, reply *string) error { +type AttrResetAccountActionTriggers struct { + Tenant string + Account string + GroupID string + UniqueID string + Executed bool +} + +func (self *ApierV1) ResetAccountActionTriggers(attr AttrResetAccountActionTriggers, reply *string) error { if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Account"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) @@ -146,11 +154,13 @@ func (self *ApierV1) ResetAccountActionTriggers(attr AttrRemoveAccountActionTrig if (attr.UniqueID == "" || at.UniqueID == attr.UniqueID) && (attr.GroupID == "" || at.ID == attr.GroupID) { // reset action trigger - at.Executed = false + at.Executed = attr.Executed } } - account.ExecuteActionTriggers(nil) + if attr.Executed == false { + account.ExecuteActionTriggers(nil) + } if err := self.AccountDb.SetAccount(account); err != nil { return 0, err } diff --git a/general_tests/tp_it_test.go b/general_tests/tp_it_test.go index 8fb9ca7b2..0a1681ec1 100644 --- a/general_tests/tp_it_test.go +++ b/general_tests/tp_it_test.go @@ -366,3 +366,32 @@ func TestTpRemActionsRefenced(t *testing.T) { t.Error("no error on ApierV2.GetActions: ", err) } } + +func TestApierResetAccountActionTriggers(t *testing.T) { + if !*testIntegration { + return + } + var acnt engine.Account + attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1005"} + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.ActionTriggers[0].Executed == true { + t.Errorf("wrong action trigger executed flag: %s", utils.ToIJSON(acnt.ActionTriggers)) + } + var reply string + if err := tpRPC.Call("ApierV2.ResetAccountActionTriggers", v1.AttrResetAccountActionTriggers{ + Tenant: "cgrates.org", + Account: "1005", + GroupID: "STANDARD_TRIGGERS", + Executed: true, + }, &reply); err != nil { + t.Error("Error on ApierV2.ResetAccountActionTriggers: ", err.Error()) + } else if reply != utils.OK { + t.Errorf("Calling ApierV2.ResetAccountActionTriggers got reply: %s", reply) + } + if err := tpRPC.Call("ApierV2.GetAccount", attrs, &acnt); err != nil { + t.Error(err) + } else if acnt.ActionTriggers[0].Executed == false { + t.Errorf("wrong action trigger executed flag: %s", utils.ToIJSON(acnt.ActionTriggers)) + } +}