From 352b58eccc5b44367e15ff0264ba7c9fa14d9c66 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 24 Jul 2013 12:54:17 +0200 Subject: [PATCH 01/17] Engine path in daemontools script --- data/daemontools/run | 4 ++-- data/pkg/debian/cgrates.apt.list | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/data/daemontools/run b/data/daemontools/run index 7de7a1b10..64a81b844 100755 --- a/data/daemontools/run +++ b/data/daemontools/run @@ -1,4 +1,4 @@ #!/bin/sh -echo Running CGRateS service -exec /usr/bin/cgr-rater -config /etc/cgrates/cgrates.cfg +echo Running CGRateS engine +exec /usr/bin/cgr-engine -config /etc/cgrates/cgrates.cfg diff --git a/data/pkg/debian/cgrates.apt.list b/data/pkg/debian/cgrates.apt.list index d0d2b3e03..2e8c3240e 100644 --- a/data/pkg/debian/cgrates.apt.list +++ b/data/pkg/debian/cgrates.apt.list @@ -1,6 +1,7 @@ ## CGRateS official APT repository. # Place this source file into your /etc/apt/sources.list.d/ folder and execute commands from bellow # wget -O - http://apt.itsyscom.com/repos/apt/conf/cgrates.gpg.key|apt-key add - -# apt-get update && apt-get install +# apt-get update && apt-get install cgrates +deb http://apt.itsyscom.com/repos/apt/debian wheezy main deb http://apt.itsyscom.com/repos/apt/debian squeeze main From 64c1cd67f0088158d71f31e3ad3e747d2ff17668 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 25 Jul 2013 12:16:15 +0300 Subject: [PATCH 02/17] implemented global rounding method and precision for get cost --- cmd/cgr-engine/cgr-engine.go | 3 ++- engine/calldesc.go | 40 ++++++++++++------------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 02a52c356..7dd2e999a 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -26,8 +26,8 @@ import ( "github.com/cgrates/cgrates/balancer2go" "github.com/cgrates/cgrates/cdrs" "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/mediator" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/mediator" "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/sessionmanager" "github.com/cgrates/cgrates/utils" @@ -252,6 +252,7 @@ func main() { } defer loggerDb.Close() engine.SetStorageLogger(loggerDb) + engine.SetRoundingMethodAndDecimals(cfg.RaterRoundingMethod, cfg.RaterRoundingDecimals) if cfg.SMDebitInterval > 0 { if dp, err := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval)); err == nil { diff --git a/engine/calldesc.go b/engine/calldesc.go index 1cdbe480c..532b658bb 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -24,7 +24,6 @@ import ( "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" "log/syslog" - "math" "strings" "time" ) @@ -51,27 +50,12 @@ var ( storageGetter, _ = NewMapStorage() //storageGetter, _ = NewMongoStorage(db_server, "27017", "cgrates_test", "", "") //storageGetter, _ = NewRedisStorage(db_server+":6379", 11, "") - storageLogger = storageGetter - debitPeriod = 10 * time.Second + storageLogger = storageGetter + debitPeriod = 10 * time.Second + roundingMethod = "*middle" + roundingDecimals = 4 ) -/* -Utility function for rounding a float to a certain number of decimals (not present in math). -*/ -func round(val float64, prec int) float64 { - - var rounder float64 - intermed := val * math.Pow(10, float64(prec)) - - if val >= 0.5 { - rounder = math.Ceil(intermed) - } else { - rounder = math.Floor(intermed) - } - - return rounder / math.Pow(10, float64(prec)) -} - /* The input stucture that contains call information. */ @@ -102,9 +86,7 @@ func (cd *CallDescriptor) GetUserBalanceKey() string { return fmt.Sprintf("%s:%s:%s", cd.Direction, cd.Tenant, subj) } -/* -Gets and caches the user balance information. -*/ +// Gets and caches the user balance information. func (cd *CallDescriptor) getUserBalance() (ub *UserBalance, err error) { if cd.userBalance == nil { cd.userBalance, err = storageGetter.GetUserBalance(cd.GetUserBalanceKey()) @@ -112,13 +94,17 @@ func (cd *CallDescriptor) getUserBalance() (ub *UserBalance, err error) { return cd.userBalance, err } -/* -Exported method to set the storage getter. -*/ +// Exported method to set the storage getter. func SetDataStorage(sg DataStorage) { storageGetter = sg } +// Sets the global rounding method and decimal precision for GetCost method +func SetRoundingMethodAndDecimals(rm string, rd int) { + roundingMethod = rm + roundingDecimals = rd +} + /* Sets the database for logging (can be de same as storage getter or different db) */ @@ -285,7 +271,7 @@ func (cd *CallDescriptor) GetCost() (*CallCost, error) { } cost += ts.getCost(cd) } - + cost = utils.Round(cost, roundingDecimals, roundingMethod) cc := &CallCost{ Direction: cd.Direction, TOR: cd.TOR, From ff9f8c82afb125c68d578cb0569d17cad4c764ce Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Thu, 25 Jul 2013 13:51:22 +0300 Subject: [PATCH 03/17] started work on generic cdr --- cdrs/cdrs.go | 26 ++++++- cdrs/fscdr.go | 60 ++++++++-------- cdrs/gencdr.go | 163 ++++++++++++++++++++++++++++++++++++++++++ config/config.go | 147 +++++++++++++++++++------------------ config/config_test.go | 98 ++++++++++++------------- config/test_data.txt | 1 + 6 files changed, 343 insertions(+), 152 deletions(-) create mode 100644 cdrs/gencdr.go diff --git a/cdrs/cdrs.go b/cdrs/cdrs.go index 1d6262c4d..75a3a92d4 100644 --- a/cdrs/cdrs.go +++ b/cdrs/cdrs.go @@ -21,8 +21,8 @@ package cdrs import ( "fmt" "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/mediator" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/mediator" "io/ioutil" "net/http" ) @@ -33,7 +33,7 @@ var ( medi *mediator.Mediator ) -func cdrHandler(w http.ResponseWriter, r *http.Request) { +func fsCdrHandler(w http.ResponseWriter, r *http.Request) { body, _ := ioutil.ReadAll(r.Body) if fsCdr, err := new(FSCdr).New(body); err == nil { storage.SetCdr(fsCdr) @@ -50,6 +50,23 @@ func cdrHandler(w http.ResponseWriter, r *http.Request) { } } +func genCdrHandler(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + if genCdr, err := new(GenCdr).New(body); err == nil { + storage.SetCdr(genCdr) + if cfg.CDRSMediator == "internal" { + errMedi := medi.MediateDBCDR(genCdr, storage) + if errMedi != nil { + engine.Logger.Err(fmt.Sprintf("Could not run mediation on CDR: %s", errMedi.Error())) + } + } else { + //TODO: use the connection to mediator + } + } else { + engine.Logger.Err(fmt.Sprintf("Could not create CDR entry: %v", err)) + } +} + type CDRS struct{} func New(s engine.DataStorage, m *mediator.Mediator, c *config.CGRConfig) *CDRS { @@ -61,7 +78,10 @@ func New(s engine.DataStorage, m *mediator.Mediator, c *config.CGRConfig) *CDRS func (cdrs *CDRS) StartCapturingCDRs() { if cfg.CDRSfsJSONEnabled { - http.HandleFunc("/freeswitch_json", cdrHandler) + http.HandleFunc("/freeswitch_json", fsCdrHandler) + } + if cfg.CDRSgenJSONEnabled { + http.HandleFunc("/generic_json", genCdrHandler) } http.ListenAndServe(cfg.CDRSListen, nil) } diff --git a/cdrs/fscdr.go b/cdrs/fscdr.go index 6e69814da..3f0a116cf 100644 --- a/cdrs/fscdr.go +++ b/cdrs/fscdr.go @@ -28,23 +28,23 @@ import ( const ( // Freswitch event property names - CDR_MAP = "variables" - DIRECTION = "direction" - ORIG_ID = "sip_call_id" //- originator_id - match cdrs - SUBJECT = "cgr_subject" - ACCOUNT = "cgr_account" - DESTINATION = "cgr_destination" - REQTYPE = "cgr_reqtype" //prepaid or postpaid - TOR = "cgr_tor" - UUID = "uuid" // -Unique ID for this call leg - CSTMID = "cgr_cstmid" - CALL_DEST_NR = "dialed_extension" - PARK_TIME = "start_epoch" - ANSWER_TIME = "answer_epoch" - HANGUP_TIME = "end_epoch" - DURATION = "billsec" - USERNAME = "user_name" - FS_IP = "sip_local_network_addr" + FS_CDR_MAP = "variables" + FS_DIRECTION = "direction" + FS_ORIG_ID = "sip_call_id" //- originator_id - match cdrs + FS_SUBJECT = "cgr_subject" + FS_ACCOUNT = "cgr_account" + FS_DESTINATION = "cgr_destination" + FS_REQTYPE = "cgr_reqtype" //prepaid or postpaid + FS_TOR = "cgr_tor" + FS_UUID = "uuid" // -Unique ID for this call leg + FS_CSTMID = "cgr_cstmid" + FS_CALL_DEST_NR = "dialed_extension" + FS_PARK_TIME = "start_epoch" + FS_ANSWER_TIME = "answer_epoch" + FS_HANGUP_TIME = "end_epoch" + FS_DURATION = "billsec" + FS_USERNAME = "user_name" + FS_IP = "sip_local_network_addr" ) type FSCdr map[string]string @@ -54,7 +54,7 @@ func (fsCdr FSCdr) New(body []byte) (utils.CDR, error) { var tmp map[string]interface{} var err error if err = json.Unmarshal(body, &tmp); err == nil { - if variables, ok := tmp[CDR_MAP]; ok { + if variables, ok := tmp[FS_CDR_MAP]; ok { if variables, ok := variables.(map[string]interface{}); ok { for k, v := range variables { fsCdr[k] = v.(string) @@ -67,10 +67,10 @@ func (fsCdr FSCdr) New(body []byte) (utils.CDR, error) { } func (fsCdr FSCdr) GetCgrId() string { - return utils.FSCgrId(fsCdr[UUID]) + return utils.FSCgrId(fsCdr[FS_UUID]) } func (fsCdr FSCdr) GetAccId() string { - return fsCdr[UUID] + return fsCdr[FS_UUID] } func (fsCdr FSCdr) GetCdrHost() string { return fsCdr[FS_IP] @@ -80,29 +80,29 @@ func (fsCdr FSCdr) GetDirection() string { return "OUT" } func (fsCdr FSCdr) GetOrigId() string { - return fsCdr[ORIG_ID] + return fsCdr[FS_ORIG_ID] } func (fsCdr FSCdr) GetSubject() string { - return utils.FirstNonEmpty(fsCdr[SUBJECT], fsCdr[USERNAME]) + return utils.FirstNonEmpty(fsCdr[FS_SUBJECT], fsCdr[FS_USERNAME]) } func (fsCdr FSCdr) GetAccount() string { - return utils.FirstNonEmpty(fsCdr[ACCOUNT], fsCdr[USERNAME]) + return utils.FirstNonEmpty(fsCdr[FS_ACCOUNT], fsCdr[FS_USERNAME]) } // Charging destination number func (fsCdr FSCdr) GetDestination() string { - return utils.FirstNonEmpty(fsCdr[DESTINATION], fsCdr[CALL_DEST_NR]) + return utils.FirstNonEmpty(fsCdr[FS_DESTINATION], fsCdr[FS_CALL_DEST_NR]) } func (fsCdr FSCdr) GetTOR() string { - return utils.FirstNonEmpty(fsCdr[TOR], cfg.DefaultTOR) + return utils.FirstNonEmpty(fsCdr[FS_TOR], cfg.DefaultTOR) } func (fsCdr FSCdr) GetTenant() string { - return utils.FirstNonEmpty(fsCdr[CSTMID], cfg.DefaultTenant) + return utils.FirstNonEmpty(fsCdr[FS_CSTMID], cfg.DefaultTenant) } func (fsCdr FSCdr) GetReqType() string { - return utils.FirstNonEmpty(fsCdr[REQTYPE], cfg.DefaultReqType) + return utils.FirstNonEmpty(fsCdr[FS_REQTYPE], cfg.DefaultReqType) } func (fsCdr FSCdr) GetExtraFields() map[string]string { extraFields := make(map[string]string, len(cfg.CDRSExtraFields)) @@ -115,19 +115,19 @@ func (fsCdr FSCdr) GetFallbackSubj() string { return cfg.DefaultSubject } func (fsCdr FSCdr) GetAnswerTime() (t time.Time, err error) { - st, err := strconv.ParseInt(fsCdr[ANSWER_TIME], 0, 64) + st, err := strconv.ParseInt(fsCdr[FS_ANSWER_TIME], 0, 64) t = time.Unix(0, st*1000) return } func (fsCdr FSCdr) GetHangupTime() (t time.Time, err error) { - st, err := strconv.ParseInt(fsCdr[HANGUP_TIME], 0, 64) + st, err := strconv.ParseInt(fsCdr[FS_HANGUP_TIME], 0, 64) t = time.Unix(0, st*1000) return } // Extracts duration as considered by the telecom switch func (fsCdr FSCdr) GetDuration() int64 { - dur, _ := strconv.ParseInt(fsCdr[DURATION], 0, 64) + dur, _ := strconv.ParseInt(fsCdr[FS_DURATION], 0, 64) return dur } diff --git a/cdrs/gencdr.go b/cdrs/gencdr.go new file mode 100644 index 000000000..8039337b3 --- /dev/null +++ b/cdrs/gencdr.go @@ -0,0 +1,163 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2013 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package cdrs + +import ( + "encoding/json" + "errors" + "github.com/cgrates/cgrates/utils" + "strconv" + "time" +) + +const ( + CDR_MAP = "variables" + DIRECTION = "direction" + ORIG_ID = "id" + SUBJECT = "subject" + ACCOUNT = "account" + DESTINATION = "destination" + REQTYPE = "reqtype" //prepaid or postpaid + TOR = "tor" + UUID = "uuid" // -Unique ID for this call leg + CSTMID = "cstmid" + CALL_DEST_NR = "dialed_extension" + PARK_TIME = "start_epoch" + ANSWER_TIME = "time_answer" + HANGUP_TIME = "time_hangup" + DURATION = "duration" + USERNAME = "user_name" + IP = "sip_local_network_addr" +) + +type GenCdr map[string]string + +func (genCdr GenCdr) New(body []byte) (utils.CDR, error) { + genCdr = make(map[string]string) + var tmp map[string]interface{} + var err error + if err = json.Unmarshal(body, &tmp); err == nil { + if variables, ok := tmp[CDR_MAP]; ok { + if variables, ok := variables.(map[string]interface{}); ok { + for k, v := range variables { + genCdr[k] = v.(string) + } + } + return genCdr, nil + } + } + return nil, err +} + +func (genCdr GenCdr) GetCgrId() string { + return utils.FSCgrId(genCdr[UUID]) +} +func (genCdr GenCdr) GetAccId() string { + return genCdr[UUID] +} +func (genCdr GenCdr) GetCdrHost() string { + return genCdr[FS_IP] +} +func (genCdr GenCdr) GetDirection() string { + //TODO: implement direction + return "OUT" +} +func (genCdr GenCdr) GetOrigId() string { + return genCdr[ORIG_ID] +} +func (genCdr GenCdr) GetSubject() string { + return utils.FirstNonEmpty(genCdr[SUBJECT], genCdr[USERNAME]) +} +func (genCdr GenCdr) GetAccount() string { + return utils.FirstNonEmpty(genCdr[ACCOUNT], genCdr[USERNAME]) +} + +// Charging destination number +func (genCdr GenCdr) GetDestination() string { + return utils.FirstNonEmpty(genCdr[DESTINATION], genCdr[CALL_DEST_NR]) +} + +func (genCdr GenCdr) GetTOR() string { + return utils.FirstNonEmpty(genCdr[TOR], cfg.DefaultTOR) +} + +func (genCdr GenCdr) GetTenant() string { + return utils.FirstNonEmpty(genCdr[CSTMID], cfg.DefaultTenant) +} +func (genCdr GenCdr) GetReqType() string { + return utils.FirstNonEmpty(genCdr[REQTYPE], cfg.DefaultReqType) +} +func (genCdr GenCdr) GetExtraFields() map[string]string { + extraFields := make(map[string]string, len(cfg.CDRSExtraFields)) + for _, field := range cfg.CDRSExtraFields { + extraFields[field] = genCdr[field] + } + return extraFields +} +func (genCdr GenCdr) GetFallbackSubj() string { + return cfg.DefaultSubject +} +func (genCdr GenCdr) GetAnswerTime() (t time.Time, err error) { + st, err := strconv.ParseInt(genCdr[ANSWER_TIME], 0, 64) + t = time.Unix(0, st*1000) + return +} +func (genCdr GenCdr) GetHangupTime() (t time.Time, err error) { + st, err := strconv.ParseInt(genCdr[HANGUP_TIME], 0, 64) + t = time.Unix(0, st*1000) + return +} + +// Extracts duration as considered by the telecom switch +func (genCdr GenCdr) GetDuration() int64 { + dur, _ := strconv.ParseInt(genCdr[DURATION], 0, 64) + return dur +} + +func (genCdr GenCdr) Store() (result string, err error) { + result += genCdr.GetCgrId() + "|" + result += genCdr.GetAccId() + "|" + result += genCdr.GetCdrHost() + "|" + result += genCdr.GetDirection() + "|" + result += genCdr.GetOrigId() + "|" + result += genCdr.GetSubject() + "|" + result += genCdr.GetAccount() + "|" + result += genCdr.GetDestination() + "|" + result += genCdr.GetTOR() + "|" + result += genCdr.GetAccId() + "|" + result += genCdr.GetTenant() + "|" + result += genCdr.GetReqType() + "|" + st, err := genCdr.GetAnswerTime() + if err != nil { + return "", err + } + result += strconv.FormatInt(st.UnixNano(), 10) + "|" + et, err := genCdr.GetHangupTime() + if err != nil { + return "", err + } + result += strconv.FormatInt(et.UnixNano(), 10) + "|" + result += strconv.FormatInt(genCdr.GetDuration(), 10) + "|" + result += genCdr.GetFallbackSubj() + "|" + return +} + +func (genCdr GenCdr) Restore(input string) error { + return errors.New("Not implemented") +} diff --git a/config/config.go b/config/config.go index 6368b7b85..9b6c2bca8 100644 --- a/config/config.go +++ b/config/config.go @@ -26,80 +26,81 @@ import ( ) const ( - DISABLED = "disabled" - INTERNAL = "internal" - JSON = "json" - GOB = "gob" - POSTGRES = "postgres" - MONGO = "mongo" - REDIS = "redis" - SAME = "same" - FS = "freeswitch" - PREPAID = "prepaid" - POSTPAID = "postpaid" + DISABLED = "disabled" + INTERNAL = "internal" + JSON = "json" + GOB = "gob" + POSTGRES = "postgres" + MONGO = "mongo" + REDIS = "redis" + SAME = "same" + FS = "freeswitch" + PREPAID = "prepaid" + POSTPAID = "postpaid" PSEUDOPREPAID = "pseudoprepaid" - RATED = "rated" + RATED = "rated" ) // Holds system configuration, defaults are overwritten with values from config file if found type CGRConfig struct { - DataDBType string - DataDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - DataDBPort string // The port to bind to. - DataDBName string // The name of the database to connect to. - DataDBUser string // The user to sign in as. - DataDBPass string // The user's password. + DataDBType string + DataDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + DataDBPort string // The port to bind to. + DataDBName string // The name of the database to connect to. + DataDBUser string // The user to sign in as. + DataDBPass string // The user's password. StorDBType string // Should reflect the database type used to store logs StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. StorDBPort string // The port to bind to. StorDBName string // The name of the database to connect to. StorDBUser string // The user to sign in as. StorDBPass string // The user's password. - RPCEncoding string // RPC encoding used on APIs: . - DefaultReqType string // Use this request type if not defined on top - DefaultTOR string // set default type of record - DefaultTenant string // set default tenant - DefaultSubject string // set default rating subject, useful in case of fallback - RaterEnabled bool // start standalone server (no balancer) - RaterBalancer string // balancer address host:port - RaterListen string // listening address host:port - RaterRoundingMethod string // Rounding method for the end price: - RaterRoundingDecimals int // Number of decimals to round end prices at - BalancerEnabled bool - BalancerListen string // Json RPC server address - SchedulerEnabled bool - CDRSListen string // CDRS's listening interface: . - CDRSfsJSONEnabled bool // Enable the handler for FreeSWITCH JSON CDRs: . - CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> - CDRSExtraFields []string //Extra fields to store in CDRs - SMEnabled bool - SMSwitchType string - SMRater string // address where to access rater. Can be internal, direct rater address or the address of a balancer - SMRaterReconnects int // Number of reconnect attempts to rater - SMDebitInterval int // the period to be debited in advanced during a call (in seconds) - MediatorEnabled bool // Starts Mediator service: . - MediatorListen string // Mediator's listening interface: . - MediatorRater string // Address where to reach the Rater: - MediatorRaterReconnects int // Number of reconnects to rater before giving up. - MediatorCDRType string // CDR type . - MediatorAccIdField string // Name of field identifying accounting id used during mediation. Use index number in case of .csv cdrs. - MediatorSubjectFields []string // Name of subject fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorReqTypeFields []string // Name of request type fields to be used during mediation. Use index number in case of .csv cdrs. - MediatorDirectionFields []string // Name of direction fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorTenantFields []string // Name of tenant fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorTORFields []string // Name of tor fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorAccountFields []string // Name of account fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorDestFields []string // Name of destination fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorTimeAnswerFields []string // Name of time_start fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorDurationFields []string // Name of duration fields to be used during mediation. Use index numbers in case of .csv cdrs. - MediatorCDRInDir string // Absolute path towards the directory where the CDRs are kept (file stored CDRs). - MediatorCDROutDir string // Absolute path towards the directory where processed CDRs will be exported (file stored CDRs). - FreeswitchServer string // freeswitch address host:port - FreeswitchPass string // FS socket password - FreeswitchReconnects int // number of times to attempt reconnect after connect fails + RPCEncoding string // RPC encoding used on APIs: . + DefaultReqType string // Use this request type if not defined on top + DefaultTOR string // set default type of record + DefaultTenant string // set default tenant + DefaultSubject string // set default rating subject, useful in case of fallback + RaterEnabled bool // start standalone server (no balancer) + RaterBalancer string // balancer address host:port + RaterListen string // listening address host:port + RaterRoundingMethod string // Rounding method for the end price: + RaterRoundingDecimals int // Number of decimals to round end prices at + BalancerEnabled bool + BalancerListen string // Json RPC server address + SchedulerEnabled bool + CDRSListen string // CDRS's listening interface: . + CDRSfsJSONEnabled bool // Enable the handler for FreeSWITCH JSON CDRs: . + CDRSgenJSONEnabled bool // Enable the handler for Generic JSON CDRs: . + CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> + CDRSExtraFields []string //Extra fields to store in CDRs + SMEnabled bool + SMSwitchType string + SMRater string // address where to access rater. Can be internal, direct rater address or the address of a balancer + SMRaterReconnects int // Number of reconnect attempts to rater + SMDebitInterval int // the period to be debited in advanced during a call (in seconds) + MediatorEnabled bool // Starts Mediator service: . + MediatorListen string // Mediator's listening interface: . + MediatorRater string // Address where to reach the Rater: + MediatorRaterReconnects int // Number of reconnects to rater before giving up. + MediatorCDRType string // CDR type . + MediatorAccIdField string // Name of field identifying accounting id used during mediation. Use index number in case of .csv cdrs. + MediatorSubjectFields []string // Name of subject fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorReqTypeFields []string // Name of request type fields to be used during mediation. Use index number in case of .csv cdrs. + MediatorDirectionFields []string // Name of direction fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorTenantFields []string // Name of tenant fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorTORFields []string // Name of tor fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorAccountFields []string // Name of account fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorDestFields []string // Name of destination fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorTimeAnswerFields []string // Name of time_start fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorDurationFields []string // Name of duration fields to be used during mediation. Use index numbers in case of .csv cdrs. + MediatorCDRInDir string // Absolute path towards the directory where the CDRs are kept (file stored CDRs). + MediatorCDROutDir string // Absolute path towards the directory where processed CDRs will be exported (file stored CDRs). + FreeswitchServer string // freeswitch address host:port + FreeswitchPass string // FS socket password + FreeswitchReconnects int // number of times to attempt reconnect after connect fails } -func ( self *CGRConfig ) setDefaults() error { +func (self *CGRConfig) setDefaults() error { self.DataDBType = REDIS self.DataDBHost = "127.0.0.1" self.DataDBPort = "6379" @@ -127,6 +128,7 @@ func ( self *CGRConfig ) setDefaults() error { self.SchedulerEnabled = false self.CDRSListen = "127.0.0.1:2022" self.CDRSfsJSONEnabled = false + self.CDRSgenJSONEnabled = false self.CDRSMediator = INTERNAL self.CDRSExtraFields = []string{} self.MediatorEnabled = false @@ -267,11 +269,14 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("cdrs", "freeswitch_json_enabled"); hasOpt { cfg.CDRSfsJSONEnabled, _ = c.GetBool("cdrs", "freeswitch_json_enabled") } + if hasOpt = c.HasOption("cdrs", "generic_json_enabled"); hasOpt { + cfg.CDRSgenJSONEnabled, _ = c.GetBool("cdrs", "generic_json_enabled") + } if hasOpt = c.HasOption("cdrs", "mediator"); hasOpt { cfg.CDRSMediator, _ = c.GetString("cdrs", "mediator") } if hasOpt = c.HasOption("cdrs", "extra_fields"); hasOpt { - if cfg.CDRSExtraFields, errParse = ConfigSlice( c, "cdrs", "extra_fields"); errParse!=nil { + if cfg.CDRSExtraFields, errParse = ConfigSlice(c, "cdrs", "extra_fields"); errParse != nil { return nil, errParse } } @@ -294,47 +299,47 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { cfg.MediatorAccIdField, _ = c.GetString("mediator", "accid_field") } if hasOpt = c.HasOption("mediator", "subject_fields"); hasOpt { - if cfg.MediatorSubjectFields, errParse = ConfigSlice( c, "mediator", "subject_fields"); errParse!=nil { + if cfg.MediatorSubjectFields, errParse = ConfigSlice(c, "mediator", "subject_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "reqtype_fields"); hasOpt { - if cfg.MediatorReqTypeFields, errParse = ConfigSlice( c, "mediator", "reqtype_fields"); errParse!=nil { + if cfg.MediatorReqTypeFields, errParse = ConfigSlice(c, "mediator", "reqtype_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "direction_fields"); hasOpt { - if cfg.MediatorDirectionFields, errParse = ConfigSlice( c, "mediator", "direction_fields"); errParse!=nil { + if cfg.MediatorDirectionFields, errParse = ConfigSlice(c, "mediator", "direction_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "tenant_fields"); hasOpt { - if cfg.MediatorTenantFields, errParse = ConfigSlice( c, "mediator", "tenant_fields"); errParse!=nil { + if cfg.MediatorTenantFields, errParse = ConfigSlice(c, "mediator", "tenant_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "tor_fields"); hasOpt { - if cfg.MediatorTORFields, errParse = ConfigSlice( c, "mediator", "tor_fields"); errParse!=nil { + if cfg.MediatorTORFields, errParse = ConfigSlice(c, "mediator", "tor_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "account_fields"); hasOpt { - if cfg.MediatorAccountFields, errParse = ConfigSlice( c, "mediator", "account_fields"); errParse!=nil { + if cfg.MediatorAccountFields, errParse = ConfigSlice(c, "mediator", "account_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "destination_fields"); hasOpt { - if cfg.MediatorDestFields, errParse = ConfigSlice( c, "mediator", "destination_fields"); errParse!=nil { + if cfg.MediatorDestFields, errParse = ConfigSlice(c, "mediator", "destination_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "time_answer_fields"); hasOpt { - if cfg.MediatorTimeAnswerFields, errParse = ConfigSlice( c, "mediator", "time_answer_fields"); errParse!=nil { + if cfg.MediatorTimeAnswerFields, errParse = ConfigSlice(c, "mediator", "time_answer_fields"); errParse != nil { return nil, errParse } } if hasOpt = c.HasOption("mediator", "duration_fields"); hasOpt { - if cfg.MediatorDurationFields, errParse = ConfigSlice( c, "mediator", "duration_fields"); errParse!=nil { + if cfg.MediatorDurationFields, errParse = ConfigSlice(c, "mediator", "duration_fields"); errParse != nil { return nil, errParse } } diff --git a/config/config_test.go b/config/config_test.go index 00a6d37ce..0911bccbf 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -20,53 +20,54 @@ package config import ( "fmt" - "testing" - "reflect" "github.com/cgrates/cgrates/utils" + "reflect" + "testing" ) // Make sure defaults did not change by mistake func TestDefaults(t *testing.T) { cfg := &CGRConfig{} errSet := cfg.setDefaults() - if errSet != nil { + if errSet != nil { t.Log(fmt.Sprintf("Coud not set defaults: %s!", errSet.Error())) t.FailNow() } eCfg := &CGRConfig{} - eCfg.DataDBType = REDIS - eCfg.DataDBHost = "127.0.0.1" - eCfg.DataDBPort = "6379" - eCfg.DataDBName = "10" - eCfg.DataDBUser = "" - eCfg.DataDBPass = "" + eCfg.DataDBType = REDIS + eCfg.DataDBHost = "127.0.0.1" + eCfg.DataDBPort = "6379" + eCfg.DataDBName = "10" + eCfg.DataDBUser = "" + eCfg.DataDBPass = "" eCfg.StorDBType = utils.MYSQL - eCfg.StorDBHost = "localhost" - eCfg.StorDBPort = "3306" - eCfg.StorDBName = "cgrates" - eCfg.StorDBUser = "cgrates" - eCfg.StorDBPass = "CGRateS.org" + eCfg.StorDBHost = "localhost" + eCfg.StorDBPort = "3306" + eCfg.StorDBName = "cgrates" + eCfg.StorDBUser = "cgrates" + eCfg.StorDBPass = "CGRateS.org" eCfg.RPCEncoding = JSON eCfg.DefaultReqType = RATED - eCfg.DefaultTOR = "0" - eCfg.DefaultTenant = "0" - eCfg.DefaultSubject = "0" - eCfg.RaterEnabled = false - eCfg.RaterBalancer = DISABLED - eCfg.RaterListen = "127.0.0.1:2012" + eCfg.DefaultTOR = "0" + eCfg.DefaultTenant = "0" + eCfg.DefaultSubject = "0" + eCfg.RaterEnabled = false + eCfg.RaterBalancer = DISABLED + eCfg.RaterListen = "127.0.0.1:2012" eCfg.RaterRoundingMethod = utils.ROUNDING_MIDDLE eCfg.RaterRoundingDecimals = 4 - eCfg.BalancerEnabled = false - eCfg.BalancerListen = "127.0.0.1:2013" - eCfg.SchedulerEnabled = false - eCfg.CDRSListen = "127.0.0.1:2022" - eCfg.CDRSfsJSONEnabled = false + eCfg.BalancerEnabled = false + eCfg.BalancerListen = "127.0.0.1:2013" + eCfg.SchedulerEnabled = false + eCfg.CDRSListen = "127.0.0.1:2022" + eCfg.CDRSfsJSONEnabled = false + eCfg.CDRSgenJSONEnabled = false eCfg.CDRSMediator = INTERNAL eCfg.CDRSExtraFields = []string{} - eCfg.MediatorEnabled = false - eCfg.MediatorListen = "127.0.0.1:2032" - eCfg.MediatorRater = "127.0.0.1:2012" - eCfg.MediatorRaterReconnects = 3 + eCfg.MediatorEnabled = false + eCfg.MediatorListen = "127.0.0.1:2032" + eCfg.MediatorRater = "127.0.0.1:2012" + eCfg.MediatorRaterReconnects = 3 eCfg.MediatorCDRType = "freeswitch_http_json" eCfg.MediatorAccIdField = "accid" eCfg.MediatorSubjectFields = []string{"subject"} @@ -78,17 +79,17 @@ func TestDefaults(t *testing.T) { eCfg.MediatorDestFields = []string{"destination"} eCfg.MediatorTimeAnswerFields = []string{"time_answer"} eCfg.MediatorDurationFields = []string{"duration"} - eCfg.MediatorCDRInDir = "/var/log/freeswitch/cdr-csv" + eCfg.MediatorCDRInDir = "/var/log/freeswitch/cdr-csv" eCfg.MediatorCDROutDir = "/var/log/cgrates/cdr/out/freeswitch/csv" - eCfg.SMEnabled = false - eCfg.SMSwitchType = FS - eCfg.SMRater = "127.0.0.1:2012" - eCfg.SMRaterReconnects = 3 - eCfg.SMDebitInterval = 10 - eCfg.FreeswitchServer = "127.0.0.1:8021" - eCfg.FreeswitchPass = "ClueCon" - eCfg.FreeswitchReconnects = 5 - if !reflect.DeepEqual(cfg ,eCfg ){ + eCfg.SMEnabled = false + eCfg.SMSwitchType = FS + eCfg.SMRater = "127.0.0.1:2012" + eCfg.SMRaterReconnects = 3 + eCfg.SMDebitInterval = 10 + eCfg.FreeswitchServer = "127.0.0.1:8021" + eCfg.FreeswitchPass = "ClueCon" + eCfg.FreeswitchReconnects = 5 + if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) t.Error("Defaults different than expected!") @@ -99,22 +100,22 @@ func TestDefaults(t *testing.T) { func TestDefaultsSanity(t *testing.T) { cfg := &CGRConfig{} errSet := cfg.setDefaults() - if errSet != nil { + if errSet != nil { t.Log(fmt.Sprintf("Coud not set defaults: %s!", errSet.Error())) t.FailNow() } - if (cfg.RaterListen != INTERNAL && - (cfg.RaterListen == cfg.BalancerListen || + if (cfg.RaterListen != INTERNAL && + (cfg.RaterListen == cfg.BalancerListen || cfg.RaterListen == cfg.CDRSListen || - cfg.RaterListen == cfg.MediatorListen )) || + cfg.RaterListen == cfg.MediatorListen)) || (cfg.BalancerListen != INTERNAL && (cfg.BalancerListen == cfg.CDRSListen || - cfg.BalancerListen == cfg.MediatorListen ))|| + cfg.BalancerListen == cfg.MediatorListen)) || (cfg.CDRSListen != INTERNAL && cfg.CDRSListen == cfg.MediatorListen) { - t.Error("Listen defaults on the same port!") + t.Error("Listen defaults on the same port!") } } -// Load config from file and make sure we have all set +// Load config from file and make sure we have all set func TestConfigFromFile(t *testing.T) { cfgPth := "test_data.txt" cfg, err := NewCGRConfig(&cfgPth) @@ -151,6 +152,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.SchedulerEnabled = true eCfg.CDRSListen = "test" eCfg.CDRSfsJSONEnabled = true + eCfg.CDRSgenJSONEnabled = true eCfg.CDRSMediator = "test" eCfg.CDRSExtraFields = []string{"test"} eCfg.MediatorEnabled = true @@ -178,9 +180,9 @@ func TestConfigFromFile(t *testing.T) { eCfg.FreeswitchServer = "test" eCfg.FreeswitchPass = "test" eCfg.FreeswitchReconnects = 99 - if !reflect.DeepEqual(cfg ,eCfg ){ + if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) t.Error("Loading of configuration from file failed!") } -} +} diff --git a/config/test_data.txt b/config/test_data.txt index 9461ce9a3..b8ed219d9 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -38,6 +38,7 @@ rounding_decimals = 99 # Number of decimals to round prices at [cdrs] listen=test # CDRS's listening interface: . freeswitch_json_enabled=true # Enable the handler for FreeSWITCH JSON CDRs: . +generic_json_enabled=true # Enable the handler for generic JSON CDRs: . mediator = test # Address where to reach the Mediator. Empty for disabling mediation. <""|internal> extra_fields = test # Extra fields to store in CDRs From 9f28fa0d24d8fd387241d0b1118303066719e3c7 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 25 Jul 2013 20:29:47 +0200 Subject: [PATCH 04/17] Partial TPCSVImporter, TPCSVFileParser --- cmd/cgr-loader/cgr-loader.go | 75 +++++------------ data/rates/prepaid1centpsec/Timings.csv | 2 +- engine/loader_csv.go | 30 ++++--- engine/loader_helpers.go | 80 ++++++++++++++---- engine/tpimporter_csv.go | 107 ++++++++++++++++++++++-- utils/consts.go | 11 +++ 6 files changed, 220 insertions(+), 85 deletions(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index fff828964..ceeb5a991 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -26,7 +26,6 @@ import ( "github.com/cgrates/cgrates/config" "log" "path" - "regexp" ) var ( @@ -50,18 +49,12 @@ var ( tpid = flag.String("tpid", "", "The tariff plan id from the database") dataPath = flag.String("path", ".", "The path containing the data files") version = flag.Bool("version", false, "Prints the application version.") + verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") fromStorDb = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb") toStorDb = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb") - sep rune ) -type validator struct { - fn string - re *regexp.Regexp - message string -} - func main() { flag.Parse() if *version { @@ -72,64 +65,42 @@ func main() { var dataDb, storDb engine.DataStorage // Init necessary db connections if *fromStorDb { - dataDb, errDataDb = engine.ConfigureDatabase(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) - storDb, errStorDb = engine.ConfigureDatabase(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) + dataDb, errDataDb = engine.ConfigureDatabase(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) + storDb, errStorDb = engine.ConfigureDatabase(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) } else if *toStorDb { // Import from csv files to storDb - storDb, errStorDb = engine.ConfigureDatabase(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) + storDb, errStorDb = engine.ConfigureDatabase(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) } else { // Default load from csv files to dataDb - dataDb, errDataDb = engine.ConfigureDatabase(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) + dataDb, errDataDb = engine.ConfigureDatabase(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) } - defer dataDb.Close() - defer storDb.Close() + // Defer databases opened to be closed when we are done + for _,db := range []engine.DataStorage{ dataDb, storDb } { + if db != nil { defer db.Close() } + } + // Stop on db errors for _,err = range []error{errDataDb, errStorDb} { if err != nil { log.Fatalf("Could not open database connection: %v", err) } } - var loader engine.TPLoader - if *fromStorDb { + if *fromStorDb { // Load Tariff Plan from storDb into dataDb loader = engine.NewDbReader(storDb, dataDb, *tpid) - } else { // Default load from csv files to dataDb - dataFilesValidators := []*validator{ - &validator{utils.DESTINATIONS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+.?\d*){1}$`), - "Tag[0-9A-Za-z_],Prefix[0-9]"}, - &validator{utils.TIMINGS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*all\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), - "Tag[0-9A-Za-z_],Years[0-9;]|*all|,Months[0-9;]|*all|,MonthDays[0-9;]|*all|,WeekDays[0-9;]|*all|,Time[0-9:]|*asap(00:00:00)"}, - &validator{utils.RATES_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), - "Tag[0-9A-Za-z_],ConnectFee[0-9.],Price[0-9.],PricedUnits[0-9.],RateIncrement[0-9.]"}, - &validator{utils.DESTINATION_RATES_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), - "Tag[0-9A-Za-z_],DestinationsTag[0-9A-Za-z_],RateTag[0-9A-Za-z_]"}, - &validator{utils.DESTRATE_TIMINGS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`), - "Tag[0-9A-Za-z_],DestinationRatesTag[0-9A-Za-z_],TimingProfile[0-9A-Za-z_],Weight[0-9.]"}, - &validator{utils.RATE_PROFILES_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\*all\s*,\s*|[\w:\.]+\s*,\s*){1}(?:\w*\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}$`), - "Tenant[0-9A-Za-z_],TOR[0-9],Direction OUT|IN,Subject[0-9A-Za-z_:.]|*all,RatesFallbackSubject[0-9A-Za-z_]|,RatesTimingTag[0-9A-Za-z_],ActivationTime[[0-9T:X]] (2012-01-01T00:00:00Z)"}, - &validator{utils.ACTIONS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:ABSOLUTE\s*,\s*|PERCENT\s*,\s*|\s*,\s*){1}(?:\d*\.?\d*\s*,?\s*){3}$`), - "Tag[0-9A-Za-z_],Action[0-9A-Za-z_],BalanceTag[0-9A-Za-z_],Direction OUT|IN,Units[0-9],DestinationTag[0-9A-Za-z_]|*all,PriceType ABSOLUT|PERCENT,PriceValue[0-9.],MinutesWeight[0-9.],Weight[0-9.]"}, - &validator{utils.ACTION_TIMINGS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`), - "Tag[0-9A-Za-z_],ActionsTag[0-9A-Za-z_],TimingTag[0-9A-Za-z_],Weight[0-9.]"}, - &validator{utils.ACTION_TRIGGERS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:MONETARY\s*,\s*|SMS\s*,\s*|MINUTES\s*,\s*|INTERNET\s*,\s*|INTERNET_TIME\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\.?\d*\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d+\.?\d*){1}$`), - "Tag[0-9A-Za-z_],BalanceTag MONETARY|SMS|MINUTES|INTERNET|INTERNET_TIME,Direction OUT|IN,ThresholdValue[0-9.],DestinationTag[0-9A-Za-z_]|*all,ActionsTag[0-9A-Za-z_],Weight[0-9.]"}, - &validator{utils.ACCOUNT_ACTIONS_CSV, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:[\w:.]+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), - "Tenant[0-9A-Za-z_],Account[0-9A-Za-z_:.],Direction OUT|IN,ActionTimingsTag[0-9A-Za-z_],ActionTriggersTag[0-9A-Za-z_]"}, + } else if *toStorDb { // Import files from a directory into storDb + if *tpid == "" { + log.Fatal("TPid required, please define it via -tpid command argument.") } - for _, v := range dataFilesValidators { - err := engine.ValidateCSVData(path.Join(*dataPath, v.fn), v.re) + csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose } + if errImport := csvImporter.Run(); errImport != nil { + log.Fatal(errImport) + } + return + } else { // Default load from csv files to dataDb + for fn, v := range engine.FileValidators { + err := engine.ValidateCSVData(path.Join(*dataPath, fn), v.Rule) if err != nil { - log.Fatal(err, "\n\t", v.message) + log.Fatal(err, "\n\t", v.Message) } } - //sep = []rune(*separator)[0] loader = engine.NewFileCSVReader(dataDb, ',', utils.DESTINATIONS_CSV, utils.TIMINGS_CSV, utils.RATES_CSV, utils.DESTINATION_RATES_CSV, utils.DESTRATE_TIMINGS_CSV, utils.RATE_PROFILES_CSV, utils.ACTIONS_CSV, utils.ACTION_TIMINGS_CSV, utils.ACTION_TRIGGERS_CSV, utils.ACCOUNT_ACTIONS_CSV) } diff --git a/data/rates/prepaid1centpsec/Timings.csv b/data/rates/prepaid1centpsec/Timings.csv index 95475f4dc..272ee70ca 100644 --- a/data/rates/prepaid1centpsec/Timings.csv +++ b/data/rates/prepaid1centpsec/Timings.csv @@ -1,3 +1,3 @@ -Tag,Years,Months,MonthDays,WeekDays,Time +#Tag,Years,Months,MonthDays,WeekDays,Time ALWAYS,*all,*all,*all,*all,00:00:00 ONE_TIME_RUN,,,,,*asap diff --git a/engine/loader_csv.go b/engine/loader_csv.go index 0ef392239..63063e005 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -33,7 +33,7 @@ import ( type CSVReader struct { sep rune storage DataStorage - readerFunc func(string, rune) (*csv.Reader, *os.File, error) + readerFunc func(string, rune, int) (*csv.Reader, *os.File, error) actions map[string][]*Action actionsTimings map[string][]*ActionTiming actionsTriggers map[string][]*ActionTrigger @@ -74,20 +74,24 @@ func NewStringCSVReader(storage DataStorage, sep rune, destinationsFn, timingsFn return c } -func openFileCSVReader(fn string, comma rune) (csvReader *csv.Reader, fp *os.File, err error) { +func openFileCSVReader(fn string, comma rune, nrFields int) (csvReader *csv.Reader, fp *os.File, err error) { fp, err = os.Open(fn) if err != nil { return } csvReader = csv.NewReader(fp) csvReader.Comma = comma + csvReader.Comment = utils.COMMENT_CHAR + csvReader.FieldsPerRecord = nrFields csvReader.TrailingComma = true return } -func openStringCSVReader(data string, comma rune) (csvReader *csv.Reader, fp *os.File, err error) { +func openStringCSVReader(data string, comma rune, nrFields int) (csvReader *csv.Reader, fp *os.File, err error) { csvReader = csv.NewReader(strings.NewReader(data)) csvReader.Comma = comma + csvReader.Comment = utils.COMMENT_CHAR + csvReader.FieldsPerRecord = nrFields csvReader.TrailingComma = true return } @@ -164,7 +168,7 @@ func (csvr *CSVReader) WriteToDatabase(flush, verbose bool) (err error) { } func (csvr *CSVReader) LoadDestinations() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.destinationsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.destinationsFn, csvr.sep, utils.DESTINATION_RATES_NRCOLS) if err != nil { log.Print("Could not load destinations file: ", err) // allow writing of the other values @@ -197,7 +201,7 @@ func (csvr *CSVReader) LoadDestinations() (err error) { } func (csvr *CSVReader) LoadTimings() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.timingsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.timingsFn, csvr.sep, utils.TIMINGS_NRCOLS) if err != nil { log.Print("Could not load timings file: ", err) // allow writing of the other values @@ -219,7 +223,7 @@ func (csvr *CSVReader) LoadTimings() (err error) { } func (csvr *CSVReader) LoadRates() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.ratesFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.ratesFn, csvr.sep, utils.RATES_NRCOLS) if err != nil { log.Print("Could not load rates file: ", err) // allow writing of the other values @@ -245,7 +249,7 @@ func (csvr *CSVReader) LoadRates() (err error) { } func (csvr *CSVReader) LoadDestinationRates() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.destinationratesFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.destinationratesFn, csvr.sep, utils.DESTINATION_RATES_NRCOLS) if err != nil { log.Print("Could not load rates file: ", err) // allow writing of the other values @@ -276,7 +280,7 @@ func (csvr *CSVReader) LoadDestinationRates() (err error) { } func (csvr *CSVReader) LoadDestinationRateTimings() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.destinationratetimingsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.destinationratetimingsFn, csvr.sep, utils.DESTRATE_TIMINGS_NRCOLS) if err != nil { log.Print("Could not load rate timings file: ", err) // allow writing of the other values @@ -313,7 +317,7 @@ func (csvr *CSVReader) LoadDestinationRateTimings() (err error) { } func (csvr *CSVReader) LoadRatingProfiles() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.ratingprofilesFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.ratingprofilesFn, csvr.sep, utils.RATE_PROFILES_NRCOLS) if err != nil { log.Print("Could not load rating profiles file: ", err) // allow writing of the other values @@ -360,7 +364,7 @@ func (csvr *CSVReader) LoadRatingProfiles() (err error) { } func (csvr *CSVReader) LoadActions() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.actionsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.actionsFn, csvr.sep, utils.ACTIONS_NRCOLS) if err != nil { log.Print("Could not load action triggers file: ", err) // allow writing of the other values @@ -429,7 +433,7 @@ func (csvr *CSVReader) LoadActions() (err error) { } func (csvr *CSVReader) LoadActionTimings() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.actiontimingsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.actiontimingsFn, csvr.sep, utils.ACTION_TIMINGS_NRCOLS) if err != nil { log.Print("Could not load action triggers file: ", err) // allow writing of the other values @@ -474,7 +478,7 @@ func (csvr *CSVReader) LoadActionTimings() (err error) { } func (csvr *CSVReader) LoadActionTriggers() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.actiontriggersFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.actiontriggersFn, csvr.sep, utils.ACTION_TRIGGERS_NRCOLS) if err != nil { log.Print("Could not load action triggers file: ", err) // allow writing of the other values @@ -513,7 +517,7 @@ func (csvr *CSVReader) LoadActionTriggers() (err error) { } func (csvr *CSVReader) LoadAccountActions() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.accountactionsFn, csvr.sep) + csvReader, fp, err := csvr.readerFunc(csvr.accountactionsFn, csvr.sep, utils.ACCOUNT_ACTIONS_NRCOLS) if err != nil { log.Print("Could not load account actions file: ", err) // allow writing of the other values diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 5ee6bb3ee..72fe59160 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -20,12 +20,14 @@ package engine import ( "bufio" + "path" "errors" "fmt" "log" "os" "regexp" "strconv" + "strings" "github.com/cgrates/cgrates/utils" ) @@ -199,41 +201,91 @@ func ValidateCSVData(fn string, re *regexp.Regexp) (err error) { return } -type TPCSVRowValidator struct { - FileName string // File name +type FileLineRegexValidator struct { + FieldsPerRecord int // Number of fields in one record, useful for crosschecks Rule *regexp.Regexp // Regexp rule - ErrMessage string // Error message + Message string // Pass this message as helper } -var TPCSVRowValidators = []*TPCSVRowValidator{ - &TPCSVRowValidator{utils.DESTINATIONS_CSV, +var FileValidators = map[string]*FileLineRegexValidator{ + utils.DESTINATIONS_CSV: &FileLineRegexValidator{ utils.DESTINATIONS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+.?\d*){1}$`), "Tag[0-9A-Za-z_],Prefix[0-9]"}, - &TPCSVRowValidator{utils.TIMINGS_CSV, + utils.TIMINGS_CSV: &FileLineRegexValidator{ utils.TIMINGS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*all\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), "Tag[0-9A-Za-z_],Years[0-9;]|*all|,Months[0-9;]|*all|,MonthDays[0-9;]|*all|,WeekDays[0-9;]|*all|,Time[0-9:]|*asap(00:00:00)"}, - &TPCSVRowValidator{utils.RATES_CSV, + utils.RATES_CSV: &FileLineRegexValidator{ utils.RATES_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), "Tag[0-9A-Za-z_],ConnectFee[0-9.],Price[0-9.],PricedUnits[0-9.],RateIncrement[0-9.]"}, - &TPCSVRowValidator{utils.DESTINATION_RATES_CSV, + utils.DESTINATION_RATES_CSV: &FileLineRegexValidator{ utils.DESTINATION_RATES_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), "Tag[0-9A-Za-z_],DestinationsTag[0-9A-Za-z_],RateTag[0-9A-Za-z_]"}, - &TPCSVRowValidator{utils.DESTRATE_TIMINGS_CSV, + utils.DESTRATE_TIMINGS_CSV: &FileLineRegexValidator{ utils.DESTRATE_TIMINGS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`), "Tag[0-9A-Za-z_],DestinationRatesTag[0-9A-Za-z_],TimingProfile[0-9A-Za-z_],Weight[0-9.]"}, - &TPCSVRowValidator{utils.RATE_PROFILES_CSV, + utils.RATE_PROFILES_CSV: &FileLineRegexValidator{ utils.RATE_PROFILES_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\*all\s*,\s*|[\w:\.]+\s*,\s*){1}(?:\w*\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}$`), "Tenant[0-9A-Za-z_],TOR[0-9],Direction OUT|IN,Subject[0-9A-Za-z_:.]|*all,RatesFallbackSubject[0-9A-Za-z_]|,RatesTimingTag[0-9A-Za-z_],ActivationTime[[0-9T:X]] (2012-01-01T00:00:00Z)"}, - &TPCSVRowValidator{utils.ACTIONS_CSV, + utils.ACTIONS_CSV: &FileLineRegexValidator{ utils.ACTIONS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:ABSOLUTE\s*,\s*|PERCENT\s*,\s*|\s*,\s*){1}(?:\d*\.?\d*\s*,?\s*){3}$`), "Tag[0-9A-Za-z_],Action[0-9A-Za-z_],BalanceTag[0-9A-Za-z_],Direction OUT|IN,Units[0-9],DestinationTag[0-9A-Za-z_]|*all,PriceType ABSOLUT|PERCENT,PriceValue[0-9.],MinutesWeight[0-9.],Weight[0-9.]"}, - &TPCSVRowValidator{utils.ACTION_TIMINGS_CSV, + utils.ACTION_TIMINGS_CSV: &FileLineRegexValidator{ utils.ACTION_TIMINGS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`), "Tag[0-9A-Za-z_],ActionsTag[0-9A-Za-z_],TimingTag[0-9A-Za-z_],Weight[0-9.]"}, - &TPCSVRowValidator{utils.ACTION_TRIGGERS_CSV, + utils.ACTION_TRIGGERS_CSV: &FileLineRegexValidator{ utils.ACTION_TRIGGERS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:MONETARY\s*,\s*|SMS\s*,\s*|MINUTES\s*,\s*|INTERNET\s*,\s*|INTERNET_TIME\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\.?\d*\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d+\.?\d*){1}$`), "Tag[0-9A-Za-z_],BalanceTag MONETARY|SMS|MINUTES|INTERNET|INTERNET_TIME,Direction OUT|IN,ThresholdValue[0-9.],DestinationTag[0-9A-Za-z_]|*all,ActionsTag[0-9A-Za-z_],Weight[0-9.]"}, - &TPCSVRowValidator{utils.ACCOUNT_ACTIONS_CSV, + utils.ACCOUNT_ACTIONS_CSV: &FileLineRegexValidator{ utils.ACCOUNT_ACTIONS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:[\w:.]+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), "Tenant[0-9A-Za-z_],Account[0-9A-Za-z_:.],Direction OUT|IN,ActionTimingsTag[0-9A-Za-z_],ActionTriggersTag[0-9A-Za-z_]"}, } + + + +func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { + validator, hasValidator := FileValidators[fileName] + if !hasValidator { + return nil, fmt.Errorf("No validator found for file <%s>", fileName) + } + // Open the file here + fin, err := os.Open( path.Join(dirPath, fileName) ) + if err != nil { + return nil, err + } + //defer fin.Close() + reader := bufio.NewReader(fin) + return &TPCSVFileParser{validator, reader}, nil +} + +// Opens the connection to a file and returns the parsed lines one by one when ParseLine() is called +type TPCSVFileParser struct { + validator *FileLineRegexValidator // Row validator + reader *bufio.Reader // Reader to the file we are interested in +} + +func (self *TPCSVFileParser) ParseNextLine() ( []string, error ) { + line, truncated, err := self.reader.ReadLine() + if err != nil { + return nil, err + } else if truncated { + return nil, errors.New("Line too long.") + } + // skip commented lines + if strings.HasPrefix(string(line), string(utils.COMMENT_CHAR)) { + return nil, errors.New("Line starts with comment character.") + } + // Validate here string line + if !self.validator.Rule.Match(line) { + return nil, fmt.Errorf("Invalid line, <%s>", self.validator.Message) + } + // Open csv reader directly on string line + csvReader, _, err := openStringCSVReader( string(line), ',', self.validator.FieldsPerRecord ) + if err != nil { + return nil, err + } + record, err := csvReader.Read() // if no errors, record should be good to go having right format and length + if err != nil { + return nil, err + } + return record, nil +} diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index b57d49489..85872e594 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -18,13 +18,110 @@ along with this program. If not, see package engine -// Import tariff plan from csv into storDb +import ( + "io" + "io/ioutil" + "log" + "github.com/cgrates/cgrates/utils" +) -type TPImporterCSV struct { - sep rune - storDb DataStorage + +// Import tariff plan from csv into storDb +type TPCSVImporter struct { + TPid string // Load data on this tpid + StorDb DataStorage // StorDb connection handle + DirPath string // Directory path to import from + Sep rune // Separator in the csv file + Verbose bool // If true will print a detailed information instead of silently discarding it } -func (self *TPImporterCSV) ProcessFolder (fPath string) (err error) { +func (self *TPCSVImporter) Run() error { + + // Maps csv file to handler which should process it + fileHandlers := map[string]func(string)error{ + utils.TIMINGS_CSV: self.importTimings, + utils.DESTINATIONS_CSV: self.importDestinations, + utils.RATES_CSV: self.importRates, + utils.DESTINATION_RATES_CSV: self.importDestinationRates, + utils.DESTRATE_TIMINGS_CSV: self.importDestRateTimings, + utils.RATE_PROFILES_CSV: self.importRatingProfiles, + utils.ACTIONS_CSV: self.importActions, + utils.ACTION_TIMINGS_CSV: self.importActionTimings, + utils.ACTION_TRIGGERS_CSV: self.importActionTriggers, + utils.ACCOUNT_ACTIONS_CSV: self.importAccountActions, + } + + files, _ := ioutil.ReadDir(self.DirPath) + for _, f := range files { + fHandler,hasName := fileHandlers[f.Name()] + if !hasName { + continue + } + fHandler( f.Name() ) + } return nil } + +// Handler importing timings from file, saved row by row to storDb +func (self *TPCSVImporter) importTimings(fn string) error { + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + tm := NewTiming( record... ) + if err := self.StorDb.SetTPTiming(self.TPid, tm); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } + return nil +} + +func (self *TPCSVImporter) importDestinations(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importRates(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importDestinationRates(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importDestRateTimings(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importRatingProfiles(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importActions(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importActionTimings(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importActionTriggers(fPath string) error { + return nil +} + +func (self *TPCSVImporter) importAccountActions(fPath string) error { + return nil +} + + diff --git a/utils/consts.go b/utils/consts.go index 00dc4a53b..103805c34 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -39,7 +39,18 @@ const ( ACTION_TIMINGS_CSV = "ActionTimings.csv" ACTION_TRIGGERS_CSV = "ActionTriggers.csv" ACCOUNT_ACTIONS_CSV = "AccountActions.csv" + TIMINGS_NRCOLS = 6 + DESTINATIONS_NRCOLS = 2 + RATES_NRCOLS = 9 + DESTINATION_RATES_NRCOLS = 3 + DESTRATE_TIMINGS_NRCOLS = 4 + RATE_PROFILES_NRCOLS = 7 + ACTIONS_NRCOLS = 11 + ACTION_TIMINGS_NRCOLS = 4 + ACTION_TRIGGERS_NRCOLS = 8 + ACCOUNT_ACTIONS_NRCOLS = 5 ROUNDING_UP = "up" ROUNDING_MIDDLE = "middle" ROUNDING_DOWN = "down" + COMMENT_CHAR = '#' ) From e5f0af9fb698e96424cdcc70080be23390e50552 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 27 Jul 2013 10:25:18 +0200 Subject: [PATCH 05/17] Refactoring validators to match new csv file structure --- cmd/cgr-loader/cgr-loader.go | 2 +- .../fs_germany_prep1/AccountActions.csv | 6 +++ .../fs_germany_prep1/ActionTimings.csv | 2 + .../fs_germany_prep1/ActionTriggers.csv | 4 ++ data/tariffplans/fs_germany_prep1/Actions.csv | 2 + .../DestinationRateTimngs.csv | 6 +++ .../fs_germany_prep1/DestinationRates.csv | 6 +++ .../fs_germany_prep1/Destinations.csv | 6 +++ data/tariffplans/fs_germany_prep1/README.md | 15 ++++++++ data/tariffplans/fs_germany_prep1/Rates.csv | 7 ++++ .../fs_germany_prep1/RatingProfiles.csv | 2 + data/tariffplans/fs_germany_prep1/Timings.csv | 7 ++++ engine/loader_csv_test.go | 22 +++++------ engine/loader_helpers.go | 38 +++++++++---------- 14 files changed, 94 insertions(+), 31 deletions(-) create mode 100644 data/tariffplans/fs_germany_prep1/AccountActions.csv create mode 100644 data/tariffplans/fs_germany_prep1/ActionTimings.csv create mode 100644 data/tariffplans/fs_germany_prep1/ActionTriggers.csv create mode 100644 data/tariffplans/fs_germany_prep1/Actions.csv create mode 100644 data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv create mode 100644 data/tariffplans/fs_germany_prep1/DestinationRates.csv create mode 100644 data/tariffplans/fs_germany_prep1/Destinations.csv create mode 100644 data/tariffplans/fs_germany_prep1/README.md create mode 100644 data/tariffplans/fs_germany_prep1/Rates.csv create mode 100644 data/tariffplans/fs_germany_prep1/RatingProfiles.csv create mode 100644 data/tariffplans/fs_germany_prep1/Timings.csv diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index ceeb5a991..8ff242cd2 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -87,7 +87,7 @@ func main() { loader = engine.NewDbReader(storDb, dataDb, *tpid) } else if *toStorDb { // Import files from a directory into storDb if *tpid == "" { - log.Fatal("TPid required, please define it via -tpid command argument.") + log.Fatal("TPid required, please define it via *-tpid* command argument.") } csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose } if errImport := csvImporter.Run(); errImport != nil { diff --git a/data/tariffplans/fs_germany_prep1/AccountActions.csv b/data/tariffplans/fs_germany_prep1/AccountActions.csv new file mode 100644 index 000000000..cf7bc4026 --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/AccountActions.csv @@ -0,0 +1,6 @@ +#Tenant,Account,Direction,ActionTimingsTag,ActionTriggersTag +cgrates.org,1001,*out,PREPAID_10,STANDARD_TRIGGERS +cgrates.org,1002,*out,PREPAID_10,STANDARD_TRIGGERS +cgrates.org,1003,*out,PREPAID_10,STANDARD_TRIGGERS +cgrates.org,1004,*out,PREPAID_10,STANDARD_TRIGGERS +cgrates.org,1005,*out,PREPAID_10,STANDARD_TRIGGERS diff --git a/data/tariffplans/fs_germany_prep1/ActionTimings.csv b/data/tariffplans/fs_germany_prep1/ActionTimings.csv new file mode 100644 index 000000000..5eade6423 --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/ActionTimings.csv @@ -0,0 +1,2 @@ +#Tag,ActionsTag,TimingTag,Weight +TOPUP_10,PREPAID_10,ASAP,10 diff --git a/data/tariffplans/fs_germany_prep1/ActionTriggers.csv b/data/tariffplans/fs_germany_prep1/ActionTriggers.csv new file mode 100644 index 000000000..b2cdce3df --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/ActionTriggers.csv @@ -0,0 +1,4 @@ +#Tag,BalanceTag,Direction,ThresholdType,ThresholdValue,DestinationTag,ActionsTag,Weight +STANDARD_TRIGGERS,*monetary,*out,*min_balance,2,,LOG_BALANCE,10 +STANDARD_TRIGGERS,*monetary,*out,*max_balance,20,,LOG_BALANCE,10 +STANDARD_TRIGGERS,*monetary,*out,*max_counter,15,FS_USERS,LOG_BALANCE,10 diff --git a/data/tariffplans/fs_germany_prep1/Actions.csv b/data/tariffplans/fs_germany_prep1/Actions.csv new file mode 100644 index 000000000..a68faa34e --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/Actions.csv @@ -0,0 +1,2 @@ +#ActionsTag,Action,BalanceType,Direction,Units,ExpirationDate,DestinationTag,RateType,RateValue,MinutesWeight,Weight +PREPAID_10,*topup_reset,*monetary,*out,10,*unlimited,*any,,,,10 diff --git a/data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv b/data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv new file mode 100644 index 000000000..c47aeafd6 --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv @@ -0,0 +1,6 @@ +#Tag,DestinationRatesTag,TimingTag,Weight +RETAIL1,DR_RETAIL_PEAK,PEAK,10 +RETAIL1,DR_RETAIL_OFFPEAK,OFFPEAK_MORNING,10 +RETAIL1,DR_RETAIL_OFFPEAK,OFFPEAK_EVENING,10 +RETAIL1,DR_RETAIL_OFFPEAK,WEEKEND,10 +RETAIL1,DR_FREESWITCH_USERS,ALWAYS,10 diff --git a/data/tariffplans/fs_germany_prep1/DestinationRates.csv b/data/tariffplans/fs_germany_prep1/DestinationRates.csv new file mode 100644 index 000000000..1812d2efd --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/DestinationRates.csv @@ -0,0 +1,6 @@ +#Tag,DestinationsTag,RatesTag +DR_RETAIL_PEAK,GERMANY,LANDLINE_PEAK +DR_RETAIL_PEAK,GERMANY_MOBILE,MOBILE_PEAK +DR_RETAIL_OFFPEAK,GERMANY,LANDLINE_OFFPEAK +DR_RETAIL_OFFPEAK,GERMANY_MOBILE,MOBILE_OFFPEAK +DR_FREESWITCH_USERS,FS_USERS,RT_FS_USERS diff --git a/data/tariffplans/fs_germany_prep1/Destinations.csv b/data/tariffplans/fs_germany_prep1/Destinations.csv new file mode 100644 index 000000000..8ee6689cb --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/Destinations.csv @@ -0,0 +1,6 @@ +#Tag,Prefix +GERMANY,+49 +GERMANY_MOBILE,+4915 +GERMANY_MOBILE,+4916 +GERMANY_MOBILE,+4917 +FS_USERS,10 diff --git a/data/tariffplans/fs_germany_prep1/README.md b/data/tariffplans/fs_germany_prep1/README.md new file mode 100644 index 000000000..32ff027ac --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/README.md @@ -0,0 +1,15 @@ +CGRateS - FSGermanyPrep1 +========================== + +Scenario: +--------- + +* Create the necessary timings (always, peak, offpeak, asap). +* Configure 3 different destinations: GERMANY, GERMANY_MOBILE and FS_USERS. +* Calls to landline and mobile numbers in Germany will be charged time based (structured in peak and offpeak profiles). Calls to landline during peak times are charged using different rate slots: first minute charged as a whole at one rate, next minutes charged per second at another rate. +* Calls to FreeSWITCH users will be free and time independent. +* This rating profile will be valid for any rating subject. + +* Create 5 prepaid accounts (equivalent of 5 FreeSWITCH default test users - 1001, 1002, 1003, 1004, 1005). +* Add to each of the accounts a monetary balance of 10 units. +* For each balance created, attach 3 triggers to control the balance: log on balance=2, log on balance=20, log on 15 mins talked towards FS_USERS destination. diff --git a/data/tariffplans/fs_germany_prep1/Rates.csv b/data/tariffplans/fs_germany_prep1/Rates.csv new file mode 100644 index 000000000..1023fdbd5 --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/Rates.csv @@ -0,0 +1,7 @@ +#Tag,ConnectFee,Rate,RatedUnits,RateIncrements,GroupInterval,RoundingMethod,RoundingDecimals,Weight +LANDLINE_PEAK,0.02,0.02,60,60,0,*up,4,10 +LANDLINE_PEAK,0.02,0.01,1,1,60,*up,4,10 +MOBILE_PEAK,0.02,0.14,60,60,0,*up,4,10 +LANDLINE_OFFPEAK,1,0,60,60,0,*up,4,10 +MOBILE_OFFPEAK,0.02,0.1,60,60,0,*up,4,10 +RT_FS_USERS,0,0,60,60,0,*up,0,10 diff --git a/data/tariffplans/fs_germany_prep1/RatingProfiles.csv b/data/tariffplans/fs_germany_prep1/RatingProfiles.csv new file mode 100644 index 000000000..7cea6c691 --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/RatingProfiles.csv @@ -0,0 +1,2 @@ +#Tenant,TOR,Direction,Subject,ActivationTime,DestinationRateTimingTag,RatesFallbackSubject +cgrates.org,call,*out,*any,2012-01-01T00:00:00Z,RETAIL1, diff --git a/data/tariffplans/fs_germany_prep1/Timings.csv b/data/tariffplans/fs_germany_prep1/Timings.csv new file mode 100644 index 000000000..404c6fcbd --- /dev/null +++ b/data/tariffplans/fs_germany_prep1/Timings.csv @@ -0,0 +1,7 @@ +#Tag,Years,Months,MonthDays,WeekDays,Time +ALWAYS,*any,*any,*any,*any,00:00:00 +ASAP,*any,*any,*any,*any,*asap +OFFPEAK_MORNING,*any,*any,*any,1;2;3;4;5,00:00:00 +PEAK,*any,*any,*any,1;2;3;4;5,08:00:00 +OFFPEAK_EVENING,*any,*any,*any,1;2;3;4;5,20:00:00 +WEEKENDS,*any,*any,*any,6;7,00:00:00 diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index b9a4ea399..5c47796a0 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -72,17 +72,17 @@ EVENING,P2,WORKDAYS_18,10 EVENING,P2,WEEKENDS,10 ` ratingProfiles = ` -CUSTOMER_1,0,*out,rif:from:tm,danb,PREMIUM,2012-01-01T00:00:00Z -CUSTOMER_1,0,*out,rif:from:tm,danb,STANDARD,2012-02-28T00:00:00Z -CUSTOMER_2,0,*out,danb:87.139.12.167,danb,STANDARD,2012-01-01T00:00:00Z -CUSTOMER_1,0,*out,danb,,PREMIUM,2012-01-01T00:00:00Z -vdf,0,*out,rif,,EVENING,2012-01-01T00:00:00Z -vdf,0,*out,rif,,EVENING,2012-02-28T00:00:00Z -vdf,0,*out,minu,,EVENING,2012-01-01T00:00:00Z -vdf,0,*out,*any,,EVENING,2012-02-28T00:00:00Z -vdf,0,*out,one,,STANDARD,2012-02-28T00:00:00Z -vdf,0,*out,inf,inf,STANDARD,2012-02-28T00:00:00Z -vdf,0,*out,fall,one|rif,PREMIUM,2012-02-28T00:00:00Z +CUSTOMER_1,0,*out,rif:from:tm,2012-01-01T00:00:00Z,PREMIUM,danb +CUSTOMER_1,0,*out,rif:from:tm,2012-02-28T00:00:00Z,STANDARD,danb +CUSTOMER_2,0,*out,danb:87.139.12.167,2012-01-01T00:00:00Z,STANDARD,danb +CUSTOMER_1,0,*out,danb,2012-01-01T00:00:00Z,PREMIUM, +vdf,0,*out,rif,2012-01-01T00:00:00Z,EVENING, +vdf,0,*out,rif,2012-02-28T00:00:00Z,EVENING, +vdf,0,*out,minu,2012-01-01T00:00:00Z,EVENING, +vdf,0,*out,*any,2012-02-28T00:00:00Z,EVENING, +vdf,0,*out,one,2012-02-28T00:00:00Z,STANDARD, +vdf,0,*out,inf,2012-02-28T00:00:00Z,STANDARD,inf +vdf,0,*out,fall,one,2012-02-28T00:00:00Z,PREMIUM|rif ` actions = ` MINI,TOPUP,MINUTES,*out,100,1374239002,NAT,*absolute,0,10,10 diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 72fe59160..919ebb120 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -209,35 +209,35 @@ type FileLineRegexValidator struct { var FileValidators = map[string]*FileLineRegexValidator{ utils.DESTINATIONS_CSV: &FileLineRegexValidator{ utils.DESTINATIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+.?\d*){1}$`), - "Tag[0-9A-Za-z_],Prefix[0-9]"}, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\+?\d+.?\d*){1}$`), + "Tag([0-9A-Za-z_]),Prefix([0-9])"}, utils.TIMINGS_CSV: &FileLineRegexValidator{ utils.TIMINGS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*all\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), - "Tag[0-9A-Za-z_],Years[0-9;]|*all|,Months[0-9;]|*all|,MonthDays[0-9;]|*all|,WeekDays[0-9;]|*all|,Time[0-9:]|*asap(00:00:00)"}, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*any\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), + "Tag([0-9A-Za-z_]),Years([0-9;]|*all|),Months([0-9;]|*all|),MonthDays([0-9;]|*all|),WeekDays([0-9;]|*all|),Time([0-9:]|*asap)"}, utils.RATES_CSV: &FileLineRegexValidator{ utils.RATES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), - "Tag[0-9A-Za-z_],ConnectFee[0-9.],Price[0-9.],PricedUnits[0-9.],RateIncrement[0-9.]"}, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\.?\d*,){5}(?:\*\w+,){1}(?:\d+\.?\d*,?){2}$`), + "Tag([0-9A-Za-z_]),ConnectFee([0-9.]),Rate([0-9.]),RatedUnits([0-9.]),RateIncrement([0-9.])"}, utils.DESTINATION_RATES_CSV: &FileLineRegexValidator{ utils.DESTINATION_RATES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`), - "Tag[0-9A-Za-z_],DestinationsTag[0-9A-Za-z_],RateTag[0-9A-Za-z_]"}, + regexp.MustCompile(`(?:\w+\s*,?\s*){3}$`), + "Tag([0-9A-Za-z_]),DestinationsTag([0-9A-Za-z_]),RateTag([0-9A-Za-z_])"}, utils.DESTRATE_TIMINGS_CSV: &FileLineRegexValidator{ utils.DESTRATE_TIMINGS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`), - "Tag[0-9A-Za-z_],DestinationRatesTag[0-9A-Za-z_],TimingProfile[0-9A-Za-z_],Weight[0-9.]"}, + "Tag([0-9A-Za-z_]),DestinationRatesTag([0-9A-Za-z_]),TimingProfile([0-9A-Za-z_]),Weight([0-9.])"}, utils.RATE_PROFILES_CSV: &FileLineRegexValidator{ utils.RATE_PROFILES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\*all\s*,\s*|[\w:\.]+\s*,\s*){1}(?:\w*\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}$`), - "Tenant[0-9A-Za-z_],TOR[0-9],Direction OUT|IN,Subject[0-9A-Za-z_:.]|*all,RatesFallbackSubject[0-9A-Za-z_]|,RatesTimingTag[0-9A-Za-z_],ActivationTime[[0-9T:X]] (2012-01-01T00:00:00Z)"}, + regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\*out\s*,\s*){1}(?:\*any\s*,\s*|\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}(?:\w*\s*,?\s*){2}$`), + "Tenant([0-9A-Za-z_]),TOR([0-9A-Za-z_]),Direction(*out),Subject([0-9A-Za-z_]|*all),RatesFallbackSubject([0-9A-Za-z_]|),RatesTimingTag([0-9A-Za-z_]),ActivationTime([0-9T:X])"}, utils.ACTIONS_CSV: &FileLineRegexValidator{ utils.ACTIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:ABSOLUTE\s*,\s*|PERCENT\s*,\s*|\s*,\s*){1}(?:\d*\.?\d*\s*,?\s*){3}$`), - "Tag[0-9A-Za-z_],Action[0-9A-Za-z_],BalanceTag[0-9A-Za-z_],Direction OUT|IN,Units[0-9],DestinationTag[0-9A-Za-z_]|*all,PriceType ABSOLUT|PERCENT,PriceValue[0-9.],MinutesWeight[0-9.],Weight[0-9.]"}, + regexp.MustCompile(`(?:\w+\s*),(?:\*\w+\s*),(?:\*\w+\s*),(?:\*out\s*),(?:\d+\s*),(?:\*\w+\s*|\+\d+[smh]\s*|\d+\s*),(?:\*any|\w+\s*),(?:\*\w+\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)$`), + "Tag([0-9A-Za-z_]),Action([0-9A-Za-z_]),BalanceType([*a-z_]),Direction(*out),Units([0-9]),ExpiryTime(*[a-z_]|+[0-9][smh]|[0-9])DestinationTag([0-9A-Za-z_]|*all),RateType(*[a-z_]),RateValue([0-9.]),MinutesWeight([0-9.]),Weight([0-9.])"}, utils.ACTION_TIMINGS_CSV: &FileLineRegexValidator{ utils.ACTION_TIMINGS_NRCOLS, regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`), - "Tag[0-9A-Za-z_],ActionsTag[0-9A-Za-z_],TimingTag[0-9A-Za-z_],Weight[0-9.]"}, + "Tag([0-9A-Za-z_]),ActionsTag([0-9A-Za-z_]),TimingTag([0-9A-Za-z_]),Weight([0-9.])"}, utils.ACTION_TRIGGERS_CSV: &FileLineRegexValidator{ utils.ACTION_TRIGGERS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:MONETARY\s*,\s*|SMS\s*,\s*|MINUTES\s*,\s*|INTERNET\s*,\s*|INTERNET_TIME\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\.?\d*\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d+\.?\d*){1}$`), - "Tag[0-9A-Za-z_],BalanceTag MONETARY|SMS|MINUTES|INTERNET|INTERNET_TIME,Direction OUT|IN,ThresholdValue[0-9.],DestinationTag[0-9A-Za-z_]|*all,ActionsTag[0-9A-Za-z_],Weight[0-9.]"}, + regexp.MustCompile(`(?:\w+),(?:\*\w+),(?:\*out),(?:\*\w+),(?:\d+\.?\d*),(?:\w+|\*any)?,(?:\w+),(?:\d+\.?\d*)$`), + "Tag([0-9A-Za-z_]),BalanceType(*[a-z_]),Direction(*out),ThresholdType(*[a-z_]),ThresholdValue([0-9]+),DestinationTag([0-9A-Za-z_]|*all),ActionsTag([0-9A-Za-z_]),Weight([0-9]+)"}, utils.ACCOUNT_ACTIONS_CSV: &FileLineRegexValidator{ utils.ACCOUNT_ACTIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:[\w:.]+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), - "Tenant[0-9A-Za-z_],Account[0-9A-Za-z_:.],Direction OUT|IN,ActionTimingsTag[0-9A-Za-z_],ActionTriggersTag[0-9A-Za-z_]"}, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\*out\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), + "Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"}, } @@ -257,7 +257,7 @@ func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { return &TPCSVFileParser{validator, reader}, nil } -// Opens the connection to a file and returns the parsed lines one by one when ParseLine() is called +// Opens the connection to a file and returns the parsed lines one by one when ParseNextLine() is called type TPCSVFileParser struct { validator *FileLineRegexValidator // Row validator reader *bufio.Reader // Reader to the file we are interested in From 26d270c7c13cf65eb001a1461a1e23b3025f621d Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 27 Jul 2013 11:39:31 +0200 Subject: [PATCH 06/17] Updating cgr-loader to dataDb --- cmd/cgr-loader/cgr-loader.go | 2 +- .../fs_germany_prep1/ActionTimings.csv | 2 +- ...eTimngs.csv => DestinationRateTimings.csv} | 0 data/tariffplans/fs_germany_prep1/Timings.csv | 2 +- engine/loader_csv.go | 77 +++++-------------- 5 files changed, 21 insertions(+), 62 deletions(-) rename data/tariffplans/fs_germany_prep1/{DestinationRateTimngs.csv => DestinationRateTimings.csv} (100%) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 8ff242cd2..d4b9c3de9 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -146,7 +146,7 @@ func main() { } // write maps to database - if err := loader.WriteToDatabase(*flush, true); err != nil { + if err := loader.WriteToDatabase(*flush, *verbose); err != nil { log.Fatal("Could not write to database: ", err) } } diff --git a/data/tariffplans/fs_germany_prep1/ActionTimings.csv b/data/tariffplans/fs_germany_prep1/ActionTimings.csv index 5eade6423..24f63c8a8 100644 --- a/data/tariffplans/fs_germany_prep1/ActionTimings.csv +++ b/data/tariffplans/fs_germany_prep1/ActionTimings.csv @@ -1,2 +1,2 @@ #Tag,ActionsTag,TimingTag,Weight -TOPUP_10,PREPAID_10,ASAP,10 +PREPAID_10,PREPAID_10,ASAP,10 diff --git a/data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv b/data/tariffplans/fs_germany_prep1/DestinationRateTimings.csv similarity index 100% rename from data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv rename to data/tariffplans/fs_germany_prep1/DestinationRateTimings.csv diff --git a/data/tariffplans/fs_germany_prep1/Timings.csv b/data/tariffplans/fs_germany_prep1/Timings.csv index 404c6fcbd..9b24e6975 100644 --- a/data/tariffplans/fs_germany_prep1/Timings.csv +++ b/data/tariffplans/fs_germany_prep1/Timings.csv @@ -4,4 +4,4 @@ ASAP,*any,*any,*any,*any,*asap OFFPEAK_MORNING,*any,*any,*any,1;2;3;4;5,00:00:00 PEAK,*any,*any,*any,1;2;3;4;5,08:00:00 OFFPEAK_EVENING,*any,*any,*any,1;2;3;4;5,20:00:00 -WEEKENDS,*any,*any,*any,6;7,00:00:00 +WEEKEND,*any,*any,*any,6;7,00:00:00 diff --git a/engine/loader_csv.go b/engine/loader_csv.go index 63063e005..5d9fead11 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -168,7 +168,7 @@ func (csvr *CSVReader) WriteToDatabase(flush, verbose bool) (err error) { } func (csvr *CSVReader) LoadDestinations() (err error) { - csvReader, fp, err := csvr.readerFunc(csvr.destinationsFn, csvr.sep, utils.DESTINATION_RATES_NRCOLS) + csvReader, fp, err := csvr.readerFunc(csvr.destinationsFn, csvr.sep, utils.DESTINATIONS_NRCOLS) if err != nil { log.Print("Could not load destinations file: ", err) // allow writing of the other values @@ -177,13 +177,10 @@ func (csvr *CSVReader) LoadDestinations() (err error) { if fp != nil { defer fp.Close() } + fmt.Println("Should start loading destinations") for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { - + fmt.Println("Reading destination line", record) tag := record[0] - if tag == "Tag" { - // skip header line - continue - } var dest *Destination for _, d := range csvr.destinations { if d.Id == tag { @@ -212,11 +209,6 @@ func (csvr *CSVReader) LoadTimings() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } - csvr.timings[tag] = NewTiming(record...) } return @@ -234,10 +226,6 @@ func (csvr *CSVReader) LoadRates() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } var r *Rate r, err = NewRate(record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8]) if err != nil { @@ -260,14 +248,11 @@ func (csvr *CSVReader) LoadDestinationRates() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } r, exists := csvr.rates[record[2]] if !exists { - return errors.New(fmt.Sprintf("Could not get rating for tag %v", record[2])) + return errors.New(fmt.Sprintf("Could not get rates for tag %v", record[2])) } + //ToDo: Not checking presence of destinations? dr := &DestinationRate{ Tag: tag, DestinationsTag: record[1], @@ -291,11 +276,6 @@ func (csvr *CSVReader) LoadDestinationRateTimings() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } - t, exists := csvr.timings[record[2]] if !exists { return errors.New(fmt.Sprintf("Could not get timing for tag %v", record[2])) @@ -327,18 +307,10 @@ func (csvr *CSVReader) LoadRatingProfiles() (err error) { defer fp.Close() } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { - tag := record[0] - if tag == "Tenant" { - // skip header line - continue - } - if len(record) != 7 { - return errors.New(fmt.Sprintf("Malformed rating profile: %v", record)) - } - tenant, tor, direction, subject, fallbacksubject := record[0], record[1], record[2], record[3], record[4] - at, err := time.Parse(time.RFC3339, record[6]) + tenant, tor, direction, subject, fallbacksubject := record[0], record[1], record[2], record[3], record[6] + at, err := time.Parse(time.RFC3339, record[4]) if err != nil { - return errors.New(fmt.Sprintf("Cannot parse activation time from %v", record[6])) + return errors.New(fmt.Sprintf("Cannot parse activation time from %v", record[4])) } key := fmt.Sprintf("%s:%s:%s:%s", direction, tenant, tor, subject) rp, ok := csvr.ratingProfiles[key] @@ -375,19 +347,17 @@ func (csvr *CSVReader) LoadActions() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } units, err := strconv.ParseFloat(record[4], 64) if err != nil { return errors.New(fmt.Sprintf("Could not parse action units: %v", err)) } - unix, err := strconv.ParseInt(record[5], 10, 64) - if err != nil { - return errors.New(fmt.Sprintf("Could not parse expiration date: %v", err)) + var expiryTime time.Time // Empty initialized time represents never expire + if record[5] != "*unlimited" { // ToDo: Expand here for other meta tags or go way of adding time for expiry + expiryTime, err = time.Parse(time.RFC3339, record[5]) + if err != nil { + return errors.New(fmt.Sprintf("Could not parse expiry time: %v", err)) + } } - expDate := time.Unix(unix, 0) var a *Action if record[2] != MINUTES { a = &Action{ @@ -395,7 +365,7 @@ func (csvr *CSVReader) LoadActions() (err error) { BalanceId: record[2], Direction: record[3], Units: units, - ExpirationDate: expDate, + ExpirationDate: expiryTime, //ToDo: Fix ExpirationDate as string to have ability of storing + reported on run time } } else { value, err := strconv.ParseFloat(record[8], 64) @@ -416,14 +386,14 @@ func (csvr *CSVReader) LoadActions() (err error) { BalanceId: record[2], Direction: record[3], Weight: weight, - ExpirationDate: expDate, + ExpirationDate: expiryTime, MinuteBucket: &MinuteBucket{ Seconds: units, Weight: minutesWeight, Price: value, PriceType: record[7], DestinationId: record[6], - ExpirationDate: expDate, + ExpirationDate: expiryTime, }, } } @@ -444,10 +414,6 @@ func (csvr *CSVReader) LoadActionTimings() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } _, exists := csvr.actions[record[1]] if !exists { return errors.New(fmt.Sprintf("ActionTiming: Could not load the action for tag: %v", record[1])) @@ -489,10 +455,6 @@ func (csvr *CSVReader) LoadActionTriggers() (err error) { } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := record[0] - if tag == "Tag" { - // skip header line - continue - } value, err := strconv.ParseFloat(record[4], 64) if err != nil { return errors.New(fmt.Sprintf("Could not parse action trigger value: %v", err)) @@ -527,10 +489,7 @@ func (csvr *CSVReader) LoadAccountActions() (err error) { defer fp.Close() } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { - if record[0] == "Tenant" { - continue - } - tag := fmt.Sprintf("%s:%s:%s", record[2], record[0], record[1]) + tag := fmt.Sprintf("%s:%s:%s", record[2][1:], record[0], record[1]) // Ignore * in front of direction aTriggers, exists := csvr.actionsTriggers[record[4]] if record[4] != "" && !exists { // only return error if there was something ther for the tag From 0be3f8b3424a220b30bc9eba1699a96659ae7176 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 27 Jul 2013 12:09:07 +0200 Subject: [PATCH 07/17] Small fixups cgr-loader --- cmd/cgr-loader/cgr-loader.go | 4 ++-- engine/loader_csv.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index d4b9c3de9..d60d6f60d 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -50,8 +50,8 @@ var ( dataPath = flag.String("path", ".", "The path containing the data files") version = flag.Bool("version", false, "Prints the application version.") verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") - fromStorDb = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb") - toStorDb = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb") + fromStorDb = flag.Bool("from-stordb", false, "Load the tariff plan from storDb to dataDb") + toStorDb = flag.Bool("to-stordb", false, "Import the tariff plan from files to storDb") ) diff --git a/engine/loader_csv.go b/engine/loader_csv.go index 5d9fead11..96c4c7da3 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -177,9 +177,7 @@ func (csvr *CSVReader) LoadDestinations() (err error) { if fp != nil { defer fp.Close() } - fmt.Println("Should start loading destinations") for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { - fmt.Println("Reading destination line", record) tag := record[0] var dest *Destination for _, d := range csvr.destinations { @@ -489,7 +487,7 @@ func (csvr *CSVReader) LoadAccountActions() (err error) { defer fp.Close() } for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { - tag := fmt.Sprintf("%s:%s:%s", record[2][1:], record[0], record[1]) // Ignore * in front of direction + tag := fmt.Sprintf("%s:%s:%s", record[2], record[0], record[1]) aTriggers, exists := csvr.actionsTriggers[record[4]] if record[4] != "" && !exists { // only return error if there was something ther for the tag From ee15ce50560a0a31bd8c5dbacb22e43704bd20f4 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 27 Jul 2013 12:26:42 +0200 Subject: [PATCH 08/17] Tests fix --- engine/loader_csv_test.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 5c47796a0..97c27463b 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -25,7 +25,7 @@ import ( var ( destinations = ` -Tag,Prefix +#Tag,Prefix GERMANY,49 GERMANY_O2,41 GERMANY_PREMIUM,43 @@ -82,17 +82,17 @@ vdf,0,*out,minu,2012-01-01T00:00:00Z,EVENING, vdf,0,*out,*any,2012-02-28T00:00:00Z,EVENING, vdf,0,*out,one,2012-02-28T00:00:00Z,STANDARD, vdf,0,*out,inf,2012-02-28T00:00:00Z,STANDARD,inf -vdf,0,*out,fall,one,2012-02-28T00:00:00Z,PREMIUM|rif +vdf,0,*out,fall,2012-02-28T00:00:00Z,PREMIUM,rif ` actions = ` -MINI,TOPUP,MINUTES,*out,100,1374239002,NAT,*absolute,0,10,10 +MINI,TOPUP,MINUTES,*out,100,2013-07-19T13:03:22Z,NAT,*absolute,0,10,10 ` actionTimings = ` MORE_MINUTES,MINI,ONE_TIME_RUN,10 ` actionTriggers = ` -STANDARD_TRIGGER,MINUTES,*out,COUNTER,10,GERMANY_O2,SOME_1,10 -STANDARD_TRIGGER,MINUTES,*out,BALANCE,200,GERMANY,SOME_2,10 +STANDARD_TRIGGER,MINUTES,*out,*min_counter,10,GERMANY_O2,SOME_1,10 +STANDARD_TRIGGER,MINUTES,*out,*max_balance,200,GERMANY,SOME_2,10 ` accountActions = ` vdf,minitsboy,*out,MORE_MINUTES,STANDARD_TRIGGER @@ -144,9 +144,6 @@ func TestLoadDestinationRateTimings(t *testing.T) { if len(csvr.activationPeriods) != 4 { t.Error("Failed to load rate timings: ", csvr.activationPeriods) } - //for _, ap := range csvr.activationPeriods { - //log.Print(ap.Intervals[0].Prices[1]) - //} } func TestLoadRatingProfiles(t *testing.T) { From 63d9612932b7660419a6edda08d6e3b253de5d9d Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 27 Jul 2013 13:29:34 +0200 Subject: [PATCH 09/17] Another way of mapping csvtpimport function, trying to fix tests on 1.0.3 --- engine/tpimporter_csv.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 85872e594..877f4467b 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -35,21 +35,23 @@ type TPCSVImporter struct { Verbose bool // If true will print a detailed information instead of silently discarding it } +var fileHandlers = map[string]func(*TPCSVImporter,string) error{ + utils.TIMINGS_CSV: (*TPCSVImporter).importTimings, + utils.DESTINATIONS_CSV: (*TPCSVImporter).importDestinations, + utils.RATES_CSV: (*TPCSVImporter).importRates, + utils.DESTINATION_RATES_CSV: (*TPCSVImporter).importDestinationRates, + utils.DESTRATE_TIMINGS_CSV: (*TPCSVImporter).importDestRateTimings, + utils.RATE_PROFILES_CSV: (*TPCSVImporter).importRatingProfiles, + utils.ACTIONS_CSV: (*TPCSVImporter).importActions, + utils.ACTION_TIMINGS_CSV: (*TPCSVImporter).importActionTimings, + utils.ACTION_TRIGGERS_CSV: (*TPCSVImporter).importActionTriggers, + utils.ACCOUNT_ACTIONS_CSV: (*TPCSVImporter).importAccountActions, + } + func (self *TPCSVImporter) Run() error { // Maps csv file to handler which should process it - fileHandlers := map[string]func(string)error{ - utils.TIMINGS_CSV: self.importTimings, - utils.DESTINATIONS_CSV: self.importDestinations, - utils.RATES_CSV: self.importRates, - utils.DESTINATION_RATES_CSV: self.importDestinationRates, - utils.DESTRATE_TIMINGS_CSV: self.importDestRateTimings, - utils.RATE_PROFILES_CSV: self.importRatingProfiles, - utils.ACTIONS_CSV: self.importActions, - utils.ACTION_TIMINGS_CSV: self.importActionTimings, - utils.ACTION_TRIGGERS_CSV: self.importActionTriggers, - utils.ACCOUNT_ACTIONS_CSV: self.importAccountActions, - } + files, _ := ioutil.ReadDir(self.DirPath) for _, f := range files { @@ -57,7 +59,7 @@ func (self *TPCSVImporter) Run() error { if !hasName { continue } - fHandler( f.Name() ) + fHandler( self, f.Name() ) } return nil } From 1a1403ace1c628ce638d8f5ee8d49edf76dfefee Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 29 Jul 2013 12:40:23 +0200 Subject: [PATCH 10/17] Few more methods on TPCSVImporter, API TPRates modifications to include GroupInterval --- apier/tprates.go | 8 ++- .../mysql/create_tariffplan_tables.sql | 3 +- docs/api_tprates.rst | 46 +++++++------ engine/action.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo.go | 2 +- engine/storage_redis.go | 2 +- engine/storage_sql.go | 31 ++++++--- engine/tpimporter_csv.go | 64 +++++++++++++++++-- utils/apitpdata.go | 1 + 11 files changed, 120 insertions(+), 43 deletions(-) diff --git a/apier/tprates.go b/apier/tprates.go index 56b9c8595..41790aa6b 100644 --- a/apier/tprates.go +++ b/apier/tprates.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) // Creates a new rate within a tariff plan @@ -36,7 +37,12 @@ func (self *Apier) SetTPRate(attrs utils.TPRate, reply *string) error { } else if exists { return errors.New(utils.ERR_DUPLICATE) } - if err := self.StorDb.SetTPRate(&attrs); err != nil { + rts := make([]*engine.Rate, len(attrs.RateSlots)) + for idx,rtSlot := range attrs.RateSlots { + rts[idx] = &engine.Rate{attrs.RateId, rtSlot.ConnectFee, rtSlot.Rate, float64(rtSlot.RatedUnits), + float64(rtSlot.RateIncrements), float64(rtSlot.GroupInterval), rtSlot.RoundingMethod, rtSlot.RoundingDecimals, rtSlot.Weight} + } + if err := self.StorDb.SetTPRates( attrs.TPid, map[string][]*engine.Rate{ attrs.RateId: rts } ); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index ae8fd575a..bead88034 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -43,11 +43,12 @@ CREATE TABLE `tp_rates` ( `rate` decimal(5,4) NOT NULL, `rated_units` int(11) NOT NULL, `rate_increments` int(11) NOT NULL, + `group_interval` int(11) NOT NULL, `rounding_method` varchar(255) NOT NULL, `rounding_decimals` tinyint(4) NOT NULL, `weight` decimal(5,2) NOT NULL, PRIMARY KEY (`id`), - UNIQUE KEY `tpid_tag_rate_weight` (`tpid`,`tag`,`weight`), + UNIQUE KEY `unique_tprate` (`tpid`,`tag`,`group_interval`), KEY `tpid` (`tpid`), KEY `tpid_tag` (`tpid`,`tag`) ); diff --git a/docs/api_tprates.rst b/docs/api_tprates.rst index ab957e9e1..98155f34e 100644 --- a/docs/api_tprates.rst +++ b/docs/api_tprates.rst @@ -15,13 +15,14 @@ Creates a new rate within a tariff plan. } type RateSlot struct { - ConnectFee float64 // ConnectFee applied once the call is answered - Rate float64 // Rate applied - RatedUnits int // Number of billing units this rate applies to - RateIncrements int // This rate will apply in increments of duration - RoundingMethod string // Use this method to round the cost - RoundingDecimals int // Round the cost number of decimals - Weight float64 // Rate's priority when dealing with grouped rates + ConnectFee float64 // ConnectFee applied once the call is answered + Rate float64 // Rate applied + RatedUnits int // Number of billing units this rate applies to + RateIncrements int // This rate will apply in increments of duration + GroupInterval int // Group position + RoundingMethod string // Use this method to round the cost + RoundingDecimals int // Round the cost number of decimals + Weight float64 // Rate's priority when dealing with grouped rates } Mandatory parameters: ``[]string{"TPid", "RateId", "ConnectFee", "RateSlots"}`` @@ -39,9 +40,10 @@ Creates a new rate within a tariff plan. { "ConnectFee": 0.2, "Rate": 2, - "RateIncrements": 1, + "RateIncrements": 60, "RatedUnits": 1, - "RoundingDecimals": 2, + "RoundingDecimals": 2, + "GroupInterval": 0, "RoundingMethod": "*up", "Weight": 10.0 }, @@ -50,7 +52,8 @@ Creates a new rate within a tariff plan. "Rate": 2.1, "RateIncrements": 1, "RatedUnits": 1, - "RoundingDecimals": 2, + "RoundingDecimals": 2, + "GroupInterval": 60, "RoundingMethod": "*up", "Weight": 20.0 } @@ -131,13 +134,14 @@ Queries specific rate on tariff plan. } type RateSlot struct { - ConnectFee float64 // ConnectFee applied once the call is answered - Rate float64 // Rate applied - RatedUnits int // Number of billing units this rate applies to - RateIncrements int // This rate will apply in increments of duration - RoundingMethod string // Use this method to round the cost - RoundingDecimals int // Round the cost number of decimals - Weight float64 // Rate's priority when dealing with grouped rates + ConnectFee float64 // ConnectFee applied once the call is answered + Rate float64 // Rate applied + RatedUnits int // Number of billing units this rate applies to + RateIncrements int // This rate will apply in increments of duration + GroupInterval int // Group position + RoundingMethod string // Use this method to round the cost + RoundingDecimals int // Round the cost number of decimals + Weight float64 // Rate's priority when dealing with grouped rates } *JSON sample*: @@ -152,9 +156,10 @@ Queries specific rate on tariff plan. { "ConnectFee": 0.2, "Rate": 2, - "RateIncrements": 1, + "RateIncrements": 60, "RatedUnits": 1, - "RoundingDecimals": 2, + "RoundingDecimals": 2, + "GroupInterval": 0, "RoundingMethod": "*up", "Weight": 10 }, @@ -163,7 +168,8 @@ Queries specific rate on tariff plan. "Rate": 2.1, "RateIncrements": 1, "RatedUnits": 1, - "RoundingDecimals": 2, + "RoundingDecimals": 2, + "GroupInterval": 60, "RoundingMethod": "*up", "Weight": 20 } diff --git a/engine/action.go b/engine/action.go index 7dd7c3f80..227277e56 100644 --- a/engine/action.go +++ b/engine/action.go @@ -116,7 +116,7 @@ func topupResetAction(ub *UserBalance, a *Action) (err error) { if a.BalanceId == MINUTES { ub.MinuteBuckets = make([]*MinuteBucket, 0) } else { - ub.BalanceMap[a.BalanceId+a.Direction] = BalanceChain{&Balance{Value: 0}} + ub.BalanceMap[a.BalanceId+a.Direction] = BalanceChain{&Balance{Value: 0}} // ToDo: can ub be empty here? } genericMakeNegative(a) genericDebit(ub, a) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index cd9b4f963..d3f8374c0 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -68,7 +68,7 @@ type DataStorage interface { GetTPDestination(string, string) (*Destination, error) GetTPDestinationIds(string) ([]string, error) ExistsTPRate(string, string) (bool, error) - SetTPRate(*utils.TPRate) error + SetTPRates(string, map[string][]*Rate) error GetTPRate(string, string) (*utils.TPRate, error) GetTPRateIds(string) ([]string, error) ExistsTPDestinationRate(string, string) (bool, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index f6aaeb4a4..78cdf78e6 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -113,7 +113,7 @@ func (ms *MapStorage) ExistsTPRate(tpid, rtId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPRate(rt *utils.TPRate) error { +func (ms *MapStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 2ecff5575..c6bf13c8b 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -188,7 +188,7 @@ func (ms *MongoStorage) ExistsTPRate(tpid, rtId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPRate(rt *utils.TPRate) error { +func (ms *MongoStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index cc539f419..9725764a5 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -143,7 +143,7 @@ func (rs *RedisStorage) ExistsTPRate(tpid, rtId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPRate(rt *utils.TPRate) error { +func (rs *RedisStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 3242b7c35..20c7458f4 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -210,19 +210,29 @@ func (self *SQLStorage) ExistsTPRate(tpid, rtId string) (bool, error) { return exists, nil } -func (self *SQLStorage) SetTPRate(rt *utils.TPRate) error { - for _, rtSlot := range rt.RateSlots { - if _, err := self.Db.Exec(fmt.Sprintf("INSERT INTO %s (tpid, tag, connect_fee, rate, rated_units, rate_increments, rounding_method, rounding_decimals, weight) VALUES ('%s', '%s', %f, %f, %d, %d,'%s', %d, %f)", - utils.TBL_TP_RATES, rt.TPid, rt.RateId, rtSlot.ConnectFee, rtSlot.Rate, rtSlot.RatedUnits, rtSlot.RateIncrements, - rtSlot.RoundingMethod, rtSlot.RoundingDecimals, rtSlot.Weight)); err != nil { - return err +func (self *SQLStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { + if len(rts) == 0 { + return nil //Nothing to set + } + qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, connect_fee, rate, rated_units, rate_increments, group_interval, rounding_method, rounding_decimals, weight) VALUES ", utils.TBL_TP_RATES) + for rtId, rtRows := range rts { + for idx, rt := range rtRows { + if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + qry += "," + } + qry += fmt.Sprintf("('%s', '%s', %f, %f, %d, %d,%d,'%s', %d, %f)", + tpid, rtId, rt.ConnectFee, rt.Price, int(rt.PricedUnits), int(rt.RateIncrements), int(rt.GroupInterval), + rt.RoundingMethod, rt.RoundingDecimals, rt.Weight) } } + if _, err := self.Db.Exec(qry); err != nil { + return err + } return nil } func (self *SQLStorage) GetTPRate(tpid, rtId string) (*utils.TPRate, error) { - rows, err := self.Db.Query(fmt.Sprintf("SELECT connect_fee, rate, rated_units, rate_increments, rounding_method, rounding_decimals, weight FROM %s WHERE tpid='%s' AND tag='%s'", utils.TBL_TP_RATES, tpid, rtId)) + rows, err := self.Db.Query(fmt.Sprintf("SELECT connect_fee, rate, rated_units, rate_increments, group_interval, rounding_method, rounding_decimals, weight FROM %s WHERE tpid='%s' AND tag='%s'", utils.TBL_TP_RATES, tpid, rtId)) if err != nil { return nil, err } @@ -232,13 +242,14 @@ func (self *SQLStorage) GetTPRate(tpid, rtId string) (*utils.TPRate, error) { for rows.Next() { i++ //Keep here a reference so we know we got at least one prefix var connectFee, rate, weight float64 - var ratedUnits, rateIncrements, roundingDecimals int + var ratedUnits, rateIncrements, roundingDecimals, groupInterval int var roundingMethod string - err = rows.Scan(&connectFee, &rate, &ratedUnits, &rateIncrements, &roundingMethod, &roundingDecimals, &weight) + err = rows.Scan(&connectFee, &rate, &ratedUnits, &rateIncrements, &groupInterval, &roundingMethod, &roundingDecimals, &weight) if err != nil { return nil, err } - rt.RateSlots = append(rt.RateSlots, utils.RateSlot{connectFee, rate, ratedUnits, rateIncrements, roundingMethod, roundingDecimals, weight}) + rt.RateSlots = append(rt.RateSlots, utils.RateSlot{connectFee, rate, ratedUnits, rateIncrements, groupInterval, + roundingMethod, roundingDecimals, weight}) } if i == 0 { return nil, nil diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 877f4467b..b268ba347 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -35,6 +35,8 @@ type TPCSVImporter struct { Verbose bool // If true will print a detailed information instead of silently discarding it } +// Maps csv file to handler which should process it. Defined like this since tests on 1.0.3 were failing on Travis. +// Change it to func(string) error as soon as Travis updates. var fileHandlers = map[string]func(*TPCSVImporter,string) error{ utils.TIMINGS_CSV: (*TPCSVImporter).importTimings, utils.DESTINATIONS_CSV: (*TPCSVImporter).importDestinations, @@ -49,10 +51,6 @@ var fileHandlers = map[string]func(*TPCSVImporter,string) error{ } func (self *TPCSVImporter) Run() error { - - // Maps csv file to handler which should process it - - files, _ := ioutil.ReadDir(self.DirPath) for _, f := range files { fHandler,hasName := fileHandlers[f.Name()] @@ -66,6 +64,9 @@ func (self *TPCSVImporter) Run() error { // Handler importing timings from file, saved row by row to storDb func (self *TPCSVImporter) importTimings(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } fParser, err := NewTPCSVFileParser( self.DirPath, fn ) if err!=nil { return err @@ -90,11 +91,62 @@ func (self *TPCSVImporter) importTimings(fn string) error { return nil } -func (self *TPCSVImporter) importDestinations(fPath string) error { +func (self *TPCSVImporter) importDestinations(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + dst := &Destination{record[0], []string{record[1]}} + if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } -func (self *TPCSVImporter) importRates(fPath string) error { +func (self *TPCSVImporter) importRates(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + rt, err := NewRate(record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8]) + if err != nil { + return err + } + if err := self.StorDb.SetTPRates( self.TPid, map[string][]*Rate{ record[0]: []*Rate{rt} } ); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index c389db1cf..89d396b33 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -31,6 +31,7 @@ type RateSlot struct { Rate float64 // Rate applied RatedUnits int // Number of billing units this rate applies to RateIncrements int // This rate will apply in increments of duration + GroupInterval int // Group position RoundingMethod string // Use this method to round the cost RoundingDecimals int // Round the cost number of decimals Weight float64 // Rate's priority when dealing with grouped rates From f1d94aebaf86dcb8a92118571f07b0cebb35312f Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 29 Jul 2013 14:18:50 +0200 Subject: [PATCH 11/17] TPCSVImporter - DestinationRates implementation --- apier/tpdestinationrates.go | 7 ++++++- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo.go | 2 +- engine/storage_redis.go | 2 +- engine/storage_sql.go | 16 +++++++++------- engine/tpimporter_csv.go | 30 +++++++++++++++++++++++++++++- 7 files changed, 48 insertions(+), 13 deletions(-) diff --git a/apier/tpdestinationrates.go b/apier/tpdestinationrates.go index 2878b2d66..e649cb8c5 100644 --- a/apier/tpdestinationrates.go +++ b/apier/tpdestinationrates.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) // Creates a new DestinationRate profile within a tariff plan @@ -36,7 +37,11 @@ func (self *Apier) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *st } else if exists { return errors.New(utils.ERR_DUPLICATE) } - if err := self.StorDb.SetTPDestinationRate(&attrs); err != nil { + drs := make([]*engine.DestinationRate, len(attrs.DestinationRates)) + for idx,dr := range attrs.DestinationRates { + drs[idx] = &engine.DestinationRate{attrs.DestinationRateId, dr.DestinationId, dr.RateId, nil} + } + if err := self.StorDb.SetTPDestinationRates( attrs.TPid, map[string][]*engine.DestinationRate{ attrs.DestinationRateId: drs } ); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/engine/storage_interface.go b/engine/storage_interface.go index d3f8374c0..15ebaea40 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -72,7 +72,7 @@ type DataStorage interface { GetTPRate(string, string) (*utils.TPRate, error) GetTPRateIds(string) ([]string, error) ExistsTPDestinationRate(string, string) (bool, error) - SetTPDestinationRate(*utils.TPDestinationRate) error + SetTPDestinationRates(string, map[string][]*DestinationRate) error GetTPDestinationRate(string, string) (*utils.TPDestinationRate, error) GetTPDestinationRateIds(string) ([]string, error) ExistsTPDestRateTiming(string, string) (bool, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 78cdf78e6..6be87a9ea 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -129,7 +129,7 @@ func (ms *MapStorage) ExistsTPDestinationRate(tpid, drId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPDestinationRate(dr *utils.TPDestinationRate) error { +func (ms *MapStorage) SetTPDestinationRates(tpid string, drs map[string][]*DestinationRate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index c6bf13c8b..9c86f6b7a 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -204,7 +204,7 @@ func (ms *MongoStorage) ExistsTPDestinationRate(tpid, drId string) (bool, error) return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPDestinationRate(dr *utils.TPDestinationRate) error { +func (ms *MongoStorage) SetTPDestinationRates(tpid string, drs map[string][]*DestinationRate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 9725764a5..ea2d2895a 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -159,7 +159,7 @@ func (rs *RedisStorage) ExistsTPDestinationRate(tpid, drId string) (bool, error) return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPDestinationRate(dr *utils.TPDestinationRate) error { +func (rs *RedisStorage) SetTPDestinationRates(tpid string, drs map[string][]*DestinationRate) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 20c7458f4..64b336fe9 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -289,17 +289,19 @@ func (self *SQLStorage) ExistsTPDestinationRate(tpid, drId string) (bool, error) return exists, nil } -func (self *SQLStorage) SetTPDestinationRate(dr *utils.TPDestinationRate) error { - if len(dr.DestinationRates) == 0 { +func (self *SQLStorage) SetTPDestinationRates(tpid string, drs map[string][]*DestinationRate) error { + if len(drs) == 0 { return nil //Nothing to set } - // Using multiple values in query to spare some network processing time qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, destinations_tag, rates_tag) VALUES ", utils.TBL_TP_DESTINATION_RATES) - for idx, drPair := range dr.DestinationRates { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator - qry += "," + for drId, drRows := range drs { + for idx, dr := range drRows { + if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + qry += "," + } + qry += fmt.Sprintf("('%s','%s','%s','%s')", + tpid, drId, dr.DestinationsTag, dr.RateTag) } - qry += fmt.Sprintf("('%s','%s','%s','%s')", dr.TPid, dr.DestinationRateId, drPair.DestinationId, drPair.RateId) } if _, err := self.Db.Exec(qry); err != nil { return err diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index b268ba347..f3f4d9102 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -150,7 +150,35 @@ func (self *TPCSVImporter) importRates(fn string) error { return nil } -func (self *TPCSVImporter) importDestinationRates(fPath string) error { +func (self *TPCSVImporter) importDestinationRates(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + dr := &DestinationRate{record[0], record[1], record[2], nil} + if err != nil { + return err + } + if err := self.StorDb.SetTPDestinationRates( self.TPid, + map[string][]*DestinationRate{ dr.Tag: []*DestinationRate{dr} } ); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } From 9887799492c2c00739b448476c9e2f532adb34b5 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 29 Jul 2013 15:32:07 +0200 Subject: [PATCH 12/17] TPCSVImporter DestRateTiming, store refactoring --- apier/tpdestratetimings.go | 11 ++++++++- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo.go | 2 +- engine/storage_redis.go | 2 +- engine/storage_sql.go | 16 ++++++------ engine/tpimporter_csv.go | 49 ++++++++++++++++++++++++++++++------- 7 files changed, 63 insertions(+), 21 deletions(-) diff --git a/apier/tpdestratetimings.go b/apier/tpdestratetimings.go index 6d387754a..16ae3bd24 100644 --- a/apier/tpdestratetimings.go +++ b/apier/tpdestratetimings.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) // Creates a new DestinationRateTiming profile within a tariff plan @@ -36,7 +37,15 @@ func (self *Apier) SetTPDestRateTiming(attrs utils.TPDestRateTiming, reply *stri } else if exists { return errors.New(utils.ERR_DUPLICATE) } - if err := self.StorDb.SetTPDestRateTiming(&attrs); err != nil { + drts := make([]*engine.DestinationRateTiming, len(attrs.DestRateTimings)) + for idx,drt := range attrs.DestRateTimings { + drts[idx] = &engine.DestinationRateTiming{Tag: attrs.DestRateTimingId, + DestinationRatesTag: drt.DestRatesId, + Weight: drt.Weight, + TimingsTag: drt.TimingId, + } + } + if err := self.StorDb.SetTPDestRateTimings( attrs.TPid, map[string][]*engine.DestinationRateTiming{ attrs.DestRateTimingId: drts } ); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 15ebaea40..8845cab2d 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -76,7 +76,7 @@ type DataStorage interface { GetTPDestinationRate(string, string) (*utils.TPDestinationRate, error) GetTPDestinationRateIds(string) ([]string, error) ExistsTPDestRateTiming(string, string) (bool, error) - SetTPDestRateTiming(*utils.TPDestRateTiming) error + SetTPDestRateTimings(string, map[string][]*DestinationRateTiming) error GetTPDestRateTiming(string, string) (*utils.TPDestRateTiming, error) GetTPDestRateTimingIds(string) ([]string, error) ExistsTPRateProfile(string, string) (bool, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 6be87a9ea..4fef94931 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -145,7 +145,7 @@ func (ms *MapStorage) ExistsTPDestRateTiming(tpid, drtId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPDestRateTiming(drt *utils.TPDestRateTiming) error { +func (ms *MapStorage) SetTPDestRateTimings(tpid string, drts map[string][]*DestinationRateTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 9c86f6b7a..c4867abc9 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -220,7 +220,7 @@ func (ms *MongoStorage) ExistsTPDestRateTiming(tpid, drtId string) (bool, error) return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPDestRateTiming(drt *utils.TPDestRateTiming) error { +func (ms *MongoStorage) SetTPDestRateTimings(tpid string, drts map[string][]*DestinationRateTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ea2d2895a..3e65dc567 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -175,7 +175,7 @@ func (rs *RedisStorage) ExistsTPDestRateTiming(tpid, drtId string) (bool, error) return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPDestRateTiming(drt *utils.TPDestRateTiming) error { +func (rs *RedisStorage) SetTPDestRateTimings(tpid string, drts map[string][]*DestinationRateTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 64b336fe9..cd29f7992 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -364,17 +364,19 @@ func (self *SQLStorage) ExistsTPDestRateTiming(tpid, drtId string) (bool, error) return exists, nil } -func (self *SQLStorage) SetTPDestRateTiming(drt *utils.TPDestRateTiming) error { - if len(drt.DestRateTimings) == 0 { +func (self *SQLStorage) SetTPDestRateTimings(tpid string, drts map[string][]*DestinationRateTiming) error { + if len(drts) == 0 { return nil //Nothing to set } - // Using multiple values in query to spare some network processing time qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, destrates_tag, timing_tag, weight) VALUES ", utils.TBL_TP_DESTRATE_TIMINGS) - for idx, drtPair := range drt.DestRateTimings { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator - qry += "," + for drtId, drtRows := range drts { + for idx, drt := range drtRows { + if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + qry += "," + } + qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", + tpid, drtId, drt.DestinationRatesTag, drt.TimingsTag, drt.Weight) } - qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", drt.TPid, drt.DestRateTimingId, drtPair.DestRatesId, drtPair.TimingId, drtPair.Weight) } if _, err := self.Db.Exec(qry); err != nil { return err diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index f3f4d9102..9f45e32ae 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -22,6 +22,7 @@ import ( "io" "io/ioutil" "log" + "strconv" "github.com/cgrates/cgrates/utils" ) @@ -171,9 +172,6 @@ func (self *TPCSVImporter) importDestinationRates(fn string) error { continue } dr := &DestinationRate{record[0], record[1], record[2], nil} - if err != nil { - return err - } if err := self.StorDb.SetTPDestinationRates( self.TPid, map[string][]*DestinationRate{ dr.Tag: []*DestinationRate{dr} } ); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) @@ -182,27 +180,60 @@ func (self *TPCSVImporter) importDestinationRates(fn string) error { return nil } -func (self *TPCSVImporter) importDestRateTimings(fPath string) error { +func (self *TPCSVImporter) importDestRateTimings(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + weight, err := strconv.ParseFloat(record[3], 64) + if err != nil { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + continue + } + drt := &DestinationRateTiming{Tag: record[0], + DestinationRatesTag: record[1], + Weight: weight, + TimingsTag: record[2], + } + if err := self.StorDb.SetTPDestRateTimings( self.TPid, map[string][]*DestinationRateTiming{drt.Tag:[]*DestinationRateTiming{drt}}); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } -func (self *TPCSVImporter) importRatingProfiles(fPath string) error { +func (self *TPCSVImporter) importRatingProfiles(fn string) error { return nil } -func (self *TPCSVImporter) importActions(fPath string) error { +func (self *TPCSVImporter) importActions(fn string) error { return nil } -func (self *TPCSVImporter) importActionTimings(fPath string) error { +func (self *TPCSVImporter) importActionTimings(fn string) error { return nil } -func (self *TPCSVImporter) importActionTriggers(fPath string) error { +func (self *TPCSVImporter) importActionTriggers(fn string) error { return nil } -func (self *TPCSVImporter) importAccountActions(fPath string) error { +func (self *TPCSVImporter) importAccountActions(fn string) error { return nil } From ddc2240c67d1a2a8b5e083199ef864ba940e7fa7 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 29 Jul 2013 18:21:13 +0200 Subject: [PATCH 13/17] TPCSVImporter RatingProfiles, changed RateProfile into RatingProfile in APIs for consistency --- ...{tprateprofiles.go => tpratingprofiles.go} | 41 +++++++---- cmd/cgr-loader/cgr-loader.go | 4 +- .../mysql/create_tariffplan_tables.sql | 2 +- ...eprofiles.rst => api_tpratingprofiles.rst} | 52 ++++++------- docs/apicalls.rst | 6 +- engine/loader_db.go | 14 ++-- engine/ratingprofile.go | 6 +- engine/storage_interface.go | 8 +- engine/storage_map.go | 8 +- engine/storage_mongo.go | 8 +- engine/storage_redis.go | 8 +- engine/storage_sql.go | 73 +++++++++++-------- engine/tpimporter_csv.go | 43 +++++++++++ utils/apitpdata.go | 6 +- utils/consts.go | 2 +- 15 files changed, 175 insertions(+), 106 deletions(-) rename apier/{tprateprofiles.go => tpratingprofiles.go} (54%) rename docs/{api_tprateprofiles.rst => api_tpratingprofiles.rst} (76%) diff --git a/apier/tprateprofiles.go b/apier/tpratingprofiles.go similarity index 54% rename from apier/tprateprofiles.go rename to apier/tpratingprofiles.go index 4ec4eefd7..ca6a6908c 100644 --- a/apier/tprateprofiles.go +++ b/apier/tpratingprofiles.go @@ -24,36 +24,49 @@ import ( "errors" "fmt" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) -// Creates a new RateProfile within a tariff plan -func (self *Apier) SetTPRateProfile(attrs utils.TPRateProfile, reply *string) error { - if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateProfileId", "Tenant", "TOR", "Direction", "Subject", "RatingActivations"}); len(missing) != 0 { +// Creates a new RatingProfile within a tariff plan +func (self *Apier) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error { + if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId", "Tenant", "TOR", "Direction", "Subject", "RatingActivations"}); len(missing) != 0 { return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } - if exists, err := self.StorDb.ExistsTPRateProfile(attrs.TPid, attrs.RateProfileId); err != nil { + if exists, err := self.StorDb.ExistsTPRatingProfile(attrs.TPid, attrs.RatingProfileId); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } else if exists { return errors.New(utils.ERR_DUPLICATE) } - if err := self.StorDb.SetTPRateProfile(&attrs); err != nil { + rps := make([]*engine.RatingProfile, len(attrs.RatingActivations)) + for idx,ra := range attrs.RatingActivations { + rps[idx] = &engine.RatingProfile{Tag: attrs.RatingProfileId, + Tenant: attrs.Tenant, + TOR: attrs.TOR, + Direction: attrs.Direction, + Subject: attrs.Subject, + ActivationTime: ra.ActivationTime, + DestRatesTimingTag: ra.DestRateTimingId, + RatesFallbackSubject: attrs.RatesFallbackSubject, + } + } + if err := self.StorDb.SetTPRatingProfiles( attrs.TPid, map[string][]*engine.RatingProfile{ attrs.RatingProfileId: rps } ); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" return nil } -type AttrGetTPRateProfile struct { +type AttrGetTPRatingProfile struct { TPid string // Tariff plan id - RateProfileId string // RateProfile id + RatingProfileId string // RatingProfile id } -// Queries specific RateProfile on tariff plan -func (self *Apier) GetTPRateProfile(attrs AttrGetTPRateProfile, reply *utils.TPRateProfile) error { - if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateProfileId"}); len(missing) != 0 { //Params missing +// Queries specific RatingProfile on tariff plan +func (self *Apier) GetTPRatingProfile(attrs AttrGetTPRatingProfile, reply *utils.TPRatingProfile) error { + if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } - if dr, err := self.StorDb.GetTPRateProfile(attrs.TPid, attrs.RateProfileId); err != nil { + if dr, err := self.StorDb.GetTPRatingProfile(attrs.TPid, attrs.RatingProfileId); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } else if dr == nil { return errors.New(utils.ERR_NOT_FOUND) @@ -63,12 +76,12 @@ func (self *Apier) GetTPRateProfile(attrs AttrGetTPRateProfile, reply *utils.TPR return nil } -// Queries RateProfile identities on specific tariff plan. -func (self *Apier) GetTPRateProfileIds(attrs utils.AttrTPRateProfileIds, reply *[]string) error { +// Queries RatingProfile identities on specific tariff plan. +func (self *Apier) GetTPRatingProfileIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } - if ids, err := self.StorDb.GetTPRateProfileIds(&attrs); err != nil { + if ids, err := self.StorDb.GetTPRatingProfileIds(&attrs); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } else if ids == nil { return errors.New(utils.ERR_NOT_FOUND) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index d60d6f60d..1a958f733 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -52,7 +52,7 @@ var ( verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") fromStorDb = flag.Bool("from-stordb", false, "Load the tariff plan from storDb to dataDb") toStorDb = flag.Bool("to-stordb", false, "Import the tariff plan from files to storDb") - + runId = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields") ) func main() { @@ -89,7 +89,7 @@ func main() { if *tpid == "" { log.Fatal("TPid required, please define it via *-tpid* command argument.") } - csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose } + csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose, *runId } if errImport := csvImporter.Run(); errImport != nil { log.Fatal(errImport) } diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index bead88034..4a7fa78b0 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -90,7 +90,7 @@ CREATE TABLE `tp_destrate_timings` ( -- Table structure for table `tp_rate_profiles` -- -CREATE TABLE `tp_rate_profiles` ( +CREATE TABLE `tp_rating_profiles` ( `id` int(11) NOT NULL AUTO_INCREMENT, `tpid` char(40) NOT NULL, `tag` varchar(24) NOT NULL, diff --git a/docs/api_tprateprofiles.rst b/docs/api_tpratingprofiles.rst similarity index 76% rename from docs/api_tprateprofiles.rst rename to docs/api_tpratingprofiles.rst index 1015fdf32..376a9370e 100644 --- a/docs/api_tprateprofiles.rst +++ b/docs/api_tpratingprofiles.rst @@ -1,16 +1,16 @@ -Apier.SetTPRateProfile -++++++++++++++++++++++ +Apier.SetTPRatingProfile +++++++++++++++++++++++++ -Creates a new RateProfile within a tariff plan. +Creates a new RatingProfile within a tariff plan. **Request**: Data: :: - type TPRateProfile struct { + type TPRatingProfile struct { TPid string // Tariff plan id - RateProfileId string // RateProfile id + RatingProfileId string // RatingProfile id Tenant string // Tenant's Id TOR string // TypeOfRecord Direction string // Traffic direction, OUT is the only one supported for now @@ -24,18 +24,18 @@ Creates a new RateProfile within a tariff plan. DestRateTimingId string // Id of DestRateTiming profile } - Mandatory parameters: ``[]string{"TPid", "RateProfileId", "Tenant", "TOR", "Direction", "Subject", "RatingActivations"}`` + Mandatory parameters: ``[]string{"TPid", "RatingProfileId", "Tenant", "TOR", "Direction", "Subject", "RatingActivations"}`` *JSON sample*: :: { "id": 3, - "method": "Apier.SetTPRateProfile", + "method": "Apier.SetTPRatingProfile", "params": [ { "Direction": "OUT", - "RateProfileId": "SAMPLE_RP_2", + "RatingProfileId": "SAMPLE_RP_2", "RatingActivations": [ { "ActivationTime": 1373609003, @@ -79,35 +79,35 @@ Creates a new RateProfile within a tariff plan. ``SERVER_ERROR`` - Server error occurred. - ``DUPLICATE`` - The specified combination of TPid/RateProfileId already exists in StorDb. + ``DUPLICATE`` - The specified combination of TPid/RatingProfileId already exists in StorDb. -Apier.GetTPRateProfile -++++++++++++++++++++++ +Apier.GetTPRatingProfile +++++++++++++++++++++++++ -Queries specific RateProfile on tariff plan. +Queries specific RatingProfile on tariff plan. **Request**: Data: :: - type AttrGetTPRateProfile struct { + type AttrGetTPRatingProfile struct { TPid string // Tariff plan id - RateProfileId string // RateProfile id + RatingProfileId string // RatingProfile id } - Mandatory parameters: ``[]string{"TPid", "RateProfileId"}`` + Mandatory parameters: ``[]string{"TPid", "RatingProfileId"}`` *JSON sample*: :: { "id": 0, - "method": "Apier.GetTPRateProfile", + "method": "Apier.GetTPRatingProfile", "params": [ { - "RateProfileId": "SAMPLE_RP_2", + "RatingProfileId": "SAMPLE_RP_2", "TPid": "SAMPLE_TP" } ] @@ -118,9 +118,9 @@ Queries specific RateProfile on tariff plan. Data: :: - type TPRateProfile struct { + type TPRatingProfile struct { TPid string // Tariff plan id - RateProfileId string // RateProfile id + RatingProfileId string // RatingProfile id Tenant string // Tenant's Id TOR string // TypeOfRecord Direction string // Traffic direction, OUT is the only one supported for now @@ -142,7 +142,7 @@ Queries specific RateProfile on tariff plan. "id": 0, "result": { "Direction": "OUT", - "RateProfileId": "SAMPLE_RP_2", + "RatingProfileId": "SAMPLE_RP_2", "RatesFallbackSubject": "", "RatingActivations": [ { @@ -167,20 +167,20 @@ Queries specific RateProfile on tariff plan. ``SERVER_ERROR`` - Server error occurred. - ``NOT_FOUND`` - Requested RateProfile profile not found. + ``NOT_FOUND`` - Requested RatingProfile profile not found. -Apier.GetTPRateProfileIds -+++++++++++++++++++++++++ +Apier.GetTPRatingProfileIds ++++++++++++++++++++++++++++ -Queries specific RateProfile on tariff plan. Attribute parameters used as extra filters. +Queries specific RatingProfile on tariff plan. Attribute parameters used as extra filters. **Request**: Data: :: - type AttrTPRateProfileIds struct { + type AttrTPRatingProfileIds struct { TPid string // Tariff plan id Tenant string // Tenant's Id TOR string // TypeOfRecord @@ -195,7 +195,7 @@ Queries specific RateProfile on tariff plan. Attribute parameters used as extra { "id": 0, - "method": "Apier.GetTPRateProfileIds", + "method": "Apier.GetTPRatingProfileIds", "params": [ { "Subject": "dan", diff --git a/docs/apicalls.rst b/docs/apicalls.rst index 4afedbeb5..dc015cd7e 100644 --- a/docs/apicalls.rst +++ b/docs/apicalls.rst @@ -169,13 +169,13 @@ DestinationRateTimings api_tpdestratetimings -RateProfiles -~~~~~~~~~~~~ +RatingProfiles +~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 - api_tprateprofiles + api_tpratingprofiles Actions ~~~~~~~ diff --git a/engine/loader_db.go b/engine/loader_db.go index 90213e278..0d7287fda 100644 --- a/engine/loader_db.go +++ b/engine/loader_db.go @@ -184,11 +184,11 @@ func (dbr *DbReader) LoadRatingProfiles() error { return err } for _, rp := range rpfs { - at := time.Unix(rp.activationTime, 0) + at := time.Unix(rp.ActivationTime, 0) for _, d := range dbr.destinations { - ap, exists := dbr.activationPeriods[rp.destRatesTimingTag] + ap, exists := dbr.activationPeriods[rp.DestRatesTimingTag] if !exists { - return errors.New(fmt.Sprintf("Could not load rating timing for tag: %v", rp.destRatesTimingTag)) + return errors.New(fmt.Sprintf("Could not load rating timing for tag: %v", rp.DestRatesTimingTag)) } newAP := &ActivationPeriod{ActivationTime: at} //copy(newAP.Intervals, ap.Intervals) @@ -211,12 +211,12 @@ func (dbr *DbReader) LoadRatingProfileByTag(tag string) error { } for _, ratingProfile := range rpm { resultRatingProfile.FallbackKey = ratingProfile.FallbackKey // it will be the last fallback key - at := time.Unix(ratingProfile.activationTime, 0) - drtm, err := dbr.storDb.GetTpDestinationRateTimings(dbr.tpid, ratingProfile.destRatesTimingTag) + at := time.Unix(ratingProfile.ActivationTime, 0) + drtm, err := dbr.storDb.GetTpDestinationRateTimings(dbr.tpid, ratingProfile.DestRatesTimingTag) if err != nil { return err } else if len(drtm) == 0 { - return fmt.Errorf("No DestRateTimings profile with id: %s", ratingProfile.destRatesTimingTag) + return fmt.Errorf("No DestRateTimings profile with id: %s", ratingProfile.DestRatesTimingTag) } for _, destrateTiming := range drtm { tm, err := dbr.storDb.GetTpTimings(dbr.tpid, destrateTiming.TimingsTag) @@ -249,7 +249,7 @@ func (dbr *DbReader) LoadRatingProfileByTag(tag string) error { return err } for _, destination := range dm { - ap := activationPeriods[ratingProfile.destRatesTimingTag] + ap := activationPeriods[ratingProfile.DestRatesTimingTag] newAP := &ActivationPeriod{ActivationTime: at} newAP.Intervals = append(newAP.Intervals, ap.Intervals...) resultRatingProfile.AddActivationPeriodIfNotPresent(destination.Id, newAP) diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 969d29349..8105e72e4 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -30,10 +30,10 @@ const ( type RatingProfile struct { Id string - FallbackKey string + FallbackKey string // FallbackKey is used as complete combination of Tenant:TOR:Direction:Subject DestinationMap map[string][]*ActivationPeriod - tag, destRatesTimingTag string // used only for loading - activationTime int64 + Tag, Tenant, TOR, Direction, Subject, DestRatesTimingTag, RatesFallbackSubject string // used only for loading + ActivationTime int64 } // Adds an activation period that applyes to current rating profile if not already present. diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 8845cab2d..dd7570213 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -79,10 +79,10 @@ type DataStorage interface { SetTPDestRateTimings(string, map[string][]*DestinationRateTiming) error GetTPDestRateTiming(string, string) (*utils.TPDestRateTiming, error) GetTPDestRateTimingIds(string) ([]string, error) - ExistsTPRateProfile(string, string) (bool, error) - SetTPRateProfile(*utils.TPRateProfile) error - GetTPRateProfile(string, string) (*utils.TPRateProfile, error) - GetTPRateProfileIds(*utils.AttrTPRateProfileIds) ([]string, error) + ExistsTPRatingProfile(string, string) (bool, error) + SetTPRatingProfiles(string, map[string][]*RatingProfile) error + GetTPRatingProfile(string, string) (*utils.TPRatingProfile, error) + GetTPRatingProfileIds(*utils.AttrTPRatingProfileIds) ([]string, error) ExistsTPActions(string, string) (bool, error) SetTPActions(*utils.TPActions) error GetTPActions(string, string) (*utils.TPActions, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 4fef94931..051b10745 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -157,19 +157,19 @@ func (ms *MapStorage) GetTPDestRateTimingIds(tpid string) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) ExistsTPRateProfile(tpid, rpId string) (bool, error) { +func (ms *MapStorage) ExistsTPRatingProfile(tpid, rpId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPRateProfile(rp *utils.TPRateProfile) error { +func (ms *MapStorage) SetTPRatingProfiles(tpid string, rps map[string][]*RatingProfile) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) GetTPRateProfile(tpid, rpId string) (*utils.TPRateProfile, error) { +func (ms *MapStorage) GetTPRatingProfile(tpid, rpId string) (*utils.TPRatingProfile, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) GetTPRateProfileIds(filters *utils.AttrTPRateProfileIds) ([]string, error) { +func (ms *MapStorage) GetTPRatingProfileIds(filters *utils.AttrTPRatingProfileIds) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index c4867abc9..a73bb6c0a 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -232,19 +232,19 @@ func (ms *MongoStorage) GetTPDestRateTimingIds(tpid string) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) ExistsTPRateProfile(tpid, rpId string) (bool, error) { +func (ms *MongoStorage) ExistsTPRatingProfile(tpid, rpId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPRateProfile(rp *utils.TPRateProfile) error { +func (ms *MongoStorage) SetTPRatingProfiles(tpid string, rps map[string][]*RatingProfile) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) GetTPRateProfile(tpid, rpId string) (*utils.TPRateProfile, error) { +func (ms *MongoStorage) GetTPRatingProfile(tpid, rpId string) (*utils.TPRatingProfile, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) GetTPRateProfileIds(filters *utils.AttrTPRateProfileIds) ([]string, error) { +func (ms *MongoStorage) GetTPRatingProfileIds(filters *utils.AttrTPRatingProfileIds) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 3e65dc567..ada155c96 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -187,19 +187,19 @@ func (rs *RedisStorage) GetTPDestRateTimingIds(tpid string) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) ExistsTPRateProfile(tpid, rpId string) (bool, error) { +func (rs *RedisStorage) ExistsTPRatingProfile(tpid, rpId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPRateProfile(rp *utils.TPRateProfile) error { +func (rs *RedisStorage) SetTPRatingProfiles(tpid string, rps map[string][]*RatingProfile) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) GetTPRateProfile(tpid, rpId string) (*utils.TPRateProfile, error) { +func (rs *RedisStorage) GetTPRatingProfile(tpid, rpId string) (*utils.TPRatingProfile, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) GetTPRateProfileIds(filters *utils.AttrTPRateProfileIds) ([]string, error) { +func (rs *RedisStorage) GetTPRatingProfileIds(filters *utils.AttrTPRatingProfileIds) ([]string, error) { return nil, errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index cd29f7992..e90b5dfbf 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -215,14 +215,16 @@ func (self *SQLStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { return nil //Nothing to set } qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, connect_fee, rate, rated_units, rate_increments, group_interval, rounding_method, rounding_decimals, weight) VALUES ", utils.TBL_TP_RATES) + i := 0 for rtId, rtRows := range rts { - for idx, rt := range rtRows { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + for _, rt := range rtRows { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s', '%s', %f, %f, %d, %d,%d,'%s', %d, %f)", tpid, rtId, rt.ConnectFee, rt.Price, int(rt.PricedUnits), int(rt.RateIncrements), int(rt.GroupInterval), rt.RoundingMethod, rt.RoundingDecimals, rt.Weight) + i++ } } if _, err := self.Db.Exec(qry); err != nil { @@ -294,13 +296,15 @@ func (self *SQLStorage) SetTPDestinationRates(tpid string, drs map[string][]*Des return nil //Nothing to set } qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, destinations_tag, rates_tag) VALUES ", utils.TBL_TP_DESTINATION_RATES) + i := 0 for drId, drRows := range drs { - for idx, dr := range drRows { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + for _, dr := range drRows { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s')", tpid, drId, dr.DestinationsTag, dr.RateTag) + i++ } } if _, err := self.Db.Exec(qry); err != nil { @@ -369,13 +373,15 @@ func (self *SQLStorage) SetTPDestRateTimings(tpid string, drts map[string][]*Des return nil //Nothing to set } qry := fmt.Sprintf("INSERT INTO %s (tpid, tag, destrates_tag, timing_tag, weight) VALUES ", utils.TBL_TP_DESTRATE_TIMINGS) + i := 0 for drtId, drtRows := range drts { - for idx, drt := range drtRows { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + for _, drt := range drtRows { + if i!=0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", tpid, drtId, drt.DestinationRatesTag, drt.TimingsTag, drt.Weight) + i++ } } if _, err := self.Db.Exec(qry); err != nil { @@ -431,7 +437,7 @@ func (self *SQLStorage) GetTPDestRateTimingIds(tpid string) ([]string, error) { return ids, nil } -func (self *SQLStorage) ExistsTPRateProfile(tpid, rpId string) (bool, error) { +func (self *SQLStorage) ExistsTPRatingProfile(tpid, rpId string) (bool, error) { var exists bool err := self.Db.QueryRow(fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE tpid='%s' AND tag='%s')", utils.TBL_TP_RATE_PROFILES, tpid, rpId)).Scan(&exists) if err != nil { @@ -440,20 +446,23 @@ func (self *SQLStorage) ExistsTPRateProfile(tpid, rpId string) (bool, error) { return exists, nil } -func (self *SQLStorage) SetTPRateProfile(rp *utils.TPRateProfile) error { - var qry string - if len(rp.RatingActivations) == 0 { // Possibility to only set fallback rate subject - qry = fmt.Sprintf("INSERT INTO %s (tpid,tag,tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', 0,'','%s')", - utils.TBL_TP_RATE_PROFILES, rp.TPid, rp.RateProfileId, rp.Tenant, rp.TOR, rp.Direction, rp.Subject, rp.RatesFallbackSubject) - } else { - qry = fmt.Sprintf("INSERT INTO %s (tpid,tag,tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject) VALUES ", utils.TBL_TP_RATE_PROFILES) - // Using multiple values in query to spare some network processing time - for idx, rpa := range rp.RatingActivations { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator +func (self *SQLStorage) SetTPRatingProfiles(tpid string, rps map[string][]*RatingProfile) error { + if len(rps) == 0 { + return nil //Nothing to set + } + qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject) VALUES ", + utils.TBL_TP_RATE_PROFILES) + i := 0 + for rpId, rp := range rps { + for _, rpa := range rp { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } - qry += fmt.Sprintf("('%s', '%s', '%s', '%s', '%s', '%s', %d,'%s','%s')", rp.TPid, rp.RateProfileId, rp.Tenant, rp.TOR, rp.Direction, rp.Subject, rpa.ActivationTime, rpa.DestRateTimingId, rp.RatesFallbackSubject) + qry += fmt.Sprintf("('%s', '%s', '%s', '%s', '%s', '%s', %d,'%s','%s')", tpid, rpId, rpa.Tenant, rpa.TOR, rpa.Direction, + rpa.Subject, rpa.ActivationTime, rpa.DestRatesTimingTag, rpa.RatesFallbackSubject) + i++ } + } if _, err := self.Db.Exec(qry); err != nil { return err @@ -461,13 +470,13 @@ func (self *SQLStorage) SetTPRateProfile(rp *utils.TPRateProfile) error { return nil } -func (self *SQLStorage) GetTPRateProfile(tpid, rpId string) (*utils.TPRateProfile, error) { +func (self *SQLStorage) GetTPRatingProfile(tpid, rpId string) (*utils.TPRatingProfile, error) { rows, err := self.Db.Query(fmt.Sprintf("SELECT tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject FROM %s WHERE tpid='%s' AND tag='%s'", utils.TBL_TP_RATE_PROFILES, tpid, rpId)) if err != nil { return nil, err } defer rows.Close() - rp := &utils.TPRateProfile{TPid: tpid, RateProfileId: rpId} + rp := &utils.TPRatingProfile{TPid: tpid, RatingProfileId: rpId} i := 0 for rows.Next() { i++ //Keep here a reference so we know we got at least one result @@ -492,7 +501,7 @@ func (self *SQLStorage) GetTPRateProfile(tpid, rpId string) (*utils.TPRateProfil return rp, nil } -func (self *SQLStorage) GetTPRateProfileIds(filters *utils.AttrTPRateProfileIds) ([]string, error) { +func (self *SQLStorage) GetTPRatingProfileIds(filters *utils.AttrTPRatingProfileIds) ([]string, error) { qry := fmt.Sprintf("SELECT DISTINCT tag FROM %s where tpid='%s'", utils.TBL_TP_RATE_PROFILES, filters.TPid) if filters.Tenant != "" { qry += fmt.Sprintf(" AND tenant='%s'", filters.Tenant) @@ -618,13 +627,15 @@ func (self *SQLStorage) SetTPActionTimings(tpid string, ats map[string][]*utils. return nil //Nothing to set } qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,actions_tag,timing_tag,weight) VALUES ", utils.TBL_TP_ACTION_TIMINGS) + i := 0 for atId, atRows := range ats { - for idx, atsRow := range atRows { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + for _, atsRow := range atRows { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", tpid, atId, atsRow.ActionsId, atsRow.TimingId, atsRow.Weight) + i++ } } if _, err := self.Db.Exec(qry); err != nil { @@ -694,14 +705,16 @@ func (self *SQLStorage) SetTPActionTriggers(tpid string, ats map[string][]*Actio } qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,balance_tag,direction,threshold_type,threshold_value,destination_tag,actions_tag,weight) VALUES ", utils.TBL_TP_ACTION_TRIGGERS) + i := 0 for atId, atRows := range ats { - for idx, atsRow := range atRows { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator + for _, atsRow := range atRows { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s','%s', %f, '%s','%s',%f)", tpid, atId, atsRow.BalanceId, atsRow.Direction, atsRow.ThresholdType, atsRow.ThresholdValue, atsRow.DestinationId, atsRow.ActionsId, atsRow.Weight) + i++ } } if _, err := self.Db.Exec(qry); err != nil { @@ -749,12 +762,12 @@ func (self *SQLStorage) SetTPAccountActions(tpid string, aa map[string]*AccountA utils.TBL_TP_ACCOUNT_ACTIONS) i := 0 for aaId, aActs := range aa { - i++ if i != 1 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s','%s','%s','%s')", tpid, aaId, aActs.Tenant, aActs.Account, aActs.Direction, aActs.ActionTimingsTag, aActs.ActionTriggersTag) + i++ } if _, err := self.Db.Exec(qry); err != nil { return err @@ -1066,12 +1079,12 @@ func (self *SQLStorage) GetTpRatingProfiles(tpid, tag string) (map[string]*Ratin } key := fmt.Sprintf("%s:%s:%s:%s", direction, tenant, tor, subject) rp, ok := rpfs[key] - if !ok || rp.tag != tag { - rp = &RatingProfile{Id: key, tag: tag} + if !ok || rp.Tag != tag { + rp = &RatingProfile{Id: key, Tag: tag} rpfs[key] = rp } - rp.destRatesTimingTag = destrates_timing_tag - rp.activationTime = activation_time + rp.DestRatesTimingTag = destrates_timing_tag + rp.ActivationTime = activation_time if fallback_subject != "" { rp.FallbackKey = fmt.Sprintf("%s:%s:%s:%s", direction, tenant, tor, fallback_subject) } diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 9f45e32ae..23d35f6bb 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "log" "strconv" + "time" "github.com/cgrates/cgrates/utils" ) @@ -34,6 +35,7 @@ type TPCSVImporter struct { DirPath string // Directory path to import from Sep rune // Separator in the csv file Verbose bool // If true will print a detailed information instead of silently discarding it + ImportId string // Use this to differentiate between imports (eg: when autogenerating fields like RatingProfileId } // Maps csv file to handler which should process it. Defined like this since tests on 1.0.3 were failing on Travis. @@ -218,6 +220,47 @@ func (self *TPCSVImporter) importDestRateTimings(fn string) error { } func (self *TPCSVImporter) importRatingProfiles(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser( self.DirPath, fn ) + if err!=nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + tenant, tor, direction, subject, destRatesTimingTag, fallbacksubject := record[0], record[1], record[2], record[3], record[5], record[6] + at, err := time.Parse(time.RFC3339, record[4]) + if err != nil { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + rpTag := "TPCSV" //Autogenerate rating profile id + if self.ImportId != "" { + rpTag += "_"+self.ImportId + } + rp := &RatingProfile{Tag: rpTag, + Tenant: tenant, + TOR: tor, + Direction: direction, + Subject: subject, + ActivationTime: at.Unix(), + DestRatesTimingTag: destRatesTimingTag, + RatesFallbackSubject: fallbacksubject, + } + if err := self.StorDb.SetTPRatingProfiles( self.TPid, map[string][]*RatingProfile{rpTag:[]*RatingProfile{rp}}); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 89d396b33..03d92d7d1 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -60,9 +60,9 @@ type DestRateTiming struct { Weight float64 // Binding priority taken into consideration when more DestinationRates are active on a time slot } -type TPRateProfile struct { +type TPRatingProfile struct { TPid string // Tariff plan id - RateProfileId string // RateProfile id + RatingProfileId string // RatingProfile id Tenant string // Tenant's Id TOR string // TypeOfRecord Direction string // Traffic direction, OUT is the only one supported for now @@ -76,7 +76,7 @@ type RatingActivation struct { DestRateTimingId string // Id of DestRateTiming profile } -type AttrTPRateProfileIds struct { +type AttrTPRatingProfileIds struct { TPid string // Tariff plan id Tenant string // Tenant's Id TOR string // TypeOfRecord diff --git a/utils/consts.go b/utils/consts.go index 103805c34..bbd905f60 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -24,7 +24,7 @@ const ( TBL_TP_RATES = "tp_rates" TBL_TP_DESTINATION_RATES = "tp_destination_rates" TBL_TP_DESTRATE_TIMINGS = "tp_destrate_timings" - TBL_TP_RATE_PROFILES = "tp_rate_profiles" + TBL_TP_RATE_PROFILES = "tp_rating_profiles" TBL_TP_ACTIONS = "tp_actions" TBL_TP_ACTION_TIMINGS = "tp_action_timings" TBL_TP_ACTION_TRIGGERS = "tp_action_triggers" From d4017890ec04402fa2a5263ecefd1a384af8345a Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 29 Jul 2013 21:18:02 +0200 Subject: [PATCH 14/17] TPCSVImporter Actions, store refactoring, small API params changes --- apier/tpaccountactions.go | 23 ++- apier/tpactions.go | 23 ++- apier/tpdestinationrates.go | 6 +- apier/tpdestratetimings.go | 16 +- apier/tprates.go | 8 +- apier/tpratingprofiles.go | 26 +-- cmd/cgr-loader/cgr-loader.go | 30 +-- config/helpers.go | 11 +- .../mysql/create_tariffplan_tables.sql | 10 +- docs/api_tpactions.rst | 28 +-- engine/action.go | 2 + engine/loader_csv.go | 2 +- engine/loader_helpers.go | 86 +++++---- engine/ratingprofile.go | 8 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo.go | 2 +- engine/storage_redis.go | 2 +- engine/storage_sql.go | 55 +++--- engine/storage_utils.go | 3 +- engine/tpimporter_csv.go | 171 ++++++++++++------ utils/apitpdata.go | 8 +- utils/consts.go | 46 ++--- 23 files changed, 323 insertions(+), 247 deletions(-) diff --git a/apier/tpaccountactions.go b/apier/tpaccountactions.go index 69681f055..459a03645 100644 --- a/apier/tpaccountactions.go +++ b/apier/tpaccountactions.go @@ -25,11 +25,10 @@ import ( "github.com/cgrates/cgrates/utils" ) - // Creates a new AccountActions profile within a tariff plan func (self *Apier) SetTPAccountActions(attrs utils.ApiTPAccountActions, reply *string) error { if missing := utils.MissingStructFields(&attrs, - []string{"TPid", "AccountActionsId","Tenant","Account","Direction","ActionTimingsId","ActionTriggersId"}); len(missing) != 0 { + []string{"TPid", "AccountActionsId", "Tenant", "Account", "Direction", "ActionTimingsId", "ActionTriggersId"}); len(missing) != 0 { return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } if exists, err := self.StorDb.ExistsTPAccountActions(attrs.TPid, attrs.AccountActionsId); err != nil { @@ -38,9 +37,9 @@ func (self *Apier) SetTPAccountActions(attrs utils.ApiTPAccountActions, reply *s return errors.New(utils.ERR_DUPLICATE) } aa := map[string]*engine.AccountAction{ - attrs.AccountActionsId: &engine.AccountAction{Tenant: attrs.Tenant, Account: attrs.Account, Direction: attrs.Direction, - ActionTimingsTag: attrs.ActionTimingsId, ActionTriggersTag: attrs.ActionTriggersId}, - } + attrs.AccountActionsId: &engine.AccountAction{Tenant: attrs.Tenant, Account: attrs.Account, Direction: attrs.Direction, + ActionTimingsTag: attrs.ActionTimingsId, ActionTriggersTag: attrs.ActionTriggersId}, + } if err := self.StorDb.SetTPAccountActions(attrs.TPid, aa); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) @@ -64,13 +63,13 @@ func (self *Apier) GetTPAccountActions(attrs AttrGetTPAccountActions, reply *uti } else if len(aa) == 0 { return errors.New(utils.ERR_NOT_FOUND) } else { - *reply = utils.ApiTPAccountActions{TPid: attrs.TPid, - AccountActionsId: attrs.AccountActionsId, - Tenant: aa[attrs.AccountActionsId].Tenant, - Account:aa[attrs.AccountActionsId].Account, - Direction: aa[attrs.AccountActionsId].Direction, - ActionTimingsId: aa[attrs.AccountActionsId].ActionTimingsTag, - ActionTriggersId: aa[attrs.AccountActionsId].ActionTriggersTag } + *reply = utils.ApiTPAccountActions{TPid: attrs.TPid, + AccountActionsId: attrs.AccountActionsId, + Tenant: aa[attrs.AccountActionsId].Tenant, + Account: aa[attrs.AccountActionsId].Account, + Direction: aa[attrs.AccountActionsId].Direction, + ActionTimingsId: aa[attrs.AccountActionsId].ActionTimingsTag, + ActionTriggersId: aa[attrs.AccountActionsId].ActionTriggersTag} } return nil } diff --git a/apier/tpactions.go b/apier/tpactions.go index 0f043ad65..6f365704d 100644 --- a/apier/tpactions.go +++ b/apier/tpactions.go @@ -21,7 +21,9 @@ package apier import ( "errors" "fmt" + "time" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) // Creates a new Actions profile within a tariff plan @@ -31,8 +33,8 @@ func (self *Apier) SetTPActions(attrs utils.TPActions, reply *string) error { } for _, action := range attrs.Actions { requiredFields := []string{"Identifier", "Weight"} - if action.BalanceId != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions - requiredFields = append(requiredFields, "Direction", "Units", "ExpirationTime") + if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions + requiredFields = append(requiredFields, "Direction", "Units", "ExpiryTime") } if missing := utils.MissingStructFields(&action, requiredFields); len(missing) != 0 { return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing) @@ -43,7 +45,22 @@ func (self *Apier) SetTPActions(attrs utils.TPActions, reply *string) error { } else if exists { return errors.New(utils.ERR_DUPLICATE) } - if err := self.StorDb.SetTPActions(&attrs); err != nil { + acts := make([]*engine.Action, len(attrs.Actions)) + for idx, act := range attrs.Actions { + acts[idx] = &engine.Action{ + ActionType: act.Identifier, + BalanceId: act.BalanceType, + Direction: act.Direction, + Units: act.Units, + ExpirationDate: time.Unix(act.ExpiryTime,0), + DestinationTag: act.DestinationId, + RateType: act.RateType, + RateValue: act.Rate, + MinutesWeight: act.MinutesWeight, + Weight: act.Weight, + } + } + if err := self.StorDb.SetTPActions(attrs.TPid, map[string][]*engine.Action{attrs.ActionsId: acts}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/apier/tpdestinationrates.go b/apier/tpdestinationrates.go index e649cb8c5..874d4de33 100644 --- a/apier/tpdestinationrates.go +++ b/apier/tpdestinationrates.go @@ -23,8 +23,8 @@ package apier import ( "errors" "fmt" - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) // Creates a new DestinationRate profile within a tariff plan @@ -38,10 +38,10 @@ func (self *Apier) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *st return errors.New(utils.ERR_DUPLICATE) } drs := make([]*engine.DestinationRate, len(attrs.DestinationRates)) - for idx,dr := range attrs.DestinationRates { + for idx, dr := range attrs.DestinationRates { drs[idx] = &engine.DestinationRate{attrs.DestinationRateId, dr.DestinationId, dr.RateId, nil} } - if err := self.StorDb.SetTPDestinationRates( attrs.TPid, map[string][]*engine.DestinationRate{ attrs.DestinationRateId: drs } ); err != nil { + if err := self.StorDb.SetTPDestinationRates(attrs.TPid, map[string][]*engine.DestinationRate{attrs.DestinationRateId: drs}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/apier/tpdestratetimings.go b/apier/tpdestratetimings.go index 16ae3bd24..f357c81e9 100644 --- a/apier/tpdestratetimings.go +++ b/apier/tpdestratetimings.go @@ -23,8 +23,8 @@ package apier import ( "errors" "fmt" - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) // Creates a new DestinationRateTiming profile within a tariff plan @@ -38,14 +38,14 @@ func (self *Apier) SetTPDestRateTiming(attrs utils.TPDestRateTiming, reply *stri return errors.New(utils.ERR_DUPLICATE) } drts := make([]*engine.DestinationRateTiming, len(attrs.DestRateTimings)) - for idx,drt := range attrs.DestRateTimings { - drts[idx] = &engine.DestinationRateTiming{Tag: attrs.DestRateTimingId, - DestinationRatesTag: drt.DestRatesId, - Weight: drt.Weight, - TimingsTag: drt.TimingId, - } + for idx, drt := range attrs.DestRateTimings { + drts[idx] = &engine.DestinationRateTiming{Tag: attrs.DestRateTimingId, + DestinationRatesTag: drt.DestRatesId, + Weight: drt.Weight, + TimingsTag: drt.TimingId, + } } - if err := self.StorDb.SetTPDestRateTimings( attrs.TPid, map[string][]*engine.DestinationRateTiming{ attrs.DestRateTimingId: drts } ); err != nil { + if err := self.StorDb.SetTPDestRateTimings(attrs.TPid, map[string][]*engine.DestinationRateTiming{attrs.DestRateTimingId: drts}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/apier/tprates.go b/apier/tprates.go index 41790aa6b..d55192cb7 100644 --- a/apier/tprates.go +++ b/apier/tprates.go @@ -23,8 +23,8 @@ package apier import ( "errors" "fmt" - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) // Creates a new rate within a tariff plan @@ -38,11 +38,11 @@ func (self *Apier) SetTPRate(attrs utils.TPRate, reply *string) error { return errors.New(utils.ERR_DUPLICATE) } rts := make([]*engine.Rate, len(attrs.RateSlots)) - for idx,rtSlot := range attrs.RateSlots { - rts[idx] = &engine.Rate{attrs.RateId, rtSlot.ConnectFee, rtSlot.Rate, float64(rtSlot.RatedUnits), + for idx, rtSlot := range attrs.RateSlots { + rts[idx] = &engine.Rate{attrs.RateId, rtSlot.ConnectFee, rtSlot.Rate, float64(rtSlot.RatedUnits), float64(rtSlot.RateIncrements), float64(rtSlot.GroupInterval), rtSlot.RoundingMethod, rtSlot.RoundingDecimals, rtSlot.Weight} } - if err := self.StorDb.SetTPRates( attrs.TPid, map[string][]*engine.Rate{ attrs.RateId: rts } ); err != nil { + if err := self.StorDb.SetTPRates(attrs.TPid, map[string][]*engine.Rate{attrs.RateId: rts}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/apier/tpratingprofiles.go b/apier/tpratingprofiles.go index ca6a6908c..928c0d367 100644 --- a/apier/tpratingprofiles.go +++ b/apier/tpratingprofiles.go @@ -23,8 +23,8 @@ package apier import ( "errors" "fmt" - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) // Creates a new RatingProfile within a tariff plan @@ -38,18 +38,18 @@ func (self *Apier) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string return errors.New(utils.ERR_DUPLICATE) } rps := make([]*engine.RatingProfile, len(attrs.RatingActivations)) - for idx,ra := range attrs.RatingActivations { - rps[idx] = &engine.RatingProfile{Tag: attrs.RatingProfileId, - Tenant: attrs.Tenant, - TOR: attrs.TOR, - Direction: attrs.Direction, - Subject: attrs.Subject, - ActivationTime: ra.ActivationTime, - DestRatesTimingTag: ra.DestRateTimingId, - RatesFallbackSubject: attrs.RatesFallbackSubject, - } + for idx, ra := range attrs.RatingActivations { + rps[idx] = &engine.RatingProfile{Tag: attrs.RatingProfileId, + Tenant: attrs.Tenant, + TOR: attrs.TOR, + Direction: attrs.Direction, + Subject: attrs.Subject, + ActivationTime: ra.ActivationTime, + DestRatesTimingTag: ra.DestRateTimingId, + RatesFallbackSubject: attrs.RatesFallbackSubject, + } } - if err := self.StorDb.SetTPRatingProfiles( attrs.TPid, map[string][]*engine.RatingProfile{ attrs.RatingProfileId: rps } ); err != nil { + if err := self.StorDb.SetTPRatingProfiles(attrs.TPid, map[string][]*engine.RatingProfile{attrs.RatingProfileId: rps}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" @@ -57,7 +57,7 @@ func (self *Apier) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string } type AttrGetTPRatingProfile struct { - TPid string // Tariff plan id + TPid string // Tariff plan id RatingProfileId string // RatingProfile id } diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 1a958f733..a520f5aa8 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -21,22 +21,22 @@ package main import ( "flag" "fmt" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/cgrates/config" "log" "path" ) var ( //separator = flag.String("separator", ",", "Default field separator") - cgrConfig,_ = config.NewDefaultCGRConfig() + cgrConfig, _ = config.NewDefaultCGRConfig() data_db_type = flag.String("datadb_type", cgrConfig.DataDBType, "The type of the dataDb database (redis|mongo|postgres|mysql)") data_db_host = flag.String("datadb_host", cgrConfig.DataDBHost, "The dataDb host to connect to.") data_db_port = flag.String("datadb_port", cgrConfig.DataDBPort, "The dataDb port to bind to.") data_db_name = flag.String("datadb_name", cgrConfig.DataDBName, "The name/number of the dataDb to connect to.") data_db_user = flag.String("datadb_user", cgrConfig.DataDBUser, "The dataDb user to sign in as.") - data_db_pass = flag.String("datadb_passwd", cgrConfig.DataDBPass, "The dataDb user's password.") + data_db_pass = flag.String("datadb_passwd", cgrConfig.DataDBPass, "The dataDb user's password.") stor_db_type = flag.String("stordb_type", cgrConfig.StorDBType, "The type of the storDb database (redis|mongo|postgres|mysql)") stor_db_host = flag.String("stordb_host", cgrConfig.StorDBHost, "The storDb host to connect to.") @@ -45,14 +45,14 @@ var ( stor_db_user = flag.String("stordb_user", cgrConfig.StorDBUser, "The storDb user to sign in as.") stor_db_pass = flag.String("stordb_passwd", cgrConfig.StorDBPass, "The storDb user's password.") - flush = flag.Bool("flush", false, "Flush the database before importing") - tpid = flag.String("tpid", "", "The tariff plan id from the database") - dataPath = flag.String("path", ".", "The path containing the data files") - version = flag.Bool("version", false, "Prints the application version.") - verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") + flush = flag.Bool("flush", false, "Flush the database before importing") + tpid = flag.String("tpid", "", "The tariff plan id from the database") + dataPath = flag.String("path", ".", "The path containing the data files") + version = flag.Bool("version", false, "Prints the application version.") + verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") fromStorDb = flag.Bool("from-stordb", false, "Load the tariff plan from storDb to dataDb") - toStorDb = flag.Bool("to-stordb", false, "Import the tariff plan from files to storDb") - runId = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields") + toStorDb = flag.Bool("to-stordb", false, "Import the tariff plan from files to storDb") + runId = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields") ) func main() { @@ -73,11 +73,13 @@ func main() { dataDb, errDataDb = engine.ConfigureDatabase(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) } // Defer databases opened to be closed when we are done - for _,db := range []engine.DataStorage{ dataDb, storDb } { - if db != nil { defer db.Close() } + for _, db := range []engine.DataStorage{dataDb, storDb} { + if db != nil { + defer db.Close() + } } // Stop on db errors - for _,err = range []error{errDataDb, errStorDb} { + for _, err = range []error{errDataDb, errStorDb} { if err != nil { log.Fatalf("Could not open database connection: %v", err) } @@ -89,7 +91,7 @@ func main() { if *tpid == "" { log.Fatal("TPid required, please define it via *-tpid* command argument.") } - csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose, *runId } + csvImporter := engine.TPCSVImporter{*tpid, storDb, *dataPath, ',', *verbose, *runId} if errImport := csvImporter.Run(); errImport != nil { log.Fatal(errImport) } diff --git a/config/helpers.go b/config/helpers.go index b76663d4e..646ebc30f 100644 --- a/config/helpers.go +++ b/config/helpers.go @@ -24,21 +24,20 @@ import ( "strings" ) - // Adds support for slice values in config -func ConfigSlice( c *conf.ConfigFile, section, valName string ) ([]string, error) { +func ConfigSlice(c *conf.ConfigFile, section, valName string) ([]string, error) { sliceStr, errGet := c.GetString(section, valName) if errGet != nil { return nil, errGet } - cfgValStrs := strings.Split(sliceStr, ",") // If need arrises, we can make the separator configurable - if len(cfgValStrs)==1 && cfgValStrs[0]=="" { // Prevents returning iterable with empty value + cfgValStrs := strings.Split(sliceStr, ",") // If need arrises, we can make the separator configurable + if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value return []string{}, nil } - for _,elm := range cfgValStrs { + for _, elm := range cfgValStrs { if elm == "" { //One empty element is presented when splitting empty string return nil, errors.New("Empty values in config slice") - + } } return cfgValStrs, nil diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 4a7fa78b0..e0051da29 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -115,10 +115,10 @@ CREATE TABLE `tp_actions` ( `tpid` char(40) NOT NULL, `tag` varchar(24) NOT NULL, `action` varchar(24) NOT NULL, - `balance_tag` varchar(24) NOT NULL, + `balance_type` varchar(24) NOT NULL, `direction` varchar(8) NOT NULL, `units` DECIMAL(5,2) NOT NULL, - `expiration_time` int(11) NOT NULL, + `expiry_time` int(16) NOT NULL, `destination_tag` varchar(24) NOT NULL, `rate_type` varchar(8) NOT NULL, `rate` DECIMAL(5,4) NOT NULL, @@ -126,7 +126,7 @@ CREATE TABLE `tp_actions` ( `weight` DECIMAL(5,2) NOT NULL, PRIMARY KEY (`id`), KEY `tpid` (`tpid`), - UNIQUE KEY `unique_action` (`tpid`,`tag`,`action`,`balance_tag`,`direction`,`expiration_time`,`destination_tag`,`rate_type`,`minutes_weight`,`weight`) + UNIQUE KEY `unique_action` (`tpid`,`tag`,`action`,`balance_type`,`direction`,`expiry_time`,`destination_tag`,`rate_type`,`minutes_weight`,`weight`) ); -- @@ -153,7 +153,7 @@ CREATE TABLE `tp_action_triggers` ( `id` int(11) NOT NULL AUTO_INCREMENT, `tpid` char(40) NOT NULL, `tag` varchar(24) NOT NULL, - `balance_tag` varchar(24) NOT NULL, + `balance_type` varchar(24) NOT NULL, `direction` varchar(8) NOT NULL, `threshold_type` char(11) NOT NULL, `threshold_value` DECIMAL(5,4) NOT NULL, @@ -162,7 +162,7 @@ CREATE TABLE `tp_action_triggers` ( `weight` DECIMAL(5,2) NOT NULL, PRIMARY KEY (`id`), KEY `tpid` (`tpid`), - UNIQUE KEY `unique_trigger_definition` (`tpid`,`tag`,`balance_tag`,`direction`,`threshold_type`,`threshold_value`,`destination_tag`,`actions_tag`) + UNIQUE KEY `unique_trigger_definition` (`tpid`,`tag`,`balance_type`,`direction`,`threshold_type`,`threshold_value`,`destination_tag`,`actions_tag`) ); -- diff --git a/docs/api_tpactions.rst b/docs/api_tpactions.rst index 8d921e989..4bcebbc46 100644 --- a/docs/api_tpactions.rst +++ b/docs/api_tpactions.rst @@ -16,16 +16,16 @@ Creates a new Actions profile within a tariff plan. type Action struct { Identifier string // Identifier mapped in the code - BalanceId string // Type of balance the action will operate on + BalanceType string // Type of balance the action will operate on Direction string // Balance direction Units float64 // Number of units to add/deduct - ExpirationTime int64 // Time when the units will expire + ExpiryTime int64 // Time when the units will expire DestinationId string // Destination profile id - RateType string // Type of price + RateType string // Type of rate <*absolute|*percent> Rate float64 // Price value MinutesWeight float64 // Minutes weight Weight float64 // Action's weight - } + } Mandatory parameters: ``[]string{"TPid", "ActionsId", "Actions", "Identifier", "Weight"}`` @@ -39,14 +39,14 @@ Creates a new Actions profile within a tariff plan. { "Actions": [ { - "BalanceId": "MONEY", + "BalanceType": "*monetary", "DestinationId": "CGRATES_NET", - "Direction": "OUT", - "ExpirationTime": 1374082259, + "Direction": "*out", + "ExpiryTime": 1374082259, "Identifier": "TOPUP_RESET", "MinutesWeight": 10, "Rate": 0.12, - "RateType": "ABSOLUTE", + "RateType": "*absolute", "Units": 10, "Weight": 10 } @@ -129,10 +129,10 @@ Queries specific Actions profile on tariff plan. type Action struct { Identifier string // Identifier mapped in the code - BalanceId string // Type of balance the action will operate on + BalanceType string // Type of balance the action will operate on Direction string // Balance direction Units float64 // Number of units to add/deduct - ExpirationTime int64 // Time when the units will expire + ExpiryTime int64 // Time when the units will expire DestinationId string // Destination profile id RateType string // Type of price Rate float64 // Price value @@ -149,14 +149,14 @@ Queries specific Actions profile on tariff plan. "result": { "Actions": [ { - "BalanceId": "MONEY", + "BalanceType": "*monetary", "DestinationId": "CGRATES_NET", - "Direction": "OUT", - "ExpirationTime": 1374082259, + "Direction": "*out", + "ExpiryTime": 1374082259, "Identifier": "TOPUP_RESET", "MinutesWeight": 10, "Rate": 0.12, - "RateType": "ABSOLUTE", + "RateType": "*absolute", "Units": 10, "Weight": 10 } diff --git a/engine/action.go b/engine/action.go index 227277e56..65e38c3d1 100644 --- a/engine/action.go +++ b/engine/action.go @@ -36,6 +36,8 @@ type Action struct { Units float64 Weight float64 MinuteBucket *MinuteBucket + DestinationTag, RateType string // From here for import/load purposes only + RateValue, MinutesWeight float64 } const ( diff --git a/engine/loader_csv.go b/engine/loader_csv.go index 96c4c7da3..ec4213b4e 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -349,7 +349,7 @@ func (csvr *CSVReader) LoadActions() (err error) { if err != nil { return errors.New(fmt.Sprintf("Could not parse action units: %v", err)) } - var expiryTime time.Time // Empty initialized time represents never expire + var expiryTime time.Time // Empty initialized time represents never expire if record[5] != "*unlimited" { // ToDo: Expand here for other meta tags or go way of adding time for expiry expiryTime, err = time.Parse(time.RFC3339, record[5]) if err != nil { diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 919ebb120..07977005e 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -20,15 +20,15 @@ package engine import ( "bufio" - "path" "errors" "fmt" + "github.com/cgrates/cgrates/utils" "log" "os" + "path" "regexp" "strconv" "strings" - "github.com/cgrates/cgrates/utils" ) type TPLoader interface { @@ -202,45 +202,43 @@ func ValidateCSVData(fn string, re *regexp.Regexp) (err error) { } type FileLineRegexValidator struct { - FieldsPerRecord int // Number of fields in one record, useful for crosschecks - Rule *regexp.Regexp // Regexp rule - Message string // Pass this message as helper + FieldsPerRecord int // Number of fields in one record, useful for crosschecks + Rule *regexp.Regexp // Regexp rule + Message string // Pass this message as helper } -var FileValidators = map[string]*FileLineRegexValidator{ - utils.DESTINATIONS_CSV: &FileLineRegexValidator{ utils.DESTINATIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\+?\d+.?\d*){1}$`), - "Tag([0-9A-Za-z_]),Prefix([0-9])"}, - utils.TIMINGS_CSV: &FileLineRegexValidator{ utils.TIMINGS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*any\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), - "Tag([0-9A-Za-z_]),Years([0-9;]|*all|),Months([0-9;]|*all|),MonthDays([0-9;]|*all|),WeekDays([0-9;]|*all|),Time([0-9:]|*asap)"}, - utils.RATES_CSV: &FileLineRegexValidator{ utils.RATES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\.?\d*,){5}(?:\*\w+,){1}(?:\d+\.?\d*,?){2}$`), - "Tag([0-9A-Za-z_]),ConnectFee([0-9.]),Rate([0-9.]),RatedUnits([0-9.]),RateIncrement([0-9.])"}, - utils.DESTINATION_RATES_CSV: &FileLineRegexValidator{ utils.DESTINATION_RATES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,?\s*){3}$`), - "Tag([0-9A-Za-z_]),DestinationsTag([0-9A-Za-z_]),RateTag([0-9A-Za-z_])"}, - utils.DESTRATE_TIMINGS_CSV: &FileLineRegexValidator{ utils.DESTRATE_TIMINGS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`), - "Tag([0-9A-Za-z_]),DestinationRatesTag([0-9A-Za-z_]),TimingProfile([0-9A-Za-z_]),Weight([0-9.])"}, - utils.RATE_PROFILES_CSV: &FileLineRegexValidator{ utils.RATE_PROFILES_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\*out\s*,\s*){1}(?:\*any\s*,\s*|\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}(?:\w*\s*,?\s*){2}$`), - "Tenant([0-9A-Za-z_]),TOR([0-9A-Za-z_]),Direction(*out),Subject([0-9A-Za-z_]|*all),RatesFallbackSubject([0-9A-Za-z_]|),RatesTimingTag([0-9A-Za-z_]),ActivationTime([0-9T:X])"}, - utils.ACTIONS_CSV: &FileLineRegexValidator{ utils.ACTIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*),(?:\*\w+\s*),(?:\*\w+\s*),(?:\*out\s*),(?:\d+\s*),(?:\*\w+\s*|\+\d+[smh]\s*|\d+\s*),(?:\*any|\w+\s*),(?:\*\w+\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)$`), - "Tag([0-9A-Za-z_]),Action([0-9A-Za-z_]),BalanceType([*a-z_]),Direction(*out),Units([0-9]),ExpiryTime(*[a-z_]|+[0-9][smh]|[0-9])DestinationTag([0-9A-Za-z_]|*all),RateType(*[a-z_]),RateValue([0-9.]),MinutesWeight([0-9.]),Weight([0-9.])"}, - utils.ACTION_TIMINGS_CSV: &FileLineRegexValidator{ utils.ACTION_TIMINGS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`), - "Tag([0-9A-Za-z_]),ActionsTag([0-9A-Za-z_]),TimingTag([0-9A-Za-z_]),Weight([0-9.])"}, - utils.ACTION_TRIGGERS_CSV: &FileLineRegexValidator{ utils.ACTION_TRIGGERS_NRCOLS, - regexp.MustCompile(`(?:\w+),(?:\*\w+),(?:\*out),(?:\*\w+),(?:\d+\.?\d*),(?:\w+|\*any)?,(?:\w+),(?:\d+\.?\d*)$`), - "Tag([0-9A-Za-z_]),BalanceType(*[a-z_]),Direction(*out),ThresholdType(*[a-z_]),ThresholdValue([0-9]+),DestinationTag([0-9A-Za-z_]|*all),ActionsTag([0-9A-Za-z_]),Weight([0-9]+)"}, - utils.ACCOUNT_ACTIONS_CSV: &FileLineRegexValidator{ utils.ACCOUNT_ACTIONS_NRCOLS, - regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\*out\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), - "Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"}, - } - - +var FileValidators = map[string]*FileLineRegexValidator{ + utils.DESTINATIONS_CSV: &FileLineRegexValidator{utils.DESTINATIONS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\+?\d+.?\d*){1}$`), + "Tag([0-9A-Za-z_]),Prefix([0-9])"}, + utils.TIMINGS_CSV: &FileLineRegexValidator{utils.TIMINGS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*any\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`), + "Tag([0-9A-Za-z_]),Years([0-9;]|*all|),Months([0-9;]|*all|),MonthDays([0-9;]|*all|),WeekDays([0-9;]|*all|),Time([0-9:]|*asap)"}, + utils.RATES_CSV: &FileLineRegexValidator{utils.RATES_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\.?\d*,){5}(?:\*\w+,){1}(?:\d+\.?\d*,?){2}$`), + "Tag([0-9A-Za-z_]),ConnectFee([0-9.]),Rate([0-9.]),RatedUnits([0-9.]),RateIncrement([0-9.])"}, + utils.DESTINATION_RATES_CSV: &FileLineRegexValidator{utils.DESTINATION_RATES_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,?\s*){3}$`), + "Tag([0-9A-Za-z_]),DestinationsTag([0-9A-Za-z_]),RateTag([0-9A-Za-z_])"}, + utils.DESTRATE_TIMINGS_CSV: &FileLineRegexValidator{utils.DESTRATE_TIMINGS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`), + "Tag([0-9A-Za-z_]),DestinationRatesTag([0-9A-Za-z_]),TimingProfile([0-9A-Za-z_]),Weight([0-9.])"}, + utils.RATE_PROFILES_CSV: &FileLineRegexValidator{utils.RATE_PROFILES_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\*out\s*,\s*){1}(?:\*any\s*,\s*|\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}(?:\w*\s*,?\s*){2}$`), + "Tenant([0-9A-Za-z_]),TOR([0-9A-Za-z_]),Direction(*out),Subject([0-9A-Za-z_]|*all),RatesFallbackSubject([0-9A-Za-z_]|),RatesTimingTag([0-9A-Za-z_]),ActivationTime([0-9T:X])"}, + utils.ACTIONS_CSV: &FileLineRegexValidator{utils.ACTIONS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*),(?:\*\w+\s*),(?:\*\w+\s*),(?:\*out\s*),(?:\d+\s*),(?:\*\w+\s*|\+\d+[smh]\s*|\d+\s*),(?:\*any|\w+\s*),(?:\*\w+\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)$`), + "Tag([0-9A-Za-z_]),Action([0-9A-Za-z_]),BalanceType([*a-z_]),Direction(*out),Units([0-9]),ExpiryTime(*[a-z_]|+[0-9][smh]|[0-9])DestinationTag([0-9A-Za-z_]|*all),RateType(*[a-z_]),RateValue([0-9.]),MinutesWeight([0-9.]),Weight([0-9.])"}, + utils.ACTION_TIMINGS_CSV: &FileLineRegexValidator{utils.ACTION_TIMINGS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`), + "Tag([0-9A-Za-z_]),ActionsTag([0-9A-Za-z_]),TimingTag([0-9A-Za-z_]),Weight([0-9.])"}, + utils.ACTION_TRIGGERS_CSV: &FileLineRegexValidator{utils.ACTION_TRIGGERS_NRCOLS, + regexp.MustCompile(`(?:\w+),(?:\*\w+),(?:\*out),(?:\*\w+),(?:\d+\.?\d*),(?:\w+|\*any)?,(?:\w+),(?:\d+\.?\d*)$`), + "Tag([0-9A-Za-z_]),BalanceType(*[a-z_]),Direction(*out),ThresholdType(*[a-z_]),ThresholdValue([0-9]+),DestinationTag([0-9A-Za-z_]|*all),ActionsTag([0-9A-Za-z_]),Weight([0-9]+)"}, + utils.ACCOUNT_ACTIONS_CSV: &FileLineRegexValidator{utils.ACCOUNT_ACTIONS_NRCOLS, + regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\*out\s*,\s*){1}(?:\w+\s*,?\s*){2}$`), + "Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"}, +} func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { validator, hasValidator := FileValidators[fileName] @@ -248,7 +246,7 @@ func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { return nil, fmt.Errorf("No validator found for file <%s>", fileName) } // Open the file here - fin, err := os.Open( path.Join(dirPath, fileName) ) + fin, err := os.Open(path.Join(dirPath, fileName)) if err != nil { return nil, err } @@ -259,11 +257,11 @@ func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { // Opens the connection to a file and returns the parsed lines one by one when ParseNextLine() is called type TPCSVFileParser struct { - validator *FileLineRegexValidator // Row validator - reader *bufio.Reader // Reader to the file we are interested in + validator *FileLineRegexValidator // Row validator + reader *bufio.Reader // Reader to the file we are interested in } -func (self *TPCSVFileParser) ParseNextLine() ( []string, error ) { +func (self *TPCSVFileParser) ParseNextLine() ([]string, error) { line, truncated, err := self.reader.ReadLine() if err != nil { return nil, err @@ -279,7 +277,7 @@ func (self *TPCSVFileParser) ParseNextLine() ( []string, error ) { return nil, fmt.Errorf("Invalid line, <%s>", self.validator.Message) } // Open csv reader directly on string line - csvReader, _, err := openStringCSVReader( string(line), ',', self.validator.FieldsPerRecord ) + csvReader, _, err := openStringCSVReader(string(line), ',', self.validator.FieldsPerRecord) if err != nil { return nil, err } diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 8105e72e4..431d4fd2a 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -29,11 +29,11 @@ const ( ) type RatingProfile struct { - Id string - FallbackKey string // FallbackKey is used as complete combination of Tenant:TOR:Direction:Subject - DestinationMap map[string][]*ActivationPeriod + Id string + FallbackKey string // FallbackKey is used as complete combination of Tenant:TOR:Direction:Subject + DestinationMap map[string][]*ActivationPeriod Tag, Tenant, TOR, Direction, Subject, DestRatesTimingTag, RatesFallbackSubject string // used only for loading - ActivationTime int64 + ActivationTime int64 } // Adds an activation period that applyes to current rating profile if not already present. diff --git a/engine/storage_interface.go b/engine/storage_interface.go index dd7570213..6f25d34bf 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -84,7 +84,7 @@ type DataStorage interface { GetTPRatingProfile(string, string) (*utils.TPRatingProfile, error) GetTPRatingProfileIds(*utils.AttrTPRatingProfileIds) ([]string, error) ExistsTPActions(string, string) (bool, error) - SetTPActions(*utils.TPActions) error + SetTPActions(string, map[string][]*Action) error GetTPActions(string, string) (*utils.TPActions, error) GetTPActionIds(string) ([]string, error) ExistsTPActionTimings(string, string) (bool, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 051b10745..d80958d1d 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -177,7 +177,7 @@ func (ms *MapStorage) ExistsTPActions(tpid, aId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPActions(ap *utils.TPActions) error { +func (ms *MapStorage) SetTPActions(tpid string, acts map[string][]*Action) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index a73bb6c0a..04c748ee3 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -252,7 +252,7 @@ func (ms *MongoStorage) ExistsTPActions(tpid, aId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPActions(ap *utils.TPActions) error { +func (ms *MongoStorage) SetTPActions(tpid string, acts map[string][]*Action) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ada155c96..39aacbdce 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -207,7 +207,7 @@ func (rs *RedisStorage) ExistsTPActions(tpid, aId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPActions(ap *utils.TPActions) error { +func (rs *RedisStorage) SetTPActions(tpid string, acts map[string][]*Action) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index e90b5dfbf..55e6986e7 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -222,7 +222,7 @@ func (self *SQLStorage) SetTPRates(tpid string, rts map[string][]*Rate) error { qry += "," } qry += fmt.Sprintf("('%s', '%s', %f, %f, %d, %d,%d,'%s', %d, %f)", - tpid, rtId, rt.ConnectFee, rt.Price, int(rt.PricedUnits), int(rt.RateIncrements), int(rt.GroupInterval), + tpid, rtId, rt.ConnectFee, rt.Price, int(rt.PricedUnits), int(rt.RateIncrements), int(rt.GroupInterval), rt.RoundingMethod, rt.RoundingDecimals, rt.Weight) i++ } @@ -250,7 +250,7 @@ func (self *SQLStorage) GetTPRate(tpid, rtId string) (*utils.TPRate, error) { if err != nil { return nil, err } - rt.RateSlots = append(rt.RateSlots, utils.RateSlot{connectFee, rate, ratedUnits, rateIncrements, groupInterval, + rt.RateSlots = append(rt.RateSlots, utils.RateSlot{connectFee, rate, ratedUnits, rateIncrements, groupInterval, roundingMethod, roundingDecimals, weight}) } if i == 0 { @@ -376,7 +376,7 @@ func (self *SQLStorage) SetTPDestRateTimings(tpid string, drts map[string][]*Des i := 0 for drtId, drtRows := range drts { for _, drt := range drtRows { - if i!=0 { //Consecutive values after the first will be prefixed with "," as separator + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", @@ -450,7 +450,7 @@ func (self *SQLStorage) SetTPRatingProfiles(tpid string, rps map[string][]*Ratin if len(rps) == 0 { return nil //Nothing to set } - qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject) VALUES ", + qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,tenant,tor,direction,subject,activation_time,destrates_timing_tag,rates_fallback_subject) VALUES ", utils.TBL_TP_RATE_PROFILES) i := 0 for rpId, rp := range rps { @@ -458,11 +458,11 @@ func (self *SQLStorage) SetTPRatingProfiles(tpid string, rps map[string][]*Ratin if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } - qry += fmt.Sprintf("('%s', '%s', '%s', '%s', '%s', '%s', %d,'%s','%s')", tpid, rpId, rpa.Tenant, rpa.TOR, rpa.Direction, + qry += fmt.Sprintf("('%s', '%s', '%s', '%s', '%s', '%s', %d,'%s','%s')", tpid, rpId, rpa.Tenant, rpa.TOR, rpa.Direction, rpa.Subject, rpa.ActivationTime, rpa.DestRatesTimingTag, rpa.RatesFallbackSubject) i++ } - + } if _, err := self.Db.Exec(qry); err != nil { return err @@ -546,19 +546,26 @@ func (self *SQLStorage) ExistsTPActions(tpid, actsId string) (bool, error) { return exists, nil } -func (self *SQLStorage) SetTPActions(acts *utils.TPActions) error { - if len(acts.Actions) == 0 { +func (self *SQLStorage) SetTPActions(tpid string, acts map[string][]*Action) error { + if len(acts) == 0 { return nil //Nothing to set } - // Using multiple values in query to spare some network processing time - qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,action,balance_tag,direction,units,expiration_time,destination_tag,rate_type,rate, minutes_weight,weight) VALUES ", utils.TBL_TP_ACTIONS) - for idx, act := range acts.Actions { - if idx != 0 { //Consecutive values after the first will be prefixed with "," as separator - qry += "," + qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,action,balance_type,direction,units,expiry_time,destination_tag,rate_type,rate, minutes_weight,weight) VALUES ", utils.TBL_TP_ACTIONS) + i := 0 + for actId, actRows := range acts { + for _, act := range actRows { + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator + qry += "," + } + var expTime int64 + if !act.ExpirationDate.IsZero() { + expTime = act.ExpirationDate.Unix() + } + qry += fmt.Sprintf("('%s','%s','%s','%s','%s',%f,%d,'%s','%s',%f,%f,%f)", + tpid, actId, act.ActionType, act.BalanceId, act.Direction, act.Units, expTime, + act.DestinationTag, act.RateType, act.RateValue, act.MinutesWeight, act.Weight) + i++ } - qry += fmt.Sprintf("('%s','%s','%s','%s','%s',%f,%d,'%s','%s',%f,%f,%f)", - acts.TPid, acts.ActionsId, act.Identifier, act.BalanceId, act.Direction, act.Units, act.ExpirationTime, - act.DestinationId, act.RateType, act.Rate, act.MinutesWeight, act.Weight) } if _, err := self.Db.Exec(qry); err != nil { return err @@ -567,7 +574,7 @@ func (self *SQLStorage) SetTPActions(acts *utils.TPActions) error { } func (self *SQLStorage) GetTPActions(tpid, actsId string) (*utils.TPActions, error) { - rows, err := self.Db.Query(fmt.Sprintf("SELECT action,balance_tag,direction,units,expiration_time,destination_tag,rate_type,rate, minutes_weight,weight FROM %s WHERE tpid='%s' AND tag='%s'", utils.TBL_TP_ACTIONS, tpid, actsId)) + rows, err := self.Db.Query(fmt.Sprintf("SELECT action,balance_type,direction,units,expiry_time,destination_tag,rate_type,rate, minutes_weight,weight FROM %s WHERE tpid='%s' AND tag='%s'", utils.TBL_TP_ACTIONS, tpid, actsId)) if err != nil { return nil, err } @@ -703,7 +710,7 @@ func (self *SQLStorage) SetTPActionTriggers(tpid string, ats map[string][]*Actio if len(ats) == 0 { return nil //Nothing to set } - qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,balance_tag,direction,threshold_type,threshold_value,destination_tag,actions_tag,weight) VALUES ", + qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,balance_type,direction,threshold_type,threshold_value,destination_tag,actions_tag,weight) VALUES ", utils.TBL_TP_ACTION_TRIGGERS) i := 0 for atId, atRows := range ats { @@ -1105,8 +1112,8 @@ func (self *SQLStorage) GetTpActions(tpid, tag string) (map[string][]*Action, er for rows.Next() { var id int var units, rate, minutes_weight, weight float64 - var tpid, tag, action, balance_tag, direction, destinations_tag, rate_type, expirationDate string - if err := rows.Scan(&id, &tpid, &tag, &action, &balance_tag, &direction, &units, &expirationDate, &destinations_tag, &rate_type, &rate, &minutes_weight, &weight); err != nil { + var tpid, tag, action, balance_type, direction, destinations_tag, rate_type, expirationDate string + if err := rows.Scan(&id, &tpid, &tag, &action, &balance_type, &direction, &units, &expirationDate, &destinations_tag, &rate_type, &rate, &minutes_weight, &weight); err != nil { return nil, err } unix, err := strconv.ParseInt(expirationDate, 10, 64) @@ -1115,10 +1122,10 @@ func (self *SQLStorage) GetTpActions(tpid, tag string) (map[string][]*Action, er } expDate := time.Unix(unix, 0) var a *Action - if balance_tag != MINUTES { + if balance_type != MINUTES { a = &Action{ ActionType: action, - BalanceId: balance_tag, + BalanceId: balance_type, Direction: direction, Units: units, ExpirationDate: expDate, @@ -1128,7 +1135,7 @@ func (self *SQLStorage) GetTpActions(tpid, tag string) (map[string][]*Action, er a = &Action{ Id: utils.GenUUID(), ActionType: action, - BalanceId: balance_tag, + BalanceId: balance_type, Direction: direction, Weight: weight, ExpirationDate: expDate, @@ -1179,7 +1186,7 @@ func (self *SQLStorage) GetTpActionTimings(tpid, tag string) (ats map[string][]* func (self *SQLStorage) GetTpActionTriggers(tpid, tag string) (map[string][]*ActionTrigger, error) { ats := make(map[string][]*ActionTrigger) - q := fmt.Sprintf("SELECT tpid,tag,balance_tag,direction,threshold_type,threshold_value,destination_tag,actions_tag,weight FROM %s WHERE tpid='%s'", + q := fmt.Sprintf("SELECT tpid,tag,balance_type,direction,threshold_type,threshold_value,destination_tag,actions_tag,weight FROM %s WHERE tpid='%s'", utils.TBL_TP_ACTION_TRIGGERS, tpid) if tag != "" { q += fmt.Sprintf(" AND tag='%s'", tag) diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 7dc1ad7b3..b791f056a 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -19,9 +19,9 @@ along with this program. If not, see package engine import ( - "strconv" "errors" "github.com/cgrates/cgrates/utils" + "strconv" ) // Various helpers to deal with database @@ -53,4 +53,3 @@ func ConfigureDatabase(db_type, host, port, name, user, pass string) (db DataSto } return db, nil } - diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 23d35f6bb..2c9fb2d0a 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -19,48 +19,47 @@ along with this program. If not, see package engine import ( + "github.com/cgrates/cgrates/utils" "io" "io/ioutil" "log" "strconv" "time" - "github.com/cgrates/cgrates/utils" ) - // Import tariff plan from csv into storDb type TPCSVImporter struct { - TPid string // Load data on this tpid - StorDb DataStorage // StorDb connection handle - DirPath string // Directory path to import from - Sep rune // Separator in the csv file - Verbose bool // If true will print a detailed information instead of silently discarding it - ImportId string // Use this to differentiate between imports (eg: when autogenerating fields like RatingProfileId + TPid string // Load data on this tpid + StorDb DataStorage // StorDb connection handle + DirPath string // Directory path to import from + Sep rune // Separator in the csv file + Verbose bool // If true will print a detailed information instead of silently discarding it + ImportId string // Use this to differentiate between imports (eg: when autogenerating fields like RatingProfileId } -// Maps csv file to handler which should process it. Defined like this since tests on 1.0.3 were failing on Travis. +// Maps csv file to handler which should process it. Defined like this since tests on 1.0.3 were failing on Travis. // Change it to func(string) error as soon as Travis updates. -var fileHandlers = map[string]func(*TPCSVImporter,string) error{ - utils.TIMINGS_CSV: (*TPCSVImporter).importTimings, - utils.DESTINATIONS_CSV: (*TPCSVImporter).importDestinations, - utils.RATES_CSV: (*TPCSVImporter).importRates, - utils.DESTINATION_RATES_CSV: (*TPCSVImporter).importDestinationRates, - utils.DESTRATE_TIMINGS_CSV: (*TPCSVImporter).importDestRateTimings, - utils.RATE_PROFILES_CSV: (*TPCSVImporter).importRatingProfiles, - utils.ACTIONS_CSV: (*TPCSVImporter).importActions, - utils.ACTION_TIMINGS_CSV: (*TPCSVImporter).importActionTimings, - utils.ACTION_TRIGGERS_CSV: (*TPCSVImporter).importActionTriggers, - utils.ACCOUNT_ACTIONS_CSV: (*TPCSVImporter).importAccountActions, - } +var fileHandlers = map[string]func(*TPCSVImporter, string) error{ + utils.TIMINGS_CSV: (*TPCSVImporter).importTimings, + utils.DESTINATIONS_CSV: (*TPCSVImporter).importDestinations, + utils.RATES_CSV: (*TPCSVImporter).importRates, + utils.DESTINATION_RATES_CSV: (*TPCSVImporter).importDestinationRates, + utils.DESTRATE_TIMINGS_CSV: (*TPCSVImporter).importDestRateTimings, + utils.RATE_PROFILES_CSV: (*TPCSVImporter).importRatingProfiles, + utils.ACTIONS_CSV: (*TPCSVImporter).importActions, + utils.ACTION_TIMINGS_CSV: (*TPCSVImporter).importActionTimings, + utils.ACTION_TRIGGERS_CSV: (*TPCSVImporter).importActionTriggers, + utils.ACCOUNT_ACTIONS_CSV: (*TPCSVImporter).importAccountActions, +} func (self *TPCSVImporter) Run() error { files, _ := ioutil.ReadDir(self.DirPath) for _, f := range files { - fHandler,hasName := fileHandlers[f.Name()] + fHandler, hasName := fileHandlers[f.Name()] if !hasName { continue } - fHandler( self, f.Name() ) + fHandler(self, f.Name()) } return nil } @@ -70,8 +69,8 @@ func (self *TPCSVImporter) importTimings(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -86,7 +85,7 @@ func (self *TPCSVImporter) importTimings(fn string) error { } continue } - tm := NewTiming( record... ) + tm := NewTiming(record...) if err := self.StorDb.SetTPTiming(self.TPid, tm); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) } @@ -98,8 +97,8 @@ func (self *TPCSVImporter) importDestinations(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -126,8 +125,8 @@ func (self *TPCSVImporter) importRates(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -146,7 +145,7 @@ func (self *TPCSVImporter) importRates(fn string) error { if err != nil { return err } - if err := self.StorDb.SetTPRates( self.TPid, map[string][]*Rate{ record[0]: []*Rate{rt} } ); err != nil { + if err := self.StorDb.SetTPRates(self.TPid, map[string][]*Rate{record[0]: []*Rate{rt}}); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) } } @@ -157,8 +156,8 @@ func (self *TPCSVImporter) importDestinationRates(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -173,9 +172,9 @@ func (self *TPCSVImporter) importDestinationRates(fn string) error { } continue } - dr := &DestinationRate{record[0], record[1], record[2], nil} - if err := self.StorDb.SetTPDestinationRates( self.TPid, - map[string][]*DestinationRate{ dr.Tag: []*DestinationRate{dr} } ); err != nil { + dr := &DestinationRate{record[0], record[1], record[2], nil} + if err := self.StorDb.SetTPDestinationRates(self.TPid, + map[string][]*DestinationRate{dr.Tag: []*DestinationRate{dr}}); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) } } @@ -186,8 +185,8 @@ func (self *TPCSVImporter) importDestRateTimings(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -207,12 +206,12 @@ func (self *TPCSVImporter) importDestRateTimings(fn string) error { log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) continue } - drt := &DestinationRateTiming{Tag: record[0], - DestinationRatesTag: record[1], - Weight: weight, - TimingsTag: record[2], - } - if err := self.StorDb.SetTPDestRateTimings( self.TPid, map[string][]*DestinationRateTiming{drt.Tag:[]*DestinationRateTiming{drt}}); err != nil { + drt := &DestinationRateTiming{Tag: record[0], + DestinationRatesTag: record[1], + Weight: weight, + TimingsTag: record[2], + } + if err := self.StorDb.SetTPDestRateTimings(self.TPid, map[string][]*DestinationRateTiming{drt.Tag: []*DestinationRateTiming{drt}}); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) } } @@ -223,8 +222,8 @@ func (self *TPCSVImporter) importRatingProfiles(fn string) error { if self.Verbose { log.Printf("Processing file: <%s> ", fn) } - fParser, err := NewTPCSVFileParser( self.DirPath, fn ) - if err!=nil { + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { return err } lineNr := 0 @@ -246,18 +245,18 @@ func (self *TPCSVImporter) importRatingProfiles(fn string) error { } rpTag := "TPCSV" //Autogenerate rating profile id if self.ImportId != "" { - rpTag += "_"+self.ImportId + rpTag += "_" + self.ImportId } - rp := &RatingProfile{Tag: rpTag, - Tenant: tenant, - TOR: tor, - Direction: direction, - Subject: subject, - ActivationTime: at.Unix(), - DestRatesTimingTag: destRatesTimingTag, - RatesFallbackSubject: fallbacksubject, - } - if err := self.StorDb.SetTPRatingProfiles( self.TPid, map[string][]*RatingProfile{rpTag:[]*RatingProfile{rp}}); err != nil { + rp := &RatingProfile{Tag: rpTag, + Tenant: tenant, + TOR: tor, + Direction: direction, + Subject: subject, + ActivationTime: at.Unix(), + DestRatesTimingTag: destRatesTimingTag, + RatesFallbackSubject: fallbacksubject, + } + if err := self.StorDb.SetTPRatingProfiles(self.TPid, map[string][]*RatingProfile{rpTag: []*RatingProfile{rp}}); err != nil { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) } } @@ -265,6 +264,62 @@ func (self *TPCSVImporter) importRatingProfiles(fn string) error { } func (self *TPCSVImporter) importActions(fn string) error { + if self.Verbose { + log.Printf("Processing file: <%s> ", fn) + } + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + actId, actionType, balanceType, direction, destTag, rateType := record[0], record[1], record[2], record[3], record[6], record[7] + units, err := strconv.ParseFloat(record[4], 64) + if err != nil { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + continue + } + var expiryTime time.Time // Empty initialized time represents never expire + if record[5] != "*unlimited" { // ToDo: Expand here for other meta tags or go way of adding time for expiry + expiryTime, err = time.Parse(time.RFC3339, record[5]) + if err != nil { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + continue + } + } + rateValue, _ := strconv.ParseFloat(record[8], 64) // Ignore errors since empty string is error, we can find out based on rateType if defined + minutesWeight, _ := strconv.ParseFloat(record[9], 64) + weight, err := strconv.ParseFloat(record[10], 64) + if err != nil { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + continue + } + act := &Action{ + ActionType: actionType, + BalanceId: balanceType, + Direction: direction, + Units: units, + ExpirationDate: expiryTime, + DestinationTag: destTag, + RateType: rateType, + RateValue: rateValue, + MinutesWeight: minutesWeight, + Weight: weight, + } + if err := self.StorDb.SetTPActions(self.TPid, map[string][]*Action{actId: []*Action{act}}); err != nil { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } return nil } @@ -279,5 +334,3 @@ func (self *TPCSVImporter) importActionTriggers(fn string) error { func (self *TPCSVImporter) importAccountActions(fn string) error { return nil } - - diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 03d92d7d1..38a7aab55 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -62,7 +62,7 @@ type DestRateTiming struct { type TPRatingProfile struct { TPid string // Tariff plan id - RatingProfileId string // RatingProfile id + RatingProfileId string // RatingProfile id Tenant string // Tenant's Id TOR string // TypeOfRecord Direction string // Traffic direction, OUT is the only one supported for now @@ -92,12 +92,12 @@ type TPActions struct { type Action struct { Identifier string // Identifier mapped in the code - BalanceId string // Type of balance the action will operate on + BalanceType string // Type of balance the action will operate on Direction string // Balance direction Units float64 // Number of units to add/deduct - ExpirationTime int64 // Time when the units will expire + ExpiryTime int64 // Time when the units will expire DestinationId string // Destination profile id - RateType string // Type of price + RateType string // Type of rate <*absolute|*percent> Rate float64 // Price value MinutesWeight float64 // Minutes weight Weight float64 // Action's weight diff --git a/utils/consts.go b/utils/consts.go index bbd905f60..944def977 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -1,11 +1,11 @@ package utils const ( - VERSION = "0.9.1rc3" - POSTGRES = "postgres" - MYSQL = "mysql" - MONGO = "mongo" - REDIS = "redis" + VERSION = "0.9.1rc3" + POSTGRES = "postgres" + MYSQL = "mysql" + MONGO = "mongo" + REDIS = "redis" LOCALHOST = "127.0.0.1" FSCDR_FILE_CSV = "freeswitch_file_csv" FSCDR_HTTP_JSON = "freeswitch_http_json" @@ -29,28 +29,28 @@ const ( TBL_TP_ACTION_TIMINGS = "tp_action_timings" TBL_TP_ACTION_TRIGGERS = "tp_action_triggers" TBL_TP_ACCOUNT_ACTIONS = "tp_account_actions" - TIMINGS_CSV = "Timings.csv" - DESTINATIONS_CSV = "Destinations.csv" - RATES_CSV = "Rates.csv" - DESTINATION_RATES_CSV = "DestinationRates.csv" - DESTRATE_TIMINGS_CSV = "DestinationRateTimings.csv" - RATE_PROFILES_CSV = "RatingProfiles.csv" - ACTIONS_CSV = "Actions.csv" - ACTION_TIMINGS_CSV = "ActionTimings.csv" - ACTION_TRIGGERS_CSV = "ActionTriggers.csv" - ACCOUNT_ACTIONS_CSV = "AccountActions.csv" - TIMINGS_NRCOLS = 6 - DESTINATIONS_NRCOLS = 2 - RATES_NRCOLS = 9 + TIMINGS_CSV = "Timings.csv" + DESTINATIONS_CSV = "Destinations.csv" + RATES_CSV = "Rates.csv" + DESTINATION_RATES_CSV = "DestinationRates.csv" + DESTRATE_TIMINGS_CSV = "DestinationRateTimings.csv" + RATE_PROFILES_CSV = "RatingProfiles.csv" + ACTIONS_CSV = "Actions.csv" + ACTION_TIMINGS_CSV = "ActionTimings.csv" + ACTION_TRIGGERS_CSV = "ActionTriggers.csv" + ACCOUNT_ACTIONS_CSV = "AccountActions.csv" + TIMINGS_NRCOLS = 6 + DESTINATIONS_NRCOLS = 2 + RATES_NRCOLS = 9 DESTINATION_RATES_NRCOLS = 3 DESTRATE_TIMINGS_NRCOLS = 4 RATE_PROFILES_NRCOLS = 7 ACTIONS_NRCOLS = 11 - ACTION_TIMINGS_NRCOLS = 4 + ACTION_TIMINGS_NRCOLS = 4 ACTION_TRIGGERS_NRCOLS = 8 ACCOUNT_ACTIONS_NRCOLS = 5 - ROUNDING_UP = "up" - ROUNDING_MIDDLE = "middle" - ROUNDING_DOWN = "down" - COMMENT_CHAR = '#' + ROUNDING_UP = "up" + ROUNDING_MIDDLE = "middle" + ROUNDING_DOWN = "down" + COMMENT_CHAR = '#' ) From d483a6669fba04509c5cfb9764695a210a9f83e3 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 30 Jul 2013 12:52:58 +0200 Subject: [PATCH 15/17] TPCSVImporter ActionTimings --- apier/tpactiontimings.go | 14 +++-- engine/action_timing.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo.go | 2 +- engine/storage_redis.go | 2 +- engine/storage_sql.go | 6 +- engine/tpimporter_csv.go | 113 +++++++++++++++++++++++++----------- 8 files changed, 98 insertions(+), 45 deletions(-) diff --git a/apier/tpactiontimings.go b/apier/tpactiontimings.go index eefa0de00..6005dc04a 100644 --- a/apier/tpactiontimings.go +++ b/apier/tpactiontimings.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/engine" ) // Creates a new ActionTimings profile within a tariff plan @@ -40,11 +41,16 @@ func (self *Apier) SetTPActionTimings(attrs utils.ApiTPActionTimings, reply *str } else if exists { return errors.New(utils.ERR_DUPLICATE) } - ats := make(map[string][]*utils.TPActionTimingsRow, 1) // Only one id will be stored in the map - for _, at := range attrs.ActionTimings { - ats[attrs.ActionTimingsId] = append(ats[attrs.ActionTimingsId], &utils.TPActionTimingsRow{at.ActionsId, at.TimingId, at.Weight}) + ats := make([]*engine.ActionTiming, len(attrs.ActionTimings)) + for idx, at := range attrs.ActionTimings { + ats[idx] = &engine.ActionTiming{ + Tag: attrs.ActionTimingsId, + ActionsTag: at.ActionsId, + TimingsTag: at.TimingId, + Weight: at.Weight, + } } - if err := self.StorDb.SetTPActionTimings(attrs.TPid, ats); err != nil { + if err := self.StorDb.SetTPActionTimings(attrs.TPid, map[string][]*engine.ActionTiming{attrs.ActionTimingsId: ats}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/engine/action_timing.go b/engine/action_timing.go index 87b0d6cbb..e8829074d 100644 --- a/engine/action_timing.go +++ b/engine/action_timing.go @@ -41,7 +41,7 @@ type ActionTiming struct { ActionsId string actions Actions stCache time.Time // cached time of the next start - actionsTag, timingsTag string // used only for loading + ActionsTag, TimingsTag string // used only for loading } type ActionTimings []*ActionTiming diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 6f25d34bf..e8e5aedb1 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -88,7 +88,7 @@ type DataStorage interface { GetTPActions(string, string) (*utils.TPActions, error) GetTPActionIds(string) ([]string, error) ExistsTPActionTimings(string, string) (bool, error) - SetTPActionTimings(string, map[string][]*utils.TPActionTimingsRow) error + SetTPActionTimings(string, map[string][]*ActionTiming) error GetTPActionTimings(string, string) (map[string][]*utils.TPActionTimingsRow, error) GetTPActionTimingIds(string) ([]string, error) ExistsTPActionTriggers(string, string) (bool, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index d80958d1d..7b1922ac3 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -193,7 +193,7 @@ func (ms *MapStorage) ExistsTPActionTimings(tpid, atId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MapStorage) SetTPActionTimings(tpid string, ats map[string][]*utils.TPActionTimingsRow) error { +func (ms *MapStorage) SetTPActionTimings(tpid string, ats map[string][]*ActionTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 04c748ee3..a555c4744 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -268,7 +268,7 @@ func (ms *MongoStorage) ExistsTPActionTimings(tpid, atId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (ms *MongoStorage) SetTPActionTimings(tpid string, ats map[string][]*utils.TPActionTimingsRow) error { +func (ms *MongoStorage) SetTPActionTimings(tpid string, ats map[string][]*ActionTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 39aacbdce..b27a6ebe7 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -223,7 +223,7 @@ func (rs *RedisStorage) ExistsTPActionTimings(tpid, atId string) (bool, error) { return false, errors.New(utils.ERR_NOT_IMPLEMENTED) } -func (rs *RedisStorage) SetTPActionTimings(tpid string, ats map[string][]*utils.TPActionTimingsRow) error { +func (rs *RedisStorage) SetTPActionTimings(tpid string, ats map[string][]*ActionTiming) error { return errors.New(utils.ERR_NOT_IMPLEMENTED) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 55e6986e7..8222afd92 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -629,19 +629,19 @@ func (self *SQLStorage) ExistsTPActionTimings(tpid, atId string) (bool, error) { } // Sets actionTimings in sqlDB. Imput is expected in form map[actionTimingId][]rows, eg a full .csv file content -func (self *SQLStorage) SetTPActionTimings(tpid string, ats map[string][]*utils.TPActionTimingsRow) error { +func (self *SQLStorage) SetTPActionTimings(tpid string, ats map[string][]*ActionTiming) error { if len(ats) == 0 { return nil //Nothing to set } qry := fmt.Sprintf("INSERT INTO %s (tpid,tag,actions_tag,timing_tag,weight) VALUES ", utils.TBL_TP_ACTION_TIMINGS) i := 0 for atId, atRows := range ats { - for _, atsRow := range atRows { + for _, at := range atRows { if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s',%f)", - tpid, atId, atsRow.ActionsId, atsRow.TimingId, atsRow.Weight) + tpid, atId, at.ActionsTag, at.TimingsTag, at.Weight) i++ } } diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 2c9fb2d0a..b2f59a523 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -66,9 +66,7 @@ func (self *TPCSVImporter) Run() error { // Handler importing timings from file, saved row by row to storDb func (self *TPCSVImporter) importTimings(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -87,16 +85,16 @@ func (self *TPCSVImporter) importTimings(fn string) error { } tm := NewTiming(record...) if err := self.StorDb.SetTPTiming(self.TPid, tm); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importDestinations(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -115,16 +113,16 @@ func (self *TPCSVImporter) importDestinations(fn string) error { } dst := &Destination{record[0], []string{record[1]}} if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importRates(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -146,16 +144,16 @@ func (self *TPCSVImporter) importRates(fn string) error { return err } if err := self.StorDb.SetTPRates(self.TPid, map[string][]*Rate{record[0]: []*Rate{rt}}); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importDestinationRates(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -175,16 +173,16 @@ func (self *TPCSVImporter) importDestinationRates(fn string) error { dr := &DestinationRate{record[0], record[1], record[2], nil} if err := self.StorDb.SetTPDestinationRates(self.TPid, map[string][]*DestinationRate{dr.Tag: []*DestinationRate{dr}}); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importDestRateTimings(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -203,7 +201,9 @@ func (self *TPCSVImporter) importDestRateTimings(fn string) error { } weight, err := strconv.ParseFloat(record[3], 64) if err != nil { - log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } continue } drt := &DestinationRateTiming{Tag: record[0], @@ -212,16 +212,16 @@ func (self *TPCSVImporter) importDestRateTimings(fn string) error { TimingsTag: record[2], } if err := self.StorDb.SetTPDestRateTimings(self.TPid, map[string][]*DestinationRateTiming{drt.Tag: []*DestinationRateTiming{drt}}); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importRatingProfiles(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -241,7 +241,9 @@ func (self *TPCSVImporter) importRatingProfiles(fn string) error { tenant, tor, direction, subject, destRatesTimingTag, fallbacksubject := record[0], record[1], record[2], record[3], record[5], record[6] at, err := time.Parse(time.RFC3339, record[4]) if err != nil { - log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } } rpTag := "TPCSV" //Autogenerate rating profile id if self.ImportId != "" { @@ -257,16 +259,16 @@ func (self *TPCSVImporter) importRatingProfiles(fn string) error { RatesFallbackSubject: fallbacksubject, } if err := self.StorDb.SetTPRatingProfiles(self.TPid, map[string][]*RatingProfile{rpTag: []*RatingProfile{rp}}); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importActions(fn string) error { - if self.Verbose { - log.Printf("Processing file: <%s> ", fn) - } + log.Printf("Processing file: <%s> ", fn) fParser, err := NewTPCSVFileParser(self.DirPath, fn) if err != nil { return err @@ -286,14 +288,18 @@ func (self *TPCSVImporter) importActions(fn string) error { actId, actionType, balanceType, direction, destTag, rateType := record[0], record[1], record[2], record[3], record[6], record[7] units, err := strconv.ParseFloat(record[4], 64) if err != nil { - log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } continue } var expiryTime time.Time // Empty initialized time represents never expire if record[5] != "*unlimited" { // ToDo: Expand here for other meta tags or go way of adding time for expiry expiryTime, err = time.Parse(time.RFC3339, record[5]) if err != nil { - log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } continue } } @@ -301,7 +307,9 @@ func (self *TPCSVImporter) importActions(fn string) error { minutesWeight, _ := strconv.ParseFloat(record[9], 64) weight, err := strconv.ParseFloat(record[10], 64) if err != nil { - log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } continue } act := &Action{ @@ -317,13 +325,52 @@ func (self *TPCSVImporter) importActions(fn string) error { Weight: weight, } if err := self.StorDb.SetTPActions(self.TPid, map[string][]*Action{actId: []*Action{act}}); err != nil { - log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } } } return nil } func (self *TPCSVImporter) importActionTimings(fn string) error { + log.Printf("Processing file: <%s> ", fn) + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + tag, actionsTag, timingTag := record[0], record[1], record[2] + weight, err := strconv.ParseFloat(record[3], 64) + if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + at := &ActionTiming{ + Tag: tag, + ActionsTag: actionsTag, + TimingsTag: timingTag, + Weight: weight, + } + if err := self.StorDb.SetTPActionTimings(self.TPid, map[string][]*ActionTiming{tag: []*ActionTiming{at}}); err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } + } return nil } From c60b26c0c4393dac1b6af8610f54624f4e844ce9 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 30 Jul 2013 17:03:16 +0200 Subject: [PATCH 16/17] TPCSVImporter ActionTriggers --- apier/tpactiontriggers.go | 8 ++-- cmd/cgr-engine/cgr-engine.go | 2 +- config/config.go | 27 +++++------ config/config_test.go | 10 ++-- config/test_data.txt | 4 +- data/conf/cgrates.cfg | 4 +- .../mysql/create_tariffplan_tables.sql | 8 ++-- docs/api_tpactions.rst | 2 +- docs/api_tpactiontriggers.rst | 12 ++--- engine/tpimporter_csv.go | 47 +++++++++++++++++++ utils/apitpdata.go | 2 +- utils/consts.go | 6 +-- utils/coreutils.go | 6 +-- utils/utils_test.go | 20 ++++---- 14 files changed, 101 insertions(+), 57 deletions(-) diff --git a/apier/tpactiontriggers.go b/apier/tpactiontriggers.go index 6b1bd4056..eff6b5c1a 100644 --- a/apier/tpactiontriggers.go +++ b/apier/tpactiontriggers.go @@ -38,12 +38,12 @@ func (self *Apier) SetTPActionTriggers(attrs utils.ApiTPActionTriggers, reply *s } aTriggers := make([]*engine.ActionTrigger, len(attrs.ActionTriggers)) for idx, at := range attrs.ActionTriggers { - requiredFields := []string{"BalanceId", "Direction", "ThresholdType", "ThresholdValue", "ActionsId", "Weight"} + requiredFields := []string{"BalanceType", "Direction", "ThresholdType", "ThresholdValue", "ActionsId", "Weight"} if missing := utils.MissingStructFields(&at, requiredFields); len(missing) != 0 { - return fmt.Errorf("%s:Balance:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.BalanceId, missing) + return fmt.Errorf("%s:Balance:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.BalanceType, missing) } at := &engine.ActionTrigger{ - BalanceId: at.BalanceId, + BalanceId: at.BalanceType, Direction: at.Direction, ThresholdType: at.ThresholdType, ThresholdValue: at.ThresholdValue, @@ -82,7 +82,7 @@ func (self *Apier) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *uti aTriggers := make([]utils.ApiActionTrigger, len(ats[attrs.ActionTriggersId])) for idx, row := range ats[attrs.ActionTriggersId] { aTriggers[idx] = utils.ApiActionTrigger{ - BalanceId: row.BalanceId, + BalanceType: row.BalanceId, Direction: row.Direction, ThresholdType: row.ThresholdType, ThresholdValue: row.ThresholdValue, diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 7dd2e999a..3f75dabe0 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -252,7 +252,7 @@ func main() { } defer loggerDb.Close() engine.SetStorageLogger(loggerDb) - engine.SetRoundingMethodAndDecimals(cfg.RaterRoundingMethod, cfg.RaterRoundingDecimals) + engine.SetRoundingMethodAndDecimals(cfg.RoundingMethod, cfg.RoundingDecimals) if cfg.SMDebitInterval > 0 { if dp, err := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval)); err == nil { diff --git a/config/config.go b/config/config.go index 9b6c2bca8..a8a835b86 100644 --- a/config/config.go +++ b/config/config.go @@ -35,10 +35,7 @@ const ( REDIS = "redis" SAME = "same" FS = "freeswitch" - PREPAID = "prepaid" - POSTPAID = "postpaid" - PSEUDOPREPAID = "pseudoprepaid" - RATED = "rated" + ) // Holds system configuration, defaults are overwritten with values from config file if found @@ -60,11 +57,11 @@ type CGRConfig struct { DefaultTOR string // set default type of record DefaultTenant string // set default tenant DefaultSubject string // set default rating subject, useful in case of fallback + RoundingMethod string // Rounding method for the end price: <*up|*middle|*down> + RoundingDecimals int // Number of decimals to round end prices at RaterEnabled bool // start standalone server (no balancer) RaterBalancer string // balancer address host:port RaterListen string // listening address host:port - RaterRoundingMethod string // Rounding method for the end price: - RaterRoundingDecimals int // Number of decimals to round end prices at BalancerEnabled bool BalancerListen string // Json RPC server address SchedulerEnabled bool @@ -114,15 +111,15 @@ func (self *CGRConfig) setDefaults() error { self.StorDBUser = "cgrates" self.StorDBPass = "CGRateS.org" self.RPCEncoding = JSON - self.DefaultReqType = "rated" + self.DefaultReqType = utils.RATED self.DefaultTOR = "0" self.DefaultTenant = "0" self.DefaultSubject = "0" + self.RoundingMethod = utils.ROUNDING_MIDDLE + self.RoundingDecimals = 4 self.RaterEnabled = false self.RaterBalancer = DISABLED self.RaterListen = "127.0.0.1:2012" - self.RaterRoundingMethod = utils.ROUNDING_MIDDLE - self.RaterRoundingDecimals = 4 self.BalancerEnabled = false self.BalancerListen = "127.0.0.1:2013" self.SchedulerEnabled = false @@ -239,6 +236,12 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("global", "default_subject"); hasOpt { cfg.DefaultSubject, _ = c.GetString("global", "default_subject") } + if hasOpt = c.HasOption("global", "rounding_method"); hasOpt { + cfg.RoundingMethod, _ = c.GetString("global", "rounding_method") + } + if hasOpt = c.HasOption("global", "rounding_decimals"); hasOpt { + cfg.RoundingDecimals, _ = c.GetInt("global", "rounding_decimals") + } if hasOpt = c.HasOption("rater", "enabled"); hasOpt { cfg.RaterEnabled, _ = c.GetBool("rater", "enabled") } @@ -248,12 +251,6 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("rater", "listen"); hasOpt { cfg.RaterListen, _ = c.GetString("rater", "listen") } - if hasOpt = c.HasOption("rater", "rounding_method"); hasOpt { - cfg.RaterRoundingMethod, _ = c.GetString("rater", "rounding_method") - } - if hasOpt = c.HasOption("rater", "rounding_decimals"); hasOpt { - cfg.RaterRoundingDecimals, _ = c.GetInt("rater", "rounding_decimals") - } if hasOpt = c.HasOption("balancer", "enabled"); hasOpt { cfg.BalancerEnabled, _ = c.GetBool("balancer", "enabled") } diff --git a/config/config_test.go b/config/config_test.go index 0911bccbf..5b1efcbda 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -47,15 +47,15 @@ func TestDefaults(t *testing.T) { eCfg.StorDBUser = "cgrates" eCfg.StorDBPass = "CGRateS.org" eCfg.RPCEncoding = JSON - eCfg.DefaultReqType = RATED + eCfg.DefaultReqType = utils.RATED eCfg.DefaultTOR = "0" eCfg.DefaultTenant = "0" eCfg.DefaultSubject = "0" + eCfg.RoundingMethod = utils.ROUNDING_MIDDLE + eCfg.RoundingDecimals = 4 eCfg.RaterEnabled = false eCfg.RaterBalancer = DISABLED eCfg.RaterListen = "127.0.0.1:2012" - eCfg.RaterRoundingMethod = utils.ROUNDING_MIDDLE - eCfg.RaterRoundingDecimals = 4 eCfg.BalancerEnabled = false eCfg.BalancerListen = "127.0.0.1:2013" eCfg.SchedulerEnabled = false @@ -142,11 +142,11 @@ func TestConfigFromFile(t *testing.T) { eCfg.DefaultTOR = "test" eCfg.DefaultTenant = "test" eCfg.DefaultSubject = "test" + eCfg.RoundingMethod = "test" + eCfg.RoundingDecimals = 99 eCfg.RaterEnabled = true eCfg.RaterBalancer = "test" eCfg.RaterListen = "test" - eCfg.RaterRoundingMethod = "test" - eCfg.RaterRoundingDecimals = 99 eCfg.BalancerEnabled = true eCfg.BalancerListen = "test" eCfg.SchedulerEnabled = true diff --git a/config/test_data.txt b/config/test_data.txt index b8ed219d9..d4ffed71e 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -19,6 +19,8 @@ default_reqtype = test # Default request type to consider when missing from r default_tor = test # Default Type of Record to consider when missing from requests. default_tenant = test # Default Tenant to consider when missing from requests. default_subject = test # Default rating Subject to consider when missing from requests. +rounding_method = test # Rounding method for floats/costs: +rounding_decimals = 99 # Number of decimals to round floats/costs at [balancer] @@ -29,8 +31,6 @@ listen = test # Balancer listen interface: . enabled = true # Enable Rater service: . balancer = test # Register to Balancer as worker: . listen = test # Rater's listening interface: . -rounding_method = test # Rounding method for the end price: -rounding_decimals = 99 # Number of decimals to round prices at [scheduler] enabled = true # Starts Scheduler service: . diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index ace34ff03..72b70a6f1 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -22,6 +22,8 @@ # default_tor = 0 # Default Type of Record to consider when missing from requests. # default_tenant = 0 # Default Tenant to consider when missing from requests. # default_subject = 0 # Default rating Subject to consider when missing from requests. +# rounding_method = *middle # Rounding method for floats/costs: <*up|*middle|*down> +# rounding_decimals = 4 # Number of decimals to round float/costs at [balancer] @@ -32,8 +34,6 @@ # enabled = false # Enable Rater service: . # balancer = disabled # Register to Balancer as worker: . # listen = 127.0.0.1:2012 # Rater's listening interface: . -# rounding_method = middle # Rounding method for the end price: -# rounding_decimals = 4 # Number of decimals to round end prices at [scheduler] # enabled = false # Starts Scheduler service: . diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index e0051da29..dffcc8c23 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -117,11 +117,11 @@ CREATE TABLE `tp_actions` ( `action` varchar(24) NOT NULL, `balance_type` varchar(24) NOT NULL, `direction` varchar(8) NOT NULL, - `units` DECIMAL(5,2) NOT NULL, + `units` DECIMAL(8,4) NOT NULL, `expiry_time` int(16) NOT NULL, `destination_tag` varchar(24) NOT NULL, `rate_type` varchar(8) NOT NULL, - `rate` DECIMAL(5,4) NOT NULL, + `rate` DECIMAL(8,4) NOT NULL, `minutes_weight` DECIMAL(5,2) NOT NULL, `weight` DECIMAL(5,2) NOT NULL, PRIMARY KEY (`id`), @@ -155,8 +155,8 @@ CREATE TABLE `tp_action_triggers` ( `tag` varchar(24) NOT NULL, `balance_type` varchar(24) NOT NULL, `direction` varchar(8) NOT NULL, - `threshold_type` char(11) NOT NULL, - `threshold_value` DECIMAL(5,4) NOT NULL, + `threshold_type` char(12) NOT NULL, + `threshold_value` DECIMAL(8,4) NOT NULL, `destination_tag` varchar(24) NOT NULL, `actions_tag` varchar(24) NOT NULL, `weight` DECIMAL(5,2) NOT NULL, diff --git a/docs/api_tpactions.rst b/docs/api_tpactions.rst index 4bcebbc46..cd7852b6c 100644 --- a/docs/api_tpactions.rst +++ b/docs/api_tpactions.rst @@ -134,7 +134,7 @@ Queries specific Actions profile on tariff plan. Units float64 // Number of units to add/deduct ExpiryTime int64 // Time when the units will expire DestinationId string // Destination profile id - RateType string // Type of price + RateType string // Type of price <*absolute|*percent> Rate float64 // Price value MinutesWeight float64 // Minutes weight Weight float64 // Action's weight diff --git a/docs/api_tpactiontriggers.rst b/docs/api_tpactiontriggers.rst index 1d1156018..35a2f4132 100644 --- a/docs/api_tpactiontriggers.rst +++ b/docs/api_tpactiontriggers.rst @@ -16,7 +16,7 @@ Creates a new ActionTriggers profile within a tariff plan. } type ApiActionTrigger struct { - BalanceId string // Id of the balance this trigger monitors + BalanceType string // Id of the balance this trigger monitors Direction string // Traffic direction ThresholdType string // This threshold type ThresholdValue float64 // Threshold @@ -25,7 +25,7 @@ Creates a new ActionTriggers profile within a tariff plan. Weight float64 // weight } - Mandatory parameters: ``[]string{"TPid", "ActionTriggersId","BalanceId", "Direction", "ThresholdType", "ThresholdValue", "ActionsId", "Weight"}`` + Mandatory parameters: ``[]string{"TPid", "ActionTriggersId","BalanceType", "Direction", "ThresholdType", "ThresholdValue", "ActionsId", "Weight"}`` *JSON sample*: :: @@ -38,7 +38,7 @@ Creates a new ActionTriggers profile within a tariff plan. "ActionTriggers": [ { "ActionsId": "ACTION_1", - "BalanceId": "MONETARY", + "BalanceType": "MONETARY", "DestinationId": "", "Direction": "OUT", "ThresholdType": "MIN_BALANCE", @@ -124,7 +124,7 @@ Queries specific ActionTriggers profile on tariff plan. } type ApiActionTrigger struct { - BalanceId string // Id of the balance this trigger monitors + BalanceType string // Id of the balance this trigger monitors Direction string // Traffic direction ThresholdType string // This threshold type ThresholdValue float64 // Threshold @@ -143,9 +143,9 @@ Queries specific ActionTriggers profile on tariff plan. "ActionTriggers": [ { "ActionsId": "ACTION_1", - "BalanceId": "MONETARY", + "BalanceType": "*monetary", "DestinationId": "", - "Direction": "OUT", + "Direction": "*out", "ThresholdType": "MIN_BALANCE", "ThresholdValue": 5, "Weight": 10 diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index b2f59a523..503e0b290 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -375,6 +375,53 @@ func (self *TPCSVImporter) importActionTimings(fn string) error { } func (self *TPCSVImporter) importActionTriggers(fn string) error { + log.Printf("Processing file: <%s> ", fn) + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + tag, balanceType, direction, thresholdType, destinationTag, actionsTag := record[0], record[1], record[2], record[3], record[5], record[6] + threshold, err := strconv.ParseFloat(record[4], 64) + if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + weight, err := strconv.ParseFloat(record[7], 64) + if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + at := &ActionTrigger{ + BalanceId: balanceType, + Direction: direction, + ThresholdType: thresholdType, + ThresholdValue: threshold, + DestinationId: destinationTag, + Weight: weight, + ActionsId: actionsTag, + } + if err := self.StorDb.SetTPActionTriggers(self.TPid, map[string][]*ActionTrigger{tag: []*ActionTrigger{at}}); err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } + } return nil } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 38a7aab55..fc784e15d 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -123,7 +123,7 @@ type ApiTPActionTriggers struct { } type ApiActionTrigger struct { - BalanceId string // Id of the balance this trigger monitors + BalanceType string // Type of balance this trigger monitors Direction string // Traffic direction ThresholdType string // This threshold type ThresholdValue float64 // Threshold diff --git a/utils/consts.go b/utils/consts.go index 944def977..9ea654ec8 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -49,8 +49,8 @@ const ( ACTION_TIMINGS_NRCOLS = 4 ACTION_TRIGGERS_NRCOLS = 8 ACCOUNT_ACTIONS_NRCOLS = 5 - ROUNDING_UP = "up" - ROUNDING_MIDDLE = "middle" - ROUNDING_DOWN = "down" + ROUNDING_UP = "*up" + ROUNDING_MIDDLE = "*middle" + ROUNDING_DOWN = "*down" COMMENT_CHAR = '#' ) diff --git a/utils/coreutils.go b/utils/coreutils.go index defc797c0..51a588610 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -78,11 +78,11 @@ func Round(x float64, prec int, method string) float64 { _, frac := math.Modf(intermed) switch method { - case "*up": + case ROUNDING_UP: rounder = math.Ceil(intermed) - case "*down": + case ROUNDING_DOWN: rounder = math.Floor(intermed) - case "*middle": + case ROUNDING_MIDDLE: if frac >= 0.5 { rounder = math.Ceil(intermed) } else { diff --git a/utils/utils_test.go b/utils/utils_test.go index fc9a2f2ed..e55775f10 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -41,7 +41,7 @@ func TestUUID(t *testing.T) { } func TestRoundUp(t *testing.T) { - result := Round(12.52, 0, "*middle") + result := Round(12.52, 0, ROUNDING_UP) expected := 13.0 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -49,7 +49,7 @@ func TestRoundUp(t *testing.T) { } func TestRoundUpMiddle(t *testing.T) { - result := Round(12.5, 0, "*middle") + result := Round(12.5, 0, ROUNDING_UP) expected := 13.0 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -57,7 +57,7 @@ func TestRoundUpMiddle(t *testing.T) { } func TestRoundDown(t *testing.T) { - result := Round(12.49, 0, "*middle") + result := Round(12.49, 0, ROUNDING_MIDDLE) expected := 12.0 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -65,7 +65,7 @@ func TestRoundDown(t *testing.T) { } func TestRoundPrec(t *testing.T) { - result := Round(12.49, 1, "*middle") + result := Round(12.49, 1, ROUNDING_UP) expected := 12.5 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -73,7 +73,7 @@ func TestRoundPrec(t *testing.T) { } func TestRoundPrecNothing(t *testing.T) { - result := Round(12.49, 2, "*middle") + result := Round(12.49, 2, ROUNDING_MIDDLE) expected := 12.49 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -89,7 +89,7 @@ func TestRoundPrecNoTouch(t *testing.T) { } func TestRoundByMethodUp1(t *testing.T) { - result := Round(12.49, 1, "*up") + result := Round(12.49, 1, ROUNDING_UP) expected := 12.5 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -97,7 +97,7 @@ func TestRoundByMethodUp1(t *testing.T) { } func TestRoundByMethodUp2(t *testing.T) { - result := Round(12.21, 1, "*up") + result := Round(12.21, 1, ROUNDING_UP) expected := 12.3 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) @@ -105,15 +105,15 @@ func TestRoundByMethodUp2(t *testing.T) { } func TestRoundByMethodDown1(t *testing.T) { - result := Round(12.49, 1, "*down") + result := Round(12.49, 1, ROUNDING_DOWN) expected := 12.4 if result != expected { - t.Errorf("Error rounding up: sould be %v was %v", expected, result) + t.Errorf("Error rounding down: sould be %v was %v", expected, result) } } func TestRoundByMethodDown2(t *testing.T) { - result := Round(12.21, 1, "*down") + result := Round(12.21, 1, ROUNDING_DOWN) expected := 12.2 if result != expected { t.Errorf("Error rounding up: sould be %v was %v", expected, result) From 2f733525b215e608478e1cddf2b001fb92fb8cbd Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 30 Jul 2013 19:35:20 +0200 Subject: [PATCH 17/17] Finalizing TPCSVImporter with AccountActions method --- engine/storage_sql.go | 2 +- engine/tpimporter_csv.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 8222afd92..db05c6fa7 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -769,7 +769,7 @@ func (self *SQLStorage) SetTPAccountActions(tpid string, aa map[string]*AccountA utils.TBL_TP_ACCOUNT_ACTIONS) i := 0 for aaId, aActs := range aa { - if i != 1 { //Consecutive values after the first will be prefixed with "," as separator + if i != 0 { //Consecutive values after the first will be prefixed with "," as separator qry += "," } qry += fmt.Sprintf("('%s','%s','%s','%s','%s','%s','%s')", diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 503e0b290..8a466bb51 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -426,5 +426,37 @@ func (self *TPCSVImporter) importActionTriggers(fn string) error { } func (self *TPCSVImporter) importAccountActions(fn string) error { + log.Printf("Processing file: <%s> ", fn) + fParser, err := NewTPCSVFileParser(self.DirPath, fn) + if err != nil { + return err + } + lineNr := 0 + for { + lineNr++ + record, err := fParser.ParseNextLine() + if err == io.EOF { // Reached end of file + break + } else if err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, warning: <%s> ", lineNr, err.Error()) + } + continue + } + tenant, account, direction, actionTimingsTag, actionTriggersTag := record[0], record[1], record[2], record[3], record[4] + tag := "TPCSV" //Autogenerate account actions profile id + if self.ImportId != "" { + tag += "_" + self.ImportId + } + aa := map[string]*AccountAction{ + tag: &AccountAction{Tenant: tenant, Account: account, Direction: direction, + ActionTimingsTag: actionTimingsTag, ActionTriggersTag: actionTriggersTag}, + } + if err := self.StorDb.SetTPAccountActions(self.TPid, aa); err != nil { + if self.Verbose { + log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) + } + } + } return nil }