From 77316a16e45fa9832734d401f6afc706caee13ba Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 18 Oct 2015 19:42:56 +0200 Subject: [PATCH] First cut of SureTax implementation, with integration tests --- config/config_defaults.go | 2 +- config/config_json_test.go | 2 +- data/conf/cgrates/cgrates.json | 59 ++++++--- engine/cdrs.go | 22 ++++ engine/suretax.go | 8 +- general_tests/suretax_it_test.go | 175 +++++++++++++++++++++++++++ general_tests/tutorial_local_test.go | 1 - 7 files changed, 251 insertions(+), 18 deletions(-) create mode 100644 general_tests/suretax_it_test.go diff --git a/config/config_defaults.go b/config/config_defaults.go index f59ec3f98..0a8b4754c 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -290,7 +290,7 @@ const CGRATES_CFG_JSON = ` "response_type": "D4", // determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response "regulatory_code": "03", // provider type "client_tracking": "CgrId", // template extracting client information out of StoredCdr; <$RSRFields> - "customer_number": "CustomerNumber", // template extracting customer number out of StoredCdr; <$RSRFields> + "customer_number": "Subject", // template extracting customer number out of StoredCdr; <$RSRFields> "orig_number": "Subject", // template extracting origination number out of StoredCdr; <$RSRFields> "term_number": "Destination", // template extracting termination number out of StoredCdr; <$RSRFields> "bill_to_number": "", // template extracting billed to number out of StoredCdr; <$RSRFields> diff --git a/config/config_json_test.go b/config/config_json_test.go index 13a52de2b..fc5cbd75f 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -469,7 +469,7 @@ func TestDfSureTaxJsonCfg(t *testing.T) { Response_type: utils.StringPointer("D4"), Regulatory_code: utils.StringPointer("03"), Client_tracking: utils.StringPointer("CgrId"), - Customer_number: utils.StringPointer("CustomerNumber"), + Customer_number: utils.StringPointer("Subject"), Orig_number: utils.StringPointer("Subject"), Term_number: utils.StringPointer("Destination"), Bill_to_number: utils.StringPointer(""), diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 98269c127..70a7ad4b6 100644 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -1,5 +1,6 @@ { + // Real-time Charging System for Telecom & ISP environments // Copyright (C) ITsysCOM GmbH // @@ -8,19 +9,20 @@ //"general": { -// "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate -// "rounding_decimals": 10, // system level precision for floats -// "dbdata_encoding": "msgpack", // encoding used to store object data in strings: -// "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans -// "default_reqtype": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> -// "default_category": "call", // default Type of Record to consider when missing from requests -// "default_tenant": "cgrates.org", // default Tenant to consider when missing from requests -// "default_subject": "cgrates", // default rating Subject to consider when missing from requests -// "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> -// "connect_attempts": 3, // initial server connect attempts -// "response_cache_ttl": "3s", // the life span of a cached response -// "reconnects": -1, // number of retries in case of connection lost -// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up +// "http_skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate +// "rounding_decimals": 10, // system level precision for floats +// "dbdata_encoding": "msgpack", // encoding used to store object data in strings: +// "tpexport_dir": "/var/log/cgrates/tpe", // path towards export folder for offline Tariff Plans +// "http_failed_dir": "/var/log/cgrates/http_failed", // directory path where we store failed http requests +// "default_reqtype": "*rated", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> +// "default_category": "call", // default Type of Record to consider when missing from requests +// "default_tenant": "cgrates.org", // default Tenant to consider when missing from requests +// "default_subject": "cgrates", // default rating Subject to consider when missing from requests +// "default_timezone": "Local", // default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> +// "connect_attempts": 3, // initial server connect attempts +// "reconnects": -1, // number of retries in case of connection lost +// "response_cache_ttl": "3s", // the life span of a cached response +// "internal_ttl": "2m", // maximum duration to wait for internal connections before giving up //}, @@ -31,7 +33,7 @@ //}, -//"tariffplan_db": { // database used to store active tariff plan configuration +//"tariffplan_db": { // database used to store active tariff plan configuration // "db_type": "redis", // tariffplan_db type: // "db_host": "127.0.0.1", // tariffplan_db host address // "db_port": 6379, // port to reach the tariffplan_db @@ -258,4 +260,33 @@ //}, +//"suretax": { +// "url": "", // API url +// "client_number": "", // client number, provided by SureTax +// "validation_key": "", // validation key provided by SureTax +// "timezone": "Local", // convert the time of the events to this timezone before sending request out +// "include_local_cost": false, // sum local calculated cost with tax one in final cost +// "return_file_code": "0", // default or Quote purposes <0|Q> +// "response_group": "03", // determines how taxes are grouped for the response <03|13> +// "response_type": "D4", // determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response +// "regulatory_code": "03", // provider type +// "client_tracking": "CgrId", // template extracting client information out of StoredCdr; <$RSRFields> +// "customer_number": "CustomerNumber", // template extracting customer number out of StoredCdr; <$RSRFields> +// "orig_number": "Subject", // template extracting origination number out of StoredCdr; <$RSRFields> +// "term_number": "Destination", // template extracting termination number out of StoredCdr; <$RSRFields> +// "bill_to_number": "", // template extracting billed to number out of StoredCdr; <$RSRFields> +// "zipcode": "", // template extracting billing zip code out of StoredCdr; <$RSRFields> +// "plus4": "", // template extracting billing zip code extension out of StoredCdr; <$RSRFields> +// "p2pzipcode": "", // template extracting secondary zip code out of StoredCdr; <$RSRFields> +// "p2pplus4": "", // template extracting secondary zip code extension out of StoredCdr; <$RSRFields> +// "units": "^1", // template extracting number of “lines” or unique charges contained within the revenue out of StoredCdr; <$RSRFields> +// "unit_type": "^00", // template extracting number of unique access lines out of StoredCdr; <$RSRFields> +// "tax_included": "^0", // template extracting tax included in revenue out of StoredCdr; <$RSRFields> +// "tax_situs_rule": "^04", // template extracting tax situs rule out of StoredCdr; <$RSRFields> +// "trans_type_code": "^010101", // template extracting transaction type indicator out of StoredCdr; <$RSRFields> +// "sales_type_code": "^R", // template extracting sales type code out of StoredCdr; <$RSRFields> +// "tax_exemption_code_list": "", // template extracting tax exemption code list out of StoredCdr; <$RSRFields> +//}, + + } \ No newline at end of file diff --git a/engine/cdrs.go b/engine/cdrs.go index 18c429ce5..90236e8ea 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -202,6 +202,22 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(storedCdr *StoredCdr) error return err } for _, cdr := range cdrRuns { + if cdr.MediationRunId != utils.META_DEFAULT { // Process Aliases and Users for derived CDRs + if err := LoadAlias(&AttrMatchingAlias{ + Destination: cdr.Destination, + Direction: cdr.Direction, + Tenant: cdr.Tenant, + Category: cdr.Category, + Account: cdr.Account, + Subject: cdr.Subject, + Context: utils.ALIAS_CONTEXT_RATING, + }, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound { + return err + } + if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil { + return err + } + } // Rate CDR if self.rater != nil && !cdr.Rated { if err := self.rateCDR(cdr); err != nil { @@ -209,6 +225,12 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(storedCdr *StoredCdr) error cdr.ExtraInfo = err.Error() } } + if cdr.MediationRunId == utils.META_SURETAX { // Request should be processed by SureTax + if err := SureTaxProcessCdr(cdr); err != nil { + cdr.Cost = -1.0 + cdr.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo + } + } if self.cgrCfg.CDRSStoreCdrs { // Store CDRs // Store RatedCDR if err := self.cdrDb.SetRatedCdr(cdr); err != nil { diff --git a/engine/suretax.go b/engine/suretax.go index 9de873b6c..20b30d4dd 100644 --- a/engine/suretax.go +++ b/engine/suretax.go @@ -38,6 +38,9 @@ var sureTaxClient *http.Client // Cache the client here if in use // Init a new request to be sent out to SureTax func NewSureTaxRequest(cdr *StoredCdr, stCfg *config.SureTaxCfg) (*SureTaxRequest, error) { + if stCfg == nil { + return nil, errors.New("Invalid SureTax config.") + } aTimeLoc := cdr.AnswerTime.In(stCfg.Timezone) revenue := utils.Round(cdr.Cost, 4, utils.ROUNDING_MIDDLE) unts, err := strconv.ParseInt(cdr.FieldsAsString(stCfg.Units), 10, 64) @@ -175,7 +178,7 @@ type STTaxItem struct { func SureTaxProcessCdr(cdr *StoredCdr) error { stCfg := config.CgrConfig().SureTaxCfg() if stCfg == nil { - return errors.New("SureTax configuration missing") + return errors.New("Invalid SureTax configuration") } if sureTaxClient == nil { // First time used, init the client here tr := &http.Transport{ @@ -187,10 +190,12 @@ func SureTaxProcessCdr(cdr *StoredCdr) error { if err != nil { return err } + body, err := json.Marshal(req) if err != nil { return err } + utils.Logger.Debug(fmt.Sprintf("###SureTax NewSureTaxRequest: %+v, ItemList: %+v\n", req, req.ItemList[0])) resp, err := sureTaxClient.Post(stCfg.Url, "application/json", bytes.NewBuffer(body)) if err != nil { return err @@ -207,6 +212,7 @@ func SureTaxProcessCdr(cdr *StoredCdr) error { if err := json.Unmarshal(respBody, &stResp); err != nil { return err } + utils.Logger.Debug(fmt.Sprintf("###SureTax received response: %+v\n", stResp)) if stResp.ResponseCode != 9999 { cdr.ExtraInfo = stResp.HeaderMessage return nil // No error because the request was processed by SureTax, error will be in the ExtraInfo diff --git a/general_tests/suretax_it_test.go b/general_tests/suretax_it_test.go new file mode 100644 index 000000000..5ade9bce0 --- /dev/null +++ b/general_tests/suretax_it_test.go @@ -0,0 +1,175 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2015 ITsysCOM GmbH + +This program is free software: you can Storagetribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITH*out ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package general_tests + +import ( + "flag" + "net/rpc" + "net/rpc/jsonrpc" + "reflect" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +/* +Integration tests with SureTax platform. +Configuration file is kept outside of CGRateS repository since it contains sensitive customer information +*/ + +var testSureTax = flag.Bool("suretax", false, "Pefrom SureTax integration tests when this flag is activated") +var configDir = flag.String("config_dir", "", "CGR config dir path here") +var tpDir = flag.String("tp_dir", "", "CGR config dir path here") + +var stiCfg *config.CGRConfig +var stiRpc *rpc.Client +var stiLoadInst engine.LoadInstance + +func TestSTIInitCfg(t *testing.T) { + if !*testSureTax { + return + } + // Init config first + var err error + stiCfg, err = config.NewCGRConfigFromFolder(*configDir) + if err != nil { + t.Error(err) + } +} + +// Remove data in both rating and accounting db +func TestSTIResetDataDb(t *testing.T) { + if !*testSureTax { + return + } + if err := engine.InitDataDb(stiCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func TestSTIResetStorDb(t *testing.T) { + if !*testSureTax { + return + } + if err := engine.InitStorDb(stiCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func TestSTIStartEngine(t *testing.T) { + if !*testSureTax { + return + } + if _, err := engine.StopStartEngine(*configDir, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func TestSTIRpcConn(t *testing.T) { + if !*testSureTax { + return + } + var err error + stiRpc, err = jsonrpc.Dial("tcp", stiCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +// Load the tariff plan, creating accounts and their balances +func TestSTILoadTariffPlanFromFolder(t *testing.T) { + if !*testSureTax { + return + } + attrs := &utils.AttrLoadTpFromFolder{FolderPath: *tpDir} + if err := stiRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &stiLoadInst); err != nil { + t.Error(err) + } else if stiLoadInst.LoadId == "" { + t.Error("Empty loadId received, loadInstance: ", stiLoadInst) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups +} + +// Check loaded stats +func TestSTICacheStats(t *testing.T) { + if !*testSureTax { + return + } + var rcvStats *utils.CacheStats + expectedStats := &utils.CacheStats{Destinations: 1, RatingPlans: 1, RatingProfiles: 1, DerivedChargers: 1, + LastLoadId: stiLoadInst.LoadId, LastLoadTime: stiLoadInst.LoadTime.Format(time.RFC3339)} + var args utils.AttrCacheStats + if err := stiRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil { + t.Error("Got error on ApierV2.GetCacheStats: ", err.Error()) + } else if !reflect.DeepEqual(expectedStats, rcvStats) { + t.Errorf("Calling ApierV2.GetCacheStats expected: %+v, received: %+v", expectedStats, rcvStats) + } +} + +// Test CDR from external sources +func TestSTIProcessExternalCdr(t *testing.T) { + if !*testSureTax { + return + } + cdr := &engine.ExternalCdr{TOR: utils.VOICE, + AccId: "teststicdr1", CdrHost: "192.168.1.1", CdrSource: "STI_TEST", ReqType: utils.META_RATED, Direction: utils.OUT, + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "+14082342500", Destination: "+16268412300", Supplier: "SUPPL1", + SetupTime: "2015-10-18T13:00:00Z", AnswerTime: "2015-10-18T13:00:00Z", + Usage: "15s", Pdd: "7.0", ExtraFields: map[string]string{"ClientNumber": "000000534", "": "valextr2"}, + } + var reply string + if err := stiRpc.Call("CdrsV2.ProcessExternalCdr", cdr, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + time.Sleep(time.Duration(*waitRater) * time.Millisecond) +} + +func TestSTIGetCdrs(t *testing.T) { + if !*testSureTax { + return + } + var cdrs []*engine.ExternalCdr + req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}, Accounts: []string{"1001"}} + if err := stiRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Error("Unexpected number of CDRs returned: ", len(cdrs)) + } else { + if cdrs[0].Cost != 0.012 { + t.Errorf("Unexpected Cost for CDR: %+v", cdrs[0]) + } + } +} + +func TestSTIStopCgrEngine(t *testing.T) { + if !*testSureTax { + return + } + if err := engine.KillEngine(100); err != nil { + t.Error(err) + } +} diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 78eb2205f..d4e404915 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -638,7 +638,6 @@ func TestTutLocalCostErrors(t *testing.T) { } else if reply != utils.OK { t.Error("Unexpected reply received: ", reply) } - var cdrs []*engine.ExternalCdr req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}, Accounts: []string{cdr.Account}, DestPrefixes: []string{cdr.Destination}} if err := tutLocalRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil {