First cut of SureTax implementation, with integration tests

config/config_defaults.go

@@ -290,7 +290,7 @@ const CGRATES_CFG_JSON = `
 	"response_type": "D4",					// determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response
 	"regulatory_code": "03",				// provider type
 	"client_tracking": "CgrId",				// template extracting client information out of StoredCdr; <$RSRFields>
-	"customer_number": "CustomerNumber",	// template extracting customer number out of StoredCdr; <$RSRFields>
+	"customer_number": "Subject",			// template extracting customer number out of StoredCdr; <$RSRFields>
 	"orig_number":  "Subject", 				// template extracting origination number out of StoredCdr; <$RSRFields>
 	"term_number": "Destination",			// template extracting termination number out of StoredCdr; <$RSRFields>
 	"bill_to_number": "",					// template extracting billed to number out of StoredCdr; <$RSRFields>

config/config_json_test.go

@@ -469,7 +469,7 @@ func TestDfSureTaxJsonCfg(t *testing.T) {
 		Response_type:           utils.StringPointer("D4"),
 		Regulatory_code:         utils.StringPointer("03"),
 		Client_tracking:         utils.StringPointer("CgrId"),
-		Customer_number:         utils.StringPointer("CustomerNumber"),
+		Customer_number:         utils.StringPointer("Subject"),
 		Orig_number:             utils.StringPointer("Subject"),
 		Term_number:             utils.StringPointer("Destination"),
 		Bill_to_number:          utils.StringPointer(""),

data/conf/cgrates/cgrates.json

@@ -1,5 +1,6 @@
 {
 
+
 // Real-time Charging System for Telecom & ISP environments
 // Copyright (C) ITsysCOM GmbH
 //
@@ -8,19 +9,20 @@
 
 
 //"general": {
-//	"http_skip_tls_verify": false,			// if enabled Http Client will accept any TLS certificate
-//	"rounding_decimals": 10,				// system level precision for floats
-//	"dbdata_encoding": "msgpack",			// encoding used to store object data in strings: <msgpack|json>
-//	"tpexport_dir": "/var/log/cgrates/tpe",	// path towards export folder for offline Tariff Plans
-//	"default_reqtype": "*rated",			// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
-//	"default_category": "call",				// default Type of Record to consider when missing from requests
-//	"default_tenant": "cgrates.org",		// default Tenant to consider when missing from requests
-//	"default_subject": "cgrates",			// default rating Subject to consider when missing from requests
-//	"default_timezone": "Local",			// default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>
-//	"connect_attempts": 3,					// initial server connect attempts
-//	"response_cache_ttl": "3s",				// the life span of a cached response
-//	"reconnects": -1,						// number of retries in case of connection lost
-//	"internal_ttl": "2m",					// maximum duration to wait for internal connections before giving up
+//	"http_skip_tls_verify": false,						// if enabled Http Client will accept any TLS certificate
+//	"rounding_decimals": 10,							// system level precision for floats
+//	"dbdata_encoding": "msgpack",						// encoding used to store object data in strings: <msgpack|json>
+//	"tpexport_dir": "/var/log/cgrates/tpe",				// path towards export folder for offline Tariff Plans
+//	"http_failed_dir": "/var/log/cgrates/http_failed",	// directory path where we store failed http requests
+//	"default_reqtype": "*rated",						// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+//	"default_category": "call",							// default Type of Record to consider when missing from requests
+//	"default_tenant": "cgrates.org",					// default Tenant to consider when missing from requests
+//	"default_subject": "cgrates",						// default rating Subject to consider when missing from requests
+//	"default_timezone": "Local",						// default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>
+//	"connect_attempts": 3,								// initial server connect attempts
+//	"reconnects": -1,									// number of retries in case of connection lost
+//	"response_cache_ttl": "3s",							// the life span of a cached response
+//	"internal_ttl": "2m",								// maximum duration to wait for internal connections before giving up
 //},
 
 
@@ -31,7 +33,7 @@
 //},
 
 
-//"tariffplan_db": {						// database used to store active tariff plan configuration
+//"tariffplan_db": {							// database used to store active tariff plan configuration
 //	"db_type": "redis",						// tariffplan_db type: <redis>
 //	"db_host": "127.0.0.1",					// tariffplan_db host address
 //	"db_port": 6379, 						// port to reach the tariffplan_db
@@ -258,4 +260,33 @@
 //},
 
 
+//"suretax": {
+//	"url": "",								// API url
+//	"client_number": "",					// client number, provided by SureTax
+//	"validation_key": "",					// validation key provided by SureTax
+//	"timezone": "Local",					// convert the time of the events to this timezone before sending request out <UTC|Local|$IANA_TZ_DB>
+//	"include_local_cost": false,			// sum local calculated cost with tax one in final cost
+//	"return_file_code": "0",				// default or Quote purposes <0|Q>
+//	"response_group": "03",					// determines how taxes are grouped for the response <03|13>
+//	"response_type": "D4",					// determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response
+//	"regulatory_code": "03",				// provider type
+//	"client_tracking": "CgrId",				// template extracting client information out of StoredCdr; <$RSRFields>
+//	"customer_number": "CustomerNumber",	// template extracting customer number out of StoredCdr; <$RSRFields>
+//	"orig_number":  "Subject", 				// template extracting origination number out of StoredCdr; <$RSRFields>
+//	"term_number": "Destination",			// template extracting termination number out of StoredCdr; <$RSRFields>
+//	"bill_to_number": "",					// template extracting billed to number out of StoredCdr; <$RSRFields>
+//	"zipcode": "",							// template extracting billing zip code out of StoredCdr; <$RSRFields>
+//	"plus4": "",							// template extracting billing zip code extension out of StoredCdr; <$RSRFields>
+//	"p2pzipcode": "",						// template extracting secondary zip code out of StoredCdr; <$RSRFields>
+//	"p2pplus4": "",							// template extracting secondary zip code extension out of StoredCdr; <$RSRFields>
+//	"units": "^1",							// template extracting number of “lines” or unique charges contained within the revenue out of StoredCdr; <$RSRFields>
+//	"unit_type": "^00",						// template extracting number of unique access lines out of StoredCdr; <$RSRFields>
+//	"tax_included": "^0",					// template extracting tax included in revenue out of StoredCdr; <$RSRFields>
+//	"tax_situs_rule": "^04",				// template extracting tax situs rule out of StoredCdr; <$RSRFields>
+//	"trans_type_code": "^010101",			// template extracting transaction type indicator out of StoredCdr; <$RSRFields>
+//	"sales_type_code": "^R",				// template extracting sales type code out of StoredCdr; <$RSRFields>
+//	"tax_exemption_code_list": "",			// template extracting tax exemption code list out of StoredCdr; <$RSRFields>
+//},
+
+
 }

engine/cdrs.go

@@ -202,6 +202,22 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(storedCdr *StoredCdr) error
 		return err
 	}
 	for _, cdr := range cdrRuns {
+		if cdr.MediationRunId != utils.META_DEFAULT { // Process Aliases and Users for derived CDRs
+			if err := LoadAlias(&AttrMatchingAlias{
+				Destination: cdr.Destination,
+				Direction:   cdr.Direction,
+				Tenant:      cdr.Tenant,
+				Category:    cdr.Category,
+				Account:     cdr.Account,
+				Subject:     cdr.Subject,
+				Context:     utils.ALIAS_CONTEXT_RATING,
+			}, cdr, utils.EXTRA_FIELDS); err != nil && err != utils.ErrNotFound {
+				return err
+			}
+			if err := LoadUserProfile(cdr, utils.EXTRA_FIELDS); err != nil {
+				return err
+			}
+		}
 		// Rate CDR
 		if self.rater != nil && !cdr.Rated {
 			if err := self.rateCDR(cdr); err != nil {
@@ -209,6 +225,12 @@ func (self *CdrServer) deriveRateStoreStatsReplicate(storedCdr *StoredCdr) error
 				cdr.ExtraInfo = err.Error()
 			}
 		}
+		if cdr.MediationRunId == utils.META_SURETAX { // Request should be processed by SureTax
+			if err := SureTaxProcessCdr(cdr); err != nil {
+				cdr.Cost = -1.0
+				cdr.ExtraInfo = err.Error() // Something failed, write the error in the ExtraInfo
+			}
+		}
 		if self.cgrCfg.CDRSStoreCdrs { // Store CDRs
 			// Store RatedCDR
 			if err := self.cdrDb.SetRatedCdr(cdr); err != nil {

engine/suretax.go

@@ -38,6 +38,9 @@ var sureTaxClient *http.Client // Cache the client here if in use
 
 // Init a new request to be sent out to SureTax
 func NewSureTaxRequest(cdr *StoredCdr, stCfg *config.SureTaxCfg) (*SureTaxRequest, error) {
+	if stCfg == nil {
+		return nil, errors.New("Invalid SureTax config.")
+	}
 	aTimeLoc := cdr.AnswerTime.In(stCfg.Timezone)
 	revenue := utils.Round(cdr.Cost, 4, utils.ROUNDING_MIDDLE)
 	unts, err := strconv.ParseInt(cdr.FieldsAsString(stCfg.Units), 10, 64)
@@ -175,7 +178,7 @@ type STTaxItem struct {
 func SureTaxProcessCdr(cdr *StoredCdr) error {
 	stCfg := config.CgrConfig().SureTaxCfg()
 	if stCfg == nil {
-		return errors.New("SureTax configuration missing")
+		return errors.New("Invalid SureTax configuration")
 	}
 	if sureTaxClient == nil { // First time used, init the client here
 		tr := &http.Transport{
@@ -187,10 +190,12 @@ func SureTaxProcessCdr(cdr *StoredCdr) error {
 	if err != nil {
 		return err
 	}
+
 	body, err := json.Marshal(req)
 	if err != nil {
 		return err
 	}
+	utils.Logger.Debug(fmt.Sprintf("###SureTax NewSureTaxRequest: %+v, ItemList: %+v\n", req, req.ItemList[0]))
 	resp, err := sureTaxClient.Post(stCfg.Url, "application/json", bytes.NewBuffer(body))
 	if err != nil {
 		return err
@@ -207,6 +212,7 @@ func SureTaxProcessCdr(cdr *StoredCdr) error {
 	if err := json.Unmarshal(respBody, &stResp); err != nil {
 		return err
 	}
+	utils.Logger.Debug(fmt.Sprintf("###SureTax received response: %+v\n", stResp))
 	if stResp.ResponseCode != 9999 {
 		cdr.ExtraInfo = stResp.HeaderMessage
 		return nil // No error because the request was processed by SureTax, error will be in the ExtraInfo

general_tests/suretax_it_test.go

@@ -0,0 +1,175 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package general_tests
+
+import (
+	"flag"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+/*
+Integration tests with SureTax platform.
+Configuration file is kept outside of CGRateS repository since it contains sensitive customer information
+*/
+
+var testSureTax = flag.Bool("suretax", false, "Pefrom SureTax integration tests when this flag is activated")
+var configDir = flag.String("config_dir", "", "CGR config dir path here")
+var tpDir = flag.String("tp_dir", "", "CGR config dir path here")
+
+var stiCfg *config.CGRConfig
+var stiRpc *rpc.Client
+var stiLoadInst engine.LoadInstance
+
+func TestSTIInitCfg(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	// Init config first
+	var err error
+	stiCfg, err = config.NewCGRConfigFromFolder(*configDir)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+// Remove data in both rating and accounting db
+func TestSTIResetDataDb(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	if err := engine.InitDataDb(stiCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Wipe out the cdr database
+func TestSTIResetStorDb(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	if err := engine.InitStorDb(stiCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Start CGR Engine
+func TestSTIStartEngine(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	if _, err := engine.StopStartEngine(*configDir, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestSTIRpcConn(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	var err error
+	stiRpc, err = jsonrpc.Dial("tcp", stiCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Load the tariff plan, creating accounts and their balances
+func TestSTILoadTariffPlanFromFolder(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	attrs := &utils.AttrLoadTpFromFolder{FolderPath: *tpDir}
+	if err := stiRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &stiLoadInst); err != nil {
+		t.Error(err)
+	} else if stiLoadInst.LoadId == "" {
+		t.Error("Empty loadId received, loadInstance: ", stiLoadInst)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups
+}
+
+// Check loaded stats
+func TestSTICacheStats(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	var rcvStats *utils.CacheStats
+	expectedStats := &utils.CacheStats{Destinations: 1, RatingPlans: 1, RatingProfiles: 1, DerivedChargers: 1,
+		LastLoadId: stiLoadInst.LoadId, LastLoadTime: stiLoadInst.LoadTime.Format(time.RFC3339)}
+	var args utils.AttrCacheStats
+	if err := stiRpc.Call("ApierV2.GetCacheStats", args, &rcvStats); err != nil {
+		t.Error("Got error on ApierV2.GetCacheStats: ", err.Error())
+	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
+		t.Errorf("Calling ApierV2.GetCacheStats expected: %+v, received: %+v", expectedStats, rcvStats)
+	}
+}
+
+// Test CDR from external sources
+func TestSTIProcessExternalCdr(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	cdr := &engine.ExternalCdr{TOR: utils.VOICE,
+		AccId: "teststicdr1", CdrHost: "192.168.1.1", CdrSource: "STI_TEST", ReqType: utils.META_RATED, Direction: utils.OUT,
+		Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "+14082342500", Destination: "+16268412300", Supplier: "SUPPL1",
+		SetupTime: "2015-10-18T13:00:00Z", AnswerTime: "2015-10-18T13:00:00Z",
+		Usage: "15s", Pdd: "7.0", ExtraFields: map[string]string{"ClientNumber": "000000534", "": "valextr2"},
+	}
+	var reply string
+	if err := stiRpc.Call("CdrsV2.ProcessExternalCdr", cdr, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply received: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
+}
+
+func TestSTIGetCdrs(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	var cdrs []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}, Accounts: []string{"1001"}}
+	if err := stiRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(cdrs) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", len(cdrs))
+	} else {
+		if cdrs[0].Cost != 0.012 {
+			t.Errorf("Unexpected Cost for CDR: %+v", cdrs[0])
+		}
+	}
+}
+
+func TestSTIStopCgrEngine(t *testing.T) {
+	if !*testSureTax {
+		return
+	}
+	if err := engine.KillEngine(100); err != nil {
+		t.Error(err)
+	}
+}

general_tests/tutorial_local_test.go

@@ -638,7 +638,6 @@ func TestTutLocalCostErrors(t *testing.T) {
 	} else if reply != utils.OK {
 		t.Error("Unexpected reply received: ", reply)
 	}
-
 	var cdrs []*engine.ExternalCdr
 	req := utils.RpcCdrsFilter{RunIds: []string{utils.META_DEFAULT}, Accounts: []string{cdr.Account}, DestPrefixes: []string{cdr.Destination}}
 	if err := tutLocalRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil {