Updated configs for cdre

config/config_defaults.go

@@ -746,10 +746,10 @@ const CGRATES_CFG_JSON = `
 	"response_group": "03",					// determines how taxes are grouped for the response <03|13>
 	"response_type": "D4",					// determines the granularity of taxes and (optionally) the decimal precision for the tax calculations and amounts in the response
 	"regulatory_code": "03",				// provider type
-	"client_tracking": "~CGRID",			// template extracting client information out of StoredCdr; <RSRParsers>
-	"customer_number": "~Subject",			// template extracting customer number out of StoredCdr; <RSRParsers>
-	"orig_number":  "~Subject", 			// template extracting origination number out of StoredCdr; <RSRParsers>
-	"term_number": "~Destination",			// template extracting termination number out of StoredCdr; <RSRParsers>
+	"client_tracking": "~*req.CGRID",		// template extracting client information out of StoredCdr; <RSRParsers>
+	"customer_number": "~*req.Subject",		// template extracting customer number out of StoredCdr; <RSRParsers>
+	"orig_number":  "~*req.Subject", 		// template extracting origination number out of StoredCdr; <RSRParsers>
+	"term_number": "~*req.Destination",		// template extracting termination number out of StoredCdr; <RSRParsers>
 	"bill_to_number": "",					// template extracting billed to number out of StoredCdr; <RSRParsers>
 	"zipcode": "",							// template extracting billing zip code out of StoredCdr; <RSRParsers>
 	"plus4": "",							// template extracting billing zip code extension out of StoredCdr; <RSRParsers>

config/config_json_test.go

@@ -1423,10 +1423,10 @@ func TestDfSureTaxJsonCfg(t *testing.T) {
 		Response_group:          utils.StringPointer("03"),
 		Response_type:           utils.StringPointer("D4"),
 		Regulatory_code:         utils.StringPointer("03"),
-		Client_tracking:         utils.StringPointer(utils.DynamicDataPrefix + utils.CGRID),
-		Customer_number:         utils.StringPointer("~Subject"),
-		Orig_number:             utils.StringPointer("~Subject"),
-		Term_number:             utils.StringPointer("~Destination"),
+		Client_tracking:         utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.CGRID),
+		Customer_number:         utils.StringPointer("~*req.Subject"),
+		Orig_number:             utils.StringPointer("~*req.Subject"),
+		Term_number:             utils.StringPointer("~*req.Destination"),
 		Bill_to_number:          utils.StringPointer(""),
 		Zipcode:                 utils.StringPointer(""),
 		Plus4:                   utils.StringPointer(""),

config/config_test.go

@@ -917,10 +917,10 @@ func TestCgrCfgJSONDefaultsSureTax(t *testing.T) {
 		ResponseGroup:        "03",
 		ResponseType:         "D4",
 		RegulatoryCode:       "03",
-		ClientTracking:       NewRSRParsersMustCompile("~CGRID", true, utils.INFIELD_SEP),
-		CustomerNumber:       NewRSRParsersMustCompile("~Subject", true, utils.INFIELD_SEP),
-		OrigNumber:           NewRSRParsersMustCompile("~Subject", true, utils.INFIELD_SEP),
-		TermNumber:           NewRSRParsersMustCompile("~Destination", true, utils.INFIELD_SEP),
+		ClientTracking:       NewRSRParsersMustCompile("~*req.CGRID", true, utils.INFIELD_SEP),
+		CustomerNumber:       NewRSRParsersMustCompile("~*req.Subject", true, utils.INFIELD_SEP),
+		OrigNumber:           NewRSRParsersMustCompile("~*req.Subject", true, utils.INFIELD_SEP),
+		TermNumber:           NewRSRParsersMustCompile("~*req.Destination", true, utils.INFIELD_SEP),
 		BillToNumber:         NewRSRParsersMustCompile("", true, utils.INFIELD_SEP),
 		Zipcode:              NewRSRParsersMustCompile("", true, utils.INFIELD_SEP),
 		P2PZipcode:           NewRSRParsersMustCompile("", true, utils.INFIELD_SEP),

data/conf/samples/cdrsonexpmaster_mongo/cdrsreplicationmaster.json

@@ -44,7 +44,7 @@
 	"http_localhost": {
 		"export_format": "*http_post",
 		"export_path": "http://127.0.0.1:12080/cdr_http",
-		"content_fields": [								// template of the exported content fields
+		"fields": [								// template of the exported content fields
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 			{"path": "*exp.RunID", "type": "*composed", "value": "~*req.RunID"},
 			{"path": "*exp.ToR", "type": "*composed", "value": "~*req.ToR"},
@@ -66,7 +66,7 @@
 		"export_format": "*amqp_json_map",
 		"export_path": "amqp://guest:guest@localhost:5672/?queue_id=cgrates_cdrs&exchange=exchangename&exchange_type=fanout&routing_key=cgr_cdrs",
 		"attempts": 3,
-		"content_fields": [								// template of the exported content fields
+		"fields": [								// template of the exported content fields
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 			{"path": "*exp.RunID", "type": "*composed", "value": "~*req.RunID"},
 			{"path": "*exp.ToR", "type": "*composed", "value": "~*req.ToR"},
@@ -87,14 +87,14 @@
 	"http_test_file": {
 		"export_format": "*http_post",
 		"export_path": "http://127.0.0.1:12080/invalid",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.OriginID", "type": "*composed", "value": "~*req.OriginID"},
 		],
 	},
 	"aws_test_file": {
 		"export_format": "*amqpv1_json_map",
 		"export_path": "amqps://guest:guest@localhost:25672/?queue_id=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
@@ -102,21 +102,21 @@
 		"export_format": "*sqs_json_map",
 		// export_path for sqs:  "endpoint?aws_region=region&aws_key=IDkey&aws_secret=secret&aws_token=sessionToken&queue_id=cgrates-cdrs"  
 		"export_path": "http://sqs.eu-west-2.amazonaws.com/?aws_region=eu-west-2&aws_key=testkey&aws_secret=testsecret&queue_id=cgrates-cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
 	"amqp_test_file": {
 		"export_format": "*amqp_json_map",
 		"export_path": "amqp://guest:guest@localhost:25672/?queue_id=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
 	"kafka_localhost": {
 		"export_format": "*kafka_json_map",
 		"export_path": "localhost:9092?topic=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
@@ -124,7 +124,7 @@
 		"export_format": "*s3_json_map",
 		// export_path for s3:  "endpoint?aws_region=region&aws_key=IDkey&aws_secret=secret&aws_token=sessionToken&queue_id=cgrates-cdrs"  
 		"export_path": "http://s3.us-east-2.amazonaws.com/?aws_region=eu-west-2&aws_key=testkey&aws_secret=testsecret&queue_id=cgrates-cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},

data/conf/samples/cdrsonexpmaster_mysql/cdrsreplicationmaster.json

@@ -42,7 +42,7 @@
 	"http_localhost": {
 		"export_format": "*http_post",
 		"export_path": "http://127.0.0.1:12080/cdr_http",
-		"content_fields": [								// template of the exported content fields
+		"fields": [								// template of the exported content fields
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 			{"path": "*exp.RunID", "type": "*composed", "value": "~*req.RunID"},
 			{"path": "*exp.ToR", "type": "*composed", "value": "~*req.ToR"},
@@ -64,7 +64,7 @@
 		"export_format": "*amqp_json_map",
 		"export_path": "amqp://guest:guest@localhost:5672/?queue_id=cgrates_cdrs&exchange=exchangename&exchange_type=fanout&routing_key=cgr_cdrs",
 		"attempts": 3,
-		"content_fields": [								// template of the exported content fields
+		"fields": [								// template of the exported content fields
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 			{"path": "*exp.RunID", "type": "*composed", "value": "~*req.RunID"},
 			{"path": "*exp.ToR", "type": "*composed", "value": "~*req.ToR"},
@@ -85,14 +85,14 @@
 	"http_test_file": {
 		"export_format": "*http_post",
 		"export_path": "http://127.0.0.1:12080/invalid",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.OriginID", "type": "*composed", "value": "~*req.OriginID"},
 		],
 	},
 	"aws_test_file": {
 		"export_format": "*amqpv1_json_map",
 		"export_path": "amqps://guest:guest@localhost:25672/?queue_id=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
@@ -100,21 +100,21 @@
 		"export_format": "*sqs_json_map",
 		// export_path for sqs:  "endpoint?aws_region=region&aws_key=IDkey&aws_secret=secret&aws_token=sessionToken&queue_id=cgrates-cdrs"  
 		"export_path": "http://sqs.eu-west-2.amazonaws.com/?aws_region=eu-west-2&aws_key=testkey&aws_secret=testsecret&queue_id=cgrates-cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
 	"amqp_test_file": {
 		"export_format": "*amqp_json_map",
 		"export_path": "amqp://guest:guest@localhost:25672/?queue_id=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
 	"kafka_localhost": {
 		"export_format": "*kafka_json_map",
 		"export_path": "localhost:9092?topic=cgrates_cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},
@@ -122,7 +122,7 @@
 		"export_format": "*s3_json_map",
 		// export_path for s3:  "endpoint?aws_region=region&aws_key=IDkey&aws_secret=secret&aws_token=sessionToken&queue_id=cgrates-cdrs"  
 		"export_path": "http://s3.us-east-2.amazonaws.com/?aws_region=eu-west-2&aws_key=testkey&aws_secret=testsecret&queue_id=cgrates-cdrs",
-		"content_fields": [
+		"fields": [
 			{"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"},
 		],
 	},

engine/cdr.go

@@ -162,67 +162,14 @@ func (cdr *CDR) FieldAsString(rsrPrs *config.RSRParser) (parsed string, err erro
 
 // FieldsAsString concatenates values of multiple fields defined in template, used eg in CDR templates
 func (cdr *CDR) FieldsAsString(rsrFlds config.RSRParsers) string {
-	outVal, err := rsrFlds.ParseDataProviderWithInterfaces(config.NewNavigableMap(cdr.AsMapStringIface()), utils.NestingSep)
+	outVal, err := rsrFlds.ParseDataProviderWithInterfaces(
+		config.NewNavigableMap(map[string]interface{}{utils.MetaReq: cdr.AsMapStringIface()}), utils.NestingSep)
 	if err != nil {
 		return ""
 	}
 	return outVal
 }
 
-// ParseFieldValue populates the field with id from value; strings are appended to original one
-func (cdr *CDR) ParseFieldValue(fieldId, fieldVal, timezone string) error {
-	var err error
-	switch fieldId {
-	case utils.OrderID:
-		if cdr.OrderID, err = strconv.ParseInt(fieldVal, 10, 64); err != nil {
-			return err
-		}
-	case utils.OriginHost: // overwrite if originHost is given from template
-		cdr.OriginHost = fieldVal
-	case utils.ToR:
-		cdr.ToR = fieldVal
-	case utils.RunID:
-		cdr.RunID = fieldVal
-	case utils.OriginID:
-		cdr.OriginID = fieldVal
-	case utils.RequestType:
-		cdr.RequestType = fieldVal
-	case utils.Tenant:
-		cdr.Tenant = fieldVal
-	case utils.Category:
-		cdr.Category = fieldVal
-	case utils.Account:
-		cdr.Account = fieldVal
-	case utils.Subject:
-		cdr.Subject = fieldVal
-	case utils.Destination:
-		cdr.Destination = fieldVal
-	case utils.PreRated:
-		cdr.PreRated, _ = strconv.ParseBool(fieldVal)
-	case utils.SetupTime:
-		if cdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
-			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
-		}
-	case utils.AnswerTime:
-		if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
-			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
-		}
-	case utils.Usage:
-		if cdr.Usage, err = utils.ParseDurationWithNanosecs(fieldVal); err != nil {
-			return fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
-		}
-	case utils.COST:
-		if cdr.Cost, err = strconv.ParseFloat(fieldVal, 64); err != nil {
-			return fmt.Errorf("Cannot parse cost field with value: %s, err: %s", fieldVal, err.Error())
-		}
-	case utils.Partial:
-		cdr.Partial, _ = strconv.ParseBool(fieldVal)
-	default: // Extra fields will not match predefined so they all show up here
-		cdr.ExtraFields[fieldId] = fieldVal
-	}
-	return nil
-}
-
 func (cdr *CDR) Clone() *CDR {
 	if cdr == nil {
 		return nil
@@ -483,8 +430,7 @@ func (cdr *CDR) AsExportRecord(exportFields []*config.FCTemplate,
 func (cdr *CDR) AsExportMap(exportFields []*config.FCTemplate, httpSkipTLSCheck bool,
 	groupedCDRs []*CDR, filterS *FilterS) (expMap map[string]string, err error) {
 	expMap = make(map[string]string)
-	nM := config.NewNavigableMap(nil)
-	nM.Set([]string{utils.MetaReq}, cdr.AsMapStringIface(), false, false)
+	nM := config.NewNavigableMap(map[string]interface{}{utils.MetaReq: cdr.AsMapStringIface()})
 	for _, cfgFld := range exportFields {
 		if !strings.HasPrefix(cfgFld.Path, utils.MetaExp+utils.NestingSep) {
 			continue

engine/cdr_test.go

@@ -227,8 +227,8 @@ func TestFieldsAsString(t *testing.T) {
 	}
 	eVal := "call_from_1001"
 	if val := cdr.FieldsAsString(
-		config.NewRSRParsersMustCompile("~Category;_from_;~Account", true, utils.INFIELD_SEP)); val != eVal {
-		t.Errorf("Expecting : %s, received: %s", eVal, val)
+		config.NewRSRParsersMustCompile("~*req.Category;_from_;~*req.Account", true, utils.INFIELD_SEP)); val != eVal {
+		t.Errorf("Expecting : %s, received: %q", eVal, val)
 	}
 }
 
@@ -451,25 +451,6 @@ func TestUsageReqAsCD(t *testing.T) {
 	}
 }
 
-func TestCDRParseFieldValue(t *testing.T) {
-	cdr := new(CDR)
-	if err := cdr.ParseFieldValue(utils.Partial, "true", ""); err != nil {
-		t.Error(err)
-	} else if !cdr.Partial {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-	if err := cdr.ParseFieldValue(utils.OrderID, "5", ""); err != nil {
-		t.Error(err)
-	} else if cdr.OrderID != 5 {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-	if err := cdr.ParseFieldValue(utils.RunID, utils.MetaDefault, ""); err != nil {
-		t.Error(err)
-	} else if cdr.RunID != utils.MetaDefault {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-}
-
 func TestCdrClone(t *testing.T) {
 	cdr := &CDR{}
 	eOut := &CDR{}
@@ -1109,25 +1090,6 @@ func TestCDRUpdateFromCGREvent(t *testing.T) {
 	}
 }
 
-func TestCDRParseFieldValue2(t *testing.T) {
-	cdr := new(CDR)
-	if err := cdr.ParseFieldValue(utils.RunID, utils.MetaDefault, ""); err != nil {
-		t.Error(err)
-	} else if cdr.RunID != utils.MetaDefault {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-	if err := cdr.ParseFieldValue(utils.OriginID, "FirstID", ""); err != nil {
-		t.Error(err)
-	} else if cdr.OriginID != "FirstID" {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-	if err := cdr.ParseFieldValue(utils.OriginID, "SecondID", ""); err != nil {
-		t.Error(err)
-	} else if cdr.OriginID != "SecondID" {
-		t.Errorf("Received cdr: %+v", cdr)
-	}
-}
-
 func TestCDRAddDefaults(t *testing.T) {
 	cdr := &CDR{
 		OriginID:   "dsafdsaf",

engine/suretax.go

@@ -38,7 +38,7 @@ var sureTaxClient *http.Client // Cache the client here if in use
 // Init a new request to be sent out to SureTax
 func NewSureTaxRequest(cdr *CDR, stCfg *config.SureTaxCfg) (*SureTaxRequest, error) {
 	if stCfg == nil {
-		return nil, errors.New("Invalid SureTax config.")
+		return nil, errors.New("invalid SureTax config")
 	}
 	aTimeLoc := cdr.AnswerTime.In(stCfg.Timezone)
 	revenue := utils.Round(cdr.Cost, 4, utils.ROUNDING_MIDDLE)

general_tests/cdrs_onlexp_it_test.go

@@ -365,7 +365,7 @@ func testCDRsOnExpAMQPReplication(t *testing.T) {
 	if err := exec.Command("service", "rabbitmq-server", "restart").Run(); err != nil {
 		t.Error(err)
 	}
-	time.Sleep(time.Duration(5 * time.Second))
+	time.Sleep(5 * time.Second)
 	testCdr := &engine.CDR{
 		CGRID:       amqpCGRID,
 		ToR:         utils.VOICE,
@@ -469,7 +469,7 @@ func checkContent(ev *engine.ExportEvents, content []interface{}) error {
 }
 
 func testCDRsOnExpFileFailover(t *testing.T) {
-	time.Sleep(time.Duration(5 * time.Second))
+	time.Sleep(5 * time.Second)
 	v1 := url.Values{}
 	v2 := url.Values{}
 	v1.Set("OriginID", "httpjsonrpc1")