CDRS auto-generating missing CGRID in processCDR, rejecting SMCosts when CGRID is missing, adding export tests to tutorial integration tests

engine/cdr.go

@@ -41,7 +41,7 @@ func NewCDRFromExternalCDR(extCdr *ExternalCDR, timezone string) (*CDR, error) {
 		}
 	}
 	if len(cdr.CGRID) == 0 { // Populate CGRID if not present
-		cdr.CGRID = utils.Sha1(cdr.OriginID, cdr.SetupTime.UTC().String())
+		cdr.ComputeCGRID()
 	}
 	if extCdr.AnswerTime != "" {
 		if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(extCdr.AnswerTime, timezone); err != nil {
@@ -113,6 +113,10 @@ func (cdr *CDR) CostDetailsJson() string {
 	return string(mrshled)
 }
 
+func (cdr *CDR) ComputeCGRID() {
+	cdr.CGRID = utils.Sha1(cdr.OriginID, cdr.SetupTime.UTC().String())
+}
+
 // Used to multiply usage on export
 func (cdr *CDR) UsageMultiply(multiplyFactor float64, roundDecimals int) {
 	cdr.Usage = time.Duration(int(utils.Round(float64(cdr.Usage.Nanoseconds())*multiplyFactor, roundDecimals, utils.ROUNDING_MIDDLE))) // Rounding down could introduce a slight loss here but only at nanoseconds level

engine/cdrs.go

@@ -175,6 +175,9 @@ func (self *CdrServer) processCdr(cdr *CDR) (err error) {
 	if !cdr.Rated { // Enforce the RunID if CDR is not rated
 		cdr.RunID = utils.MetaRaw
 	}
+	if cdr.RunID == utils.MetaRaw {
+		cdr.Cost = -1.0
+	}
 	if self.cgrCfg.CDRSStoreCdrs { // Store RawCDRs, this we do sync so we can reply with the status
 		if cdr.CostDetails != nil {
 			cdr.CostDetails.UpdateCost()
@@ -534,6 +537,9 @@ func (self *CdrServer) RateCDRs(cdrFltr *utils.CDRsFilter, sendToStats bool) err
 // Internally used and called from CDRSv1
 // Cached requests for HA setups
 func (self *CdrServer) V1ProcessCDR(cdr *CDR, reply *string) error {
+	if len(cdr.CGRID) == 0 { // Populate CGRID if not present
+		cdr.ComputeCGRID()
+	}
 	cacheKey := "V1ProcessCDR" + cdr.CGRID + cdr.RunID
 	if item, err := self.getCache().Get(cacheKey); err == nil && item != nil {
 		if item.Value != nil {
@@ -552,6 +558,11 @@ func (self *CdrServer) V1ProcessCDR(cdr *CDR, reply *string) error {
 
 // RPC method, differs from storeSMCost through it's signature
 func (self *CdrServer) V1StoreSMCost(attr AttrCDRSStoreSMCost, reply *string) error {
+	if attr.Cost.CGRID == "" {
+		return utils.NewCGRError(utils.CDRSCtx,
+			utils.MandatoryIEMissingCaps, fmt.Sprintf("%s: CGRID", utils.MandatoryInfoMissing),
+			"SMCost: %+v with empty CGRID")
+	}
 	cacheKey := "V1StoreSMCost" + attr.Cost.CGRID + attr.Cost.RunID + attr.Cost.OriginID
 	if item, err := self.getCache().Get(cacheKey); err == nil && item != nil {
 		if item.Value != nil {

general_tests/tutorial_it_test.go

@@ -21,6 +21,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package general_tests
 
 import (
+	"io/ioutil"
 	"net/rpc"
 	"net/rpc/jsonrpc"
 	"path"
@@ -1347,6 +1348,53 @@ func TestTutITPrepaidCDRWithoutSMCost(t *testing.T) {
 	*/
 }
 
+func TestTutITExportCDR(t *testing.T) {
+	cdr := &engine.CDR{ToR: utils.VOICE, OriginID: "testexportcdr1", OriginHost: "192.168.1.1", Source: "TestTutITExportCDR", RequestType: utils.META_RATED,
+		Direction: utils.OUT, Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1003",
+		SetupTime: time.Date(2016, 11, 30, 17, 5, 24, 0, time.UTC), AnswerTime: time.Date(2016, 11, 30, 17, 6, 4, 0, time.UTC),
+		Usage: time.Duration(98) * time.Second, Supplier: "suppl1",
+		ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}}
+	cdr.ComputeCGRID()
+	var reply string
+	if err := tutLocalRpc.Call("CdrsV1.ProcessCdr", cdr, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply received: ", reply)
+	}
+	time.Sleep(time.Duration(50) * time.Millisecond) // Give time for CDR to be processed
+	var cdrs []*engine.ExternalCDR
+	req := utils.RPCCDRsFilter{RunIDs: []string{utils.META_DEFAULT}, CGRIDs: []string{cdr.CGRID}}
+	if err := tutLocalRpc.Call("ApierV2.GetCdrs", req, &cdrs); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(cdrs) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", len(cdrs))
+	} else {
+		if cdrs[0].OriginID != cdr.OriginID {
+			t.Errorf("Unexpected OriginID for Cdr received: %+v", cdrs[0])
+		}
+		if cdrs[0].Cost != 1.3334 {
+			t.Errorf("Unexpected Cost for Cdr received: %+v", cdrs[0])
+		}
+	}
+	var replyExport utils.ExportedFileCdrs
+	exportArgs := utils.AttrExportCdrsToFile{ExportDirectory: utils.StringPointer("/tmp"),
+		ExportFileName: utils.StringPointer("TestTutITExportCDR.csv"),
+		RPCCDRsFilter:  utils.RPCCDRsFilter{CGRIDs: []string{cdr.CGRID}}}
+	if err := tutLocalRpc.Call("ApierV2.ExportCdrsToFile", exportArgs, &replyExport); err != nil {
+		t.Error(err)
+	}
+	eExportContent := `f0a92222a7d21b4d9f72744aabe82daef52e20d8,*default,*voice,testexportcdr1,*rated,*out,cgrates.org,call,1001,1001,1003,2016-11-30T18:05:24+01:00,2016-11-30T18:06:04+01:00,98,1.3334
+f0a92222a7d21b4d9f72744aabe82daef52e20d8,*raw,*voice,testexportcdr1,*rated,*out,cgrates.org,call,1001,1001,1003,2016-11-30T18:05:24+01:00,2016-11-30T18:06:04+01:00,98,-1.0000
+f0a92222a7d21b4d9f72744aabe82daef52e20d8,derived_run1,*voice,testexportcdr1,*rated,*out,cgrates.org,call,1001,1002,1003,2016-11-30T18:05:24+01:00,2016-11-30T18:06:04+01:00,98,1.3334
+`
+	if expContent, err := ioutil.ReadFile(path.Join(*exportArgs.ExportDirectory, *exportArgs.ExportFileName)); err != nil {
+		t.Error(err)
+	} else if eExportContent != string(expContent) {
+		t.Errorf("Expecting: <%q>, received: <%q>", eExportContent, string(expContent))
+	}
+
+}
+
 func TestTutITStopCgrEngine(t *testing.T) {
 	if err := engine.KillEngine(100); err != nil {
 		t.Error(err)

utils/apitpdata.go

@@ -1090,7 +1090,7 @@ type AttrExportCdrsToFile struct {
 	ExportID                   *string  // Optional exportid
 	ExportDirectory            *string  // If provided it overwrites the configured export directory
 	ExportFileName             *string  // If provided the output filename will be set to this
-	ExportTemplate             *string  // Exported fields template  <""|fld1,fld2|*xml:instance_name>
+	ExportTemplate             *string  // Exported fields template  <""|fld1,fld2|>
 	DataUsageMultiplyFactor    *float64 // Multiply data usage before export (eg: convert from KBytes to Bytes)
 	SMSUsageMultiplyFactor     *float64 // Multiply sms usage before export (eg: convert from SMS unit to call duration for some billing systems)
 	MMSUsageMultiplyFactor     *float64 // Multiply mms usage before export (eg: convert from MMS unit to call duration for some billing systems)

utils/consts.go

@@ -311,4 +311,6 @@ const (
 	ServerErrorCaps              = "SERVER_ERROR"
 	MandatoryIEMissingCaps       = "MANDATORY_IE_MISSING"
 	UnsupportedCachePrefix       = "unsupported cache prefix"
+	CDRSCtx                      = "cdrs"
+	MandatoryInfoMissing         = "mandatory information missing"
 )