From f436346873200131cbb25a2621dc2d3a1750a6a3 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 3 Jun 2014 19:05:36 +0200 Subject: [PATCH] Refactored CDRExporter to merge common exports into one mechanism, added CdreConfig type to collect configuration for CDRE from more sources --- apier/apier_local_test.go | 19 +- apier/cdre.go | 191 ++++----- apier/tutfscsv_local_test.go | 2 +- apier/tutfsjson_local_test.go | 2 +- cdre/cdrexporter.go | 393 +++++++++++++++++- cdre/csv.go | 85 ---- cdre/csv_test.go | 27 +- cdre/fixedwidth.go | 331 --------------- cdre/fixedwidth_test.go | 145 ++++--- cdre/{libfixedwidth.go => libcdre.go} | 0 ...{libfixedwidth_test.go => libcdre_test.go} | 0 config/cdreconfig.go | 223 ++++++++++ config/cdreconfig_test.go | 240 +++++++++++ config/config.go | 73 +--- config/config_local_test.go | 2 +- config/config_test.go | 72 ++-- config/test_data.txt | 16 +- config/xmlcdre.go | 113 ++++- config/xmlcdre_test.go | 250 ++++++++--- config/xmlconfig.go | 64 +-- data/conf/cgrates.cfg | 10 +- data/conf/samples/cgr_addconfig.xml | 41 +- data/conf/samples/multiplecdrc_fwexport.xml | 107 ++--- utils/apitpdata.go | 56 +-- utils/consts.go | 1 + utils/storedcdr.go | 28 +- utils/storedcdr_test.go | 34 +- utils/utils_local_test.go | 2 +- 28 files changed, 1586 insertions(+), 941 deletions(-) delete mode 100644 cdre/csv.go delete mode 100644 cdre/fixedwidth.go rename cdre/{libfixedwidth.go => libcdre.go} (100%) rename cdre/{libfixedwidth_test.go => libcdre_test.go} (100%) create mode 100644 config/cdreconfig.go create mode 100644 config/cdreconfig_test.go diff --git a/apier/apier_local_test.go b/apier/apier_local_test.go index c2f1202f8..d43291b81 100644 --- a/apier/apier_local_test.go +++ b/apier/apier_local_test.go @@ -31,7 +31,7 @@ import ( "path" "reflect" "sort" - "strings" + //"strings" "testing" "time" @@ -74,7 +74,7 @@ func TestCreateDirs(t *testing.T) { if !*testLocal { return } - for _, pathDir := range []string{cfg.CdreDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} { + for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} { if err := os.RemoveAll(pathDir); err != nil { t.Fatal("Error removing folder: ", pathDir, err) } @@ -1391,16 +1391,18 @@ func TestCdrServer(t *testing.T) { } } +/* func TestExportCdrsToFile(t *testing.T) { if !*testLocal { return } var reply *utils.ExportedFileCdrs req := utils.AttrExpFileCdrs{} - if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) { - t.Error("Failed to detect missing parameter") - } - req.CdrFormat = utils.CDRE_DRYRUN + //if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) { + // t.Error("Failed to detect missing parameter") + //} + dryRun := utils.CDRE_DRYRUN + req.CdrFormat = &dryRun tm1, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:22Z") tm2, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:23Z") expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCgrIds: []string{utils.Sha1("dsafdsaf", tm1.String()), @@ -1410,7 +1412,7 @@ func TestExportCdrsToFile(t *testing.T) { } else if !reflect.DeepEqual(reply, expectReply) { t.Errorf("Unexpected reply: %v", reply) } - /* Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN + Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN req.RemoveFromDb = true if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil { t.Error(err.Error()) @@ -1423,8 +1425,9 @@ func TestExportCdrsToFile(t *testing.T) { } else if !reflect.DeepEqual(reply, expectReply) { t.Errorf("Unexpected reply: %v", reply) } - */ + } +*/ func TestLocalGetCdrs(t *testing.T) { if !*testLocal { diff --git a/apier/cdre.go b/apier/cdre.go index 5652d8e14..4866b7bcf 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -19,9 +19,11 @@ along with this program. If not, see package apier import ( + "encoding/csv" "fmt" "github.com/cgrates/cgrates/cdre" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "os" "path" @@ -34,10 +36,7 @@ import ( func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error { var tStart, tEnd time.Time var err error - cdrFormat := strings.ToLower(attr.CdrFormat) - if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) { - return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat") - } + engine.Logger.Debug(fmt.Sprintf("ExportCdrsToFile: %+v", attr)) if len(attr.TimeStart) != 0 { if tStart, err = utils.ParseTimeDetectLayout(attr.TimeStart); err != nil { return err @@ -48,27 +47,76 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E return err } } - exportDir := attr.ExportDir - fileName := attr.ExportFileName - exportId := attr.ExportId - if len(exportId) == 0 { - exportId = strconv.FormatInt(time.Now().Unix(), 10) + exportTemplate := self.Config.CdreDefaultInstance + if attr.ExportTemplate != nil { // XML Template defined, can be field names or xml reference + if strings.HasPrefix(*attr.ExportTemplate, utils.XML_PROFILE_PREFIX) { + if self.Config.XmlCfgDocument == nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, "XmlDocumentNotLoaded") + } + expTplStr := *attr.ExportTemplate + if xmlTemplates := self.Config.XmlCfgDocument.GetCdreCfgs(expTplStr[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates == nil { + return fmt.Errorf("%s:ExportTemplate", utils.ERR_NOT_FOUND) + } else { + exportTemplate = xmlTemplates[expTplStr[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig() + } + } else { + exportTemplate, _ = config.NewDefaultCdreConfig() + if contentFlds, err := config.NewCdreCdrFieldsFromIds(strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } else { + exportTemplate.ContentFields = contentFlds + } + } } - costShiftDigits := self.Config.CdreCostShiftDigits - if attr.CostShiftDigits != -1 { // -1 enables system general config - costShiftDigits = attr.CostShiftDigits + if exportTemplate == nil { + return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING) } - roundDecimals := self.Config.RoundingDecimals - if attr.RoundDecimals != -1 { // -1 enables system default config - roundDecimals = attr.RoundDecimals + cdrFormat := exportTemplate.CdrFormat + if attr.CdrFormat != nil { + cdrFormat = strings.ToLower(*attr.CdrFormat) } - maskDestId := attr.MaskDestinationId - if len(maskDestId) == 0 { - maskDestId = self.Config.CdreMaskDestId + if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) { + return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat") } - maskLen := self.Config.CdreMaskLength - if attr.MaskLength != -1 { - maskLen = attr.MaskLength + exportDir := exportTemplate.ExportDir + if attr.ExportDir != nil { + exportDir = *attr.ExportDir + } + exportId := strconv.FormatInt(time.Now().Unix(), 10) + if attr.ExportId != nil { + exportId = exportId + } + fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat) + if attr.ExportFileName != nil { + fileName = *attr.ExportFileName + } + filePath := path.Join(exportDir, fileName) + if cdrFormat == utils.CDRE_DRYRUN { + filePath = utils.CDRE_DRYRUN + } + dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor + if attr.DataUsageMultiplyFactor != nil { + dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor + } + costMultiplyFactor := exportTemplate.CostMultiplyFactor + if attr.CostMultiplyFactor != nil { + costMultiplyFactor = *attr.CostMultiplyFactor + } + costShiftDigits := exportTemplate.CostShiftDigits + if attr.CostShiftDigits != nil { + costShiftDigits = *attr.CostShiftDigits + } + roundingDecimals := exportTemplate.CostRoundingDecimals + if attr.RoundDecimals != nil { + roundingDecimals = *attr.RoundDecimals + } + maskDestId := exportTemplate.MaskDestId + if attr.MaskDestinationId != nil { + maskDestId = *attr.MaskDestinationId + } + maskLen := exportTemplate.MaskLength + if attr.MaskLength != nil { + maskLen = *attr.MaskLength } cdrs, err := self.CdrDb.GetStoredCdrs(attr.CgrIds, attr.MediationRunId, attr.TOR, attr.CdrHost, attr.CdrSource, attr.ReqType, attr.Direction, attr.Tenant, attr.Category, attr.Account, attr.Subject, attr.DestinationPrefix, attr.OrderIdStart, attr.OrderIdEnd, tStart, tEnd, attr.SkipErrors, attr.SkipRated, false) @@ -78,86 +126,29 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} return nil } - switch cdrFormat { - case utils.CDRE_DRYRUN: - exportedIds := make([]string, len(cdrs)) - for idxCdr, cdr := range cdrs { - exportedIds[idxCdr] = cdr.CgrId - } - *reply = utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds} - case utils.CSV: - if len(exportDir) == 0 { - exportDir = path.Join(self.Config.CdreDir, utils.CSV) - } - if len(fileName) == 0 { - fileName = fmt.Sprintf("cdre_%s.csv", exportId) - } - exportedFields := self.Config.CdreExportedFields - if len(attr.ExportTemplate) != 0 { - if exportedFields, err = config.ParseRSRFields(attr.ExportTemplate); err != nil { - return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) - } - } - if len(exportedFields) == 0 { - return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING) - } - filePath := path.Join(exportDir, fileName) - fileOut, err := os.Create(filePath) - if err != nil { - return err - } - defer fileOut.Close() - csvWriter := cdre.NewCsvCdrWriter(fileOut, costShiftDigits, roundDecimals, maskDestId, maskLen, exportedFields) - exportedIds := make([]string, 0) - unexportedIds := make(map[string]string) - for _, cdr := range cdrs { - if err := csvWriter.WriteCdr(cdr); err != nil { - unexportedIds[cdr.CgrId] = err.Error() - } else { - exportedIds = append(exportedIds, cdr.CgrId) - } - } - csvWriter.Close() - *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: csvWriter.TotalCost(), - ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds, - FirstOrderId: csvWriter.FirstOrderId(), LastOrderId: csvWriter.LastOrderId()} - case utils.CDRE_FIXED_WIDTH: - if len(exportDir) == 0 { - exportDir = path.Join(self.Config.CdreDir, utils.CDRE_FIXED_WIDTH) - } - if len(fileName) == 0 { - fileName = fmt.Sprintf("cdre_%s.fwv", exportId) - } - exportTemplate := self.Config.CdreFWXmlTemplate - if len(attr.ExportTemplate) != 0 && self.Config.XmlCfgDocument != nil { - if xmlTemplate := self.Config.XmlCfgDocument.GetCdreFWCfgs(attr.ExportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplate != nil { - exportTemplate = xmlTemplate[attr.ExportTemplate[len(utils.XML_PROFILE_PREFIX):]] - } - } - if exportTemplate == nil { - return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING) - } - filePath := path.Join(exportDir, fileName) - fileOut, err := os.Create(filePath) - if err != nil { - return err - } - defer fileOut.Close() - fww, _ := cdre.NewFWCdrWriter(self.LogDb, fileOut, exportTemplate, exportId, costShiftDigits, roundDecimals, maskDestId, maskLen) - exportedIds := make([]string, 0) - unexportedIds := make(map[string]string) - for _, cdr := range cdrs { - if err := fww.WriteCdr(cdr); err != nil { - unexportedIds[cdr.CgrId] = err.Error() - } else { - exportedIds = append(exportedIds, cdr.CgrId) - } - } - fww.Close() - *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: fww.TotalCost(), - ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds, - FirstOrderId: fww.FirstOrderId(), LastOrderId: fww.LastOrderId()} + fileOut, err := os.Create(filePath) + if err != nil { + return err } + defer fileOut.Close() + cdrexp, err := cdre.NewCdrExporter(cdrs, self.LogDb, exportTemplate, exportId, + dataUsageMultiplyFactor, costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen) + if err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } + switch cdrFormat { + case utils.CDRE_FIXED_WIDTH: + if err := cdrexp.WriteOut(fileOut); err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } + case utils.CSV: + csvWriter := csv.NewWriter(fileOut) + if err := cdrexp.WriteCsv(csvWriter); err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } + } + *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), + ExportedCgrIds: cdrexp.PositiveExports(), UnexportedCgrIds: cdrexp.NegativeExports(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()} return nil } diff --git a/apier/tutfscsv_local_test.go b/apier/tutfscsv_local_test.go index 81316137f..4eefb69a4 100644 --- a/apier/tutfscsv_local_test.go +++ b/apier/tutfscsv_local_test.go @@ -46,7 +46,7 @@ func TestFsCsvRemoveDirs(t *testing.T) { if !*testLocal { return } - for _, pathDir := range []string{cfg.CdreDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} { + for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} { if err := os.RemoveAll(pathDir); err != nil { t.Fatal("Error removing folder: ", pathDir, err) } diff --git a/apier/tutfsjson_local_test.go b/apier/tutfsjson_local_test.go index 8bbb4c1f0..721de1893 100644 --- a/apier/tutfsjson_local_test.go +++ b/apier/tutfsjson_local_test.go @@ -49,7 +49,7 @@ func TestFsJsonRemoveDirs(t *testing.T) { if !*testLocal { return } - for _, pathDir := range []string{fsjsonCfg.CdreDir, fsjsonCfg.HistoryDir} { + for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, fsjsonCfg.HistoryDir} { if err := os.RemoveAll(pathDir); err != nil { t.Fatal("Error removing folder: ", pathDir, err) } diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index 0f6c9c416..e845ffdb7 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -2,13 +2,13 @@ Real-time Charging System for Telecom & ISP environments Copyright (C) 2012-2014 ITsysCOM GmbH -This program is free software: you can Storagetribute it and/or modify +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, -but WITH*out ANY WARRANTY; without even the implied warranty of +but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -19,13 +19,390 @@ along with this program. If not, see package cdre import ( + "encoding/csv" + "encoding/json" + "fmt" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" + "io" + "strconv" + "strings" + "time" ) -type CdrWriter interface { - FirstOrderId() int64 - LastOrderId() int64 - TotalCost() float64 - WriteCdr(cdr *utils.StoredCdr) string - Close() +const ( + COST_DETAILS = "cost_details" + FILLER = "filler" + CONSTANT = "constant" + METATAG = "metatag" + CONCATENATED_CDRFIELD = "concatenated_cdrfield" + META_EXPORTID = "export_id" + META_TIMENOW = "time_now" + META_FIRSTCDRATIME = "first_cdr_atime" + META_LASTCDRATIME = "last_cdr_atime" + META_NRCDRS = "cdrs_number" + META_DURCDRS = "cdrs_duration" + META_COSTCDRS = "cdrs_cost" + META_MASKDESTINATION = "mask_destination" + META_FORMATCOST = "format_cost" +) + +var err error + +func NewCdrExporter(cdrs []*utils.StoredCdr, logDb engine.LogStorage, exportTpl *config.CdreConfig, exportId string, + dataUsageMultiplyFactor, costMultiplyFactor float64, costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int) (*CdrExporter, error) { + if len(cdrs) == 0 { // Nothing to export + return nil, nil + } + cdre := &CdrExporter{ + cdrs: cdrs, + logDb: logDb, + exportTemplate: exportTpl, + exportId: exportId, + dataUsageMultiplyFactor: dataUsageMultiplyFactor, + costMultiplyFactor: costMultiplyFactor, + costShiftDigits: costShiftDigits, + roundDecimals: roundDecimals, + cgrPrecision: cgrPrecision, + maskDestId: maskDestId, + maskLen: maskLen, + negativeExports: make(map[string]string), + } + if err := cdre.processCdrs(); err != nil { + return nil, err + } + return cdre, nil +} + +type CdrExporter struct { + cdrs []*utils.StoredCdr + logDb engine.LogStorage // Used to extract cost_details if these are requested + exportTemplate *config.CdreConfig + exportId string // Unique identifier or this export + dataUsageMultiplyFactor, costMultiplyFactor float64 + costShiftDigits, roundDecimals, cgrPrecision int + maskDestId string + maskLen int + header, trailer []string // Header and Trailer fields + content [][]string // Rows of cdr fields + firstCdrATime, lastCdrATime time.Time + numberOfRecords int + totalDuration time.Duration + totalCost float64 + firstExpOrderId, lastExpOrderId int64 + positiveExports []string // CGRIds of successfully exported CDRs + negativeExports map[string]string // CgrIds of failed exports +} + +// Return Json marshaled callCost attached to +// Keep it separately so we test only this part in local tests +func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) { + cc, err := cdre.logDb.GetCallCostLog(cgrId, "", runId) + if err != nil { + return "", err + } else if cc == nil { + return "", nil + } + ccJson, _ := json.Marshal(cc) + return string(ccJson), nil +} + +// Check if the destination should be masked in output +func (cdre *CdrExporter) maskedDestination(destination string) bool { + if len(cdre.maskDestId) != 0 && engine.CachedDestHasPrefix(cdre.maskDestId, destination) { + return true + } + return false +} + +// Extracts the value specified by cfgHdr out of cdr +func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, rsrFld *utils.RSRField, layout string) (string, error) { + if rsrFld == nil { + return "", nil + } + var cdrVal string + switch rsrFld.Id { + case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb + if cdrVal, err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil { + return "", err + } + case utils.COST: + cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals) + case utils.USAGE: + cdrVal = cdr.FormatUsage(layout) + case utils.SETUP_TIME: + cdrVal = cdr.SetupTime.Format(layout) + case utils.ANSWER_TIME: // Format time based on layout + cdrVal = cdr.AnswerTime.Format(layout) + case utils.DESTINATION: + cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}) + if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) { + cdrVal = MaskDestination(cdrVal, cdre.maskLen) + } + default: + cdrVal = cdr.FieldAsString(rsrFld) + } + return rsrFld.ParseValue(cdrVal), nil +} + +// Handle various meta functions used in header/trailer +func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) { + switch tag { + case META_EXPORTID: + return cdre.exportId, nil + case META_TIMENOW: + return time.Now().Format(arg), nil + case META_FIRSTCDRATIME: + return cdre.firstCdrATime.Format(arg), nil + case META_LASTCDRATIME: + return cdre.lastCdrATime.Format(arg), nil + case META_NRCDRS: + return strconv.Itoa(cdre.numberOfRecords), nil + case META_DURCDRS: + return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil + case META_COSTCDRS: + return strconv.FormatFloat(utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil + case META_MASKDESTINATION: + if cdre.maskedDestination(arg) { + return "1", nil + } + return "0", nil + default: + return "", fmt.Errorf("Unsupported METATAG: %s", tag) + } + return "", nil +} + +// Compose and cache the header +func (cdre *CdrExporter) composeHeader() error { + for _, cfgFld := range cdre.exportTemplate.HeaderFields { + var outVal string + switch cfgFld.Type { + case FILLER: + outVal = cfgFld.Value + cfgFld.Padding = "right" + case CONSTANT: + outVal = cfgFld.Value + case METATAG: + outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + default: + return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) + } + if err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, error: %s", err.Error())) + return err + } + fmtOut := outVal + if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH { + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, error: %s", err.Error())) + return err + } + } + cdre.header = append(cdre.header, fmtOut) + } + return nil +} + +// Compose and cache the trailer +func (cdre *CdrExporter) composeTrailer() error { + for _, cfgFld := range cdre.exportTemplate.TrailerFields { + var outVal string + switch cfgFld.Type { + case FILLER: + outVal = cfgFld.Value + cfgFld.Padding = "right" + case CONSTANT: + outVal = cfgFld.Value + case METATAG: + outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + default: + return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) + } + if err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, error: %s", err.Error())) + return err + } + fmtOut := outVal + if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH { + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, error: %s", err.Error())) + return err + } + } + cdre.trailer = append(cdre.trailer, fmtOut) + } + return nil +} + +// Write individual cdr into content buffer, build stats +func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error { + if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs + return nil + } + if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA { + cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision) + } + if cdre.costMultiplyFactor != 0.0 { + cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision) + } + var err error + cdrRow := make([]string, len(cdre.exportTemplate.ContentFields)) + for idx, cfgFld := range cdre.exportTemplate.ContentFields { + var outVal string + switch cfgFld.Type { + case FILLER: + outVal = cfgFld.Value + cfgFld.Padding = "right" + case CONSTANT: + outVal = cfgFld.Value + case utils.CDRFIELD: + outVal, err = cdre.cdrFieldValue(cdr, cfgFld.ValueAsRSRField(), cfgFld.Layout) + case CONCATENATED_CDRFIELD: + for _, fld := range strings.Split(cfgFld.Value, ",") { + if fldOut, err := cdre.cdrFieldValue(cdr, &utils.RSRField{Id: fld}, cfgFld.Layout); err != nil { + break // The error will be reported bellow + } else { + outVal += fldOut + } + } + case METATAG: + if cfgFld.Value == META_MASKDESTINATION { + outVal, err = cdre.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})) + } else { + outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout) + } + } + if err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error())) + return err + } + fmtOut := outVal + if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH { + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error())) + return err + } + } + cdrRow[idx] += fmtOut + } + if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined + return nil + } else { + cdre.content = append(cdre.content, cdrRow) + } + // Done with writing content, compute stats here + if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) { + cdre.firstCdrATime = cdr.AnswerTime + } + if cdr.AnswerTime.After(cdre.lastCdrATime) { + cdre.lastCdrATime = cdr.AnswerTime + } + cdre.numberOfRecords += 1 + if !utils.IsSliceMember([]string{utils.DATA, utils.SMS}, cdr.TOR) { // Only count duration for non data cdrs + cdre.totalDuration += cdr.Usage + } + cdre.totalCost += cdr.Cost + cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) + if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 { + cdre.firstExpOrderId = cdr.OrderId + } + if cdre.lastExpOrderId < cdr.OrderId { + cdre.lastExpOrderId = cdr.OrderId + } + return nil +} + +// Builds header, content and trailers +func (cdre *CdrExporter) processCdrs() error { + if cdre.exportTemplate.HeaderFields != nil { + if err := cdre.composeHeader(); err != nil { + return err + } + } + if cdre.exportTemplate.TrailerFields != nil { + if err := cdre.composeTrailer(); err != nil { + return err + } + } + for _, cdr := range cdre.cdrs { + if err := cdre.processCdr(cdr); err != nil { + cdre.negativeExports[cdr.CgrId] = err.Error() + } else { + cdre.positiveExports = append(cdre.positiveExports, cdr.CgrId) + } + } + return nil +} + +func (cdre *CdrExporter) WriteCsv(csvWriter *csv.Writer) error { + if len(cdre.header) != 0 { + if err := csvWriter.Write(cdre.header); err != nil { + return err + } + } + for _, cdrContent := range cdre.content { + if err := csvWriter.Write(cdrContent); err != nil { + return err + } + } + if len(cdre.trailer) != 0 { + if err := csvWriter.Write(cdre.trailer); err != nil { + return err + } + } + csvWriter.Flush() + return nil +} + +// Write fwv content +func (cdre *CdrExporter) WriteOut(ioWriter io.Writer) error { + if len(cdre.header) != 0 { + for _, fld := range append(cdre.header, "\n") { + if _, err := io.WriteString(ioWriter, fld); err != nil { + return err + } + } + } + for _, cdrContent := range cdre.content { + for _, cdrFld := range append(cdrContent, "\n") { + if _, err := io.WriteString(ioWriter, cdrFld); err != nil { + return err + } + } + } + if len(cdre.trailer) != 0 { + for _, fld := range append(cdre.trailer, "\n") { + if _, err := io.WriteString(ioWriter, fld); err != nil { + return err + } + } + } + return nil +} + +// Return the first exported Cdr OrderId +func (cdre *CdrExporter) FirstOrderId() int64 { + return cdre.firstExpOrderId +} + +// Return the last exported Cdr OrderId +func (cdre *CdrExporter) LastOrderId() int64 { + return cdre.lastExpOrderId +} + +// Return total cost in the exported cdrs +func (cdre *CdrExporter) TotalCost() float64 { + return cdre.totalCost +} + +// Return successfully exported CgrIds +func (cdre *CdrExporter) PositiveExports() []string { + return cdre.positiveExports +} + +// Return failed exported CgrIds together with the reason +func (cdre *CdrExporter) NegativeExports() map[string]string { + return cdre.negativeExports } diff --git a/cdre/csv.go b/cdre/csv.go deleted file mode 100644 index 7dfb6ed80..000000000 --- a/cdre/csv.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Real-time Charging System for Telecom & ISP environments -Copyright (C) 2012-2014 ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdre - -import ( - "encoding/csv" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "io" -) - -type CsvCdrWriter struct { - writer *csv.Writer - costShiftDigits, roundDecimals int // Round floats like Cost using this number of decimals - maskDestId string - maskLen int - exportedFields []*utils.RSRField // The fields exported, order important - firstExpOrderId, lastExpOrderId int64 - totalCost float64 // Cummulated cost of all the -} - -func NewCsvCdrWriter(writer io.Writer, costShiftDigits, roundDecimals int, maskDestId string, maskLen int, exportedFields []*utils.RSRField) *CsvCdrWriter { - return &CsvCdrWriter{writer: csv.NewWriter(writer), costShiftDigits: costShiftDigits, roundDecimals: roundDecimals, maskDestId: maskDestId, maskLen: maskLen, exportedFields: exportedFields} -} - -// Return the first exported Cdr OrderId -func (csvwr *CsvCdrWriter) FirstOrderId() int64 { - return csvwr.firstExpOrderId -} - -func (csvwr *CsvCdrWriter) LastOrderId() int64 { - return csvwr.lastExpOrderId -} - -func (csvwr *CsvCdrWriter) TotalCost() float64 { - return csvwr.totalCost -} - -func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { - row := make([]string, len(csvwr.exportedFields)) - for idx, fld := range csvwr.exportedFields { - var fldVal string - if fld.Id == utils.COST { - fldVal = cdr.FormatCost(csvwr.costShiftDigits, csvwr.roundDecimals) - } else if fld.Id == utils.DESTINATION { - fldVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}) - if len(csvwr.maskDestId) != 0 && csvwr.maskLen > 0 && engine.CachedDestHasPrefix(csvwr.maskDestId, fldVal) { - fldVal = MaskDestination(fldVal, csvwr.maskLen) - } - } else { - fldVal = cdr.FieldAsString(fld) - } - row[idx] = fld.ParseValue(fldVal) - } - if csvwr.firstExpOrderId > cdr.OrderId || csvwr.firstExpOrderId == 0 { - csvwr.firstExpOrderId = cdr.OrderId - } - if csvwr.lastExpOrderId < cdr.OrderId { - csvwr.lastExpOrderId = cdr.OrderId - } - csvwr.totalCost += cdr.Cost - csvwr.totalCost = utils.Round(csvwr.totalCost, csvwr.roundDecimals, utils.ROUNDING_MIDDLE) - return csvwr.writer.Write(row) - -} - -func (csvwr *CsvCdrWriter) Close() { - csvwr.writer.Flush() -} diff --git a/cdre/csv_test.go b/cdre/csv_test.go index 2d27a8bd2..7a5b284d4 100644 --- a/cdre/csv_test.go +++ b/cdre/csv_test.go @@ -1,6 +1,6 @@ /* -Rating system designed to be used in VoIP Carriers World -Copyright (C) 2013 ITsysCOM +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2014 ITsysCOM GmbH This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,7 +20,9 @@ package cdre import ( "bytes" + "encoding/csv" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "strings" "testing" @@ -30,22 +32,27 @@ import ( func TestCsvCdrWriter(t *testing.T) { writer := &bytes.Buffer{} cfg, _ := config.NewDefaultCGRConfig() - exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"}) - csvCdrWriter := NewCsvCdrWriter(writer, 0, 4, "", -1, exportedFields) - ratedCdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", + logDb, _ := engine.NewMapStorage() + storedCdr1 := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01, } - csvCdrWriter.WriteCdr(ratedCdr) - csvCdrWriter.Close() - expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10000000000,1.0100,val_extra3,"",val_extra1` + cdre, err := NewCdrExporter([]*utils.StoredCdr{storedCdr1}, logDb, cfg.CdreDefaultInstance, "firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0) + if err != nil { + t.Error("Unexpected error received: ", err) + } + csvWriter := csv.NewWriter(writer) + if err := cdre.WriteCsv(csvWriter); err != nil { + t.Error("Unexpected error: ", err) + } + expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10000000000,1.0100` result := strings.TrimSpace(writer.String()) if result != expected { t.Errorf("Expected: \n%s received: \n%s.", expected, result) } - if csvCdrWriter.TotalCost() != 1.01 { - t.Error("Unexpected TotalCost: ", csvCdrWriter.TotalCost()) + if cdre.TotalCost() != 1.01 { + t.Error("Unexpected TotalCost: ", cdre.TotalCost()) } } diff --git a/cdre/fixedwidth.go b/cdre/fixedwidth.go deleted file mode 100644 index 323333839..000000000 --- a/cdre/fixedwidth.go +++ /dev/null @@ -1,331 +0,0 @@ -/* -Real-time Charging System for Telecom & ISP environments -Copyright (C) 2012-2014 ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdre - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "io" - "os" - "strconv" - "strings" - "time" -) - -const ( - COST_DETAILS = "cost_details" - FILLER = "filler" - CONSTANT = "constant" - CDRFIELD = "cdrfield" - METATAG = "metatag" - CONCATENATED_CDRFIELD = "concatenated_cdrfield" - META_EXPORTID = "export_id" - META_TIMENOW = "time_now" - META_FIRSTCDRATIME = "first_cdr_atime" - META_LASTCDRATIME = "last_cdr_atime" - META_NRCDRS = "cdrs_number" - META_DURCDRS = "cdrs_duration" - META_COSTCDRS = "cdrs_cost" - META_MASKDESTINATION = "mask_destination" - META_FORMATCOST = "format_cost" -) - -var err error - -func NewFWCdrWriter(logDb engine.LogStorage, outFile *os.File, exportTpl *config.CgrXmlCdreFwCfg, exportId string, - costShiftDigits, roundDecimals int, maskDestId string, maskLen int) (*FixedWidthCdrWriter, error) { - return &FixedWidthCdrWriter{ - logDb: logDb, - writer: outFile, - exportTemplate: exportTpl, - exportId: exportId, - costShiftDigits: costShiftDigits, - roundDecimals: roundDecimals, - maskDestId: maskDestId, - maskLen: maskLen, - header: &bytes.Buffer{}, - content: &bytes.Buffer{}, - trailer: &bytes.Buffer{}}, nil -} - -type FixedWidthCdrWriter struct { - logDb engine.LogStorage // Used to extract cost_details if these are requested - writer io.Writer - exportTemplate *config.CgrXmlCdreFwCfg - exportId string // Unique identifier or this export - costShiftDigits, roundDecimals int - maskDestId string - maskLen int - header, content, trailer *bytes.Buffer - firstCdrATime, lastCdrATime time.Time - numberOfRecords int - totalDuration time.Duration - totalCost float64 - firstExpOrderId, lastExpOrderId int64 -} - -// Return Json marshaled callCost attached to -// Keep it separately so we test only this part in local tests -func (fwv *FixedWidthCdrWriter) getCdrCostDetails(cgrId, runId string) (string, error) { - cc, err := fwv.logDb.GetCallCostLog(cgrId, "", runId) - if err != nil { - return "", err - } else if cc == nil { - return "", nil - } - ccJson, _ := json.Marshal(cc) - return string(ccJson), nil -} - -func (fwv *FixedWidthCdrWriter) maskedDestination(destination string) bool { - if len(fwv.maskDestId) != 0 && engine.CachedDestHasPrefix(fwv.maskDestId, destination) { - return true - } - return false -} - -// Extracts the value specified by cfgHdr out of cdr -func (fwv *FixedWidthCdrWriter) cdrFieldValue(cdr *utils.StoredCdr, cfgHdr, layout string) (string, error) { - rsrField, err := utils.NewRSRField(cfgHdr) - if err != nil { - return "", err - } else if rsrField == nil { - return "", nil - } - var cdrVal string - switch rsrField.Id { - case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb - if cdrVal, err = fwv.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil { - return "", err - } - case utils.COST: - cdrVal = cdr.FormatCost(fwv.costShiftDigits, fwv.roundDecimals) - case utils.USAGE: - cdrVal = cdr.FormatUsage(layout) - case utils.SETUP_TIME: - cdrVal = cdr.SetupTime.Format(layout) - case utils.ANSWER_TIME: // Format time based on layout - cdrVal = cdr.AnswerTime.Format(layout) - case utils.DESTINATION: - cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}) - if fwv.maskLen != -1 && fwv.maskedDestination(cdrVal) { - cdrVal = MaskDestination(cdrVal, fwv.maskLen) - } - default: - cdrVal = cdr.FieldAsString(rsrField) - } - return rsrField.ParseValue(cdrVal), nil -} - -func (fwv *FixedWidthCdrWriter) metaHandler(tag, arg string) (string, error) { - switch tag { - case META_EXPORTID: - return fwv.exportId, nil - case META_TIMENOW: - return time.Now().Format(arg), nil - case META_FIRSTCDRATIME: - return fwv.firstCdrATime.Format(arg), nil - case META_LASTCDRATIME: - return fwv.lastCdrATime.Format(arg), nil - case META_NRCDRS: - return strconv.Itoa(fwv.numberOfRecords), nil - case META_DURCDRS: - return strconv.FormatFloat(fwv.totalDuration.Seconds(), 'f', -1, 64), nil - case META_COSTCDRS: - return strconv.FormatFloat(utils.Round(fwv.totalCost, fwv.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil - case META_MASKDESTINATION: - if fwv.maskedDestination(arg) { - return "1", nil - } - return "0", nil - default: - return "", fmt.Errorf("Unsupported METATAG: %s", tag) - } - return "", nil -} - -// Return the first exported Cdr OrderId -func (fwv *FixedWidthCdrWriter) FirstOrderId() int64 { - return fwv.firstExpOrderId -} - -// Return the last exported Cdr OrderId -func (fwv *FixedWidthCdrWriter) LastOrderId() int64 { - return fwv.lastExpOrderId -} - -func (fwv *FixedWidthCdrWriter) TotalCost() float64 { - return fwv.totalCost -} - -// Writes the header into it's buffer -func (fwv *FixedWidthCdrWriter) ComposeHeader() error { - header := "" - for _, cfgFld := range fwv.exportTemplate.Header.Fields { - var outVal string - switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value - cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value - case METATAG: - outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout) - default: - return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) - } - if err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, error: %s", err.Error())) - return err - } - if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, error: %s", err.Error())) - return err - } else { - header += fmtOut - } - } - if len(header) == 0 { // No header data, most likely no configuration fields defined - return nil - } - header += "\n" // Done with cdr, postpend new line char - fwv.header.WriteString(header) - return nil -} - -// Writes the trailer into it's buffer -func (fwv *FixedWidthCdrWriter) ComposeTrailer() error { - trailer := "" - for _, cfgFld := range fwv.exportTemplate.Trailer.Fields { - var outVal string - switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value - cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value - case METATAG: - outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout) - default: - return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) - } - if err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, error: %s", err.Error())) - return err - } - if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, error: %s", err.Error())) - return err - } else { - trailer += fmtOut - } - } - if len(trailer) == 0 { // No header data, most likely no configuration fields defined - return nil - } - trailer += "\n" // Done with cdr, postpend new line char - fwv.trailer.WriteString(trailer) - return nil -} - -// Write individual cdr into content buffer, build stats -func (fwv *FixedWidthCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { - if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs - return nil - } - var err error - cdrRow := "" - for _, cfgFld := range fwv.exportTemplate.Content.Fields { - var outVal string - switch cfgFld.Type { - case FILLER: - outVal = cfgFld.Value - cfgFld.Padding = "right" - case CONSTANT: - outVal = cfgFld.Value - case CDRFIELD: - outVal, err = fwv.cdrFieldValue(cdr, cfgFld.Value, cfgFld.Layout) - case CONCATENATED_CDRFIELD: - for _, fld := range strings.Split(cfgFld.Value, ",") { - if fldOut, err := fwv.cdrFieldValue(cdr, fld, cfgFld.Layout); err != nil { - break // The error will be reported bellow - } else { - outVal += fldOut - } - } - case METATAG: - if cfgFld.Value == META_MASKDESTINATION { - outVal, err = fwv.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})) - } else { - outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout) - } - } - if err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error())) - return err - } - if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error())) - return err - } else { - cdrRow += fmtOut - } - } - if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined - return nil - } - cdrRow += "\n" // Done with cdr, postpend new line char - fwv.content.WriteString(cdrRow) - // Done with writing content, compute stats here - if fwv.firstCdrATime.IsZero() || cdr.AnswerTime.Before(fwv.firstCdrATime) { - fwv.firstCdrATime = cdr.AnswerTime - } - if cdr.AnswerTime.After(fwv.lastCdrATime) { - fwv.lastCdrATime = cdr.AnswerTime - } - fwv.numberOfRecords += 1 - if !utils.IsSliceMember([]string{utils.DATA, utils.SMS}, cdr.TOR) { // Only count duration for non data cdrs - fwv.totalDuration += cdr.Usage - } - fwv.totalCost += cdr.Cost - fwv.totalCost = utils.Round(fwv.totalCost, fwv.roundDecimals, utils.ROUNDING_MIDDLE) - if fwv.firstExpOrderId > cdr.OrderId || fwv.firstExpOrderId == 0 { - fwv.firstExpOrderId = cdr.OrderId - } - if fwv.lastExpOrderId < cdr.OrderId { - fwv.lastExpOrderId = cdr.OrderId - } - return nil -} - -func (fwv *FixedWidthCdrWriter) Close() { - if fwv.exportTemplate.Header != nil { - fwv.ComposeHeader() - } - if fwv.exportTemplate.Trailer != nil { - fwv.ComposeTrailer() - } - for _, buf := range []*bytes.Buffer{fwv.header, fwv.content, fwv.trailer} { - fwv.writer.Write(buf.Bytes()) - } -} diff --git a/cdre/fixedwidth_test.go b/cdre/fixedwidth_test.go index 68647ed78..21b9c3b0f 100644 --- a/cdre/fixedwidth_test.go +++ b/cdre/fixedwidth_test.go @@ -21,6 +21,7 @@ package cdre import ( "bytes" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" "math" "testing" @@ -40,24 +41,24 @@ var hdrCfgFlds = []*config.CgrXmlCfgCdrField{ var contentCfgFlds = []*config.CgrXmlCfgCdrField{ &config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "20", Width: 2}, - &config.CgrXmlCfgCdrField{Name: "Account", Type: CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Subject", Type: CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "CLI", Type: CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Destination", Type: CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "Account", Type: utils.CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "Subject", Type: utils.CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "CLI", Type: utils.CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "Destination", Type: utils.CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"}, &config.CgrXmlCfgCdrField{Name: "TOR", Type: CONSTANT, Value: "02", Width: 2}, &config.CgrXmlCfgCdrField{Name: "SubtypeTOR", Type: CONSTANT, Value: "11", Width: 4, Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "SetupTime", Type: CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"}, - &config.CgrXmlCfgCdrField{Name: "Duration", Type: CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS}, + &config.CgrXmlCfgCdrField{Name: "SetupTime", Type: utils.CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"}, + &config.CgrXmlCfgCdrField{Name: "Duration", Type: utils.CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS}, &config.CgrXmlCfgCdrField{Name: "DataVolume", Type: FILLER, Width: 6}, &config.CgrXmlCfgCdrField{Name: "TaxCode", Type: CONSTANT, Value: "1", Width: 1}, - &config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "ProductId", Type: CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: utils.CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "ProductId", Type: utils.CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"}, &config.CgrXmlCfgCdrField{Name: "NetworkId", Type: CONSTANT, Value: "3", Width: 1}, - &config.CgrXmlCfgCdrField{Name: "CallId", Type: CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"}, + &config.CgrXmlCfgCdrField{Name: "CallId", Type: utils.CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"}, &config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8}, &config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8}, &config.CgrXmlCfgCdrField{Name: "TerminationCode", Type: CONCATENATED_CDRFIELD, Value: "operator,product", Width: 5, Strip: "right", Padding: "right"}, - &config.CgrXmlCfgCdrField{Name: "Cost", Type: CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"}, + &config.CgrXmlCfgCdrField{Name: "Cost", Type: utils.CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"}, &config.CgrXmlCfgCdrField{Name: "DestinationPrivacy", Type: METATAG, Value: META_MASKDESTINATION, Width: 1}, } @@ -76,35 +77,33 @@ var trailerCfgFlds = []*config.CgrXmlCfgCdrField{ // Write one CDR and test it's results only for content buffer func TestWriteCdr(t *testing.T) { wrBuf := &bytes.Buffer{} - exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds}, - Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds}, - Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, + logDb, _ := engine.NewMapStorage() + cfg, _ := config.NewDefaultCGRConfig() + fixedWidth := utils.CDRE_FIXED_WIDTH + exportTpl := &config.CgrXmlCdreCfg{ + CdrFormat: &fixedWidth, + Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds}, + Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds}, + Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, } - fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}} - cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1", + ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, } - if err := fwWriter.WriteCdr(cdr); err != nil { + cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr}, logDb, exportTpl.AsCdreConfig(), "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1) + if err != nil { t.Error(err) } - eContentOut := "201001 1001 1002 0211 07111308420010 1 3dsafdsaf 0002.34570\n" - contentOut := fwWriter.content.String() - if len(contentOut) != 145 { - t.Error("Unexpected content length", len(contentOut)) - } else if contentOut != eContentOut { - t.Errorf("Content out different than expected. Have <%s>, expecting: <%s>", contentOut, eContentOut) - } eHeader := "10 VOI0000007111308420024031415390001 \n" + eContentOut := "201001 1001 1002 0211 07111308420010 1 3dsafdsaf 0002.34570\n" eTrailer := "90 VOI0000000000100000010071113084260071113084200 \n" - outBeforeWrite := "" - if wrBuf.String() != outBeforeWrite { - t.Errorf("Output buffer should be empty before write") + if err := cdre.WriteOut(wrBuf); err != nil { + t.Error(err) } - fwWriter.Close() allOut := wrBuf.String() eAllOut := eHeader + eContentOut + eTrailer if math.Mod(float64(len(allOut)), 145) != 0 { @@ -113,35 +112,38 @@ func TestWriteCdr(t *testing.T) { t.Errorf("Output does not match expected length. Have output %q, expecting: %q", allOut, eAllOut) } // Test stats - if !fwWriter.firstCdrATime.Equal(cdr.AnswerTime) { - t.Error("Unexpected firstCdrATime in stats: ", fwWriter.firstCdrATime) - } else if !fwWriter.lastCdrATime.Equal(cdr.AnswerTime) { - t.Error("Unexpected lastCdrATime in stats: ", fwWriter.lastCdrATime) - } else if fwWriter.numberOfRecords != 1 { - t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords) - } else if fwWriter.totalDuration != cdr.Usage { - t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration) - } else if fwWriter.totalCost != utils.Round(cdr.Cost, fwWriter.roundDecimals, utils.ROUNDING_MIDDLE) { - t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost) + if !cdre.firstCdrATime.Equal(cdr.AnswerTime) { + t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime) + } else if !cdre.lastCdrATime.Equal(cdr.AnswerTime) { + t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime) + } else if cdre.numberOfRecords != 1 { + t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords) + } else if cdre.totalDuration != cdr.Usage { + t.Error("Unexpected total duration in the stats: ", cdre.totalDuration) + } else if cdre.totalCost != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) { + t.Error("Unexpected total cost in the stats: ", cdre.totalCost) } - if fwWriter.FirstOrderId() != 1 { - t.Error("Unexpected FirstOrderId", fwWriter.FirstOrderId()) + if cdre.FirstOrderId() != 1 { + t.Error("Unexpected FirstOrderId", cdre.FirstOrderId()) } - if fwWriter.LastOrderId() != 1 { - t.Error("Unexpected LastOrderId", fwWriter.LastOrderId()) + if cdre.LastOrderId() != 1 { + t.Error("Unexpected LastOrderId", cdre.LastOrderId()) } - if fwWriter.TotalCost() != utils.Round(cdr.Cost, fwWriter.roundDecimals, utils.ROUNDING_MIDDLE) { - t.Error("Unexpected TotalCost: ", fwWriter.TotalCost()) + if cdre.TotalCost() != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) { + t.Error("Unexpected TotalCost: ", cdre.TotalCost()) } } func TestWriteCdrs(t *testing.T) { wrBuf := &bytes.Buffer{} - exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds}, - Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds}, - Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, + logDb, _ := engine.NewMapStorage() + fixedWidth := utils.CDRE_FIXED_WIDTH + exportTpl := &config.CgrXmlCdreCfg{ + CdrFormat: &fixedWidth, + Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds}, + Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds}, + Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, } - fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}} cdr1 := &utils.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1010", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), @@ -164,45 +166,40 @@ func TestWriteCdrs(t *testing.T) { Usage: time.Duration(20) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567, ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"}, } - for _, cdr := range []*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4} { - if err := fwWriter.WriteCdr(cdr); err != nil { - t.Error(err) - } - contentOut := fwWriter.content.String() - if math.Mod(float64(len(contentOut)), 145) != 0 { // Rest must be 0 always, so content is always multiple of 145 which is our row fixLength - t.Error("Unexpected content length", len(contentOut)) - } + cfg, _ := config.NewDefaultCGRConfig() + cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4}, logDb, exportTpl.AsCdreConfig(), "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1) + if err != nil { + t.Error(err) } - if len(wrBuf.String()) != 0 { - t.Errorf("Output buffer should be empty before write") + if err := cdre.WriteOut(wrBuf); err != nil { + t.Error(err) } - fwWriter.Close() if len(wrBuf.String()) != 725 { t.Error("Output buffer does not contain expected info. Expecting len: 725, got: ", len(wrBuf.String())) } // Test stats - if !fwWriter.firstCdrATime.Equal(cdr2.AnswerTime) { - t.Error("Unexpected firstCdrATime in stats: ", fwWriter.firstCdrATime) + if !cdre.firstCdrATime.Equal(cdr2.AnswerTime) { + t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime) } - if !fwWriter.lastCdrATime.Equal(cdr4.AnswerTime) { - t.Error("Unexpected lastCdrATime in stats: ", fwWriter.lastCdrATime) + if !cdre.lastCdrATime.Equal(cdr4.AnswerTime) { + t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime) } - if fwWriter.numberOfRecords != 3 { - t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords) + if cdre.numberOfRecords != 3 { + t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords) } - if fwWriter.totalDuration != time.Duration(330)*time.Second { - t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration) + if cdre.totalDuration != time.Duration(330)*time.Second { + t.Error("Unexpected total duration in the stats: ", cdre.totalDuration) } - if fwWriter.totalCost != 5.9957 { - t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost) + if cdre.totalCost != 5.9957 { + t.Error("Unexpected total cost in the stats: ", cdre.totalCost) } - if fwWriter.FirstOrderId() != 2 { - t.Error("Unexpected FirstOrderId", fwWriter.FirstOrderId()) + if cdre.FirstOrderId() != 2 { + t.Error("Unexpected FirstOrderId", cdre.FirstOrderId()) } - if fwWriter.LastOrderId() != 4 { - t.Error("Unexpected LastOrderId", fwWriter.LastOrderId()) + if cdre.LastOrderId() != 4 { + t.Error("Unexpected LastOrderId", cdre.LastOrderId()) } - if fwWriter.TotalCost() != 5.9957 { - t.Error("Unexpected TotalCost: ", fwWriter.TotalCost()) + if cdre.TotalCost() != 5.9957 { + t.Error("Unexpected TotalCost: ", cdre.TotalCost()) } } diff --git a/cdre/libfixedwidth.go b/cdre/libcdre.go similarity index 100% rename from cdre/libfixedwidth.go rename to cdre/libcdre.go diff --git a/cdre/libfixedwidth_test.go b/cdre/libcdre_test.go similarity index 100% rename from cdre/libfixedwidth_test.go rename to cdre/libcdre_test.go diff --git a/config/cdreconfig.go b/config/cdreconfig.go new file mode 100644 index 000000000..4f8ad96f6 --- /dev/null +++ b/config/cdreconfig.go @@ -0,0 +1,223 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2014 ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package config + +import ( + "errors" + "github.com/cgrates/cgrates/utils" +) + +// Converts a list of field identifiers into proper CDR field content +func NewCdreCdrFieldsFromIds(fldsIds ...string) ([]*CdreCdrField, error) { + cdrFields := make([]*CdreCdrField, len(fldsIds)) + for idx, fldId := range fldsIds { + if parsedRsr, err := utils.NewRSRField(fldId); err != nil { + return nil, err + } else { + cdrFld := &CdreCdrField{Name: fldId, Type: utils.CDRFIELD, Value: fldId, valueAsRsrField: parsedRsr} + if err := cdrFld.setDefaultFixedWidthProperties(); err != nil { // Set default fixed width properties to be used later if needed + return nil, err + } + cdrFields[idx] = cdrFld + } + } + return cdrFields, nil +} + +func NewDefaultCdreConfig() (*CdreConfig, error) { + cdreCfg := new(CdreConfig) + if err := cdreCfg.setDefaults(); err != nil { + return nil, err + } + return cdreCfg, nil +} + +// One instance of CdrExporter +type CdreConfig struct { + CdrFormat string + DataUsageMultiplyFactor float64 + CostMultiplyFactor float64 + CostRoundingDecimals int + CostShiftDigits int + MaskDestId string + MaskLength int + ExportDir string + HeaderFields []*CdreCdrField + ContentFields []*CdreCdrField + TrailerFields []*CdreCdrField +} + +// Set here defaults +func (cdreCfg *CdreConfig) setDefaults() error { + cdreCfg.CdrFormat = utils.CSV + cdreCfg.DataUsageMultiplyFactor = 0.0 + cdreCfg.CostMultiplyFactor = 0.0 + cdreCfg.CostRoundingDecimals = -1 + cdreCfg.CostShiftDigits = 0 + cdreCfg.MaskDestId = "" + cdreCfg.MaskLength = 0 + cdreCfg.ExportDir = "/var/log/cgrates/cdre" + if flds, err := NewCdreCdrFieldsFromIds(utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, + utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, utils.COST); err != nil { + return err + } else { + cdreCfg.ContentFields = flds + } + return nil +} + +type CdreCdrField struct { + Name string + Type string + Value string + Width int + Strip string + Padding string + Layout string + Mandatory bool + valueAsRsrField *utils.RSRField // Cached if the need arrises +} + +func (cdrField *CdreCdrField) ValueAsRSRField() *utils.RSRField { + return cdrField.valueAsRsrField +} + +// Should be called on .fwv configuration without providing default values for fixed with parameters +func (cdrField *CdreCdrField) setDefaultFixedWidthProperties() error { + if cdrField.valueAsRsrField == nil { + return errors.New("Missing valueAsRsrField") + } + switch cdrField.valueAsRsrField.Id { + case utils.CGRID: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.ORDERID: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.TOR: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.ACCID: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.CDRHOST: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.CDRSOURCE: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.REQTYPE: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.DIRECTION: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.TENANT: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.CATEGORY: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.ACCOUNT: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.SUBJECT: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.DESTINATION: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.SETUP_TIME: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "2006-01-02T15:04:05Z07:00" + cdrField.Mandatory = true + case utils.ANSWER_TIME: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "2006-01-02T15:04:05Z07:00" + cdrField.Mandatory = true + case utils.USAGE: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.MEDI_RUNID: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + case utils.COST: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + default: + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "" + cdrField.Layout = "" + cdrField.Mandatory = true + } + return nil +} diff --git a/config/cdreconfig_test.go b/config/cdreconfig_test.go new file mode 100644 index 000000000..8a2f8d6ba --- /dev/null +++ b/config/cdreconfig_test.go @@ -0,0 +1,240 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2014 ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package config + +import ( + "github.com/cgrates/cgrates/utils" + "reflect" + "testing" +) + +func TestCdreCfgNewCdreCdrFieldsFromIds(t *testing.T) { + expectedFlds := []*CdreCdrField{ + &CdreCdrField{ + Name: utils.CGRID, + Type: utils.CDRFIELD, + Value: utils.CGRID, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + }, + &CdreCdrField{ + Name: "extra1", + Type: utils.CDRFIELD, + Value: "extra1", + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: "extra1"}, + }, + } + if cdreFlds, err := NewCdreCdrFieldsFromIds(utils.CGRID, "extra1"); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expectedFlds, cdreFlds) { + t.Errorf("Expected: %v, received: %v", expectedFlds, cdreFlds) + } +} + +func TestCdreCfgValueAsRSRField(t *testing.T) { + cdreCdrFld := &CdreCdrField{ + Name: utils.CGRID, + Type: utils.CDRFIELD, + Value: utils.CGRID, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + } + if rsrVal := cdreCdrFld.ValueAsRSRField(); rsrVal != cdreCdrFld.valueAsRsrField { + t.Error("Unexpected value received: ", rsrVal) + } +} +func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { + cdreCdrFld := &CdreCdrField{ + valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + } + eCdreCdrFld := &CdreCdrField{ + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + } + if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { + t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) + } +} + +func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { + eCdreCfg := new(CdreConfig) + eCdreCfg.CdrFormat = utils.CSV + eCdreCfg.DataUsageMultiplyFactor = 0.0 + eCdreCfg.CostMultiplyFactor = 0.0 + eCdreCfg.CostRoundingDecimals = -1 + eCdreCfg.CostShiftDigits = 0 + eCdreCfg.MaskDestId = "" + eCdreCfg.MaskLength = 0 + eCdreCfg.ExportDir = "/var/log/cgrates/cdre" + eCdreCfg.ContentFields = []*CdreCdrField{ + &CdreCdrField{ + Name: utils.CGRID, + Type: utils.CDRFIELD, + Value: utils.CGRID, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, + }, + &CdreCdrField{ + Name: utils.MEDI_RUNID, + Type: utils.CDRFIELD, + Value: utils.MEDI_RUNID, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, + }, + &CdreCdrField{ + Name: utils.TOR, + Type: utils.CDRFIELD, + Value: utils.TOR, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.TOR}, + }, + &CdreCdrField{ + Name: utils.ACCID, + Type: utils.CDRFIELD, + Value: utils.ACCID, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, + }, + &CdreCdrField{ + Name: utils.REQTYPE, + Type: utils.CDRFIELD, + Value: utils.REQTYPE, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, + }, + &CdreCdrField{ + Name: utils.DIRECTION, + Type: utils.CDRFIELD, + Value: utils.DIRECTION, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, + }, + &CdreCdrField{ + Name: utils.TENANT, + Type: utils.CDRFIELD, + Value: utils.TENANT, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, + }, + &CdreCdrField{ + Name: utils.CATEGORY, + Type: utils.CDRFIELD, + Value: utils.CATEGORY, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, + }, + &CdreCdrField{ + Name: utils.ACCOUNT, + Type: utils.CDRFIELD, + Value: utils.ACCOUNT, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, + }, + &CdreCdrField{ + Name: utils.SUBJECT, + Type: utils.CDRFIELD, + Value: utils.SUBJECT, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, + }, + &CdreCdrField{ + Name: utils.DESTINATION, + Type: utils.CDRFIELD, + Value: utils.DESTINATION, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, + }, + &CdreCdrField{ + Name: utils.SETUP_TIME, + Type: utils.CDRFIELD, + Value: utils.SETUP_TIME, + Width: 10, + Strip: "xright", + Layout: "2006-01-02T15:04:05Z07:00", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, + }, + &CdreCdrField{ + Name: utils.ANSWER_TIME, + Type: utils.CDRFIELD, + Value: utils.ANSWER_TIME, + Width: 10, + Strip: "xright", + Layout: "2006-01-02T15:04:05Z07:00", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, + }, + &CdreCdrField{ + Name: utils.USAGE, + Type: utils.CDRFIELD, + Value: utils.USAGE, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, + }, + &CdreCdrField{ + Name: utils.COST, + Type: utils.CDRFIELD, + Value: utils.COST, + Width: 10, + Strip: "xright", + Mandatory: true, + valueAsRsrField: &utils.RSRField{Id: utils.COST}, + }, + } + if cdreCfg, err := NewDefaultCdreConfig(); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eCdreCfg, cdreCfg) { + t.Errorf("Expecting: %v, received: %v", eCdreCfg, cdreCfg) + } +} diff --git a/config/config.go b/config/config.go index b7b11b2b1..410613a87 100644 --- a/config/config.go +++ b/config/config.go @@ -90,13 +90,7 @@ type CGRConfig struct { CDRSEnabled bool // Enable CDR Server service CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> - CdreCdrFormat string // Format of the exported CDRs. - CdreMaskDestId string // Id of the destination list to be masked in CDRs - CdreMaskLength int // Number of digits to mask in the destination suffix if destination is in the MaskDestinationdsId - CdreCostShiftDigits int // Shift digits in the cost on export (eg: convert from EUR to cents) - CdreDir string // Path towards exported cdrs directory - CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs - CdreFWXmlTemplate *CgrXmlCdreFwCfg // Use this configuration as export template in case of fixed fields length + CdreDefaultInstance *CdreConfig // Will be used in the case no specific one selected by API CdrcEnabled bool // Enable CDR client functionality CdrcCdrs string // Address where to reach CDR server CdrcRunDelay time.Duration // Sleep interval between consecutive runs, 0 to use automation via inotify @@ -167,11 +161,7 @@ func (self *CGRConfig) setDefaults() error { self.CDRSEnabled = false self.CDRSExtraFields = []*utils.RSRField{} self.CDRSMediator = "" - self.CdreCdrFormat = "csv" - self.CdreMaskDestId = "" - self.CdreMaskLength = 0 - self.CdreCostShiftDigits = 0 - self.CdreDir = "/var/log/cgrates/cdre" + self.CdreDefaultInstance, _ = NewDefaultCdreConfig() self.CdrcEnabled = false self.CdrcCdrs = utils.INTERNAL self.CdrcRunDelay = time.Duration(0) @@ -217,35 +207,10 @@ func (self *CGRConfig) setDefaults() error { self.MailerAuthUser = "cgrates" self.MailerAuthPass = "CGRateS.org" self.MailerFromAddr = "cgr-mailer@localhost.localdomain" - self.CdreExportedFields = []*utils.RSRField{ - &utils.RSRField{Id: utils.CGRID}, - &utils.RSRField{Id: utils.MEDI_RUNID}, - &utils.RSRField{Id: utils.TOR}, - &utils.RSRField{Id: utils.ACCID}, - &utils.RSRField{Id: utils.REQTYPE}, - &utils.RSRField{Id: utils.DIRECTION}, - &utils.RSRField{Id: utils.TENANT}, - &utils.RSRField{Id: utils.CATEGORY}, - &utils.RSRField{Id: utils.ACCOUNT}, - &utils.RSRField{Id: utils.SUBJECT}, - &utils.RSRField{Id: utils.DESTINATION}, - &utils.RSRField{Id: utils.SETUP_TIME}, - &utils.RSRField{Id: utils.ANSWER_TIME}, - &utils.RSRField{Id: utils.USAGE}, - &utils.RSRField{Id: utils.COST}, - } return nil } func (self *CGRConfig) checkConfigSanity() error { - // Cdre sanity check for fixed_width - if self.CdreCdrFormat == utils.CDRE_FIXED_WIDTH { - if self.XmlCfgDocument == nil { - return errors.New("Need XmlConfigurationDocument for fixed_width cdr export") - } else if self.CdreFWXmlTemplate == nil { - return errors.New("Need XmlTemplate for fixed_width cdr export") - } - } if self.CdrcEnabled { if len(self.CdrcCdrFields) == 0 { return errors.New("CdrC enabled but no fields to be processed defined!") @@ -253,7 +218,6 @@ func (self *CGRConfig) checkConfigSanity() error { if self.CdrcCdrType == utils.CSV { for _, rsrFld := range self.CdrcCdrFields { if _, errConv := strconv.Atoi(rsrFld.Id); errConv != nil { - fmt.Println("5") return fmt.Errorf("CDR fields must be indices in case of .csv files, have instead: %s", rsrFld.Id) } } @@ -426,33 +390,42 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { cfg.CDRSMediator, _ = c.GetString("cdrs", "mediator") } if hasOpt = c.HasOption("cdre", "cdr_format"); hasOpt { - cfg.CdreCdrFormat, _ = c.GetString("cdre", "cdr_format") + cfg.CdreDefaultInstance.CdrFormat, _ = c.GetString("cdre", "cdr_format") } if hasOpt = c.HasOption("cdre", "mask_destination_id"); hasOpt { - cfg.CdreMaskDestId, _ = c.GetString("cdre", "mask_destination_id") + cfg.CdreDefaultInstance.MaskDestId, _ = c.GetString("cdre", "mask_destination_id") } if hasOpt = c.HasOption("cdre", "mask_length"); hasOpt { - cfg.CdreMaskLength, _ = c.GetInt("cdre", "mask_length") + cfg.CdreDefaultInstance.MaskLength, _ = c.GetInt("cdre", "mask_length") + } + if hasOpt = c.HasOption("cdre", "data_usage_multiply_factor"); hasOpt { + cfg.CdreDefaultInstance.DataUsageMultiplyFactor, _ = c.GetFloat64("cdre", "data_usage_multiply_factor") + } + if hasOpt = c.HasOption("cdre", "cost_multiply_factor"); hasOpt { + cfg.CdreDefaultInstance.CostMultiplyFactor, _ = c.GetFloat64("cdre", "cost_multiply_factor") + } + if hasOpt = c.HasOption("cdre", "cost_rounding_decimals"); hasOpt { + cfg.CdreDefaultInstance.CostRoundingDecimals, _ = c.GetInt("cdre", "cost_rounding_decimals") } if hasOpt = c.HasOption("cdre", "cost_shift_digits"); hasOpt { - cfg.CdreCostShiftDigits, _ = c.GetInt("cdre", "cost_shift_digits") + cfg.CdreDefaultInstance.CostShiftDigits, _ = c.GetInt("cdre", "cost_shift_digits") } if hasOpt = c.HasOption("cdre", "export_template"); hasOpt { // Load configs for csv normally from template, fixed_width from xml file exportTemplate, _ := c.GetString("cdre", "export_template") - if cfg.CdreCdrFormat != utils.CDRE_FIXED_WIDTH { // Csv most likely - if extraFields, err := ParseRSRFields(exportTemplate); err != nil { + if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) { + if xmlTemplates := cfg.XmlCfgDocument.GetCdreCfgs(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates != nil { + cfg.CdreDefaultInstance = xmlTemplates[exportTemplate[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig() + } + } else { // Not loading out of template + if flds, err := NewCdreCdrFieldsFromIds(strings.Split(exportTemplate, string(utils.CSV_SEP))...); err != nil { return nil, err } else { - cfg.CdreExportedFields = extraFields - } - } else if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) { - if xmlTemplate := cfg.XmlCfgDocument.GetCdreFWCfgs(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplate != nil { - cfg.CdreFWXmlTemplate = xmlTemplate[exportTemplate[len(utils.XML_PROFILE_PREFIX):]] + cfg.CdreDefaultInstance.ContentFields = flds } } } if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt { - cfg.CdreDir, _ = c.GetString("cdre", "export_dir") + cfg.CdreDefaultInstance.ExportDir, _ = c.GetString("cdre", "export_dir") } if hasOpt = c.HasOption("cdrc", "enabled"); hasOpt { cfg.CdrcEnabled, _ = c.GetBool("cdrc", "enabled") diff --git a/config/config_local_test.go b/config/config_local_test.go index 5461126af..a643337fa 100644 --- a/config/config_local_test.go +++ b/config/config_local_test.go @@ -39,7 +39,7 @@ func TestLoadXmlCfg(t *testing.T) { if cfg.XmlCfgDocument == nil { t.Error("Did not load the XML Config Document") } - if cdreFWCfg := cfg.XmlCfgDocument.GetCdreFWCfgs("CDREFW-A"); cdreFWCfg == nil { + if cdreFWCfg := cfg.XmlCfgDocument.GetCdreCfgs("CDREFW-A"); cdreFWCfg == nil { t.Error("Could not retrieve CDRExporter FixedWidth config instance") } } diff --git a/config/config_test.go b/config/config_test.go index 02ffe5392..1297bd5bb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,6 @@ /* -Rating system designed to be used in VoIP Carriers World -Copyright (C) 2013 ITsysCOM +Real-time Charging System for Telecom & ISP environments +Copyright (C) 2012-2014 ITsysCOM GmbH This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -78,14 +78,10 @@ func TestDefaults(t *testing.T) { eCfg.RaterBalancer = "" eCfg.BalancerEnabled = false eCfg.SchedulerEnabled = false + eCfg.CdreDefaultInstance, _ = NewDefaultCdreConfig() eCfg.CDRSEnabled = false eCfg.CDRSExtraFields = []*utils.RSRField{} eCfg.CDRSMediator = "" - eCfg.CdreCdrFormat = "csv" - eCfg.CdreMaskDestId = "" - eCfg.CdreMaskLength = 0 - eCfg.CdreCostShiftDigits = 0 - eCfg.CdreDir = "/var/log/cgrates/cdre" eCfg.CdrcEnabled = false eCfg.CdrcCdrs = utils.INTERNAL eCfg.CdrcRunDelay = time.Duration(0) @@ -131,23 +127,6 @@ func TestDefaults(t *testing.T) { eCfg.MailerAuthUser = "cgrates" eCfg.MailerAuthPass = "CGRateS.org" eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain" - eCfg.CdreExportedFields = []*utils.RSRField{ - &utils.RSRField{Id: utils.CGRID}, - &utils.RSRField{Id: utils.MEDI_RUNID}, - &utils.RSRField{Id: utils.TOR}, - &utils.RSRField{Id: utils.ACCID}, - &utils.RSRField{Id: utils.REQTYPE}, - &utils.RSRField{Id: utils.DIRECTION}, - &utils.RSRField{Id: utils.TENANT}, - &utils.RSRField{Id: utils.CATEGORY}, - &utils.RSRField{Id: utils.ACCOUNT}, - &utils.RSRField{Id: utils.SUBJECT}, - &utils.RSRField{Id: utils.DESTINATION}, - &utils.RSRField{Id: utils.SETUP_TIME}, - &utils.RSRField{Id: utils.ANSWER_TIME}, - &utils.RSRField{Id: utils.USAGE}, - &utils.RSRField{Id: utils.COST}, - } if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) @@ -165,10 +144,6 @@ func TestSanityCheck(t *testing.T) { t.Error("Invalid defaults: ", err) } cfg = &CGRConfig{} - cfg.CdreCdrFormat = utils.CDRE_FIXED_WIDTH - if err := cfg.checkConfigSanity(); err == nil { - t.Error("Failed to detect fixed_width dependency on xml configuration") - } cfg.CdrcEnabled = true if err := cfg.checkConfigSanity(); err == nil { t.Error("Failed to detect missing CDR fields definition") @@ -228,12 +203,16 @@ func TestConfigFromFile(t *testing.T) { eCfg.CDRSEnabled = true eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}} eCfg.CDRSMediator = "test" - eCfg.CdreCdrFormat = "test" - eCfg.CdreMaskDestId = "test" - eCfg.CdreMaskLength = 99 - eCfg.CdreCostShiftDigits = 99 - eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}} - eCfg.CdreDir = "test" + eCfg.CdreDefaultInstance = &CdreConfig{ + CdrFormat: "test", + DataUsageMultiplyFactor: 99.0, + CostMultiplyFactor: 99.0, + CostRoundingDecimals: 99, + CostShiftDigits: 99, + MaskDestId: "test", + MaskLength: 99, + ExportDir: "test"} + eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds("test") eCfg.CdrcEnabled = true eCfg.CdrcCdrs = "test" eCfg.CdrcRunDelay = time.Duration(99) * time.Second @@ -326,17 +305,32 @@ func TestCdreExtraFields(t *testing.T) { cdr_format = csv export_template = cgrid,mediation_runid,accid `) + expectedFlds := []*CdreCdrField{ + &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 10, Strip: "xright", Mandatory: true}, + &CdreCdrField{Name: "mediation_runid", Type: utils.CDRFIELD, Value: "mediation_runid", valueAsRsrField: &utils.RSRField{Id: "mediation_runid"}, + Width: 10, Strip: "xright", Mandatory: true}, + &CdreCdrField{Name: "accid", Type: utils.CDRFIELD, Value: "accid", valueAsRsrField: &utils.RSRField{Id: "accid"}, Width: 10, Strip: "xright", Mandatory: true}, + } + expCdreCfg := &CdreConfig{CdrFormat: utils.CSV, CostRoundingDecimals: -1, ExportDir: "/var/log/cgrates/cdre", ContentFields: expectedFlds} if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { t.Error("Could not parse the config", err.Error()) - } else if !reflect.DeepEqual(cfg.CdreExportedFields, []*utils.RSRField{&utils.RSRField{Id: "cgrid"}, &utils.RSRField{Id: "mediation_runid"}, &utils.RSRField{Id: "accid"}}) { - t.Errorf("Unexpected value for CdrsExtraFields: %v", cfg.CDRSExtraFields) + } else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) { + t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance) } eFieldsCfg = []byte(`[cdre] cdr_format = csv -export_template = cgrid,mediation_runid,accid, +export_template = cgrid,~effective_caller_id_number:s/(\d+)/+$1/ `) - if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil { - t.Error("Failed to detect empty field in the end of export_template defition") + rsrField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`) + expectedFlds = []*CdreCdrField{ + &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 10, Strip: "xright", Mandatory: true}, + &CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`, valueAsRsrField: rsrField, + Width: 10, Strip: "xright", Mandatory: true}} + expCdreCfg.ContentFields = expectedFlds + if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { + t.Error("Could not parse the config", err.Error()) + } else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) { + t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance) } eFieldsCfg = []byte(`[cdre] cdr_format = csv diff --git a/config/test_data.txt b/config/test_data.txt index 6ee120180..94095517b 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -30,7 +30,6 @@ default_tenant = test # Default Tenant to consider when missing from requests. default_subject = test # Default rating Subject to consider when missing from requests. rounding_decimals = 99 # Number of decimals to round floats/costs at - [balancer] enabled = true # Start Balancer service: . @@ -47,12 +46,15 @@ extra_fields = test # Extra fields to scategorye in CDRs mediator = test # Address where to reach the Mediacategory. Empty for disabling mediation. <""|internal> [cdre] -cdr_format = test # Exported CDRs format -mask_destination_id = test # Destination id containing called addresses to be masked on export -mask_length = 99 # Length of the destination suffix to be masked -cost_shift_digits = 99 # Shift the number of cost -export_dir = test # Path where the exported CDRs will be placed -export_template = test # List of fields in the exported CDRs +cdr_format = test # Exported CDRs format +data_usage_multiply_factor = 99.0 # Multiply data usage before export (eg: convert from KBytes to Bytes) +cost_multiply_factor = 99.0 # Multiply cost before export (0.0 to disable), eg: add VAT +cost_rounding_decimals = 99 # Rounding decimals for Cost values. -1 to disable rounding +cost_shift_digits = 99 # Shift digits in the cost on export (eg: convert from EUR to cents) +mask_destination_id = test # Destination id containing called addresses to be masked on export +mask_length = 99 # Length of the destination suffix to be masked +export_dir = test # Path where the exported CDRs will be placed +export_template = test # List of fields in the exported CDRs [cdrc] enabled = true # Enable CDR client functionality diff --git a/config/xmlcdre.go b/config/xmlcdre.go index a1a68da82..bd0be8cea 100644 --- a/config/xmlcdre.go +++ b/config/xmlcdre.go @@ -20,13 +20,69 @@ package config import ( "encoding/xml" + "github.com/cgrates/cgrates/utils" ) -// The CdrExporter Fixed Width configuration instance -type CgrXmlCdreFwCfg struct { - Header *CgrXmlCfgCdrHeader `xml:"header"` - Content *CgrXmlCfgCdrContent `xml:"content"` - Trailer *CgrXmlCfgCdrTrailer `xml:"trailer"` +// The CdrExporter configuration instance +type CgrXmlCdreCfg struct { + CdrFormat *string `xml:"cdr_format"` + DataUsageMultiplyFactor *float64 `xml:"data_usage_multiply_factor"` + CostMultiplyFactor *float64 `xml:"cost_multiply_factor"` + CostRoundingDecimals *int `xml:"cost_rounding_decimals"` + CostShiftDigits *int `xml:"cost_shift_digits"` + MaskDestId *string `xml:"mask_destination_id"` + MaskLength *int `xml:"mask_length"` + ExportDir *string `xml:"export_dir"` + Header *CgrXmlCfgCdrHeader `xml:"export_template>header"` + Content *CgrXmlCfgCdrContent `xml:"export_template>content"` + Trailer *CgrXmlCfgCdrTrailer `xml:"export_template>trailer"` +} + +func (xmlCdreCfg *CgrXmlCdreCfg) AsCdreConfig() *CdreConfig { + cdreCfg, _ := NewDefaultCdreConfig() + if xmlCdreCfg.CdrFormat != nil { + cdreCfg.CdrFormat = *xmlCdreCfg.CdrFormat + } + if xmlCdreCfg.DataUsageMultiplyFactor != nil { + cdreCfg.DataUsageMultiplyFactor = *xmlCdreCfg.DataUsageMultiplyFactor + } + if xmlCdreCfg.CostMultiplyFactor != nil { + cdreCfg.CostMultiplyFactor = *xmlCdreCfg.CostMultiplyFactor + } + if xmlCdreCfg.CostRoundingDecimals != nil { + cdreCfg.CostRoundingDecimals = *xmlCdreCfg.CostRoundingDecimals + } + if xmlCdreCfg.CostShiftDigits != nil { + cdreCfg.CostShiftDigits = *xmlCdreCfg.CostShiftDigits + } + if xmlCdreCfg.MaskDestId != nil { + cdreCfg.MaskDestId = *xmlCdreCfg.MaskDestId + } + if xmlCdreCfg.MaskLength != nil { + cdreCfg.MaskLength = *xmlCdreCfg.MaskLength + } + if xmlCdreCfg.ExportDir != nil { + cdreCfg.ExportDir = *xmlCdreCfg.ExportDir + } + if xmlCdreCfg.Header != nil { + cdreCfg.HeaderFields = make([]*CdreCdrField, len(xmlCdreCfg.Header.Fields)) + for idx, xmlFld := range xmlCdreCfg.Header.Fields { + cdreCfg.HeaderFields[idx] = xmlFld.AsCdreCdrField() + } + } + if xmlCdreCfg.Content != nil { + cdreCfg.ContentFields = make([]*CdreCdrField, len(xmlCdreCfg.Content.Fields)) + for idx, xmlFld := range xmlCdreCfg.Content.Fields { + cdreCfg.ContentFields[idx] = xmlFld.AsCdreCdrField() + } + } + if xmlCdreCfg.Trailer != nil { + cdreCfg.TrailerFields = make([]*CdreCdrField, len(xmlCdreCfg.Trailer.Fields)) + for idx, xmlFld := range xmlCdreCfg.Trailer.Fields { + cdreCfg.TrailerFields[idx] = xmlFld.AsCdreCdrField() + } + } + return cdreCfg } // CDR header @@ -49,13 +105,42 @@ type CgrXmlCfgCdrTrailer struct { // CDR field type CgrXmlCfgCdrField struct { - XMLName xml.Name `xml:"field"` - Name string `xml:"name,attr"` - Type string `xml:"type,attr"` - Value string `xml:"value,attr"` - Width int `xml:"width,attr"` // Field width - Strip string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright> - Padding string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right> - Layout string `xml:"layout,attr"` // Eg. time format layout - Mandatory bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported + XMLName xml.Name `xml:"field"` + Name string `xml:"name,attr"` + Type string `xml:"type,attr"` + Value string `xml:"value,attr"` + Width int `xml:"width,attr"` // Field width + Strip string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright> + Padding string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right> + Layout string `xml:"layout,attr"` // Eg. time format layout + Mandatory bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported + valueAsRsrField *utils.RSRField // Cached if the need arrises +} + +func (cdrFld *CgrXmlCfgCdrField) populateRSRField() (err error) { + if cdrFld.Type != utils.CDRFIELD { // We only need rsrField in case of cdrfield type + return nil + } + if cdrFld.valueAsRsrField, err = utils.NewRSRField(cdrFld.Value); err != nil { + return err + } + return nil +} + +func (cdrFld *CgrXmlCfgCdrField) ValueAsRSRField() *utils.RSRField { + return cdrFld.valueAsRsrField +} + +func (cdrFld *CgrXmlCfgCdrField) AsCdreCdrField() *CdreCdrField { + return &CdreCdrField{ + Name: cdrFld.Name, + Type: cdrFld.Type, + Value: cdrFld.Value, + Width: cdrFld.Width, + Strip: cdrFld.Strip, + Padding: cdrFld.Padding, + Layout: cdrFld.Layout, + Mandatory: cdrFld.Mandatory, + valueAsRsrField: cdrFld.valueAsRsrField, + } } diff --git a/config/xmlcdre_test.go b/config/xmlcdre_test.go index d6568d1d4..287261b70 100644 --- a/config/xmlcdre_test.go +++ b/config/xmlcdre_test.go @@ -19,65 +19,96 @@ along with this program. If not, see package config import ( + "github.com/cgrates/cgrates/utils" + "reflect" "strings" "testing" ) var cfgDoc *CgrXmlCfgDocument // Will be populated by first test -func TestParseXmlConfig(t *testing.T) { - cfgXmlStr := ` +func TestXmlCdreCfgPopulateCdreRSRFIeld(t *testing.T) { + cdreField := CgrXmlCfgCdrField{Name: "TEST1", Type: "cdrfield", Value: `~effective_caller_id_number:s/(\d+)/+$1/`} + if err := cdreField.populateRSRField(); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if cdreField.valueAsRsrField == nil { + t.Error("Failed loading the RSRField") + } + valRSRField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`) + if recv := cdreField.ValueAsRSRField(); !reflect.DeepEqual(valRSRField, recv) { + t.Errorf("Expecting %v, received %v", valRSRField, recv) + } + cdreField = CgrXmlCfgCdrField{Name: "TEST1", Type: "constant", Value: `someval`} + if err := cdreField.populateRSRField(); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if cdreField.valueAsRsrField != nil { + t.Error("Should not load the RSRField") + } +} + +func TestXmlCdreCfgParseXmlConfig(t *testing.T) { + cfgXmlStr := ` -
- - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + fwv + 0.0 + 0.0 + -1 + 0 + MASKED_DESTINATIONS + 0 + /var/log/cgrates/cdre + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ` var err error @@ -87,13 +118,13 @@ func TestParseXmlConfig(t *testing.T) { } else if cfgDoc == nil { t.Fatal("Could not parse xml configuration document") } - if len(cfgDoc.cdrefws) != 1 { + if len(cfgDoc.cdres) != 1 { t.Error("Did not cache") } } -func TestGetCdreFWCfg(t *testing.T) { - cdreFWCfg := cfgDoc.GetCdreFWCfgs("CDRE-FW1") +func TestXmlCdreCfgGetCdreCfg(t *testing.T) { + cdreFWCfg := cfgDoc.GetCdreCfgs("CDRE-FW1") if cdreFWCfg == nil { t.Error("Could not parse CdreFw instance") } @@ -107,3 +138,114 @@ func TestGetCdreFWCfg(t *testing.T) { t.Error("Unexpected number of trailer fields parsed", len(cdreFWCfg["CDRE-FW1"].Trailer.Fields)) } } + +func TestXmlCdreCfgAsCdreConfig(t *testing.T) { + cfgXmlStr := ` + + + fwv + 1024.0 + 1.19 + -1 + -3 + MASKED_DESTINATIONS + 1 + /var/log/cgrates/cdre + +
+ + + + +
+ + + + + + + + + + + + + + + +` + var err error + reader := strings.NewReader(cfgXmlStr) + if cfgDoc, err = ParseCgrXmlConfig(reader); err != nil { + t.Error(err.Error()) + } else if cfgDoc == nil { + t.Fatal("Could not parse xml configuration document") + } + xmlCdreCfgs := cfgDoc.GetCdreCfgs("CDRE-FW2") + if xmlCdreCfgs == nil { + t.Error("Could not parse XmlCdre instance") + } + eCdreCfg := &CdreConfig{ + CdrFormat: "fwv", + DataUsageMultiplyFactor: 1024.0, + CostMultiplyFactor: 1.19, + CostRoundingDecimals: -1, + CostShiftDigits: -3, + MaskDestId: "MASKED_DESTINATIONS", + MaskLength: 1, + ExportDir: "/var/log/cgrates/cdre", + } + eCdreCfg.HeaderFields = []*CdreCdrField{ + &CdreCdrField{ + Name: "TypeOfRecord", + Type: "constant", + Value: "10", + Width: 2}, + &CdreCdrField{ + Name: "LastCdr", + Type: "metatag", + Value: "last_cdr_time", + Layout: "020106150400", + Width: 12}, + } + eCdreCfg.ContentFields = []*CdreCdrField{ + &CdreCdrField{ + Name: "OperatorCode", + Type: "cdrfield", + Value: "operator", + Width: 2, + valueAsRsrField: &utils.RSRField{Id: "operator"}, + }, + &CdreCdrField{ + Name: "ProductId", + Type: "cdrfield", + Value: "productid", + Width: 5, + valueAsRsrField: &utils.RSRField{Id: "productid"}, + }, + &CdreCdrField{ + Name: "NetworkId", + Type: "constant", + Value: "3", + Width: 1, + }, + } + eCdreCfg.TrailerFields = []*CdreCdrField{ + &CdreCdrField{ + Name: "DistributorCode", + Type: "constant", + Value: "VOI", + Width: 3, + }, + &CdreCdrField{ + Name: "FileSeqNr", + Type: "metatag", + Value: "export_id", + Width: 5, + Padding: "zeroleft", + }, + } + if rcvCdreCfg := xmlCdreCfgs["CDRE-FW2"].AsCdreConfig(); !reflect.DeepEqual(rcvCdreCfg, eCdreCfg) { + t.Errorf("Expecting: %v, received: %v", eCdreCfg, rcvCdreCfg) + } +} diff --git a/config/xmlconfig.go b/config/xmlconfig.go index be9137447..fbaedb7d7 100644 --- a/config/xmlconfig.go +++ b/config/xmlconfig.go @@ -40,11 +40,11 @@ func ParseCgrXmlConfig(reader io.Reader) (*CgrXmlCfgDocument, error) { // Define a format for configuration file, one doc contains more configuration instances, identified by section, type and id type CgrXmlCfgDocument struct { - XMLName xml.Name `xml:"document"` - Type string `xml:"type,attr"` - Configurations []*CgrXmlConfiguration `xml:"configuration"` - cdrefws map[string]*CgrXmlCdreFwCfg // Cache for processed fixed width config instances, key will be the id of the instance + XMLName xml.Name `xml:"document"` + Type string `xml:"type,attr"` + Configurations []*CgrXmlConfiguration `xml:"configuration"` cdrcs map[string]*CgrXmlCdrcCfg + cdres map[string]*CgrXmlCdreCfg // Cahe cdrexporter instances, key will be the ID } // Storage for raw configuration @@ -63,7 +63,7 @@ func (cfgInst *CgrXmlConfiguration) rawConfigElement() []byte { } func (xmlCfg *CgrXmlCfgDocument) cacheAll() error { - for _, cacheFunc := range []func() error{xmlCfg.cacheCdreFWCfgs, xmlCfg.cacheCdrcCfgs} { + for _, cacheFunc := range []func() error{xmlCfg.cacheCdrcCfgs, xmlCfg.cacheCdreCfgs} { if err := cacheFunc(); err != nil { return err } @@ -71,26 +71,6 @@ func (xmlCfg *CgrXmlCfgDocument) cacheAll() error { return nil } -// Avoid building from raw config string always, so build cache here -func (xmlCfg *CgrXmlCfgDocument) cacheCdreFWCfgs() error { - xmlCfg.cdrefws = make(map[string]*CgrXmlCdreFwCfg) - for _, cfgInst := range xmlCfg.Configurations { - if cfgInst.Section == utils.CDRE || cfgInst.Type == utils.CDRE_FIXED_WIDTH { - cdrefwCfg := new(CgrXmlCdreFwCfg) - rawConfig := append([]byte(""), cfgInst.RawConfig...) // Encapsulate the rawConfig in one element so we can Unmarshall into one struct - rawConfig = append(rawConfig, []byte("")...) - if err := xml.Unmarshal(rawConfig, cdrefwCfg); err != nil { - return err - } else if cdrefwCfg == nil { - return fmt.Errorf("Could not unmarshal CgrXmlCdreFwCfg: %s", cfgInst.Id) - } else { // All good, cache the config instance - xmlCfg.cdrefws[cfgInst.Id] = cdrefwCfg - } - } - } - return nil -} - // Avoid building from raw config string always, so build cache here func (xmlCfg *CgrXmlCfgDocument) cacheCdrcCfgs() error { xmlCfg.cdrcs = make(map[string]*CgrXmlCdrcCfg) @@ -116,16 +96,42 @@ func (xmlCfg *CgrXmlCfgDocument) cacheCdrcCfgs() error { return nil } +// Avoid building from raw config string always, so build cache here +func (xmlCfg *CgrXmlCfgDocument) cacheCdreCfgs() error { + xmlCfg.cdres = make(map[string]*CgrXmlCdreCfg) + for _, cfgInst := range xmlCfg.Configurations { + if cfgInst.Section != utils.CDRE { + continue + } + cdreCfg := new(CgrXmlCdreCfg) + if err := xml.Unmarshal(cfgInst.rawConfigElement(), cdreCfg); err != nil { + return err + } else if cdreCfg == nil { + return fmt.Errorf("Could not unmarshal CgrXmlCdreCfg: %s", cfgInst.Id) + } + if cdreCfg.Content != nil { + // Cache rsr fields + for _, fld := range cdreCfg.Content.Fields { + if err := fld.populateRSRField(); err != nil { + return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error()) + } + } + } + xmlCfg.cdres[cfgInst.Id] = cdreCfg + } + return nil +} + // Return instances or filtered instance of cdrefw configuration -func (xmlCfg *CgrXmlCfgDocument) GetCdreFWCfgs(instName string) map[string]*CgrXmlCdreFwCfg { +func (xmlCfg *CgrXmlCfgDocument) GetCdreCfgs(instName string) map[string]*CgrXmlCdreCfg { if len(instName) != 0 { - if cfg, hasIt := xmlCfg.cdrefws[instName]; !hasIt { + if cfg, hasIt := xmlCfg.cdres[instName]; !hasIt { return nil } else { - return map[string]*CgrXmlCdreFwCfg{instName: cfg} + return map[string]*CgrXmlCdreCfg{instName: cfg} } } - return xmlCfg.cdrefws + return xmlCfg.cdres } // Return instances or filtered instance of cdrc configuration diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index d853f134e..7a0a47a5e 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -28,11 +28,10 @@ # rpc_gob_listen = 127.0.0.1:2013 # RPC GOB listening address # http_listen = 127.0.0.1:2080 # HTTP listening address # default_reqtype = rated # Default request type to consider when missing from requests: <""|prepaid|postpaid|pseudoprepaid|rated>. -# default_category = call # Default Type of Record to consider when missing from requests. +# default_category = call # Default Type of Record to consider when missing from requests. # default_tenant = cgrates.org # Default Tenant to consider when missing from requests. # default_subject = cgrates # Default rating Subject to consider when missing from requests. -# rounding_method = *middle # Rounding method for floats/costs: <*up|*middle|*down> -# rounding_decimals = 10 # Number of decimals to round float/costs at +# rounding_decimals = 10 # System level precision for floats # xmlcfg_path = # Path towards additional config defined in xml file [balancer] @@ -52,9 +51,12 @@ [cdre] # cdr_format = csv # Exported CDRs format +# data_usage_multiply_factor = 0.0 # Multiply data usage before export (eg: convert from KBytes to Bytes) +# cost_multiply_factor = 0.0 # Multiply cost before export (0.0 to disable), eg: add VAT +# cost_rounding_decimals = -1 # Rounding decimals for Cost values. -1 to disable rounding +# cost_shift_digits = 0 # Shift digits in the cost on export (eg: convert from EUR to cents) # mask_destination_id = # Destination id containing called addresses to be masked on export # mask_length = 0 # Length of the destination suffix to be masked -# cost_shift_digits = 0 # Shift cost on export with the number of digits digits defined here (eg: convert from Eur to cent). # export_dir = /var/log/cgrates/cdre # Path where the exported CDRs will be placed # export_template = cgrid,mediation_runid,tor,accid,reqtype,direction,tenant,category,account,subject,destination,setup_time,answer_time,usage,cost # Exported fields template <""|fld1,fld2|*xml:instance_name> diff --git a/data/conf/samples/cgr_addconfig.xml b/data/conf/samples/cgr_addconfig.xml index 6bc5260e0..415abb7ed 100644 --- a/data/conf/samples/cgr_addconfig.xml +++ b/data/conf/samples/cgr_addconfig.xml @@ -1,20 +1,29 @@ - + -
- - - -
- - - - - - - - - - + fwv + 0.0 + 0.0 + 0 + MASKED_DESTINATIONS + 0 + /var/log/cgrates/cdre + +
+ + + +
+ + + + + + + + + + + diff --git a/data/conf/samples/multiplecdrc_fwexport.xml b/data/conf/samples/multiplecdrc_fwexport.xml index 693e56557..fc34de389 100644 --- a/data/conf/samples/multiplecdrc_fwexport.xml +++ b/data/conf/samples/multiplecdrc_fwexport.xml @@ -49,54 +49,63 @@ -
- - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + fwv + 0.0 + 0.0 + 0 + MASKED_DESTINATIONS + 0 + /var/log/cgrates/cdre + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/utils/apitpdata.go b/utils/apitpdata.go index be639e5df..ad33714cc 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -327,33 +327,35 @@ type CachedItemAge struct { } type AttrExpFileCdrs struct { - CdrFormat string // Cdr output file format - ExportId string // Optional exportid - ExportDir string // If provided it overwrites the configured export directory - ExportFileName string // If provided the output filename will be set to this - ExportTemplate string // Exported fields template <""|fld1,fld2|*xml:instance_name> - CostShiftDigits int // If defined it will shift cost digits before applying rouding (eg: convert from Eur->cents), -1 to use general config ones - RoundDecimals int // Overwrite configured roundDecimals with this dynamically, -1 to use general config ones - MaskDestinationId string // Overwrite configured MaskDestId - MaskLength int // Overwrite configured MaskLength, -1 to use general config ones - CgrIds []string // If provided, it will filter based on the cgrids present in list - MediationRunId []string // If provided, it will filter on mediation runid - TOR []string // If provided, filter on TypeOfRecord - CdrHost []string // If provided, it will filter cdrhost - CdrSource []string // If provided, it will filter cdrsource - ReqType []string // If provided, it will fiter reqtype - Direction []string // If provided, it will fiter direction - Tenant []string // If provided, it will filter tenant - Category []string // If provided, it will filter çategory - Account []string // If provided, it will filter account - Subject []string // If provided, it will filter the rating subject - DestinationPrefix []string // If provided, it will filter on destination prefix - OrderIdStart int64 // Export from this order identifier - OrderIdEnd int64 // Export smaller than this order identifier - TimeStart string // If provided, it will represent the starting of the CDRs interval (>=) - TimeEnd string // If provided, it will represent the end of the CDRs interval (<) - SkipErrors bool // Do not export errored CDRs - SkipRated bool // Do not export rated CDRs + CdrFormat *string // Cdr output file format + ExportId *string // Optional exportid + ExportDir *string // If provided it overwrites the configured export directory + ExportFileName *string // If provided the output filename will be set to this + ExportTemplate *string // Exported fields template <""|fld1,fld2|*xml:instance_name> + DataUsageMultiplyFactor *float64 // Multiply data usage before export (eg: convert from KBytes to Bytes) + CostMultiplyFactor *float64 // Multiply the cost before export, eg: apply VAT + CostShiftDigits *int // If defined it will shift cost digits before applying rouding (eg: convert from Eur->cents), -1 to use general config ones + RoundDecimals *int // Overwrite configured roundDecimals with this dynamically, -1 to use general config ones + MaskDestinationId *string // Overwrite configured MaskDestId + MaskLength *int // Overwrite configured MaskLength, -1 to use general config ones + CgrIds []string // If provided, it will filter based on the cgrids present in list + MediationRunId []string // If provided, it will filter on mediation runid + TOR []string // If provided, filter on TypeOfRecord + CdrHost []string // If provided, it will filter cdrhost + CdrSource []string // If provided, it will filter cdrsource + ReqType []string // If provided, it will fiter reqtype + Direction []string // If provided, it will fiter direction + Tenant []string // If provided, it will filter tenant + Category []string // If provided, it will filter çategory + Account []string // If provided, it will filter account + Subject []string // If provided, it will filter the rating subject + DestinationPrefix []string // If provided, it will filter on destination prefix + OrderIdStart int64 // Export from this order identifier + OrderIdEnd int64 // Export smaller than this order identifier + TimeStart string // If provided, it will represent the starting of the CDRs interval (>=) + TimeEnd string // If provided, it will represent the end of the CDRs interval (<) + SkipErrors bool // Do not export errored CDRs + SkipRated bool // Do not export rated CDRs } type ExportedFileCdrs struct { diff --git a/utils/consts.go b/utils/consts.go index 263ae2023..2000e444b 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -119,6 +119,7 @@ const ( OUT = "*out" CDR_IMPORT = "cdr_import" CDR_EXPORT = "cdr_export" + CDRFIELD = "cdrfield" ) var ( diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 08fe024e8..aedd8c057 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -50,19 +50,17 @@ type StoredCdr struct { Cost float64 } -// Should only be used for display purposes, bad otherwise. -// cdrDirection: CDR_IMPORT or CDR_EXPORT -func (storedCdr *StoredCdr) MangleDataUsage(cdrDirection string) { - if IsSliceMember([]string{DATA, SMS}, storedCdr.TOR) { - if cdrDirection == CDR_IMPORT { // On import CDRs usages are converted to nanoseconds, for data we need seconds, fix it here. - storedCdr.Usage = time.Duration(storedCdr.Usage.Nanoseconds()) * time.Second - } else if cdrDirection == CDR_EXPORT { // On exports we need to show the data back in seconds instead of internally stored as nanoseconds - storedCdr.Usage = time.Duration(int(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE))) - } - } +// Used to multiply usage on export +func (storedCdr *StoredCdr) UsageMultiply(multiplyFactor float64, roundDecimals int) { + storedCdr.Usage = time.Duration(int(Round(float64(storedCdr.Usage.Nanoseconds())*multiplyFactor, roundDecimals, ROUNDING_MIDDLE))) // Rounding down could introduce a slight loss here but only at nanoseconds level } -// Return cost as string, formated with number of decimals configured +// Used to multiply cost on export +func (storedCdr *StoredCdr) CostMultiply(multiplyFactor float64, roundDecimals int) { + storedCdr.Cost = Round(storedCdr.Cost*multiplyFactor, roundDecimals, ROUNDING_MIDDLE) +} + +// Format cost as string on export func (storedCdr *StoredCdr) FormatCost(shiftDecimals, roundDecimals int) string { cost := storedCdr.Cost if shiftDecimals != 0 { @@ -71,18 +69,18 @@ func (storedCdr *StoredCdr) FormatCost(shiftDecimals, roundDecimals int) string return strconv.FormatFloat(cost, 'f', roundDecimals, 64) } -// Rounds up so 0.8 seconds will become 1 +// Formats usage on export func (storedCdr *StoredCdr) FormatUsage(layout string) string { if IsSliceMember([]string{DATA, SMS}, storedCdr.TOR) { return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64) } switch layout { case HOURS: - return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Hours()), 'f', -1, 64) + return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64) case MINUTES: - return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Minutes()), 'f', -1, 64) + return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64) case SECONDS: - return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Seconds()), 'f', -1, 64) + return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64) default: return strconv.FormatInt(storedCdr.Usage.Nanoseconds(), 10) } diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 1941e188b..4f1590703 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -76,28 +76,28 @@ func TestFieldAsString(t *testing.T) { cdr.FieldAsString(&RSRField{Id: "fieldextr2"}) != cdr.ExtraFields["fieldextr2"], cdr.FieldAsString(&RSRField{Id: "dummy_field"}) != "") } - /*cdr.TOR = DATA - if formated := cdr.FieldAsString(&RSRField{Id: USAGE}); formated != "10" { - t.Error("Wrong exported value for data field: ", formated) - }*/ } -func TestMangleDataUsage(t *testing.T) { - cdr := StoredCdr{TOR: DATA, Usage: time.Duration(1640113)} - if cdr.MangleDataUsage(CDR_IMPORT); cdr.Usage != time.Duration(1640113000000000) { - t.Error("Unexpected usage after mangling: ", cdr.Usage) +func TestUsageMultiply(t *testing.T) { + cdr := StoredCdr{Usage: time.Duration(10) * time.Second} + if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second { + t.Errorf("Unexpected usage after multiply: %v", cdr.Usage.Nanoseconds()) } - cdr = StoredCdr{TOR: VOICE, Usage: time.Duration(1640113000000000)} - if cdr.MangleDataUsage(CDR_IMPORT); cdr.Usage != time.Duration(1640113000000000) { - t.Error("Unexpected usage after mangling: ", cdr.Usage) + cdr = StoredCdr{Usage: time.Duration(10240) * time.Second} // Simulate conversion back, gives out a bit odd result but this can be rounded on export + expectDuration, _ := time.ParseDuration("10.000005120s") + if cdr.UsageMultiply(0.000976563, 0); cdr.Usage != expectDuration { + t.Errorf("Unexpected usage after multiply: %v", cdr.Usage.Nanoseconds()) } - cdr = StoredCdr{TOR: DATA, Usage: time.Duration(1640113000000000)} - if cdr.MangleDataUsage(CDR_EXPORT); cdr.Usage != time.Duration(1640113) { - t.Error("Unexpected usage after mangling: ", cdr.Usage) +} + +func TestCostMultiply(t *testing.T) { + cdr := StoredCdr{Cost: 1.01} + if cdr.CostMultiply(1.19, 4); cdr.Cost != 1.2019 { + t.Error("Unexpected cost after multiply: %v", cdr.Cost) } - cdr = StoredCdr{TOR: VOICE, Usage: time.Duration(1640113000000000)} - if cdr.MangleDataUsage(CDR_EXPORT); cdr.Usage != time.Duration(1640113000000000) { - t.Error("Unexpected usage after mangling: ", cdr.Usage) + cdr = StoredCdr{Cost: 1.01} + if cdr.CostMultiply(1000, 0); cdr.Cost != 1010 { + t.Error("Unexpected cost after multiply: %v", cdr.Cost) } } diff --git a/utils/utils_local_test.go b/utils/utils_local_test.go index cd2801a34..bffcdd663 100644 --- a/utils/utils_local_test.go +++ b/utils/utils_local_test.go @@ -32,7 +32,7 @@ func TestHttpJsonPost(t *testing.T) { return } cdrOut := &CgrCdrOut{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", - CdrSource: UNIT_TEST, ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + CdrSource: UNIT_TEST, ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "account1", Subject: "tgooiscs0014", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, Usage: 0.00000001, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, }