uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Refactored CDRExporter to merge common exports into one mechanism, added CdreConfig type to collect configuration for CDRE from more sources

Browse Source
This commit is contained in:
DanB
2014-06-03 19:05:36 +02:00
parent c063bc2a21
commit f436346873
28 changed files with 1586 additions and 941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
apier/apier_local_test.go
View File

@@ -31,7 +31,7 @@ import (
"path"
"reflect"
"sort"
"strings"
//"strings"
"testing"
"time"
@@ -74,7 +74,7 @@ func TestCreateDirs(t *testing.T) {
if !*testLocal {
return
}
for _, pathDir := range []string{cfg.CdreDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
if err := os.RemoveAll(pathDir); err != nil {
t.Fatal("Error removing folder: ", pathDir, err)
}
@@ -1391,16 +1391,18 @@ func TestCdrServer(t *testing.T) {
}
}
/*
func TestExportCdrsToFile(t *testing.T) {
if !*testLocal {
return
}
var reply *utils.ExportedFileCdrs
req := utils.AttrExpFileCdrs{}
if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) {
t.Error("Failed to detect missing parameter")
}
req.CdrFormat = utils.CDRE_DRYRUN
//if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err == nil || !strings.HasPrefix(err.Error(), utils.ERR_MANDATORY_IE_MISSING) {
// t.Error("Failed to detect missing parameter")
//}
dryRun := utils.CDRE_DRYRUN
req.CdrFormat = &dryRun
tm1, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:22Z")
tm2, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:23Z")
expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCgrIds: []string{utils.Sha1("dsafdsaf", tm1.String()),
@@ -1410,7 +1412,7 @@ func TestExportCdrsToFile(t *testing.T) {
} else if !reflect.DeepEqual(reply, expectReply) {
t.Errorf("Unexpected reply: %v", reply)
}
/* Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN
Need to implement temporary file writing in order to test removal from db, not possible on DRYRUN
req.RemoveFromDb = true
if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil {
t.Error(err.Error())
@@ -1423,8 +1425,9 @@ func TestExportCdrsToFile(t *testing.T) {
} else if !reflect.DeepEqual(reply, expectReply) {
t.Errorf("Unexpected reply: %v", reply)
}
*/
}
*/
func TestLocalGetCdrs(t *testing.T) {
if !*testLocal {

191
apier/cdre.go
View File

@@ -19,9 +19,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package apier
import (
"encoding/csv"
"fmt"
"github.com/cgrates/cgrates/cdre"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"os"
"path"
@@ -34,10 +36,7 @@ import (
func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
var tStart, tEnd time.Time
var err error
cdrFormat := strings.ToLower(attr.CdrFormat)
if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat")
}
engine.Logger.Debug(fmt.Sprintf("ExportCdrsToFile: %+v", attr))
if len(attr.TimeStart) != 0 {
if tStart, err = utils.ParseTimeDetectLayout(attr.TimeStart); err != nil {
return err
@@ -48,27 +47,76 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E
return err
}
}
exportDir := attr.ExportDir
fileName := attr.ExportFileName
exportId := attr.ExportId
if len(exportId) == 0 {
exportId = strconv.FormatInt(time.Now().Unix(), 10)
exportTemplate := self.Config.CdreDefaultInstance
if attr.ExportTemplate != nil { // XML Template defined, can be field names or xml reference
if strings.HasPrefix(*attr.ExportTemplate, utils.XML_PROFILE_PREFIX) {
if self.Config.XmlCfgDocument == nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, "XmlDocumentNotLoaded")
}
expTplStr := *attr.ExportTemplate
if xmlTemplates := self.Config.XmlCfgDocument.GetCdreCfgs(expTplStr[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates == nil {
return fmt.Errorf("%s:ExportTemplate", utils.ERR_NOT_FOUND)
} else {
exportTemplate = xmlTemplates[expTplStr[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig()
}
} else {
exportTemplate, _ = config.NewDefaultCdreConfig()
if contentFlds, err := config.NewCdreCdrFieldsFromIds(strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
} else {
exportTemplate.ContentFields = contentFlds
}
}
}
costShiftDigits := self.Config.CdreCostShiftDigits
if attr.CostShiftDigits != -1 { // -1 enables system general config
costShiftDigits = attr.CostShiftDigits
if exportTemplate == nil {
return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
}
roundDecimals := self.Config.RoundingDecimals
if attr.RoundDecimals != -1 { // -1 enables system default config
roundDecimals = attr.RoundDecimals
cdrFormat := exportTemplate.CdrFormat
if attr.CdrFormat != nil {
cdrFormat = strings.ToLower(*attr.CdrFormat)
}
maskDestId := attr.MaskDestinationId
if len(maskDestId) == 0 {
maskDestId = self.Config.CdreMaskDestId
if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat")
}
maskLen := self.Config.CdreMaskLength
if attr.MaskLength != -1 {
maskLen = attr.MaskLength
exportDir := exportTemplate.ExportDir
if attr.ExportDir != nil {
exportDir = *attr.ExportDir
}
exportId := strconv.FormatInt(time.Now().Unix(), 10)
if attr.ExportId != nil {
exportId = exportId
}
fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
if attr.ExportFileName != nil {
fileName = *attr.ExportFileName
}
filePath := path.Join(exportDir, fileName)
if cdrFormat == utils.CDRE_DRYRUN {
filePath = utils.CDRE_DRYRUN
}
dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
if attr.DataUsageMultiplyFactor != nil {
dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
}
costMultiplyFactor := exportTemplate.CostMultiplyFactor
if attr.CostMultiplyFactor != nil {
costMultiplyFactor = *attr.CostMultiplyFactor
}
costShiftDigits := exportTemplate.CostShiftDigits
if attr.CostShiftDigits != nil {
costShiftDigits = *attr.CostShiftDigits
}
roundingDecimals := exportTemplate.CostRoundingDecimals
if attr.RoundDecimals != nil {
roundingDecimals = *attr.RoundDecimals
}
maskDestId := exportTemplate.MaskDestId
if attr.MaskDestinationId != nil {
maskDestId = *attr.MaskDestinationId
}
maskLen := exportTemplate.MaskLength
if attr.MaskLength != nil {
maskLen = *attr.MaskLength
}
cdrs, err := self.CdrDb.GetStoredCdrs(attr.CgrIds, attr.MediationRunId, attr.TOR, attr.CdrHost, attr.CdrSource, attr.ReqType, attr.Direction,
attr.Tenant, attr.Category, attr.Account, attr.Subject, attr.DestinationPrefix, attr.OrderIdStart, attr.OrderIdEnd, tStart, tEnd, attr.SkipErrors, attr.SkipRated, false)
@@ -78,86 +126,29 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E
*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
return nil
}
switch cdrFormat {
case utils.CDRE_DRYRUN:
exportedIds := make([]string, len(cdrs))
for idxCdr, cdr := range cdrs {
exportedIds[idxCdr] = cdr.CgrId
}
*reply = utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds}
case utils.CSV:
if len(exportDir) == 0 {
exportDir = path.Join(self.Config.CdreDir, utils.CSV)
}
if len(fileName) == 0 {
fileName = fmt.Sprintf("cdre_%s.csv", exportId)
}
exportedFields := self.Config.CdreExportedFields
if len(attr.ExportTemplate) != 0 {
if exportedFields, err = config.ParseRSRFields(attr.ExportTemplate); err != nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
}
}
if len(exportedFields) == 0 {
return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
}
filePath := path.Join(exportDir, fileName)
fileOut, err := os.Create(filePath)
if err != nil {
return err
}
defer fileOut.Close()
csvWriter := cdre.NewCsvCdrWriter(fileOut, costShiftDigits, roundDecimals, maskDestId, maskLen, exportedFields)
exportedIds := make([]string, 0)
unexportedIds := make(map[string]string)
for _, cdr := range cdrs {
if err := csvWriter.WriteCdr(cdr); err != nil {
unexportedIds[cdr.CgrId] = err.Error()
} else {
exportedIds = append(exportedIds, cdr.CgrId)
}
}
csvWriter.Close()
*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: csvWriter.TotalCost(),
ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds,
FirstOrderId: csvWriter.FirstOrderId(), LastOrderId: csvWriter.LastOrderId()}
case utils.CDRE_FIXED_WIDTH:
if len(exportDir) == 0 {
exportDir = path.Join(self.Config.CdreDir, utils.CDRE_FIXED_WIDTH)
}
if len(fileName) == 0 {
fileName = fmt.Sprintf("cdre_%s.fwv", exportId)
}
exportTemplate := self.Config.CdreFWXmlTemplate
if len(attr.ExportTemplate) != 0 && self.Config.XmlCfgDocument != nil {
if xmlTemplate := self.Config.XmlCfgDocument.GetCdreFWCfgs(attr.ExportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplate != nil {
exportTemplate = xmlTemplate[attr.ExportTemplate[len(utils.XML_PROFILE_PREFIX):]]
}
}
if exportTemplate == nil {
return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
}
filePath := path.Join(exportDir, fileName)
fileOut, err := os.Create(filePath)
if err != nil {
return err
}
defer fileOut.Close()
fww, _ := cdre.NewFWCdrWriter(self.LogDb, fileOut, exportTemplate, exportId, costShiftDigits, roundDecimals, maskDestId, maskLen)
exportedIds := make([]string, 0)
unexportedIds := make(map[string]string)
for _, cdr := range cdrs {
if err := fww.WriteCdr(cdr); err != nil {
unexportedIds[cdr.CgrId] = err.Error()
} else {
exportedIds = append(exportedIds, cdr.CgrId)
}
}
fww.Close()
*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: fww.TotalCost(),
ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds,
FirstOrderId: fww.FirstOrderId(), LastOrderId: fww.LastOrderId()}
fileOut, err := os.Create(filePath)
if err != nil {
return err
}
defer fileOut.Close()
cdrexp, err := cdre.NewCdrExporter(cdrs, self.LogDb, exportTemplate, exportId,
dataUsageMultiplyFactor, costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen)
if err != nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
}
switch cdrFormat {
case utils.CDRE_FIXED_WIDTH:
if err := cdrexp.WriteOut(fileOut); err != nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
}
case utils.CSV:
csvWriter := csv.NewWriter(fileOut)
if err := cdrexp.WriteCsv(csvWriter); err != nil {
return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
}
}
*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(),
ExportedCgrIds: cdrexp.PositiveExports(), UnexportedCgrIds: cdrexp.NegativeExports(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
return nil
}

2
apier/tutfscsv_local_test.go
View File

@@ -46,7 +46,7 @@ func TestFsCsvRemoveDirs(t *testing.T) {
if !*testLocal {
return
}
for _, pathDir := range []string{cfg.CdreDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
if err := os.RemoveAll(pathDir); err != nil {
t.Fatal("Error removing folder: ", pathDir, err)
}

2
apier/tutfsjson_local_test.go
View File

@@ -49,7 +49,7 @@ func TestFsJsonRemoveDirs(t *testing.T) {
if !*testLocal {
return
}
for _, pathDir := range []string{fsjsonCfg.CdreDir, fsjsonCfg.HistoryDir} {
for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, fsjsonCfg.HistoryDir} {
if err := os.RemoveAll(pathDir); err != nil {
t.Fatal("Error removing folder: ", pathDir, err)
}

393
cdre/cdrexporter.go
View File

@@ -2,13 +2,13 @@
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can Storagetribute it and/or modify
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITH*out ANY WARRANTY; without even the implied warranty of
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
@@ -19,13 +19,390 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package cdre
import (
"encoding/csv"
"encoding/json"
"fmt"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"io"
"strconv"
"strings"
"time"
)
type CdrWriter interface {
FirstOrderId() int64
LastOrderId() int64
TotalCost() float64
WriteCdr(cdr *utils.StoredCdr) string
Close()
const (
COST_DETAILS = "cost_details"
FILLER = "filler"
CONSTANT = "constant"
METATAG = "metatag"
CONCATENATED_CDRFIELD = "concatenated_cdrfield"
META_EXPORTID = "export_id"
META_TIMENOW = "time_now"
META_FIRSTCDRATIME = "first_cdr_atime"
META_LASTCDRATIME = "last_cdr_atime"
META_NRCDRS = "cdrs_number"
META_DURCDRS = "cdrs_duration"
META_COSTCDRS = "cdrs_cost"
META_MASKDESTINATION = "mask_destination"
META_FORMATCOST = "format_cost"
)
var err error
func NewCdrExporter(cdrs []*utils.StoredCdr, logDb engine.LogStorage, exportTpl *config.CdreConfig, exportId string,
dataUsageMultiplyFactor, costMultiplyFactor float64, costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int) (*CdrExporter, error) {
if len(cdrs) == 0 { // Nothing to export
return nil, nil
}
cdre := &CdrExporter{
cdrs: cdrs,
logDb: logDb,
exportTemplate: exportTpl,
exportId: exportId,
dataUsageMultiplyFactor: dataUsageMultiplyFactor,
costMultiplyFactor: costMultiplyFactor,
costShiftDigits: costShiftDigits,
roundDecimals: roundDecimals,
cgrPrecision: cgrPrecision,
maskDestId: maskDestId,
maskLen: maskLen,
negativeExports: make(map[string]string),
}
if err := cdre.processCdrs(); err != nil {
return nil, err
}
return cdre, nil
}
type CdrExporter struct {
cdrs []*utils.StoredCdr
logDb engine.LogStorage // Used to extract cost_details if these are requested
exportTemplate *config.CdreConfig
exportId string // Unique identifier or this export
dataUsageMultiplyFactor, costMultiplyFactor float64
costShiftDigits, roundDecimals, cgrPrecision int
maskDestId string
maskLen int
header, trailer []string // Header and Trailer fields
content [][]string // Rows of cdr fields
firstCdrATime, lastCdrATime time.Time
numberOfRecords int
totalDuration time.Duration
totalCost float64
firstExpOrderId, lastExpOrderId int64
positiveExports []string // CGRIds of successfully exported CDRs
negativeExports map[string]string // CgrIds of failed exports
}
// Return Json marshaled callCost attached to
// Keep it separately so we test only this part in local tests
func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) {
cc, err := cdre.logDb.GetCallCostLog(cgrId, "", runId)
if err != nil {
return "", err
} else if cc == nil {
return "", nil
}
ccJson, _ := json.Marshal(cc)
return string(ccJson), nil
}
// Check if the destination should be masked in output
func (cdre *CdrExporter) maskedDestination(destination string) bool {
if len(cdre.maskDestId) != 0 && engine.CachedDestHasPrefix(cdre.maskDestId, destination) {
return true
}
return false
}
// Extracts the value specified by cfgHdr out of cdr
func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, rsrFld *utils.RSRField, layout string) (string, error) {
if rsrFld == nil {
return "", nil
}
var cdrVal string
switch rsrFld.Id {
case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
if cdrVal, err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
return "", err
}
case utils.COST:
cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
case utils.USAGE:
cdrVal = cdr.FormatUsage(layout)
case utils.SETUP_TIME:
cdrVal = cdr.SetupTime.Format(layout)
case utils.ANSWER_TIME: // Format time based on layout
cdrVal = cdr.AnswerTime.Format(layout)
case utils.DESTINATION:
cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})
if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
cdrVal = MaskDestination(cdrVal, cdre.maskLen)
}
default:
cdrVal = cdr.FieldAsString(rsrFld)
}
return rsrFld.ParseValue(cdrVal), nil
}
// Handle various meta functions used in header/trailer
func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) {
switch tag {
case META_EXPORTID:
return cdre.exportId, nil
case META_TIMENOW:
return time.Now().Format(arg), nil
case META_FIRSTCDRATIME:
return cdre.firstCdrATime.Format(arg), nil
case META_LASTCDRATIME:
return cdre.lastCdrATime.Format(arg), nil
case META_NRCDRS:
return strconv.Itoa(cdre.numberOfRecords), nil
case META_DURCDRS:
return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil
case META_COSTCDRS:
return strconv.FormatFloat(utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
case META_MASKDESTINATION:
if cdre.maskedDestination(arg) {
return "1", nil
}
return "0", nil
default:
return "", fmt.Errorf("Unsupported METATAG: %s", tag)
}
return "", nil
}
// Compose and cache the header
func (cdre *CdrExporter) composeHeader() error {
for _, cfgFld := range cdre.exportTemplate.HeaderFields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case METATAG:
outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
default:
return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, error: %s", err.Error()))
return err
}
fmtOut := outVal
if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH {
if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdrExporter> Cannot export CDR header, error: %s", err.Error()))
return err
}
}
cdre.header = append(cdre.header, fmtOut)
}
return nil
}
// Compose and cache the trailer
func (cdre *CdrExporter) composeTrailer() error {
for _, cfgFld := range cdre.exportTemplate.TrailerFields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case METATAG:
outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
default:
return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
return err
}
fmtOut := outVal
if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH {
if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
return err
}
}
cdre.trailer = append(cdre.trailer, fmtOut)
}
return nil
}
// Write individual cdr into content buffer, build stats
func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
return nil
}
if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
}
if cdre.costMultiplyFactor != 0.0 {
cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
}
var err error
cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
for idx, cfgFld := range cdre.exportTemplate.ContentFields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case utils.CDRFIELD:
outVal, err = cdre.cdrFieldValue(cdr, cfgFld.ValueAsRSRField(), cfgFld.Layout)
case CONCATENATED_CDRFIELD:
for _, fld := range strings.Split(cfgFld.Value, ",") {
if fldOut, err := cdre.cdrFieldValue(cdr, &utils.RSRField{Id: fld}, cfgFld.Layout); err != nil {
break // The error will be reported bellow
} else {
outVal += fldOut
}
}
case METATAG:
if cfgFld.Value == META_MASKDESTINATION {
outVal, err = cdre.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
} else {
outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
}
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
return err
}
fmtOut := outVal
if cdre.exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH {
if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error()))
return err
}
}
cdrRow[idx] += fmtOut
}
if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
return nil
} else {
cdre.content = append(cdre.content, cdrRow)
}
// Done with writing content, compute stats here
if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
cdre.firstCdrATime = cdr.AnswerTime
}
if cdr.AnswerTime.After(cdre.lastCdrATime) {
cdre.lastCdrATime = cdr.AnswerTime
}
cdre.numberOfRecords += 1
if !utils.IsSliceMember([]string{utils.DATA, utils.SMS}, cdr.TOR) { // Only count duration for non data cdrs
cdre.totalDuration += cdr.Usage
}
cdre.totalCost += cdr.Cost
cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 {
cdre.firstExpOrderId = cdr.OrderId
}
if cdre.lastExpOrderId < cdr.OrderId {
cdre.lastExpOrderId = cdr.OrderId
}
return nil
}
// Builds header, content and trailers
func (cdre *CdrExporter) processCdrs() error {
if cdre.exportTemplate.HeaderFields != nil {
if err := cdre.composeHeader(); err != nil {
return err
}
}
if cdre.exportTemplate.TrailerFields != nil {
if err := cdre.composeTrailer(); err != nil {
return err
}
}
for _, cdr := range cdre.cdrs {
if err := cdre.processCdr(cdr); err != nil {
cdre.negativeExports[cdr.CgrId] = err.Error()
} else {
cdre.positiveExports = append(cdre.positiveExports, cdr.CgrId)
}
}
return nil
}
func (cdre *CdrExporter) WriteCsv(csvWriter *csv.Writer) error {
if len(cdre.header) != 0 {
if err := csvWriter.Write(cdre.header); err != nil {
return err
}
}
for _, cdrContent := range cdre.content {
if err := csvWriter.Write(cdrContent); err != nil {
return err
}
}
if len(cdre.trailer) != 0 {
if err := csvWriter.Write(cdre.trailer); err != nil {
return err
}
}
csvWriter.Flush()
return nil
}
// Write fwv content
func (cdre *CdrExporter) WriteOut(ioWriter io.Writer) error {
if len(cdre.header) != 0 {
for _, fld := range append(cdre.header, "\n") {
if _, err := io.WriteString(ioWriter, fld); err != nil {
return err
}
}
}
for _, cdrContent := range cdre.content {
for _, cdrFld := range append(cdrContent, "\n") {
if _, err := io.WriteString(ioWriter, cdrFld); err != nil {
return err
}
}
}
if len(cdre.trailer) != 0 {
for _, fld := range append(cdre.trailer, "\n") {
if _, err := io.WriteString(ioWriter, fld); err != nil {
return err
}
}
}
return nil
}
// Return the first exported Cdr OrderId
func (cdre *CdrExporter) FirstOrderId() int64 {
return cdre.firstExpOrderId
}
// Return the last exported Cdr OrderId
func (cdre *CdrExporter) LastOrderId() int64 {
return cdre.lastExpOrderId
}
// Return total cost in the exported cdrs
func (cdre *CdrExporter) TotalCost() float64 {
return cdre.totalCost
}
// Return successfully exported CgrIds
func (cdre *CdrExporter) PositiveExports() []string {
return cdre.positiveExports
}
// Return failed exported CgrIds together with the reason
func (cdre *CdrExporter) NegativeExports() map[string]string {
return cdre.negativeExports
}

85
cdre/csv.go
View File

@@ -1,85 +0,0 @@
/*
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package cdre
import (
"encoding/csv"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"io"
)
type CsvCdrWriter struct {
writer *csv.Writer
costShiftDigits, roundDecimals int // Round floats like Cost using this number of decimals
maskDestId string
maskLen int
exportedFields []*utils.RSRField // The fields exported, order important
firstExpOrderId, lastExpOrderId int64
totalCost float64 // Cummulated cost of all the
}
func NewCsvCdrWriter(writer io.Writer, costShiftDigits, roundDecimals int, maskDestId string, maskLen int, exportedFields []*utils.RSRField) *CsvCdrWriter {
return &CsvCdrWriter{writer: csv.NewWriter(writer), costShiftDigits: costShiftDigits, roundDecimals: roundDecimals, maskDestId: maskDestId, maskLen: maskLen, exportedFields: exportedFields}
}
// Return the first exported Cdr OrderId
func (csvwr *CsvCdrWriter) FirstOrderId() int64 {
return csvwr.firstExpOrderId
}
func (csvwr *CsvCdrWriter) LastOrderId() int64 {
return csvwr.lastExpOrderId
}
func (csvwr *CsvCdrWriter) TotalCost() float64 {
return csvwr.totalCost
}
func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error {
row := make([]string, len(csvwr.exportedFields))
for idx, fld := range csvwr.exportedFields {
var fldVal string
if fld.Id == utils.COST {
fldVal = cdr.FormatCost(csvwr.costShiftDigits, csvwr.roundDecimals)
} else if fld.Id == utils.DESTINATION {
fldVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})
if len(csvwr.maskDestId) != 0 && csvwr.maskLen > 0 && engine.CachedDestHasPrefix(csvwr.maskDestId, fldVal) {
fldVal = MaskDestination(fldVal, csvwr.maskLen)
}
} else {
fldVal = cdr.FieldAsString(fld)
}
row[idx] = fld.ParseValue(fldVal)
}
if csvwr.firstExpOrderId > cdr.OrderId || csvwr.firstExpOrderId == 0 {
csvwr.firstExpOrderId = cdr.OrderId
}
if csvwr.lastExpOrderId < cdr.OrderId {
csvwr.lastExpOrderId = cdr.OrderId
}
csvwr.totalCost += cdr.Cost
csvwr.totalCost = utils.Round(csvwr.totalCost, csvwr.roundDecimals, utils.ROUNDING_MIDDLE)
return csvwr.writer.Write(row)
}
func (csvwr *CsvCdrWriter) Close() {
csvwr.writer.Flush()
}

27
cdre/csv_test.go
View File

@@ -1,6 +1,6 @@
/*
Rating system designed to be used in VoIP Carriers World
Copyright (C) 2013 ITsysCOM
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -20,7 +20,9 @@ package cdre
import (
"bytes"
"encoding/csv"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"strings"
"testing"
@@ -30,22 +32,27 @@ import (
func TestCsvCdrWriter(t *testing.T) {
writer := &bytes.Buffer{}
cfg, _ := config.NewDefaultCGRConfig()
exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"})
csvCdrWriter := NewCsvCdrWriter(writer, 0, 4, "", -1, exportedFields)
ratedCdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
logDb, _ := engine.NewMapStorage()
storedCdr1 := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
}
csvCdrWriter.WriteCdr(ratedCdr)
csvCdrWriter.Close()
expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10000000000,1.0100,val_extra3,"",val_extra1`
cdre, err := NewCdrExporter([]*utils.StoredCdr{storedCdr1}, logDb, cfg.CdreDefaultInstance, "firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0)
if err != nil {
t.Error("Unexpected error received: ", err)
}
csvWriter := csv.NewWriter(writer)
if err := cdre.WriteCsv(csvWriter); err != nil {
t.Error("Unexpected error: ", err)
}
expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10000000000,1.0100`
result := strings.TrimSpace(writer.String())
if result != expected {
t.Errorf("Expected: \n%s received: \n%s.", expected, result)
}
if csvCdrWriter.TotalCost() != 1.01 {
t.Error("Unexpected TotalCost: ", csvCdrWriter.TotalCost())
if cdre.TotalCost() != 1.01 {
t.Error("Unexpected TotalCost: ", cdre.TotalCost())
}
}

331
cdre/fixedwidth.go
View File

@@ -1,331 +0,0 @@
/*
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package cdre
import (
"bytes"
"encoding/json"
"fmt"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"io"
"os"
"strconv"
"strings"
"time"
)
const (
COST_DETAILS = "cost_details"
FILLER = "filler"
CONSTANT = "constant"
CDRFIELD = "cdrfield"
METATAG = "metatag"
CONCATENATED_CDRFIELD = "concatenated_cdrfield"
META_EXPORTID = "export_id"
META_TIMENOW = "time_now"
META_FIRSTCDRATIME = "first_cdr_atime"
META_LASTCDRATIME = "last_cdr_atime"
META_NRCDRS = "cdrs_number"
META_DURCDRS = "cdrs_duration"
META_COSTCDRS = "cdrs_cost"
META_MASKDESTINATION = "mask_destination"
META_FORMATCOST = "format_cost"
)
var err error
func NewFWCdrWriter(logDb engine.LogStorage, outFile *os.File, exportTpl *config.CgrXmlCdreFwCfg, exportId string,
costShiftDigits, roundDecimals int, maskDestId string, maskLen int) (*FixedWidthCdrWriter, error) {
return &FixedWidthCdrWriter{
logDb: logDb,
writer: outFile,
exportTemplate: exportTpl,
exportId: exportId,
costShiftDigits: costShiftDigits,
roundDecimals: roundDecimals,
maskDestId: maskDestId,
maskLen: maskLen,
header: &bytes.Buffer{},
content: &bytes.Buffer{},
trailer: &bytes.Buffer{}}, nil
}
type FixedWidthCdrWriter struct {
logDb engine.LogStorage // Used to extract cost_details if these are requested
writer io.Writer
exportTemplate *config.CgrXmlCdreFwCfg
exportId string // Unique identifier or this export
costShiftDigits, roundDecimals int
maskDestId string
maskLen int
header, content, trailer *bytes.Buffer
firstCdrATime, lastCdrATime time.Time
numberOfRecords int
totalDuration time.Duration
totalCost float64
firstExpOrderId, lastExpOrderId int64
}
// Return Json marshaled callCost attached to
// Keep it separately so we test only this part in local tests
func (fwv *FixedWidthCdrWriter) getCdrCostDetails(cgrId, runId string) (string, error) {
cc, err := fwv.logDb.GetCallCostLog(cgrId, "", runId)
if err != nil {
return "", err
} else if cc == nil {
return "", nil
}
ccJson, _ := json.Marshal(cc)
return string(ccJson), nil
}
func (fwv *FixedWidthCdrWriter) maskedDestination(destination string) bool {
if len(fwv.maskDestId) != 0 && engine.CachedDestHasPrefix(fwv.maskDestId, destination) {
return true
}
return false
}
// Extracts the value specified by cfgHdr out of cdr
func (fwv *FixedWidthCdrWriter) cdrFieldValue(cdr *utils.StoredCdr, cfgHdr, layout string) (string, error) {
rsrField, err := utils.NewRSRField(cfgHdr)
if err != nil {
return "", err
} else if rsrField == nil {
return "", nil
}
var cdrVal string
switch rsrField.Id {
case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
if cdrVal, err = fwv.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
return "", err
}
case utils.COST:
cdrVal = cdr.FormatCost(fwv.costShiftDigits, fwv.roundDecimals)
case utils.USAGE:
cdrVal = cdr.FormatUsage(layout)
case utils.SETUP_TIME:
cdrVal = cdr.SetupTime.Format(layout)
case utils.ANSWER_TIME: // Format time based on layout
cdrVal = cdr.AnswerTime.Format(layout)
case utils.DESTINATION:
cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})
if fwv.maskLen != -1 && fwv.maskedDestination(cdrVal) {
cdrVal = MaskDestination(cdrVal, fwv.maskLen)
}
default:
cdrVal = cdr.FieldAsString(rsrField)
}
return rsrField.ParseValue(cdrVal), nil
}
func (fwv *FixedWidthCdrWriter) metaHandler(tag, arg string) (string, error) {
switch tag {
case META_EXPORTID:
return fwv.exportId, nil
case META_TIMENOW:
return time.Now().Format(arg), nil
case META_FIRSTCDRATIME:
return fwv.firstCdrATime.Format(arg), nil
case META_LASTCDRATIME:
return fwv.lastCdrATime.Format(arg), nil
case META_NRCDRS:
return strconv.Itoa(fwv.numberOfRecords), nil
case META_DURCDRS:
return strconv.FormatFloat(fwv.totalDuration.Seconds(), 'f', -1, 64), nil
case META_COSTCDRS:
return strconv.FormatFloat(utils.Round(fwv.totalCost, fwv.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
case META_MASKDESTINATION:
if fwv.maskedDestination(arg) {
return "1", nil
}
return "0", nil
default:
return "", fmt.Errorf("Unsupported METATAG: %s", tag)
}
return "", nil
}
// Return the first exported Cdr OrderId
func (fwv *FixedWidthCdrWriter) FirstOrderId() int64 {
return fwv.firstExpOrderId
}
// Return the last exported Cdr OrderId
func (fwv *FixedWidthCdrWriter) LastOrderId() int64 {
return fwv.lastExpOrderId
}
func (fwv *FixedWidthCdrWriter) TotalCost() float64 {
return fwv.totalCost
}
// Writes the header into it's buffer
func (fwv *FixedWidthCdrWriter) ComposeHeader() error {
header := ""
for _, cfgFld := range fwv.exportTemplate.Header.Fields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case METATAG:
outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout)
default:
return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, error: %s", err.Error()))
return err
}
if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, error: %s", err.Error()))
return err
} else {
header += fmtOut
}
}
if len(header) == 0 { // No header data, most likely no configuration fields defined
return nil
}
header += "\n" // Done with cdr, postpend new line char
fwv.header.WriteString(header)
return nil
}
// Writes the trailer into it's buffer
func (fwv *FixedWidthCdrWriter) ComposeTrailer() error {
trailer := ""
for _, cfgFld := range fwv.exportTemplate.Trailer.Fields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case METATAG:
outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout)
default:
return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
return err
}
if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
return err
} else {
trailer += fmtOut
}
}
if len(trailer) == 0 { // No header data, most likely no configuration fields defined
return nil
}
trailer += "\n" // Done with cdr, postpend new line char
fwv.trailer.WriteString(trailer)
return nil
}
// Write individual cdr into content buffer, build stats
func (fwv *FixedWidthCdrWriter) WriteCdr(cdr *utils.StoredCdr) error {
if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
return nil
}
var err error
cdrRow := ""
for _, cfgFld := range fwv.exportTemplate.Content.Fields {
var outVal string
switch cfgFld.Type {
case FILLER:
outVal = cfgFld.Value
cfgFld.Padding = "right"
case CONSTANT:
outVal = cfgFld.Value
case CDRFIELD:
outVal, err = fwv.cdrFieldValue(cdr, cfgFld.Value, cfgFld.Layout)
case CONCATENATED_CDRFIELD:
for _, fld := range strings.Split(cfgFld.Value, ",") {
if fldOut, err := fwv.cdrFieldValue(cdr, fld, cfgFld.Layout); err != nil {
break // The error will be reported bellow
} else {
outVal += fldOut
}
}
case METATAG:
if cfgFld.Value == META_MASKDESTINATION {
outVal, err = fwv.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
} else {
outVal, err = fwv.metaHandler(cfgFld.Value, cfgFld.Layout)
}
}
if err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
return err
}
if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error()))
return err
} else {
cdrRow += fmtOut
}
}
if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
return nil
}
cdrRow += "\n" // Done with cdr, postpend new line char
fwv.content.WriteString(cdrRow)
// Done with writing content, compute stats here
if fwv.firstCdrATime.IsZero() || cdr.AnswerTime.Before(fwv.firstCdrATime) {
fwv.firstCdrATime = cdr.AnswerTime
}
if cdr.AnswerTime.After(fwv.lastCdrATime) {
fwv.lastCdrATime = cdr.AnswerTime
}
fwv.numberOfRecords += 1
if !utils.IsSliceMember([]string{utils.DATA, utils.SMS}, cdr.TOR) { // Only count duration for non data cdrs
fwv.totalDuration += cdr.Usage
}
fwv.totalCost += cdr.Cost
fwv.totalCost = utils.Round(fwv.totalCost, fwv.roundDecimals, utils.ROUNDING_MIDDLE)
if fwv.firstExpOrderId > cdr.OrderId || fwv.firstExpOrderId == 0 {
fwv.firstExpOrderId = cdr.OrderId
}
if fwv.lastExpOrderId < cdr.OrderId {
fwv.lastExpOrderId = cdr.OrderId
}
return nil
}
func (fwv *FixedWidthCdrWriter) Close() {
if fwv.exportTemplate.Header != nil {
fwv.ComposeHeader()
}
if fwv.exportTemplate.Trailer != nil {
fwv.ComposeTrailer()
}
for _, buf := range []*bytes.Buffer{fwv.header, fwv.content, fwv.trailer} {
fwv.writer.Write(buf.Bytes())
}
}

145
cdre/fixedwidth_test.go
View File

@@ -21,6 +21,7 @@ package cdre
import (
"bytes"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
"math"
"testing"
@@ -40,24 +41,24 @@ var hdrCfgFlds = []*config.CgrXmlCfgCdrField{
var contentCfgFlds = []*config.CgrXmlCfgCdrField{
&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "20", Width: 2},
&config.CgrXmlCfgCdrField{Name: "Account", Type: CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Subject", Type: CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "CLI", Type: CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Destination", Type: CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Account", Type: utils.CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Subject", Type: utils.CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "CLI", Type: utils.CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Destination", Type: utils.CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "TOR", Type: CONSTANT, Value: "02", Width: 2},
&config.CgrXmlCfgCdrField{Name: "SubtypeTOR", Type: CONSTANT, Value: "11", Width: 4, Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "SetupTime", Type: CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"},
&config.CgrXmlCfgCdrField{Name: "Duration", Type: CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS},
&config.CgrXmlCfgCdrField{Name: "SetupTime", Type: utils.CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"},
&config.CgrXmlCfgCdrField{Name: "Duration", Type: utils.CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS},
&config.CgrXmlCfgCdrField{Name: "DataVolume", Type: FILLER, Width: 6},
&config.CgrXmlCfgCdrField{Name: "TaxCode", Type: CONSTANT, Value: "1", Width: 1},
&config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "ProductId", Type: CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: utils.CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "ProductId", Type: utils.CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "NetworkId", Type: CONSTANT, Value: "3", Width: 1},
&config.CgrXmlCfgCdrField{Name: "CallId", Type: CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "CallId", Type: utils.CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8},
&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8},
&config.CgrXmlCfgCdrField{Name: "TerminationCode", Type: CONCATENATED_CDRFIELD, Value: "operator,product", Width: 5, Strip: "right", Padding: "right"},
&config.CgrXmlCfgCdrField{Name: "Cost", Type: CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"},
&config.CgrXmlCfgCdrField{Name: "Cost", Type: utils.CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"},
&config.CgrXmlCfgCdrField{Name: "DestinationPrivacy", Type: METATAG, Value: META_MASKDESTINATION, Width: 1},
}
@@ -76,35 +77,33 @@ var trailerCfgFlds = []*config.CgrXmlCfgCdrField{
// Write one CDR and test it's results only for content buffer
func TestWriteCdr(t *testing.T) {
wrBuf := &bytes.Buffer{}
exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
logDb, _ := engine.NewMapStorage()
cfg, _ := config.NewDefaultCGRConfig()
fixedWidth := utils.CDRE_FIXED_WIDTH
exportTpl := &config.CgrXmlCdreCfg{
CdrFormat: &fixedWidth,
Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
}
fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}}
cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
}
if err := fwWriter.WriteCdr(cdr); err != nil {
cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr}, logDb, exportTpl.AsCdreConfig(), "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1)
if err != nil {
t.Error(err)
}
eContentOut := "201001 1001 1002 0211 07111308420010 1 3dsafdsaf 0002.34570\n"
contentOut := fwWriter.content.String()
if len(contentOut) != 145 {
t.Error("Unexpected content length", len(contentOut))
} else if contentOut != eContentOut {
t.Errorf("Content out different than expected. Have <%s>, expecting: <%s>", contentOut, eContentOut)
}
eHeader := "10 VOI0000007111308420024031415390001 \n"
eContentOut := "201001 1001 1002 0211 07111308420010 1 3dsafdsaf 0002.34570\n"
eTrailer := "90 VOI0000000000100000010071113084260071113084200 \n"
outBeforeWrite := ""
if wrBuf.String() != outBeforeWrite {
t.Errorf("Output buffer should be empty before write")
if err := cdre.WriteOut(wrBuf); err != nil {
t.Error(err)
}
fwWriter.Close()
allOut := wrBuf.String()
eAllOut := eHeader + eContentOut + eTrailer
if math.Mod(float64(len(allOut)), 145) != 0 {
@@ -113,35 +112,38 @@ func TestWriteCdr(t *testing.T) {
t.Errorf("Output does not match expected length. Have output %q, expecting: %q", allOut, eAllOut)
}
// Test stats
if !fwWriter.firstCdrATime.Equal(cdr.AnswerTime) {
t.Error("Unexpected firstCdrATime in stats: ", fwWriter.firstCdrATime)
} else if !fwWriter.lastCdrATime.Equal(cdr.AnswerTime) {
t.Error("Unexpected lastCdrATime in stats: ", fwWriter.lastCdrATime)
} else if fwWriter.numberOfRecords != 1 {
t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords)
} else if fwWriter.totalDuration != cdr.Usage {
t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration)
} else if fwWriter.totalCost != utils.Round(cdr.Cost, fwWriter.roundDecimals, utils.ROUNDING_MIDDLE) {
t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost)
if !cdre.firstCdrATime.Equal(cdr.AnswerTime) {
t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
} else if !cdre.lastCdrATime.Equal(cdr.AnswerTime) {
t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
} else if cdre.numberOfRecords != 1 {
t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
} else if cdre.totalDuration != cdr.Usage {
t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
} else if cdre.totalCost != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
}
if fwWriter.FirstOrderId() != 1 {
t.Error("Unexpected FirstOrderId", fwWriter.FirstOrderId())
if cdre.FirstOrderId() != 1 {
t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
}
if fwWriter.LastOrderId() != 1 {
t.Error("Unexpected LastOrderId", fwWriter.LastOrderId())
if cdre.LastOrderId() != 1 {
t.Error("Unexpected LastOrderId", cdre.LastOrderId())
}
if fwWriter.TotalCost() != utils.Round(cdr.Cost, fwWriter.roundDecimals, utils.ROUNDING_MIDDLE) {
t.Error("Unexpected TotalCost: ", fwWriter.TotalCost())
if cdre.TotalCost() != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
t.Error("Unexpected TotalCost: ", cdre.TotalCost())
}
}
func TestWriteCdrs(t *testing.T) {
wrBuf := &bytes.Buffer{}
exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
logDb, _ := engine.NewMapStorage()
fixedWidth := utils.CDRE_FIXED_WIDTH
exportTpl := &config.CgrXmlCdreCfg{
CdrFormat: &fixedWidth,
Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
}
fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}}
cdr1 := &utils.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "1001", Subject: "1001", Destination: "1010",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
@@ -164,45 +166,40 @@ func TestWriteCdrs(t *testing.T) {
Usage: time.Duration(20) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"},
}
for _, cdr := range []*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4} {
if err := fwWriter.WriteCdr(cdr); err != nil {
t.Error(err)
}
contentOut := fwWriter.content.String()
if math.Mod(float64(len(contentOut)), 145) != 0 { // Rest must be 0 always, so content is always multiple of 145 which is our row fixLength
t.Error("Unexpected content length", len(contentOut))
}
cfg, _ := config.NewDefaultCGRConfig()
cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4}, logDb, exportTpl.AsCdreConfig(), "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1)
if err != nil {
t.Error(err)
}
if len(wrBuf.String()) != 0 {
t.Errorf("Output buffer should be empty before write")
if err := cdre.WriteOut(wrBuf); err != nil {
t.Error(err)
}
fwWriter.Close()
if len(wrBuf.String()) != 725 {
t.Error("Output buffer does not contain expected info. Expecting len: 725, got: ", len(wrBuf.String()))
}
// Test stats
if !fwWriter.firstCdrATime.Equal(cdr2.AnswerTime) {
t.Error("Unexpected firstCdrATime in stats: ", fwWriter.firstCdrATime)
if !cdre.firstCdrATime.Equal(cdr2.AnswerTime) {
t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
}
if !fwWriter.lastCdrATime.Equal(cdr4.AnswerTime) {
t.Error("Unexpected lastCdrATime in stats: ", fwWriter.lastCdrATime)
if !cdre.lastCdrATime.Equal(cdr4.AnswerTime) {
t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
}
if fwWriter.numberOfRecords != 3 {
t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords)
if cdre.numberOfRecords != 3 {
t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
}
if fwWriter.totalDuration != time.Duration(330)*time.Second {
t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration)
if cdre.totalDuration != time.Duration(330)*time.Second {
t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
}
if fwWriter.totalCost != 5.9957 {
t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost)
if cdre.totalCost != 5.9957 {
t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
}
if fwWriter.FirstOrderId() != 2 {
t.Error("Unexpected FirstOrderId", fwWriter.FirstOrderId())
if cdre.FirstOrderId() != 2 {
t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
}
if fwWriter.LastOrderId() != 4 {
t.Error("Unexpected LastOrderId", fwWriter.LastOrderId())
if cdre.LastOrderId() != 4 {
t.Error("Unexpected LastOrderId", cdre.LastOrderId())
}
if fwWriter.TotalCost() != 5.9957 {
t.Error("Unexpected TotalCost: ", fwWriter.TotalCost())
if cdre.TotalCost() != 5.9957 {
t.Error("Unexpected TotalCost: ", cdre.TotalCost())
}
}

0
cdre/libfixedwidth.go → cdre/libcdre.go
View File

0
cdre/libfixedwidth_test.go → cdre/libcdre_test.go
View File

223
config/cdreconfig.go Normal file
View File

@@ -0,0 +1,223 @@
/*
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package config
import (
"errors"
"github.com/cgrates/cgrates/utils"
)
// Converts a list of field identifiers into proper CDR field content
func NewCdreCdrFieldsFromIds(fldsIds ...string) ([]*CdreCdrField, error) {
cdrFields := make([]*CdreCdrField, len(fldsIds))
for idx, fldId := range fldsIds {
if parsedRsr, err := utils.NewRSRField(fldId); err != nil {
return nil, err
} else {
cdrFld := &CdreCdrField{Name: fldId, Type: utils.CDRFIELD, Value: fldId, valueAsRsrField: parsedRsr}
if err := cdrFld.setDefaultFixedWidthProperties(); err != nil { // Set default fixed width properties to be used later if needed
return nil, err
}
cdrFields[idx] = cdrFld
}
}
return cdrFields, nil
}
func NewDefaultCdreConfig() (*CdreConfig, error) {
cdreCfg := new(CdreConfig)
if err := cdreCfg.setDefaults(); err != nil {
return nil, err
}
return cdreCfg, nil
}
// One instance of CdrExporter
type CdreConfig struct {
CdrFormat string
DataUsageMultiplyFactor float64
CostMultiplyFactor float64
CostRoundingDecimals int
CostShiftDigits int
MaskDestId string
MaskLength int
ExportDir string
HeaderFields []*CdreCdrField
ContentFields []*CdreCdrField
TrailerFields []*CdreCdrField
}
// Set here defaults
func (cdreCfg *CdreConfig) setDefaults() error {
cdreCfg.CdrFormat = utils.CSV
cdreCfg.DataUsageMultiplyFactor = 0.0
cdreCfg.CostMultiplyFactor = 0.0
cdreCfg.CostRoundingDecimals = -1
cdreCfg.CostShiftDigits = 0
cdreCfg.MaskDestId = ""
cdreCfg.MaskLength = 0
cdreCfg.ExportDir = "/var/log/cgrates/cdre"
if flds, err := NewCdreCdrFieldsFromIds(utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT,
utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, utils.COST); err != nil {
return err
} else {
cdreCfg.ContentFields = flds
}
return nil
}
type CdreCdrField struct {
Name string
Type string
Value string
Width int
Strip string
Padding string
Layout string
Mandatory bool
valueAsRsrField *utils.RSRField // Cached if the need arrises
}
func (cdrField *CdreCdrField) ValueAsRSRField() *utils.RSRField {
return cdrField.valueAsRsrField
}
// Should be called on .fwv configuration without providing default values for fixed with parameters
func (cdrField *CdreCdrField) setDefaultFixedWidthProperties() error {
if cdrField.valueAsRsrField == nil {
return errors.New("Missing valueAsRsrField")
}
switch cdrField.valueAsRsrField.Id {
case utils.CGRID:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.ORDERID:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.TOR:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.ACCID:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.CDRHOST:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.CDRSOURCE:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.REQTYPE:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.DIRECTION:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.TENANT:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.CATEGORY:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.ACCOUNT:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.SUBJECT:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.DESTINATION:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.SETUP_TIME:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = "2006-01-02T15:04:05Z07:00"
cdrField.Mandatory = true
case utils.ANSWER_TIME:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = "2006-01-02T15:04:05Z07:00"
cdrField.Mandatory = true
case utils.USAGE:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.MEDI_RUNID:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
case utils.COST:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
default:
cdrField.Width = 10
cdrField.Strip = "xright"
cdrField.Padding = ""
cdrField.Layout = ""
cdrField.Mandatory = true
}
return nil
}

240
config/cdreconfig_test.go Normal file
View File

@@ -0,0 +1,240 @@
/*
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package config
import (
"github.com/cgrates/cgrates/utils"
"reflect"
"testing"
)
func TestCdreCfgNewCdreCdrFieldsFromIds(t *testing.T) {
expectedFlds := []*CdreCdrField{
&CdreCdrField{
Name: utils.CGRID,
Type: utils.CDRFIELD,
Value: utils.CGRID,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
},
&CdreCdrField{
Name: "extra1",
Type: utils.CDRFIELD,
Value: "extra1",
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: "extra1"},
},
}
if cdreFlds, err := NewCdreCdrFieldsFromIds(utils.CGRID, "extra1"); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expectedFlds, cdreFlds) {
t.Errorf("Expected: %v, received: %v", expectedFlds, cdreFlds)
}
}
func TestCdreCfgValueAsRSRField(t *testing.T) {
cdreCdrFld := &CdreCdrField{
Name: utils.CGRID,
Type: utils.CDRFIELD,
Value: utils.CGRID,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
}
if rsrVal := cdreCdrFld.ValueAsRSRField(); rsrVal != cdreCdrFld.valueAsRsrField {
t.Error("Unexpected value received: ", rsrVal)
}
}
func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) {
cdreCdrFld := &CdreCdrField{
valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
}
eCdreCdrFld := &CdreCdrField{
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
}
if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
}
}
func TestCdreCfgNewDefaultCdreConfig(t *testing.T) {
eCdreCfg := new(CdreConfig)
eCdreCfg.CdrFormat = utils.CSV
eCdreCfg.DataUsageMultiplyFactor = 0.0
eCdreCfg.CostMultiplyFactor = 0.0
eCdreCfg.CostRoundingDecimals = -1
eCdreCfg.CostShiftDigits = 0
eCdreCfg.MaskDestId = ""
eCdreCfg.MaskLength = 0
eCdreCfg.ExportDir = "/var/log/cgrates/cdre"
eCdreCfg.ContentFields = []*CdreCdrField{
&CdreCdrField{
Name: utils.CGRID,
Type: utils.CDRFIELD,
Value: utils.CGRID,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
},
&CdreCdrField{
Name: utils.MEDI_RUNID,
Type: utils.CDRFIELD,
Value: utils.MEDI_RUNID,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID},
},
&CdreCdrField{
Name: utils.TOR,
Type: utils.CDRFIELD,
Value: utils.TOR,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.TOR},
},
&CdreCdrField{
Name: utils.ACCID,
Type: utils.CDRFIELD,
Value: utils.ACCID,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.ACCID},
},
&CdreCdrField{
Name: utils.REQTYPE,
Type: utils.CDRFIELD,
Value: utils.REQTYPE,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE},
},
&CdreCdrField{
Name: utils.DIRECTION,
Type: utils.CDRFIELD,
Value: utils.DIRECTION,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION},
},
&CdreCdrField{
Name: utils.TENANT,
Type: utils.CDRFIELD,
Value: utils.TENANT,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.TENANT},
},
&CdreCdrField{
Name: utils.CATEGORY,
Type: utils.CDRFIELD,
Value: utils.CATEGORY,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY},
},
&CdreCdrField{
Name: utils.ACCOUNT,
Type: utils.CDRFIELD,
Value: utils.ACCOUNT,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT},
},
&CdreCdrField{
Name: utils.SUBJECT,
Type: utils.CDRFIELD,
Value: utils.SUBJECT,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT},
},
&CdreCdrField{
Name: utils.DESTINATION,
Type: utils.CDRFIELD,
Value: utils.DESTINATION,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION},
},
&CdreCdrField{
Name: utils.SETUP_TIME,
Type: utils.CDRFIELD,
Value: utils.SETUP_TIME,
Width: 10,
Strip: "xright",
Layout: "2006-01-02T15:04:05Z07:00",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME},
},
&CdreCdrField{
Name: utils.ANSWER_TIME,
Type: utils.CDRFIELD,
Value: utils.ANSWER_TIME,
Width: 10,
Strip: "xright",
Layout: "2006-01-02T15:04:05Z07:00",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME},
},
&CdreCdrField{
Name: utils.USAGE,
Type: utils.CDRFIELD,
Value: utils.USAGE,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.USAGE},
},
&CdreCdrField{
Name: utils.COST,
Type: utils.CDRFIELD,
Value: utils.COST,
Width: 10,
Strip: "xright",
Mandatory: true,
valueAsRsrField: &utils.RSRField{Id: utils.COST},
},
}
if cdreCfg, err := NewDefaultCdreConfig(); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eCdreCfg, cdreCfg) {
t.Errorf("Expecting: %v, received: %v", eCdreCfg, cdreCfg)
}
}

73
config/config.go
View File

@@ -90,13 +90,7 @@ type CGRConfig struct {
CDRSEnabled bool // Enable CDR Server service
CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs
CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal>
CdreCdrFormat string // Format of the exported CDRs. <csv>
CdreMaskDestId string // Id of the destination list to be masked in CDRs
CdreMaskLength int // Number of digits to mask in the destination suffix if destination is in the MaskDestinationdsId
CdreCostShiftDigits int // Shift digits in the cost on export (eg: convert from EUR to cents)
CdreDir string // Path towards exported cdrs directory
CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs
CdreFWXmlTemplate *CgrXmlCdreFwCfg // Use this configuration as export template in case of fixed fields length
CdreDefaultInstance *CdreConfig // Will be used in the case no specific one selected by API
CdrcEnabled bool // Enable CDR client functionality
CdrcCdrs string // Address where to reach CDR server
CdrcRunDelay time.Duration // Sleep interval between consecutive runs, 0 to use automation via inotify
@@ -167,11 +161,7 @@ func (self *CGRConfig) setDefaults() error {
self.CDRSEnabled = false
self.CDRSExtraFields = []*utils.RSRField{}
self.CDRSMediator = ""
self.CdreCdrFormat = "csv"
self.CdreMaskDestId = ""
self.CdreMaskLength = 0
self.CdreCostShiftDigits = 0
self.CdreDir = "/var/log/cgrates/cdre"
self.CdreDefaultInstance, _ = NewDefaultCdreConfig()
self.CdrcEnabled = false
self.CdrcCdrs = utils.INTERNAL
self.CdrcRunDelay = time.Duration(0)
@@ -217,35 +207,10 @@ func (self *CGRConfig) setDefaults() error {
self.MailerAuthUser = "cgrates"
self.MailerAuthPass = "CGRateS.org"
self.MailerFromAddr = "cgr-mailer@localhost.localdomain"
self.CdreExportedFields = []*utils.RSRField{
&utils.RSRField{Id: utils.CGRID},
&utils.RSRField{Id: utils.MEDI_RUNID},
&utils.RSRField{Id: utils.TOR},
&utils.RSRField{Id: utils.ACCID},
&utils.RSRField{Id: utils.REQTYPE},
&utils.RSRField{Id: utils.DIRECTION},
&utils.RSRField{Id: utils.TENANT},
&utils.RSRField{Id: utils.CATEGORY},
&utils.RSRField{Id: utils.ACCOUNT},
&utils.RSRField{Id: utils.SUBJECT},
&utils.RSRField{Id: utils.DESTINATION},
&utils.RSRField{Id: utils.SETUP_TIME},
&utils.RSRField{Id: utils.ANSWER_TIME},
&utils.RSRField{Id: utils.USAGE},
&utils.RSRField{Id: utils.COST},
}
return nil
}
func (self *CGRConfig) checkConfigSanity() error {
// Cdre sanity check for fixed_width
if self.CdreCdrFormat == utils.CDRE_FIXED_WIDTH {
if self.XmlCfgDocument == nil {
return errors.New("Need XmlConfigurationDocument for fixed_width cdr export")
} else if self.CdreFWXmlTemplate == nil {
return errors.New("Need XmlTemplate for fixed_width cdr export")
}
}
if self.CdrcEnabled {
if len(self.CdrcCdrFields) == 0 {
return errors.New("CdrC enabled but no fields to be processed defined!")
@@ -253,7 +218,6 @@ func (self *CGRConfig) checkConfigSanity() error {
if self.CdrcCdrType == utils.CSV {
for _, rsrFld := range self.CdrcCdrFields {
if _, errConv := strconv.Atoi(rsrFld.Id); errConv != nil {
fmt.Println("5")
return fmt.Errorf("CDR fields must be indices in case of .csv files, have instead: %s", rsrFld.Id)
}
}
@@ -426,33 +390,42 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) {
cfg.CDRSMediator, _ = c.GetString("cdrs", "mediator")
}
if hasOpt = c.HasOption("cdre", "cdr_format"); hasOpt {
cfg.CdreCdrFormat, _ = c.GetString("cdre", "cdr_format")
cfg.CdreDefaultInstance.CdrFormat, _ = c.GetString("cdre", "cdr_format")
}
if hasOpt = c.HasOption("cdre", "mask_destination_id"); hasOpt {
cfg.CdreMaskDestId, _ = c.GetString("cdre", "mask_destination_id")
cfg.CdreDefaultInstance.MaskDestId, _ = c.GetString("cdre", "mask_destination_id")
}
if hasOpt = c.HasOption("cdre", "mask_length"); hasOpt {
cfg.CdreMaskLength, _ = c.GetInt("cdre", "mask_length")
cfg.CdreDefaultInstance.MaskLength, _ = c.GetInt("cdre", "mask_length")
}
if hasOpt = c.HasOption("cdre", "data_usage_multiply_factor"); hasOpt {
cfg.CdreDefaultInstance.DataUsageMultiplyFactor, _ = c.GetFloat64("cdre", "data_usage_multiply_factor")
}
if hasOpt = c.HasOption("cdre", "cost_multiply_factor"); hasOpt {
cfg.CdreDefaultInstance.CostMultiplyFactor, _ = c.GetFloat64("cdre", "cost_multiply_factor")
}
if hasOpt = c.HasOption("cdre", "cost_rounding_decimals"); hasOpt {
cfg.CdreDefaultInstance.CostRoundingDecimals, _ = c.GetInt("cdre", "cost_rounding_decimals")
}
if hasOpt = c.HasOption("cdre", "cost_shift_digits"); hasOpt {
cfg.CdreCostShiftDigits, _ = c.GetInt("cdre", "cost_shift_digits")
cfg.CdreDefaultInstance.CostShiftDigits, _ = c.GetInt("cdre", "cost_shift_digits")
}
if hasOpt = c.HasOption("cdre", "export_template"); hasOpt { // Load configs for csv normally from template, fixed_width from xml file
exportTemplate, _ := c.GetString("cdre", "export_template")
if cfg.CdreCdrFormat != utils.CDRE_FIXED_WIDTH { // Csv most likely
if extraFields, err := ParseRSRFields(exportTemplate); err != nil {
if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) {
if xmlTemplates := cfg.XmlCfgDocument.GetCdreCfgs(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates != nil {
cfg.CdreDefaultInstance = xmlTemplates[exportTemplate[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig()
}
} else { // Not loading out of template
if flds, err := NewCdreCdrFieldsFromIds(strings.Split(exportTemplate, string(utils.CSV_SEP))...); err != nil {
return nil, err
} else {
cfg.CdreExportedFields = extraFields
}
} else if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) {
if xmlTemplate := cfg.XmlCfgDocument.GetCdreFWCfgs(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); xmlTemplate != nil {
cfg.CdreFWXmlTemplate = xmlTemplate[exportTemplate[len(utils.XML_PROFILE_PREFIX):]]
cfg.CdreDefaultInstance.ContentFields = flds
}
}
}
if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt {
cfg.CdreDir, _ = c.GetString("cdre", "export_dir")
cfg.CdreDefaultInstance.ExportDir, _ = c.GetString("cdre", "export_dir")
}
if hasOpt = c.HasOption("cdrc", "enabled"); hasOpt {
cfg.CdrcEnabled, _ = c.GetBool("cdrc", "enabled")

2
config/config_local_test.go
View File

@@ -39,7 +39,7 @@ func TestLoadXmlCfg(t *testing.T) {
if cfg.XmlCfgDocument == nil {
t.Error("Did not load the XML Config Document")
}
if cdreFWCfg := cfg.XmlCfgDocument.GetCdreFWCfgs("CDREFW-A"); cdreFWCfg == nil {
if cdreFWCfg := cfg.XmlCfgDocument.GetCdreCfgs("CDREFW-A"); cdreFWCfg == nil {
t.Error("Could not retrieve CDRExporter FixedWidth config instance")
}
}

72
config/config_test.go
View File

@@ -1,6 +1,6 @@
/*
Rating system designed to be used in VoIP Carriers World
Copyright (C) 2013 ITsysCOM
Real-time Charging System for Telecom & ISP environments
Copyright (C) 2012-2014 ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -78,14 +78,10 @@ func TestDefaults(t *testing.T) {
eCfg.RaterBalancer = ""
eCfg.BalancerEnabled = false
eCfg.SchedulerEnabled = false
eCfg.CdreDefaultInstance, _ = NewDefaultCdreConfig()
eCfg.CDRSEnabled = false
eCfg.CDRSExtraFields = []*utils.RSRField{}
eCfg.CDRSMediator = ""
eCfg.CdreCdrFormat = "csv"
eCfg.CdreMaskDestId = ""
eCfg.CdreMaskLength = 0
eCfg.CdreCostShiftDigits = 0
eCfg.CdreDir = "/var/log/cgrates/cdre"
eCfg.CdrcEnabled = false
eCfg.CdrcCdrs = utils.INTERNAL
eCfg.CdrcRunDelay = time.Duration(0)
@@ -131,23 +127,6 @@ func TestDefaults(t *testing.T) {
eCfg.MailerAuthUser = "cgrates"
eCfg.MailerAuthPass = "CGRateS.org"
eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain"
eCfg.CdreExportedFields = []*utils.RSRField{
&utils.RSRField{Id: utils.CGRID},
&utils.RSRField{Id: utils.MEDI_RUNID},
&utils.RSRField{Id: utils.TOR},
&utils.RSRField{Id: utils.ACCID},
&utils.RSRField{Id: utils.REQTYPE},
&utils.RSRField{Id: utils.DIRECTION},
&utils.RSRField{Id: utils.TENANT},
&utils.RSRField{Id: utils.CATEGORY},
&utils.RSRField{Id: utils.ACCOUNT},
&utils.RSRField{Id: utils.SUBJECT},
&utils.RSRField{Id: utils.DESTINATION},
&utils.RSRField{Id: utils.SETUP_TIME},
&utils.RSRField{Id: utils.ANSWER_TIME},
&utils.RSRField{Id: utils.USAGE},
&utils.RSRField{Id: utils.COST},
}
if !reflect.DeepEqual(cfg, eCfg) {
t.Log(eCfg)
t.Log(cfg)
@@ -165,10 +144,6 @@ func TestSanityCheck(t *testing.T) {
t.Error("Invalid defaults: ", err)
}
cfg = &CGRConfig{}
cfg.CdreCdrFormat = utils.CDRE_FIXED_WIDTH
if err := cfg.checkConfigSanity(); err == nil {
t.Error("Failed to detect fixed_width dependency on xml configuration")
}
cfg.CdrcEnabled = true
if err := cfg.checkConfigSanity(); err == nil {
t.Error("Failed to detect missing CDR fields definition")
@@ -228,12 +203,16 @@ func TestConfigFromFile(t *testing.T) {
eCfg.CDRSEnabled = true
eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
eCfg.CDRSMediator = "test"
eCfg.CdreCdrFormat = "test"
eCfg.CdreMaskDestId = "test"
eCfg.CdreMaskLength = 99
eCfg.CdreCostShiftDigits = 99
eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
eCfg.CdreDir = "test"
eCfg.CdreDefaultInstance = &CdreConfig{
CdrFormat: "test",
DataUsageMultiplyFactor: 99.0,
CostMultiplyFactor: 99.0,
CostRoundingDecimals: 99,
CostShiftDigits: 99,
MaskDestId: "test",
MaskLength: 99,
ExportDir: "test"}
eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds("test")
eCfg.CdrcEnabled = true
eCfg.CdrcCdrs = "test"
eCfg.CdrcRunDelay = time.Duration(99) * time.Second
@@ -326,17 +305,32 @@ func TestCdreExtraFields(t *testing.T) {
cdr_format = csv
export_template = cgrid,mediation_runid,accid
`)
expectedFlds := []*CdreCdrField{
&CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 10, Strip: "xright", Mandatory: true},
&CdreCdrField{Name: "mediation_runid", Type: utils.CDRFIELD, Value: "mediation_runid", valueAsRsrField: &utils.RSRField{Id: "mediation_runid"},
Width: 10, Strip: "xright", Mandatory: true},
&CdreCdrField{Name: "accid", Type: utils.CDRFIELD, Value: "accid", valueAsRsrField: &utils.RSRField{Id: "accid"}, Width: 10, Strip: "xright", Mandatory: true},
}
expCdreCfg := &CdreConfig{CdrFormat: utils.CSV, CostRoundingDecimals: -1, ExportDir: "/var/log/cgrates/cdre", ContentFields: expectedFlds}
if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
t.Error("Could not parse the config", err.Error())
} else if !reflect.DeepEqual(cfg.CdreExportedFields, []*utils.RSRField{&utils.RSRField{Id: "cgrid"}, &utils.RSRField{Id: "mediation_runid"}, &utils.RSRField{Id: "accid"}}) {
t.Errorf("Unexpected value for CdrsExtraFields: %v", cfg.CDRSExtraFields)
} else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) {
t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance)
}
eFieldsCfg = []byte(`[cdre]
cdr_format = csv
export_template = cgrid,mediation_runid,accid,
export_template = cgrid,~effective_caller_id_number:s/(\d+)/+$1/
`)
if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil {
t.Error("Failed to detect empty field in the end of export_template defition")
rsrField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`)
expectedFlds = []*CdreCdrField{
&CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 10, Strip: "xright", Mandatory: true},
&CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`, valueAsRsrField: rsrField,
Width: 10, Strip: "xright", Mandatory: true}}
expCdreCfg.ContentFields = expectedFlds
if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
t.Error("Could not parse the config", err.Error())
} else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) {
t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance)
}
eFieldsCfg = []byte(`[cdre]
cdr_format = csv

16
config/test_data.txt
View File

@@ -30,7 +30,6 @@ default_tenant = test # Default Tenant to consider when missing from requests.
default_subject = test # Default rating Subject to consider when missing from requests.
rounding_decimals = 99 # Number of decimals to round floats/costs at
[balancer]
enabled = true # Start Balancer service: <true|false>.
@@ -47,12 +46,15 @@ extra_fields = test # Extra fields to scategorye in CDRs
mediator = test # Address where to reach the Mediacategory. Empty for disabling mediation. <""|internal>
[cdre]
cdr_format = test # Exported CDRs format <csv>
mask_destination_id = test # Destination id containing called addresses to be masked on export
mask_length = 99 # Length of the destination suffix to be masked
cost_shift_digits = 99 # Shift the number of cost
export_dir = test # Path where the exported CDRs will be placed
export_template = test # List of fields in the exported CDRs
cdr_format = test # Exported CDRs format <csv>
data_usage_multiply_factor = 99.0 # Multiply data usage before export (eg: convert from KBytes to Bytes)
cost_multiply_factor = 99.0 # Multiply cost before export (0.0 to disable), eg: add VAT
cost_rounding_decimals = 99 # Rounding decimals for Cost values. -1 to disable rounding
cost_shift_digits = 99 # Shift digits in the cost on export (eg: convert from EUR to cents)
mask_destination_id = test # Destination id containing called addresses to be masked on export
mask_length = 99 # Length of the destination suffix to be masked
export_dir = test # Path where the exported CDRs will be placed
export_template = test # List of fields in the exported CDRs
[cdrc]
enabled = true # Enable CDR client functionality

113
config/xmlcdre.go
View File

@@ -20,13 +20,69 @@ package config
import (
"encoding/xml"
"github.com/cgrates/cgrates/utils"
)
// The CdrExporter Fixed Width configuration instance
type CgrXmlCdreFwCfg struct {
Header *CgrXmlCfgCdrHeader `xml:"header"`
Content *CgrXmlCfgCdrContent `xml:"content"`
Trailer *CgrXmlCfgCdrTrailer `xml:"trailer"`
// The CdrExporter configuration instance
type CgrXmlCdreCfg struct {
CdrFormat *string `xml:"cdr_format"`
DataUsageMultiplyFactor *float64 `xml:"data_usage_multiply_factor"`
CostMultiplyFactor *float64 `xml:"cost_multiply_factor"`
CostRoundingDecimals *int `xml:"cost_rounding_decimals"`
CostShiftDigits *int `xml:"cost_shift_digits"`
MaskDestId *string `xml:"mask_destination_id"`
MaskLength *int `xml:"mask_length"`
ExportDir *string `xml:"export_dir"`
Header *CgrXmlCfgCdrHeader `xml:"export_template>header"`
Content *CgrXmlCfgCdrContent `xml:"export_template>content"`
Trailer *CgrXmlCfgCdrTrailer `xml:"export_template>trailer"`
}
func (xmlCdreCfg *CgrXmlCdreCfg) AsCdreConfig() *CdreConfig {
cdreCfg, _ := NewDefaultCdreConfig()
if xmlCdreCfg.CdrFormat != nil {
cdreCfg.CdrFormat = *xmlCdreCfg.CdrFormat
}
if xmlCdreCfg.DataUsageMultiplyFactor != nil {
cdreCfg.DataUsageMultiplyFactor = *xmlCdreCfg.DataUsageMultiplyFactor
}
if xmlCdreCfg.CostMultiplyFactor != nil {
cdreCfg.CostMultiplyFactor = *xmlCdreCfg.CostMultiplyFactor
}
if xmlCdreCfg.CostRoundingDecimals != nil {
cdreCfg.CostRoundingDecimals = *xmlCdreCfg.CostRoundingDecimals
}
if xmlCdreCfg.CostShiftDigits != nil {
cdreCfg.CostShiftDigits = *xmlCdreCfg.CostShiftDigits
}
if xmlCdreCfg.MaskDestId != nil {
cdreCfg.MaskDestId = *xmlCdreCfg.MaskDestId
}
if xmlCdreCfg.MaskLength != nil {
cdreCfg.MaskLength = *xmlCdreCfg.MaskLength
}
if xmlCdreCfg.ExportDir != nil {
cdreCfg.ExportDir = *xmlCdreCfg.ExportDir
}
if xmlCdreCfg.Header != nil {
cdreCfg.HeaderFields = make([]*CdreCdrField, len(xmlCdreCfg.Header.Fields))
for idx, xmlFld := range xmlCdreCfg.Header.Fields {
cdreCfg.HeaderFields[idx] = xmlFld.AsCdreCdrField()
}
}
if xmlCdreCfg.Content != nil {
cdreCfg.ContentFields = make([]*CdreCdrField, len(xmlCdreCfg.Content.Fields))
for idx, xmlFld := range xmlCdreCfg.Content.Fields {
cdreCfg.ContentFields[idx] = xmlFld.AsCdreCdrField()
}
}
if xmlCdreCfg.Trailer != nil {
cdreCfg.TrailerFields = make([]*CdreCdrField, len(xmlCdreCfg.Trailer.Fields))
for idx, xmlFld := range xmlCdreCfg.Trailer.Fields {
cdreCfg.TrailerFields[idx] = xmlFld.AsCdreCdrField()
}
}
return cdreCfg
}
// CDR header
@@ -49,13 +105,42 @@ type CgrXmlCfgCdrTrailer struct {
// CDR field
type CgrXmlCfgCdrField struct {
XMLName xml.Name `xml:"field"`
Name string `xml:"name,attr"`
Type string `xml:"type,attr"`
Value string `xml:"value,attr"`
Width int `xml:"width,attr"` // Field width
Strip string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright>
Padding string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right>
Layout string `xml:"layout,attr"` // Eg. time format layout
Mandatory bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported
XMLName xml.Name `xml:"field"`
Name string `xml:"name,attr"`
Type string `xml:"type,attr"`
Value string `xml:"value,attr"`
Width int `xml:"width,attr"` // Field width
Strip string `xml:"strip,attr"` // Strip strategy in case value is bigger than field width <""|left|xleft|right|xright>
Padding string `xml:"padding,attr"` // Padding strategy in case of value is smaller than width <""left|zeroleft|right>
Layout string `xml:"layout,attr"` // Eg. time format layout
Mandatory bool `xml:"mandatory,attr"` // If field is mandatory, empty value will be considered as error and CDR will not be exported
valueAsRsrField *utils.RSRField // Cached if the need arrises
}
func (cdrFld *CgrXmlCfgCdrField) populateRSRField() (err error) {
if cdrFld.Type != utils.CDRFIELD { // We only need rsrField in case of cdrfield type
return nil
}
if cdrFld.valueAsRsrField, err = utils.NewRSRField(cdrFld.Value); err != nil {
return err
}
return nil
}
func (cdrFld *CgrXmlCfgCdrField) ValueAsRSRField() *utils.RSRField {
return cdrFld.valueAsRsrField
}
func (cdrFld *CgrXmlCfgCdrField) AsCdreCdrField() *CdreCdrField {
return &CdreCdrField{
Name: cdrFld.Name,
Type: cdrFld.Type,
Value: cdrFld.Value,
Width: cdrFld.Width,
Strip: cdrFld.Strip,
Padding: cdrFld.Padding,
Layout: cdrFld.Layout,
Mandatory: cdrFld.Mandatory,
valueAsRsrField: cdrFld.valueAsRsrField,
}
}

250
config/xmlcdre_test.go
View File

@@ -19,65 +19,96 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package config
import (
"github.com/cgrates/cgrates/utils"
"reflect"
"strings"
"testing"
)
var cfgDoc *CgrXmlCfgDocument // Will be populated by first test
func TestParseXmlConfig(t *testing.T) {
cfgXmlStr := `<?xml version="1.0" encoding="UTF-8" ?>
func TestXmlCdreCfgPopulateCdreRSRFIeld(t *testing.T) {
cdreField := CgrXmlCfgCdrField{Name: "TEST1", Type: "cdrfield", Value: `~effective_caller_id_number:s/(\d+)/+$1/`}
if err := cdreField.populateRSRField(); err != nil {
t.Error("Unexpected error: ", err.Error())
} else if cdreField.valueAsRsrField == nil {
t.Error("Failed loading the RSRField")
}
valRSRField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`)
if recv := cdreField.ValueAsRSRField(); !reflect.DeepEqual(valRSRField, recv) {
t.Errorf("Expecting %v, received %v", valRSRField, recv)
}
cdreField = CgrXmlCfgCdrField{Name: "TEST1", Type: "constant", Value: `someval`}
if err := cdreField.populateRSRField(); err != nil {
t.Error("Unexpected error: ", err.Error())
} else if cdreField.valueAsRsrField != nil {
t.Error("Should not load the RSRField")
}
}
func TestXmlCdreCfgParseXmlConfig(t *testing.T) {
cfgXmlStr := `<?xml version="1.0" encoding="UTF-8"?>
<document type="cgrates/xml">
<configuration section="cdre" type="fixed_width" id="CDRE-FW1">
<header>
<fields>
<field name="TypeOfRecord" type="constant" value="10" width="2"/>
<field name="Filler1" type="filler" width="3"/>
<field name="DistributorCode" type="constant" value="VOI" width="3"/>
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5"/>
<field name="LastCdr" type="metatag" value="last_cdr_time" layout="020106150400" width="12"/>
<field name="FileCreationfTime" type="metatag" value="time_now" layout="020106150400" width="12"/>
<field name="Version" type="constant" value="01" width="2"/>
<field name="Filler2" type="filler" width="105"/>
</fields>
</header>
<content>
<fields>
<field name="TypeOfRecord" type="constant" value="20" width="2"/>
<field name="Account" type="cdrfield" value="cgrid" width="12" mandatory="true"/>
<field name="Subject" type="cdrfield" value="subject" strip="left" padding="left" width="5"/>
<field name="CLI" type="cdrfield" value="cli" strip="xright" width="15"/>
<field name="Destination" type="cdrfield" value="destination" strip="xright" width="24"/>
<field name="TOR" type="constant" value="02" width="2"/>
<field name="SubtypeTOR" type="constant" value="11" width="4"/>
<field name="SetupTime" type="cdrfield" value="start_time" layout="020106150400" width="12"/>
<field name="Duration" type="cdrfield" value="duration" width="6"/>
<field name="DataVolume" type="filler" width="6"/>
<field name="TaxCode" type="constant" value="1" width="1"/>
<field name="OperatorCode" type="cdrfield" value="operator" width="2"/>
<field name="ProductId" type="cdrfield" value="productid" width="5"/>
<field name="NetworkId" type="constant" value="3" width="1"/>
<field name="CallId" type="cdrfield" value="accid" width="16"/>
<field name="Filler" type="filler" width="8"/>
<field name="Filler" type="filler" width="8"/>
<field name="TerminationCode" type="cdrfield" value='~cost_details:s/"MatchedDestId":".+_(\s\s\s\s\s)"/$1/' width="5"/>
<field name="Cost" type="cdrfield" value="cost" padding="zeroleft" width="9"/>
<field name="CalledMask" type="cdrfield" value="calledmask" width="1"/>
</fields>
</content>
<trailer>
<fields>
<field name="TypeOfRecord" type="constant" value="90" width="2"/>
<field name="Filler1" type="filler" width="3"/>
<field name="DistributorCode" type="constant" value="VOI" width="3"/>
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5"/>
<field name="NumberOfRecords" type="metatag" value="cdrs_number" padding="zeroleft" width="6"/>
<field name="CdrsDuration" type="metatag" value="cdrs_duration" padding="zeroleft" width="8"/>
<field name="FirstCdrTime" type="metatag" value="first_cdr_time" layout="020106150400" width="12"/>
<field name="LastCdrTime" type="metatag" value="last_cdr_time" layout="020106150400" width="12"/>
<field name="Filler1" type="filler" width="93"/>
</fields>
</trailer>
<cdr_format>fwv</cdr_format>
<data_usage_multiply_factor>0.0</data_usage_multiply_factor>
<cost_multiply_factor>0.0</cost_multiply_factor>
<cost_rounding_decimals>-1</cost_rounding_decimals>
<cost_shift_digits>0</cost_shift_digits>
<mask_destination_id>MASKED_DESTINATIONS</mask_destination_id>
<mask_length>0</mask_length>
<export_dir>/var/log/cgrates/cdre</export_dir>
<export_template>
<header>
<fields>
<field name="TypeOfRecord" type="constant" value="10" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="DistributorCode" type="constant" value="VOI" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="LastCdr" type="metatag" value="last_cdr_time" layout="020106150400" width="12" />
<field name="FileCreationfTime" type="metatag" value="time_now" layout="020106150400" width="12" />
<field name="Version" type="constant" value="01" width="2" />
<field name="Filler2" type="filler" width="105" />
</fields>
</header>
<content>
<fields>
<field name="TypeOfRecord" type="constant" value="20" width="2" />
<field name="Account" type="cdrfield" value="cgrid" width="12" mandatory="true" />
<field name="Subject" type="cdrfield" value="subject" strip="left" padding="left" width="5" />
<field name="CLI" type="cdrfield" value="cli" strip="xright" width="15" />
<field name="Destination" type="cdrfield" value="destination" strip="xright" width="24" />
<field name="TOR" type="constant" value="02" width="2" />
<field name="SubtypeTOR" type="constant" value="11" width="4" />
<field name="SetupTime" type="cdrfield" value="start_time" layout="020106150400" width="12" />
<field name="Duration" type="cdrfield" value="duration" width="6" multiply_factor_voice="1000" />
<field name="DataVolume" type="filler" width="6" />
<field name="TaxCode" type="constant" value="1" width="1" />
<field name="OperatorCode" type="cdrfield" value="operator" width="2" />
<field name="ProductId" type="cdrfield" value="productid" width="5" />
<field name="NetworkId" type="constant" value="3" width="1" />
<field name="CallId" type="cdrfield" value="accid" width="16" />
<field name="Filler" type="filler" width="8" />
<field name="Filler" type="filler" width="8" />
<field name="TerminationCode" type="cdrfield" value="~cost_details:s/&quot;MatchedDestId&quot;:&quot;.+_(\s\s\s\s\s)&quot;/$1/" width="5" />
<field name="Cost" type="cdrfield" value="cost" padding="zeroleft" width="9" />
<field name="CalledMask" type="cdrfield" value="calledmask" width="1" />
</fields>
</content>
<trailer>
<fields>
<field name="TypeOfRecord" type="constant" value="90" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="DistributorCode" type="constant" value="VOI" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="NumberOfRecords" type="metatag" value="cdrs_number" padding="zeroleft" width="6" />
<field name="CdrsDuration" type="metatag" value="cdrs_duration" padding="zeroleft" width="8" />
<field name="FirstCdrTime" type="metatag" value="first_cdr_time" layout="020106150400" width="12" />
<field name="LastCdrTime" type="metatag" value="last_cdr_time" layout="020106150400" width="12" />
<field name="Filler1" type="filler" width="93" />
</fields>
</trailer>
</export_template>
</configuration>
</document>`
var err error
@@ -87,13 +118,13 @@ func TestParseXmlConfig(t *testing.T) {
} else if cfgDoc == nil {
t.Fatal("Could not parse xml configuration document")
}
if len(cfgDoc.cdrefws) != 1 {
if len(cfgDoc.cdres) != 1 {
t.Error("Did not cache")
}
}
func TestGetCdreFWCfg(t *testing.T) {
cdreFWCfg := cfgDoc.GetCdreFWCfgs("CDRE-FW1")
func TestXmlCdreCfgGetCdreCfg(t *testing.T) {
cdreFWCfg := cfgDoc.GetCdreCfgs("CDRE-FW1")
if cdreFWCfg == nil {
t.Error("Could not parse CdreFw instance")
}
@@ -107,3 +138,114 @@ func TestGetCdreFWCfg(t *testing.T) {
t.Error("Unexpected number of trailer fields parsed", len(cdreFWCfg["CDRE-FW1"].Trailer.Fields))
}
}
func TestXmlCdreCfgAsCdreConfig(t *testing.T) {
cfgXmlStr := `<?xml version="1.0" encoding="UTF-8"?>
<document type="cgrates/xml">
<configuration section="cdre" type="fixed_width" id="CDRE-FW2">
<cdr_format>fwv</cdr_format>
<data_usage_multiply_factor>1024.0</data_usage_multiply_factor>
<cost_multiply_factor>1.19</cost_multiply_factor>
<cost_rounding_decimals>-1</cost_rounding_decimals>
<cost_shift_digits>-3</cost_shift_digits>
<mask_destination_id>MASKED_DESTINATIONS</mask_destination_id>
<mask_length>1</mask_length>
<export_dir>/var/log/cgrates/cdre</export_dir>
<export_template>
<header>
<fields>
<field name="TypeOfRecord" type="constant" value="10" width="2" />
<field name="LastCdr" type="metatag" value="last_cdr_time" layout="020106150400" width="12" />
</fields>
</header>
<content>
<fields>
<field name="OperatorCode" type="cdrfield" value="operator" width="2" />
<field name="ProductId" type="cdrfield" value="productid" width="5" />
<field name="NetworkId" type="constant" value="3" width="1" />
</fields>
</content>
<trailer>
<fields>
<field name="DistributorCode" type="constant" value="VOI" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
</fields>
</trailer>
</export_template>
</configuration>
</document>`
var err error
reader := strings.NewReader(cfgXmlStr)
if cfgDoc, err = ParseCgrXmlConfig(reader); err != nil {
t.Error(err.Error())
} else if cfgDoc == nil {
t.Fatal("Could not parse xml configuration document")
}
xmlCdreCfgs := cfgDoc.GetCdreCfgs("CDRE-FW2")
if xmlCdreCfgs == nil {
t.Error("Could not parse XmlCdre instance")
}
eCdreCfg := &CdreConfig{
CdrFormat: "fwv",
DataUsageMultiplyFactor: 1024.0,
CostMultiplyFactor: 1.19,
CostRoundingDecimals: -1,
CostShiftDigits: -3,
MaskDestId: "MASKED_DESTINATIONS",
MaskLength: 1,
ExportDir: "/var/log/cgrates/cdre",
}
eCdreCfg.HeaderFields = []*CdreCdrField{
&CdreCdrField{
Name: "TypeOfRecord",
Type: "constant",
Value: "10",
Width: 2},
&CdreCdrField{
Name: "LastCdr",
Type: "metatag",
Value: "last_cdr_time",
Layout: "020106150400",
Width: 12},
}
eCdreCfg.ContentFields = []*CdreCdrField{
&CdreCdrField{
Name: "OperatorCode",
Type: "cdrfield",
Value: "operator",
Width: 2,
valueAsRsrField: &utils.RSRField{Id: "operator"},
},
&CdreCdrField{
Name: "ProductId",
Type: "cdrfield",
Value: "productid",
Width: 5,
valueAsRsrField: &utils.RSRField{Id: "productid"},
},
&CdreCdrField{
Name: "NetworkId",
Type: "constant",
Value: "3",
Width: 1,
},
}
eCdreCfg.TrailerFields = []*CdreCdrField{
&CdreCdrField{
Name: "DistributorCode",
Type: "constant",
Value: "VOI",
Width: 3,
},
&CdreCdrField{
Name: "FileSeqNr",
Type: "metatag",
Value: "export_id",
Width: 5,
Padding: "zeroleft",
},
}
if rcvCdreCfg := xmlCdreCfgs["CDRE-FW2"].AsCdreConfig(); !reflect.DeepEqual(rcvCdreCfg, eCdreCfg) {
t.Errorf("Expecting: %v, received: %v", eCdreCfg, rcvCdreCfg)
}
}

64
config/xmlconfig.go
View File

@@ -40,11 +40,11 @@ func ParseCgrXmlConfig(reader io.Reader) (*CgrXmlCfgDocument, error) {
// Define a format for configuration file, one doc contains more configuration instances, identified by section, type and id
type CgrXmlCfgDocument struct {
XMLName xml.Name `xml:"document"`
Type string `xml:"type,attr"`
Configurations []*CgrXmlConfiguration `xml:"configuration"`
cdrefws map[string]*CgrXmlCdreFwCfg // Cache for processed fixed width config instances, key will be the id of the instance
XMLName xml.Name `xml:"document"`
Type string `xml:"type,attr"`
Configurations []*CgrXmlConfiguration `xml:"configuration"`
cdrcs map[string]*CgrXmlCdrcCfg
cdres map[string]*CgrXmlCdreCfg // Cahe cdrexporter instances, key will be the ID
}
// Storage for raw configuration
@@ -63,7 +63,7 @@ func (cfgInst *CgrXmlConfiguration) rawConfigElement() []byte {
}
func (xmlCfg *CgrXmlCfgDocument) cacheAll() error {
for _, cacheFunc := range []func() error{xmlCfg.cacheCdreFWCfgs, xmlCfg.cacheCdrcCfgs} {
for _, cacheFunc := range []func() error{xmlCfg.cacheCdrcCfgs, xmlCfg.cacheCdreCfgs} {
if err := cacheFunc(); err != nil {
return err
}
@@ -71,26 +71,6 @@ func (xmlCfg *CgrXmlCfgDocument) cacheAll() error {
return nil
}
// Avoid building from raw config string always, so build cache here
func (xmlCfg *CgrXmlCfgDocument) cacheCdreFWCfgs() error {
xmlCfg.cdrefws = make(map[string]*CgrXmlCdreFwCfg)
for _, cfgInst := range xmlCfg.Configurations {
if cfgInst.Section == utils.CDRE || cfgInst.Type == utils.CDRE_FIXED_WIDTH {
cdrefwCfg := new(CgrXmlCdreFwCfg)
rawConfig := append([]byte("<element>"), cfgInst.RawConfig...) // Encapsulate the rawConfig in one element so we can Unmarshall into one struct
rawConfig = append(rawConfig, []byte("</element>")...)
if err := xml.Unmarshal(rawConfig, cdrefwCfg); err != nil {
return err
} else if cdrefwCfg == nil {
return fmt.Errorf("Could not unmarshal CgrXmlCdreFwCfg: %s", cfgInst.Id)
} else { // All good, cache the config instance
xmlCfg.cdrefws[cfgInst.Id] = cdrefwCfg
}
}
}
return nil
}
// Avoid building from raw config string always, so build cache here
func (xmlCfg *CgrXmlCfgDocument) cacheCdrcCfgs() error {
xmlCfg.cdrcs = make(map[string]*CgrXmlCdrcCfg)
@@ -116,16 +96,42 @@ func (xmlCfg *CgrXmlCfgDocument) cacheCdrcCfgs() error {
return nil
}
// Avoid building from raw config string always, so build cache here
func (xmlCfg *CgrXmlCfgDocument) cacheCdreCfgs() error {
xmlCfg.cdres = make(map[string]*CgrXmlCdreCfg)
for _, cfgInst := range xmlCfg.Configurations {
if cfgInst.Section != utils.CDRE {
continue
}
cdreCfg := new(CgrXmlCdreCfg)
if err := xml.Unmarshal(cfgInst.rawConfigElement(), cdreCfg); err != nil {
return err
} else if cdreCfg == nil {
return fmt.Errorf("Could not unmarshal CgrXmlCdreCfg: %s", cfgInst.Id)
}
if cdreCfg.Content != nil {
// Cache rsr fields
for _, fld := range cdreCfg.Content.Fields {
if err := fld.populateRSRField(); err != nil {
return fmt.Errorf("Populating field %s, error: %s", fld.Name, err.Error())
}
}
}
xmlCfg.cdres[cfgInst.Id] = cdreCfg
}
return nil
}
// Return instances or filtered instance of cdrefw configuration
func (xmlCfg *CgrXmlCfgDocument) GetCdreFWCfgs(instName string) map[string]*CgrXmlCdreFwCfg {
func (xmlCfg *CgrXmlCfgDocument) GetCdreCfgs(instName string) map[string]*CgrXmlCdreCfg {
if len(instName) != 0 {
if cfg, hasIt := xmlCfg.cdrefws[instName]; !hasIt {
if cfg, hasIt := xmlCfg.cdres[instName]; !hasIt {
return nil
} else {
return map[string]*CgrXmlCdreFwCfg{instName: cfg}
return map[string]*CgrXmlCdreCfg{instName: cfg}
}
}
return xmlCfg.cdrefws
return xmlCfg.cdres
}
// Return instances or filtered instance of cdrc configuration

10
data/conf/cgrates.cfg
View File

@@ -28,11 +28,10 @@
# rpc_gob_listen = 127.0.0.1:2013 # RPC GOB listening address
# http_listen = 127.0.0.1:2080 # HTTP listening address
# default_reqtype = rated # Default request type to consider when missing from requests: <""|prepaid|postpaid|pseudoprepaid|rated>.
# default_category = call # Default Type of Record to consider when missing from requests.
# default_category = call # Default Type of Record to consider when missing from requests.
# default_tenant = cgrates.org # Default Tenant to consider when missing from requests.
# default_subject = cgrates # Default rating Subject to consider when missing from requests.
# rounding_method = *middle # Rounding method for floats/costs: <*up|*middle|*down>
# rounding_decimals = 10 # Number of decimals to round float/costs at
# rounding_decimals = 10 # System level precision for floats
# xmlcfg_path = # Path towards additional config defined in xml file
[balancer]
@@ -52,9 +51,12 @@
[cdre]
# cdr_format = csv # Exported CDRs format <csv>
# data_usage_multiply_factor = 0.0 # Multiply data usage before export (eg: convert from KBytes to Bytes)
# cost_multiply_factor = 0.0 # Multiply cost before export (0.0 to disable), eg: add VAT
# cost_rounding_decimals = -1 # Rounding decimals for Cost values. -1 to disable rounding
# cost_shift_digits = 0 # Shift digits in the cost on export (eg: convert from EUR to cents)
# mask_destination_id = # Destination id containing called addresses to be masked on export
# mask_length = 0 # Length of the destination suffix to be masked
# cost_shift_digits = 0 # Shift cost on export with the number of digits digits defined here (eg: convert from Eur to cent).
# export_dir = /var/log/cgrates/cdre # Path where the exported CDRs will be placed
# export_template = cgrid,mediation_runid,tor,accid,reqtype,direction,tenant,category,account,subject,destination,setup_time,answer_time,usage,cost
# Exported fields template <""|fld1,fld2|*xml:instance_name>

41
data/conf/samples/cgr_addconfig.xml
View File

@@ -1,20 +1,29 @@
<?xml version="1.0" encoding="UTF-8" ?>
<?xml version="1.0" encoding="UTF-8"?>
<document type="cgrates/xml">
<configuration section="cdre" type="fixed_width" id="CDREFW-A">
<header>
<fields>
<field name="Filler1" type="filler" width="4"/>
</fields>
</header>
<content>
<fields>
<field name="TypeOfRecord" type="constant" value="call"/>
</fields>
</content>
<trailer>
<fields>
<field name="Filler1" type="filler" width="3"/>
</fields>
</trailer>
<cdr_format>fwv</cdr_format>
<data_usage_multiply_factor>0.0</data_usage_multiply_factor>
<cost_multiply_factor>0.0</cost_multiply_factor>
<cost_shift_digits>0</cost_shift_digits>
<mask_destination_id>MASKED_DESTINATIONS</mask_destination_id>
<mask_length>0</mask_length>
<export_dir>/var/log/cgrates/cdre</export_dir>
<export_template>
<header>
<fields>
<field name="Filler1" type="filler" width="4" />
</fields>
</header>
<content>
<fields>
<field name="TypeOfRecord" type="constant" value="call" />
</fields>
</content>
<trailer>
<fields>
<field name="Filler1" type="filler" width="3" />
</fields>
</trailer>
</export_template>
</configuration>
</document>

107
data/conf/samples/multiplecdrc_fwexport.xml
View File

@@ -49,54 +49,63 @@
</fields>
</configuration>
<configuration section="cdre" type="fwv" id="CDRE-FW1">
<header>
<fields>
<field name="ToR" type="constant" value="10" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="FileType" type="constant" value="SIP" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="LastCdr" type="metatag" value="last_cdr_atime" layout="020106150405" width="12" />
<field name="FileCreationfTime" type="metatag" value="time_now" layout="020106150405" width="12" />
<field name="FileVersion" type="constant" value="01" width="2" />
<field name="Filler2" type="filler" width="105" />
</fields>
</header>
<content>
<fields>
<field name="ToR" type="constant" value="20" width="2" />
<field name="Subject" type="cdrfield" value="subject" width="12" padding="right" mandatory="true" />
<field name="ConnectionNumber" type="constant" value="00000" width="5" />
<field name="CallerId" type="cdrfield" value="~callerid:s/\+(\d+)/00$1/" strip="xright" width="15" padding="right" />
<field name="Destination" type="cdrfield" value="~destination:s/^\+311400(\d+)/$1/:s/^\+311412\d\d112/112/:s/^\+31(\d+)/0$1/:s/^\+(\d+)/00$1/" strip="xright" width="24" padding="right" mandatory="true" />
<field name="TypeOfService" type="constant" value="00" width="2" />
<field name="ServiceId" type="constant" value="11" width="4" padding="right" />
<field name="AnswerTime" type="cdrfield" value="answer_time" layout="020106150405" width="12" mandatory="true" />
<field name="Usage" type="cdrfield" value="usage" layout="seconds" width="6" padding="right" mandatory="true" />
<field name="DataCounter" type="filler" width="6" />
<field name="VatCode" type="constant" value="1" width="1" />
<field name="NetworkId" type="constant" value="S1" width="2" />
<field name="DestinationSubId" type="cdrfield" value="~cost_details:s/&quot;MatchedDestId&quot;:&quot;.+_(\w{5})&quot;/$1/:s/(\w{6})/$1/" width="5" />
<field name="NetworkSubtype" type="constant" value="3" width="1" padding="left" />
<field name="CgrId" type="cdrfield" value="cgrid" strip="xleft" width="16" paddingi="right" mandatory="true" />
<field name="FillerVolume1" type="filler" width="8" />
<field name="FillerVolume2" type="filler" width="8" />
<field name="DestinationSubId" type="cdrfield" value="~cost_details:s/&quot;MatchedDestId&quot;:&quot;.+_(\w{5})&quot;/$1/:s/(\w{6})/$1/" width="5" />
<field name="Cost" type="cdrfield" value="cost" padding="zeroleft" width="9" />
<field name="MaskDestination" type="metatag" value="mask_destination" width="1" />
</fields>
</content>
<trailer>
<fields>
<field name="ToR" type="constant" value="90" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="FileType" type="constant" value="SIP" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="TotalRecords" type="metatag" value="cdrs_number" padding="zeroleft" width="6" />
<field name="TotalDuration" type="metatag" value="cdrs_duration" padding="zeroleft" width="8" />
<field name="FirstCdrTime" type="metatag" value="first_cdr_atime" layout="020106150405" width="12" />
<field name="LastCdrTime" type="metatag" value="last_cdr_atime" layout="020106150405" width="12" />
<field name="Filler1" type="filler" width="93" />
</fields>
</trailer>
<cdr_format>fwv</cdr_format>
<data_usage_multiply_factor>0.0</data_usage_multiply_factor>
<cost_multiply_factor>0.0</cost_multiply_factor>
<cost_shift_digits>0</cost_shift_digits>
<mask_destination_id>MASKED_DESTINATIONS</mask_destination_id>
<mask_length>0</mask_length>
<export_dir>/var/log/cgrates/cdre</export_dir>
<export_template>
<header>
<fields>
<field name="ToR" type="constant" value="10" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="FileType" type="constant" value="SIP" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="LastCdr" type="metatag" value="last_cdr_atime" layout="020106150405" width="12" />
<field name="FileCreationfTime" type="metatag" value="time_now" layout="020106150405" width="12" />
<field name="FileVersion" type="constant" value="01" width="2" />
<field name="Filler2" type="filler" width="105" />
</fields>
</header>
<content>
<fields>
<field name="ToR" type="constant" value="20" width="2" />
<field name="Subject" type="cdrfield" value="subject" width="12" padding="right" mandatory="true" />
<field name="ConnectionNumber" type="constant" value="00000" width="5" />
<field name="CallerId" type="cdrfield" value="~callerid:s/\+(\d+)/00$1/" strip="xright" width="15" padding="right" />
<field name="Destination" type="cdrfield" value="~destination:s/^\+311400(\d+)/$1/:s/^\+311412\d\d112/112/:s/^\+31(\d+)/0$1/:s/^\+(\d+)/00$1/" strip="xright" width="24" padding="right" mandatory="true" />
<field name="TypeOfService" type="constant" value="00" width="2" />
<field name="ServiceId" type="constant" value="11" width="4" padding="right" />
<field name="AnswerTime" type="cdrfield" value="answer_time" layout="020106150405" width="12" mandatory="true" />
<field name="Usage" type="cdrfield" value="usage" layout="seconds" width="6" padding="right" mandatory="true" />
<field name="DataCounter" type="filler" width="6" />
<field name="VatCode" type="constant" value="1" width="1" />
<field name="NetworkId" type="constant" value="S1" width="2" />
<field name="DestinationSubId" type="cdrfield" value="~cost_details:s/&quot;MatchedDestId&quot;:&quot;.+_(\w{5})&quot;/$1/:s/(\w{6})/$1/" width="5" />
<field name="NetworkSubtype" type="constant" value="3" width="1" padding="left" />
<field name="CgrId" type="cdrfield" value="cgrid" strip="xleft" width="16" paddingi="right" mandatory="true" />
<field name="FillerVolume1" type="filler" width="8" />
<field name="FillerVolume2" type="filler" width="8" />
<field name="DestinationSubId" type="cdrfield" value="~cost_details:s/&quot;MatchedDestId&quot;:&quot;.+_(\w{5})&quot;/$1/:s/(\w{6})/$1/" width="5" />
<field name="Cost" type="cdrfield" value="cost" padding="zeroleft" width="9" />
<field name="MaskDestination" type="metatag" value="mask_destination" width="1" />
</fields>
</content>
<trailer>
<fields>
<field name="ToR" type="constant" value="90" width="2" />
<field name="Filler1" type="filler" width="3" />
<field name="FileType" type="constant" value="SIP" width="3" />
<field name="FileSeqNr" type="metatag" value="export_id" padding="zeroleft" width="5" />
<field name="TotalRecords" type="metatag" value="cdrs_number" padding="zeroleft" width="6" />
<field name="TotalDuration" type="metatag" value="cdrs_duration" padding="zeroleft" width="8" />
<field name="FirstCdrTime" type="metatag" value="first_cdr_atime" layout="020106150405" width="12" />
<field name="LastCdrTime" type="metatag" value="last_cdr_atime" layout="020106150405" width="12" />
<field name="Filler1" type="filler" width="93" />
</fields>
</trailer>
</export_template>
</configuration>
</document>

56
utils/apitpdata.go
View File

@@ -327,33 +327,35 @@ type CachedItemAge struct {
}
type AttrExpFileCdrs struct {
CdrFormat string // Cdr output file format <utils.CdreCdrFormats>
ExportId string // Optional exportid
ExportDir string // If provided it overwrites the configured export directory
ExportFileName string // If provided the output filename will be set to this
ExportTemplate string // Exported fields template <""|fld1,fld2|*xml:instance_name>
CostShiftDigits int // If defined it will shift cost digits before applying rouding (eg: convert from Eur->cents), -1 to use general config ones
RoundDecimals int // Overwrite configured roundDecimals with this dynamically, -1 to use general config ones
MaskDestinationId string // Overwrite configured MaskDestId
MaskLength int // Overwrite configured MaskLength, -1 to use general config ones
CgrIds []string // If provided, it will filter based on the cgrids present in list
MediationRunId []string // If provided, it will filter on mediation runid
TOR []string // If provided, filter on TypeOfRecord
CdrHost []string // If provided, it will filter cdrhost
CdrSource []string // If provided, it will filter cdrsource
ReqType []string // If provided, it will fiter reqtype
Direction []string // If provided, it will fiter direction
Tenant []string // If provided, it will filter tenant
Category []string // If provided, it will filter çategory
Account []string // If provided, it will filter account
Subject []string // If provided, it will filter the rating subject
DestinationPrefix []string // If provided, it will filter on destination prefix
OrderIdStart int64 // Export from this order identifier
OrderIdEnd int64 // Export smaller than this order identifier
TimeStart string // If provided, it will represent the starting of the CDRs interval (>=)
TimeEnd string // If provided, it will represent the end of the CDRs interval (<)
SkipErrors bool // Do not export errored CDRs
SkipRated bool // Do not export rated CDRs
CdrFormat *string // Cdr output file format <utils.CdreCdrFormats>
ExportId *string // Optional exportid
ExportDir *string // If provided it overwrites the configured export directory
ExportFileName *string // If provided the output filename will be set to this
ExportTemplate *string // Exported fields template <""|fld1,fld2|*xml:instance_name>
DataUsageMultiplyFactor *float64 // Multiply data usage before export (eg: convert from KBytes to Bytes)
CostMultiplyFactor *float64 // Multiply the cost before export, eg: apply VAT
CostShiftDigits *int // If defined it will shift cost digits before applying rouding (eg: convert from Eur->cents), -1 to use general config ones
RoundDecimals *int // Overwrite configured roundDecimals with this dynamically, -1 to use general config ones
MaskDestinationId *string // Overwrite configured MaskDestId
MaskLength *int // Overwrite configured MaskLength, -1 to use general config ones
CgrIds []string // If provided, it will filter based on the cgrids present in list
MediationRunId []string // If provided, it will filter on mediation runid
TOR []string // If provided, filter on TypeOfRecord
CdrHost []string // If provided, it will filter cdrhost
CdrSource []string // If provided, it will filter cdrsource
ReqType []string // If provided, it will fiter reqtype
Direction []string // If provided, it will fiter direction
Tenant []string // If provided, it will filter tenant
Category []string // If provided, it will filter çategory
Account []string // If provided, it will filter account
Subject []string // If provided, it will filter the rating subject
DestinationPrefix []string // If provided, it will filter on destination prefix
OrderIdStart int64 // Export from this order identifier
OrderIdEnd int64 // Export smaller than this order identifier
TimeStart string // If provided, it will represent the starting of the CDRs interval (>=)
TimeEnd string // If provided, it will represent the end of the CDRs interval (<)
SkipErrors bool // Do not export errored CDRs
SkipRated bool // Do not export rated CDRs
}
type ExportedFileCdrs struct {

1
utils/consts.go
View File

@@ -119,6 +119,7 @@ const (
OUT = "*out"
CDR_IMPORT = "cdr_import"
CDR_EXPORT = "cdr_export"
CDRFIELD = "cdrfield"
)
var (

28
utils/storedcdr.go
View File

@@ -50,19 +50,17 @@ type StoredCdr struct {
Cost float64
}
// Should only be used for display purposes, bad otherwise.
// cdrDirection: CDR_IMPORT or CDR_EXPORT
func (storedCdr *StoredCdr) MangleDataUsage(cdrDirection string) {
if IsSliceMember([]string{DATA, SMS}, storedCdr.TOR) {
if cdrDirection == CDR_IMPORT { // On import CDRs usages are converted to nanoseconds, for data we need seconds, fix it here.
storedCdr.Usage = time.Duration(storedCdr.Usage.Nanoseconds()) * time.Second
} else if cdrDirection == CDR_EXPORT { // On exports we need to show the data back in seconds instead of internally stored as nanoseconds
storedCdr.Usage = time.Duration(int(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE)))
}
}
// Used to multiply usage on export
func (storedCdr *StoredCdr) UsageMultiply(multiplyFactor float64, roundDecimals int) {
storedCdr.Usage = time.Duration(int(Round(float64(storedCdr.Usage.Nanoseconds())*multiplyFactor, roundDecimals, ROUNDING_MIDDLE))) // Rounding down could introduce a slight loss here but only at nanoseconds level
}
// Return cost as string, formated with number of decimals configured
// Used to multiply cost on export
func (storedCdr *StoredCdr) CostMultiply(multiplyFactor float64, roundDecimals int) {
storedCdr.Cost = Round(storedCdr.Cost*multiplyFactor, roundDecimals, ROUNDING_MIDDLE)
}
// Format cost as string on export
func (storedCdr *StoredCdr) FormatCost(shiftDecimals, roundDecimals int) string {
cost := storedCdr.Cost
if shiftDecimals != 0 {
@@ -71,18 +69,18 @@ func (storedCdr *StoredCdr) FormatCost(shiftDecimals, roundDecimals int) string
return strconv.FormatFloat(cost, 'f', roundDecimals, 64)
}
// Rounds up so 0.8 seconds will become 1
// Formats usage on export
func (storedCdr *StoredCdr) FormatUsage(layout string) string {
if IsSliceMember([]string{DATA, SMS}, storedCdr.TOR) {
return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64)
}
switch layout {
case HOURS:
return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Hours()), 'f', -1, 64)
return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64)
case MINUTES:
return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Minutes()), 'f', -1, 64)
return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64)
case SECONDS:
return strconv.FormatFloat(math.Ceil(storedCdr.Usage.Seconds()), 'f', -1, 64)
return strconv.FormatFloat(Round(storedCdr.Usage.Seconds(), 0, ROUNDING_MIDDLE), 'f', -1, 64)
default:
return strconv.FormatInt(storedCdr.Usage.Nanoseconds(), 10)
}

34
utils/storedcdr_test.go
View File

@@ -76,28 +76,28 @@ func TestFieldAsString(t *testing.T) {
cdr.FieldAsString(&RSRField{Id: "fieldextr2"}) != cdr.ExtraFields["fieldextr2"],
cdr.FieldAsString(&RSRField{Id: "dummy_field"}) != "")
}
/*cdr.TOR = DATA
if formated := cdr.FieldAsString(&RSRField{Id: USAGE}); formated != "10" {
t.Error("Wrong exported value for data field: ", formated)
}*/
}
func TestMangleDataUsage(t *testing.T) {
cdr := StoredCdr{TOR: DATA, Usage: time.Duration(1640113)}
if cdr.MangleDataUsage(CDR_IMPORT); cdr.Usage != time.Duration(1640113000000000) {
t.Error("Unexpected usage after mangling: ", cdr.Usage)
func TestUsageMultiply(t *testing.T) {
cdr := StoredCdr{Usage: time.Duration(10) * time.Second}
if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second {
t.Errorf("Unexpected usage after multiply: %v", cdr.Usage.Nanoseconds())
}
cdr = StoredCdr{TOR: VOICE, Usage: time.Duration(1640113000000000)}
if cdr.MangleDataUsage(CDR_IMPORT); cdr.Usage != time.Duration(1640113000000000) {
t.Error("Unexpected usage after mangling: ", cdr.Usage)
cdr = StoredCdr{Usage: time.Duration(10240) * time.Second} // Simulate conversion back, gives out a bit odd result but this can be rounded on export
expectDuration, _ := time.ParseDuration("10.000005120s")
if cdr.UsageMultiply(0.000976563, 0); cdr.Usage != expectDuration {
t.Errorf("Unexpected usage after multiply: %v", cdr.Usage.Nanoseconds())
}
cdr = StoredCdr{TOR: DATA, Usage: time.Duration(1640113000000000)}
if cdr.MangleDataUsage(CDR_EXPORT); cdr.Usage != time.Duration(1640113) {
t.Error("Unexpected usage after mangling: ", cdr.Usage)
}
func TestCostMultiply(t *testing.T) {
cdr := StoredCdr{Cost: 1.01}
if cdr.CostMultiply(1.19, 4); cdr.Cost != 1.2019 {
t.Error("Unexpected cost after multiply: %v", cdr.Cost)
}
cdr = StoredCdr{TOR: VOICE, Usage: time.Duration(1640113000000000)}
if cdr.MangleDataUsage(CDR_EXPORT); cdr.Usage != time.Duration(1640113000000000) {
t.Error("Unexpected usage after mangling: ", cdr.Usage)
cdr = StoredCdr{Cost: 1.01}
if cdr.CostMultiply(1000, 0); cdr.Cost != 1010 {
t.Error("Unexpected cost after multiply: %v", cdr.Cost)
}
}

2
utils/utils_local_test.go
View File

@@ -32,7 +32,7 @@ func TestHttpJsonPost(t *testing.T) {
return
}
cdrOut := &CgrCdrOut{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
CdrSource: UNIT_TEST, ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
CdrSource: UNIT_TEST, ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "account1", Subject: "tgooiscs0014", Destination: "1002",
SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID,
Usage: 0.00000001, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}