uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 10:06:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Adding full CDR template in exported CDRs, using RSRFields

Browse Source
This commit is contained in:
DanB
2014-03-16 13:11:08 +01:00
parent db433a760f
commit f6d16cecc5
13 changed files with 166 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apier/cdrs.go
View File

@@ -58,7 +58,7 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E
} else {
defer fileOut.Close()
}
csvWriter := cdrexporter.NewCsvCdrWriter(fileOut, self.Config.RoundingDecimals, self.Config.CdreExtraFields)
csvWriter := cdrexporter.NewCsvCdrWriter(fileOut, self.Config.RoundingDecimals, self.Config.CdreExportedFields)
for _, cdr := range cdrs {
if err := csvWriter.Write(cdr); err != nil {
os.Remove(fileName)

42
cdrexporter/csv.go
View File

@@ -22,38 +22,32 @@ import (
"encoding/csv"
"github.com/cgrates/cgrates/utils"
"io"
"sort"
"strconv"
)
type CsvCdrWriter struct {
writer *csv.Writer
roundDecimals int // Round floats like Cost using this number of decimals
extraFields []string // Extra fields to append after primary ones, order important
writer *csv.Writer
roundDecimals int // Round floats like Cost using this number of decimals
exportedFields []*utils.RSRField // The fields exported, order important
}
func NewCsvCdrWriter(writer io.Writer, roundDecimals int, extraFields []string) *CsvCdrWriter {
return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, extraFields}
func NewCsvCdrWriter(writer io.Writer, roundDecimals int, exportedFields []*utils.RSRField) *CsvCdrWriter {
return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, exportedFields}
}
func (dcw *CsvCdrWriter) Write(cdr *utils.StoredCdr) error {
primaryFields := []string{cdr.CgrId, cdr.MediationRunId, cdr.AccId, cdr.CdrHost, cdr.ReqType, cdr.Direction, cdr.Tenant, cdr.TOR, cdr.Account, cdr.Subject,
cdr.Destination, cdr.SetupTime.String(), cdr.AnswerTime.String(), strconv.Itoa(int(cdr.Duration)), strconv.FormatFloat(cdr.Cost, 'f', dcw.roundDecimals, 64)}
if len(dcw.extraFields) == 0 {
dcw.extraFields = utils.MapKeys(cdr.ExtraFields)
sort.Strings(dcw.extraFields) // Controlled order in case of dynamic extra fields
func (csvwr *CsvCdrWriter) Write(cdr *utils.StoredCdr) error {
row := make([]string, len(csvwr.exportedFields))
for idx, fld := range csvwr.exportedFields { // Add primary fields
var fldVal string
if fld.Id == utils.COST {
fldVal = cdr.FormatCost(csvwr.roundDecimals)
} else {
fldVal = cdr.ExportFieldValue(fld.Id)
}
row[idx] = fld.ParseValue(fldVal)
}
lenPrimary := len(primaryFields)
row := make([]string, lenPrimary+len(dcw.extraFields))
for idx, fld := range primaryFields { // Add primary fields
row[idx] = fld
}
for idx, fldKey := range dcw.extraFields { // Add extra fields
row[lenPrimary+idx] = cdr.ExtraFields[fldKey]
}
return dcw.writer.Write(row)
return csvwr.writer.Write(row)
}
func (dcw *CsvCdrWriter) Close() {
dcw.writer.Flush()
func (csvwr *CsvCdrWriter) Close() {
csvwr.writer.Flush()
}

11
cdrexporter/csv_test.go
View File

@@ -20,6 +20,7 @@ package cdrexporter
import (
"bytes"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
"strings"
"testing"
@@ -28,17 +29,19 @@ import (
func TestCsvCdrWriter(t *testing.T) {
writer := &bytes.Buffer{}
csvCdrWriter := NewCsvCdrWriter(writer, 4, []string{"extra3", "extra1"})
cfg, _ := config.NewDefaultCGRConfig()
exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"})
csvCdrWriter := NewCsvCdrWriter(writer, 4, exportedFields)
ratedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
Duration: 10, MediationRunId: utils.DEFAULT_RUNID,
Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
}
csvCdrWriter.Write(ratedCdr)
csvCdrWriter.Close()
expected := "b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,val_extra1"
expected := `b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,"",val_extra1`
result := strings.TrimSpace(writer.String())
if result != expected {
t.Errorf("Expected %s received %s.", expected, result)
t.Errorf("Expected: \n%s received: \n%s.", expected, result)
}
}

4
cdrs/fscdr.go
View File

@@ -120,9 +120,7 @@ func (fsCdr FSCdr) GetExtraFields() map[string]string {
if !foundInVars {
origFieldVal = fsCdr.searchExtraField(field.Id, fsCdr.body)
}
if len(origFieldVal) != 0 { // Found a value, parse it
extraFields[field.Id] = field.ParseValue(origFieldVal)
}
extraFields[field.Id] = field.ParseValue(origFieldVal)
}
return extraFields
}

27
config/config.go
View File

@@ -88,7 +88,7 @@ type CGRConfig struct {
CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs
CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal>
CdreCdrFormat string // Format of the exported CDRs. <csv>
CdreExtraFields []string // Extra fields list to add in exported CDRs
CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs
CdreDir string // Path towards exported cdrs directory
CdrcEnabled bool // Enable CDR client functionality
CdrcCdrs string // Address where to reach CDR server
@@ -192,7 +192,6 @@ func (self *CGRConfig) setDefaults() error {
self.CDRSExtraFields = []*utils.RSRField{}
self.CDRSMediator = ""
self.CdreCdrFormat = "csv"
self.CdreExtraFields = []string{}
self.CdreDir = "/var/log/cgrates/cdr/cdrexport/csv"
self.CdrcEnabled = false
self.CdrcCdrs = utils.INTERNAL
@@ -257,6 +256,23 @@ func (self *CGRConfig) setDefaults() error {
self.MailerAuthUser = "cgrates"
self.MailerAuthPass = "CGRateS.org"
self.MailerFromAddr = "cgr-mailer@localhost.localdomain"
self.CdreExportedFields = []*utils.RSRField{
&utils.RSRField{Id: utils.CGRID},
&utils.RSRField{Id: utils.MEDI_RUNID},
&utils.RSRField{Id: utils.ACCID},
&utils.RSRField{Id: utils.CDRHOST},
&utils.RSRField{Id: utils.REQTYPE},
&utils.RSRField{Id: utils.DIRECTION},
&utils.RSRField{Id: utils.TENANT},
&utils.RSRField{Id: utils.TOR},
&utils.RSRField{Id: utils.ACCOUNT},
&utils.RSRField{Id: utils.SUBJECT},
&utils.RSRField{Id: utils.DESTINATION},
&utils.RSRField{Id: utils.SETUP_TIME},
&utils.RSRField{Id: utils.ANSWER_TIME},
&utils.RSRField{Id: utils.DURATION},
&utils.RSRField{Id: utils.COST},
}
return nil
}
@@ -448,9 +464,12 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) {
if hasOpt = c.HasOption("cdre", "cdr_format"); hasOpt {
cfg.CdreCdrFormat, _ = c.GetString("cdre", "cdr_format")
}
if hasOpt = c.HasOption("cdre", "extra_fields"); hasOpt {
if cfg.CdreExtraFields, errParse = ConfigSlice(c, "cdre", "extra_fields"); errParse != nil {
if hasOpt = c.HasOption("cdre", "exported_fields"); hasOpt {
extraFieldsStr, _ := c.GetString("cdre", "exported_fields")
if extraFields, err := ParseRSRFields(extraFieldsStr); err != nil {
return nil, errParse
} else {
cfg.CdreExportedFields = extraFields
}
}
if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt {

20
config/config_test.go
View File

@@ -81,7 +81,6 @@ func TestDefaults(t *testing.T) {
eCfg.CDRSExtraFields = []*utils.RSRField{}
eCfg.CDRSMediator = ""
eCfg.CdreCdrFormat = "csv"
eCfg.CdreExtraFields = []string{}
eCfg.CdreDir = "/var/log/cgrates/cdr/cdrexport/csv"
eCfg.CdrcEnabled = false
eCfg.CdrcCdrs = utils.INTERNAL
@@ -146,6 +145,23 @@ func TestDefaults(t *testing.T) {
eCfg.MailerAuthUser = "cgrates"
eCfg.MailerAuthPass = "CGRateS.org"
eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain"
eCfg.CdreExportedFields = []*utils.RSRField{
&utils.RSRField{Id: utils.CGRID},
&utils.RSRField{Id: utils.MEDI_RUNID},
&utils.RSRField{Id: utils.ACCID},
&utils.RSRField{Id: utils.CDRHOST},
&utils.RSRField{Id: utils.REQTYPE},
&utils.RSRField{Id: utils.DIRECTION},
&utils.RSRField{Id: utils.TENANT},
&utils.RSRField{Id: utils.TOR},
&utils.RSRField{Id: utils.ACCOUNT},
&utils.RSRField{Id: utils.SUBJECT},
&utils.RSRField{Id: utils.DESTINATION},
&utils.RSRField{Id: utils.SETUP_TIME},
&utils.RSRField{Id: utils.ANSWER_TIME},
&utils.RSRField{Id: utils.DURATION},
&utils.RSRField{Id: utils.COST},
}
if !reflect.DeepEqual(cfg, eCfg) {
t.Log(eCfg)
t.Log(cfg)
@@ -214,7 +230,7 @@ func TestConfigFromFile(t *testing.T) {
eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
eCfg.CDRSMediator = "test"
eCfg.CdreCdrFormat = "test"
eCfg.CdreExtraFields = []string{"test"}
eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
eCfg.CdreDir = "test"
eCfg.CdrcEnabled = true
eCfg.CdrcCdrs = "test"

2
config/test_data.txt
View File

@@ -49,8 +49,8 @@ mediator = test # Address where to reach the Mediator. Empty for disabling me
[cdre]
cdr_format = test # Exported CDRs format <csv>
extra_fields = test # List of extra fields to be exported out in CDRs
export_dir = test # Path where the exported CDRs will be placed
exported_fields = test # List of fields in the exported CDRs
[cdrc]
enabled = true # Enable CDR client functionality

3
data/conf/cgrates.cfg
View File

@@ -51,8 +51,9 @@
[cdre]
# cdr_format = csv # Exported CDRs format <csv>
# extra_fields = # List of extra fields to be exported out in CDRs
# export_dir = /var/log/cgrates/cdr/cdrexport/csv # Path where the exported CDRs will be placed
# exported_fields = cgrid,mediation_runid,accid,cdrhost,reqtype,direction,tenant,tor,account,subject,destination,setup_time,answer_time,duration,cost
# List of fields in the exported CDRs
[cdrc]
# enabled = false # Enable CDR client functionality

2
utils/consts.go
View File

@@ -82,6 +82,8 @@ const (
SETUP_TIME = "setup_time"
ANSWER_TIME = "answer_time"
DURATION = "duration"
MEDI_RUNID = "mediation_runid"
COST = "cost"
DEFAULT_RUNID = "default"
STATIC_VALUE_PREFIX = "^"
CDRE_CSV = "csv"

9
utils/researchreplace.go
View File

@@ -28,9 +28,12 @@ type ReSearchReplace struct {
ReplaceTemplate string
}
func (self *ReSearchReplace) Process(source string) string {
func (rsr *ReSearchReplace) Process(source string) string {
if rsr.SearchRegexp == nil {
return ""
}
res := []byte{}
match := self.SearchRegexp.FindStringSubmatchIndex(source)
res = self.SearchRegexp.ExpandString(res, self.ReplaceTemplate, source, match)
match := rsr.SearchRegexp.FindStringSubmatchIndex(source)
res = rsr.SearchRegexp.ExpandString(res, rsr.ReplaceTemplate, source, match)
return string(res)
}

3
utils/rsrfield.go
View File

@@ -25,6 +25,9 @@ type RSRField struct {
// Parse the field value from a string
func (rsrf *RSRField) ParseValue(value string) string {
if len(value) == 0 {
return value
}
if rsrf.RSRule != nil {
value = rsrf.RSRule.Process(value)
}

45
utils/storedcdr.go
View File

@@ -134,6 +134,11 @@ func (storedCdr *StoredCdr) GetExtraFields() map[string]string {
return storedCdr.ExtraFields
}
// Return cost as string, formated with number of decimals configured
func (storedCdr *StoredCdr) FormatCost(roundDecimals int) string {
return strconv.FormatFloat(storedCdr.Cost, 'f', roundDecimals, 64)
}
func (storedCdr *StoredCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld string, extraFlds []string, fieldsMandatory bool) (*StoredCdr, error) {
return storedCdr, nil
}
@@ -159,3 +164,43 @@ func (storedCdr *StoredCdr) AsRawCdrHttpForm() url.Values {
}
return v
}
// Used to export fields as string, primary fields are const labeled
func (storedCdr *StoredCdr) ExportFieldValue(fldName string) string {
switch fldName {
case CGRID:
return storedCdr.CgrId
case ACCID:
return storedCdr.AccId
case CDRHOST:
return storedCdr.CdrHost
case CDRSOURCE:
return storedCdr.CdrSource
case REQTYPE:
return storedCdr.ReqType
case DIRECTION:
return storedCdr.Direction
case TENANT:
return storedCdr.Tenant
case TOR:
return storedCdr.TOR
case ACCOUNT:
return storedCdr.Account
case SUBJECT:
return storedCdr.Subject
case DESTINATION:
return storedCdr.Destination
case SETUP_TIME:
return storedCdr.SetupTime.String()
case ANSWER_TIME:
return storedCdr.AnswerTime.String()
case DURATION:
return strconv.FormatFloat(storedCdr.Duration.Seconds(), 'f', -1, 64)
case MEDI_RUNID:
return storedCdr.MediationRunId
case COST:
return strconv.FormatFloat(storedCdr.Cost, 'f', -1, 64) // Recommended to use FormatCost
default:
return storedCdr.ExtraFields[fldName]
}
}

39
utils/storedcdr_test.go
View File

@@ -148,3 +148,42 @@ func TestAsRawCdrHttpForm(t *testing.T) {
t.Errorf("Expected: %s, received: %s", ratedCdr.ExtraFields["fieldextr2"], cdrForm.Get("fieldextr2"))
}
}
func TestExportFieldValue(t *testing.T) {
cdr := StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID,
Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if cdr.ExportFieldValue(CGRID) != cdr.CgrId ||
cdr.ExportFieldValue(ACCID) != cdr.AccId ||
cdr.ExportFieldValue(CDRHOST) != cdr.CdrHost ||
cdr.ExportFieldValue(CDRSOURCE) != cdr.CdrSource ||
cdr.ExportFieldValue(REQTYPE) != cdr.ReqType ||
cdr.ExportFieldValue(DIRECTION) != cdr.Direction ||
cdr.ExportFieldValue(TENANT) != cdr.Tenant ||
cdr.ExportFieldValue(TOR) != cdr.TOR ||
cdr.ExportFieldValue(ACCOUNT) != cdr.Account ||
cdr.ExportFieldValue(SUBJECT) != cdr.Subject ||
cdr.ExportFieldValue(DESTINATION) != cdr.Destination ||
cdr.ExportFieldValue(SETUP_TIME) != "0001-01-01 00:00:00 +0000 UTC" ||
cdr.ExportFieldValue(ANSWER_TIME) != cdr.AnswerTime.String() ||
cdr.ExportFieldValue(DURATION) != "10" ||
cdr.ExportFieldValue(MEDI_RUNID) != cdr.MediationRunId ||
cdr.ExportFieldValue(COST) != "1.01" ||
cdr.ExportFieldValue("field_extr1") != cdr.ExtraFields["field_extr1"] ||
cdr.ExportFieldValue("fieldextr2") != cdr.ExtraFields["fieldextr2"] ||
cdr.ExportFieldValue("dummy_field") != "" {
t.Error("Unexpected filed value received")
}
}
func TestFormatCost(t *testing.T) {
cdr := StoredCdr{Cost: 1.01}
if cdr.FormatCost(4) != "1.0100" {
t.Error("Unexpected format of the cost: ", cdr.FormatCost(4))
}
cdr = StoredCdr{Cost: 1.01001}
if cdr.FormatCost(4) != "1.0100" {
t.Error("Unexpected format of the cost: ", cdr.FormatCost(4))
}
}