Combine cdrc csv export profiles into a single config.json

This commit is contained in:
TeoV
2018-09-03 10:29:44 -04:00
committed by Dan Christian Bogos
parent d9cb9c25ae
commit e3086a6e15
5 changed files with 92 additions and 150 deletions

View File

@@ -173,7 +173,7 @@ accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;"
func TestCsvIT2InitConfig(t *testing.T) {
var err error
csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsvwithfilter")
csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv")
if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil {
t.Fatal("Got config error: ", err.Error())
}
@@ -223,14 +223,14 @@ func TestCsvIT2HandleCdr2File(t *testing.T) {
if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_2), 0644); err != nil {
t.Fatal(err.Error())
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctestswithfilters/csvit1/in", fileName)); err != nil {
if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter1/csvit1/in", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func TestCsvIT2ProcessedFiles(t *testing.T) {
time.Sleep(time.Duration(2**waitRater) * time.Millisecond)
if outContent2, err := ioutil.ReadFile("/tmp/cdrctestswithfilters/csvit1/out/file1.csv"); err != nil {
if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter1/csvit1/out/file1.csv"); err != nil {
t.Error(err)
} else if fileContent1_2 != string(outContent2) {
t.Errorf("Expecting: %q, received: %q", fileContent1_2, string(outContent2))
@@ -255,6 +255,7 @@ func TestCsvIT2KillEngine(t *testing.T) {
if err := engine.KillEngine(*waitRater); err != nil {
t.Error(err)
}
time.Sleep(time.Duration(2**waitRater) * time.Millisecond)
}
// Begin tests for cdrc csv with new filters
@@ -264,7 +265,7 @@ accid23;*prepaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra3
func TestCsvIT3InitConfig(t *testing.T) {
var err error
csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsvwithfilter")
csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv")
if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil {
t.Fatal("Got config error: ", err.Error())
}
@@ -314,14 +315,14 @@ func TestCsvIT3HandleCdr2File(t *testing.T) {
if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_3), 0644); err != nil {
t.Fatal(err.Error())
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctestswithfilters/csvit2/in", fileName)); err != nil {
if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter2/csvit2/in", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func TestCsvIT3ProcessedFiles(t *testing.T) {
time.Sleep(time.Duration(2**waitRater) * time.Millisecond)
if outContent2, err := ioutil.ReadFile("/tmp/cdrctestswithfilters/csvit2/out/file1.csv"); err != nil {
if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter2/csvit2/out/file1.csv"); err != nil {
t.Error(err)
} else if fileContent1_3 != string(outContent2) {
t.Errorf("Expecting: %q, received: %q", fileContent1_3, string(outContent2))
@@ -341,6 +342,7 @@ func TestCsvIT3KillEngine(t *testing.T) {
if err := engine.KillEngine(*waitRater); err != nil {
t.Error(err)
}
time.Sleep(time.Duration(2**waitRater) * time.Millisecond)
}
// Begin tests for cdrc csv with new filters
@@ -350,7 +352,7 @@ accid23;*postpaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra
func TestCsvIT4InitConfig(t *testing.T) {
var err error
csvCfgPath = path.Join(*dataDir, "conf", "samples", "csvwithfieldfilter")
csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv")
if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil {
t.Fatal("Got config error: ", err.Error())
}
@@ -400,14 +402,14 @@ func TestCsvIT4HandleCdr2File(t *testing.T) {
if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_4), 0644); err != nil {
t.Fatal(err.Error())
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfielfilter/csvit2/in", fileName)); err != nil {
if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter3/csvit3/in", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func TestCsvIT4ProcessedFiles(t *testing.T) {
time.Sleep(time.Duration(2**waitRater) * time.Millisecond)
if outContent4, err := ioutil.ReadFile("/tmp/csvwithfielfilter/csvit2/out/file1.csv"); err != nil {
if outContent4, err := ioutil.ReadFile("/tmp/csvwithfilter3/csvit3/out/file1.csv"); err != nil {
t.Error(err)
} else if fileContent1_4 != string(outContent4) {
t.Errorf("Expecting: %q, received: %q", fileContent1_4, string(outContent4))

View File

@@ -639,7 +639,6 @@ func TestCgrCfgJSONDefaultsCdreProfiles(t *testing.T) {
Value: NewRSRParsersMustCompile("~Cost", true),
RoundingDecimals: 4},
}
//
eCdreCfg := map[string]*CdreConfig{
"*default": {
ExportFormat: utils.MetaFileCSV,

View File

@@ -11,17 +11,18 @@
"db_password": "CGRateS.org", // password to use when connecting to stordb
},
"rals": {
"enabled": true // so we can query CDRs
},
"cdrs": {
"enabled": true,
"rals_conns": [], // no rating support, just *raw CDR testing
},
"cdrc": [
{
"id": "*default",
@@ -55,6 +56,85 @@
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
],
},
{
"id": "*CSVWithFilter1", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/csvwithfilter1/csvit1/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/csvwithfilter1/csvit1/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit1", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:3:1002"], //filter Account to be 1002
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
],
},
{
"id": "*CSVWithFilter2", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/csvwithfilter2/csvit2/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/csvwithfilter2/csvit2/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:3:1002","*string:1:*prepaid","*gte:6:70"], //filter Account to be 1002 and RequestType *prepaid
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
],
},
{
"id": "*CSVit4", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/csvwithfilter3/csvit3/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/csvwithfilter3/csvit3/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit4", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:1:*postpaid"], //filter Account to be 1002
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:3:1001"]},
],
},
],

View File

@@ -1,81 +0,0 @@
{
// Real-time Charging System for Telecom & ISP environments
// Copyright (C) ITsysCOM GmbH
//
// This file contains the default configuration hardcoded into CGRateS.
// This is what you get when you load CGRateS with an empty configuration file.
"stor_db": { // database used to store offline tariff plans and CDRs
"db_password": "CGRateS.org", // password to use when connecting to stordb
},
"rals": {
"enabled": true // so we can query CDRs
},
"cdrs": {
"enabled": true,
"rals_conns": [], // no rating support, just *raw CDR testing
},
"cdrc": [
{
"id": "*CSVWithFilter1", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/cdrctestswithfilters/csvit1/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/cdrctestswithfilters/csvit1/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit1", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:3:1002"], //filter Account to be 1002
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
],
},
{
"id": "*CSVWithFilter2", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/cdrctestswithfilters/csvit2/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/cdrctestswithfilters/csvit2/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:3:1002","*string:1:*prepaid","*gte:6:70"], //filter Account to be 1002 and RequestType *prepaid
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
],
},
],
}

View File

@@ -1,58 +0,0 @@
{
// Real-time Charging System for Telecom & ISP environments
// Copyright (C) ITsysCOM GmbH
//
// This file contains the default configuration hardcoded into CGRateS.
// This is what you get when you load CGRateS with an empty configuration file.
"general": {
"log_level": 7,
},
"stor_db": { // database used to store offline tariff plans and CDRs
"db_password": "CGRateS.org", // password to use when connecting to stordb
},
"rals": {
"enabled": true // so we can query CDRs
},
"cdrs": {
"enabled": true,
"rals_conns": [], // no rating support, just *raw CDR testing
},
"cdrc": [
{
"id": "*CSVit4", // identifier of the CDRC runner
"enabled": true, // enable CDR client functionality
"field_separator": ";",
"cdr_in_dir": "/tmp/csvwithfielfilter/csvit2/in", // absolute path towards the directory where the CDRs are stored
"cdr_out_dir": "/tmp/csvwithfielfilter/csvit2/out", // absolute path towards the directory where processed CDRs will be moved
"cdr_source_id": "csvit4", // free form field, tag identifying the source of the CDRs within CDRS database
"filters":["*string:1:*postpaid"], //filter Account to be 1002
"content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
{"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true},
{"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true},
{"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true},
{"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true},
{"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true},
{"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true},
{"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true},
{"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true},
{"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true},
{"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true},
{"id": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:3:1001"]},
],
},
],
}