diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go index eba5d4093..af4bd5a4f 100644 --- a/cdrc/csv_it_test.go +++ b/cdrc/csv_it_test.go @@ -173,7 +173,7 @@ accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;" func TestCsvIT2InitConfig(t *testing.T) { var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsvwithfilter") + csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil { t.Fatal("Got config error: ", err.Error()) } @@ -223,14 +223,14 @@ func TestCsvIT2HandleCdr2File(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_2), 0644); err != nil { t.Fatal(err.Error()) } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctestswithfilters/csvit1/in", fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter1/csvit1/in", fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } func TestCsvIT2ProcessedFiles(t *testing.T) { time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent2, err := ioutil.ReadFile("/tmp/cdrctestswithfilters/csvit1/out/file1.csv"); err != nil { + if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter1/csvit1/out/file1.csv"); err != nil { t.Error(err) } else if fileContent1_2 != string(outContent2) { t.Errorf("Expecting: %q, received: %q", fileContent1_2, string(outContent2)) @@ -255,6 +255,7 @@ func TestCsvIT2KillEngine(t *testing.T) { if err := engine.KillEngine(*waitRater); err != nil { t.Error(err) } + time.Sleep(time.Duration(2**waitRater) * time.Millisecond) } // Begin tests for cdrc csv with new filters @@ -264,7 +265,7 @@ accid23;*prepaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra3 func TestCsvIT3InitConfig(t *testing.T) { var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsvwithfilter") + csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil { t.Fatal("Got config error: ", err.Error()) } @@ -314,14 +315,14 @@ func TestCsvIT3HandleCdr2File(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_3), 0644); err != nil { t.Fatal(err.Error()) } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctestswithfilters/csvit2/in", fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter2/csvit2/in", fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } func TestCsvIT3ProcessedFiles(t *testing.T) { time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent2, err := ioutil.ReadFile("/tmp/cdrctestswithfilters/csvit2/out/file1.csv"); err != nil { + if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter2/csvit2/out/file1.csv"); err != nil { t.Error(err) } else if fileContent1_3 != string(outContent2) { t.Errorf("Expecting: %q, received: %q", fileContent1_3, string(outContent2)) @@ -341,6 +342,7 @@ func TestCsvIT3KillEngine(t *testing.T) { if err := engine.KillEngine(*waitRater); err != nil { t.Error(err) } + time.Sleep(time.Duration(2**waitRater) * time.Millisecond) } // Begin tests for cdrc csv with new filters @@ -350,7 +352,7 @@ accid23;*postpaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra func TestCsvIT4InitConfig(t *testing.T) { var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "csvwithfieldfilter") + csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") if csvCfg, err = config.NewCGRConfigFromFolder(csvCfgPath); err != nil { t.Fatal("Got config error: ", err.Error()) } @@ -400,14 +402,14 @@ func TestCsvIT4HandleCdr2File(t *testing.T) { if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_4), 0644); err != nil { t.Fatal(err.Error()) } - if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfielfilter/csvit2/in", fileName)); err != nil { + if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter3/csvit3/in", fileName)); err != nil { t.Fatal("Error moving file to processing directory: ", err) } } func TestCsvIT4ProcessedFiles(t *testing.T) { time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent4, err := ioutil.ReadFile("/tmp/csvwithfielfilter/csvit2/out/file1.csv"); err != nil { + if outContent4, err := ioutil.ReadFile("/tmp/csvwithfilter3/csvit3/out/file1.csv"); err != nil { t.Error(err) } else if fileContent1_4 != string(outContent4) { t.Errorf("Expecting: %q, received: %q", fileContent1_4, string(outContent4)) diff --git a/config/config_test.go b/config/config_test.go index 837fce17f..d0fb4f400 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -639,7 +639,6 @@ func TestCgrCfgJSONDefaultsCdreProfiles(t *testing.T) { Value: NewRSRParsersMustCompile("~Cost", true), RoundingDecimals: 4}, } - // eCdreCfg := map[string]*CdreConfig{ "*default": { ExportFormat: utils.MetaFileCSV, diff --git a/data/conf/samples/cdrccsv/cgrates.json b/data/conf/samples/cdrccsv/cgrates.json index efeb34b06..468a1e21b 100644 --- a/data/conf/samples/cdrccsv/cgrates.json +++ b/data/conf/samples/cdrccsv/cgrates.json @@ -11,17 +11,18 @@ "db_password": "CGRateS.org", // password to use when connecting to stordb }, + "rals": { "enabled": true // so we can query CDRs }, + "cdrs": { "enabled": true, "rals_conns": [], // no rating support, just *raw CDR testing }, - "cdrc": [ { "id": "*default", @@ -55,6 +56,85 @@ {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, ], }, + { + "id": "*CSVWithFilter1", // identifier of the CDRC runner + "enabled": true, // enable CDR client functionality + "field_separator": ";", + "cdr_in_dir": "/tmp/csvwithfilter1/csvit1/in", // absolute path towards the directory where the CDRs are stored + "cdr_out_dir": "/tmp/csvwithfilter1/csvit1/out", // absolute path towards the directory where processed CDRs will be moved + "cdr_source_id": "csvit1", // free form field, tag identifying the source of the CDRs within CDRS database + "filters":["*string:3:1002"], //filter Account to be 1002 + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, + {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, + {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, + {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, + {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, + {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, + {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, + ], + }, + { + "id": "*CSVWithFilter2", // identifier of the CDRC runner + "enabled": true, // enable CDR client functionality + "field_separator": ";", + "cdr_in_dir": "/tmp/csvwithfilter2/csvit2/in", // absolute path towards the directory where the CDRs are stored + "cdr_out_dir": "/tmp/csvwithfilter2/csvit2/out", // absolute path towards the directory where processed CDRs will be moved + "cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database + "filters":["*string:3:1002","*string:1:*prepaid","*gte:6:70"], //filter Account to be 1002 and RequestType *prepaid + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, + {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, + {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, + {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, + {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, + {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, + {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, + ], + }, + { + "id": "*CSVit4", // identifier of the CDRC runner + "enabled": true, // enable CDR client functionality + "field_separator": ";", + "cdr_in_dir": "/tmp/csvwithfilter3/csvit3/in", // absolute path towards the directory where the CDRs are stored + "cdr_out_dir": "/tmp/csvwithfilter3/csvit3/out", // absolute path towards the directory where processed CDRs will be moved + "cdr_source_id": "csvit4", // free form field, tag identifying the source of the CDRs within CDRS database + "filters":["*string:1:*postpaid"], //filter Account to be 1002 + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, + {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, + {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, + {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, + {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, + {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, + {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, + {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, + {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, + {"id": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:3:1001"]}, + ], + }, ], diff --git a/data/conf/samples/cdrccsvwithfilter/cgrates.json b/data/conf/samples/cdrccsvwithfilter/cgrates.json deleted file mode 100755 index d92f4ac2c..000000000 --- a/data/conf/samples/cdrccsvwithfilter/cgrates.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. - - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - "rals": { - "enabled": true // so we can query CDRs - }, - - "cdrs": { - "enabled": true, - "rals_conns": [], // no rating support, just *raw CDR testing -}, - - - - "cdrc": [ - { - "id": "*CSVWithFilter1", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_dir": "/tmp/cdrctestswithfilters/csvit1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_dir": "/tmp/cdrctestswithfilters/csvit1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit1", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:3:1002"], //filter Account to be 1002 - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, - {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, - {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, - {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, - {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, - ], - }, - { - "id": "*CSVWithFilter2", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_dir": "/tmp/cdrctestswithfilters/csvit2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_dir": "/tmp/cdrctestswithfilters/csvit2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:3:1002","*string:1:*prepaid","*gte:6:70"], //filter Account to be 1002 and RequestType *prepaid - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, - {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, - {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, - {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, - {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, - ], - }, -], - - -} \ No newline at end of file diff --git a/data/conf/samples/csvwithfieldfilter/cgrates.json b/data/conf/samples/csvwithfieldfilter/cgrates.json deleted file mode 100755 index 14a55d991..000000000 --- a/data/conf/samples/csvwithfieldfilter/cgrates.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. -"general": { - "log_level": 7, -}, - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - "rals": { - "enabled": true // so we can query CDRs - }, - - "cdrs": { - "enabled": true, - "rals_conns": [], // no rating support, just *raw CDR testing -}, - - - - "cdrc": [ - { - "id": "*CSVit4", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_dir": "/tmp/csvwithfielfilter/csvit2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_dir": "/tmp/csvwithfielfilter/csvit2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit4", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:1:*postpaid"], //filter Account to be 1002 - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"id": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"id": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~0", "mandatory": true}, - {"id": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~1", "mandatory": true}, - {"id": "Direction", "field_id": "Direction", "type": "*composed", "value": "*out", "mandatory": true}, - {"id": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~2", "mandatory": true}, - {"id": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"id": "Account", "field_id": "Account", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Subject", "field_id": "Subject", "type": "*composed", "value": "~3", "mandatory": true}, - {"id": "Destination", "field_id": "Destination", "type": "*composed", "value": "~4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"id": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~5", "mandatory": true}, - {"id": "Usage", "field_id": "Usage", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~6", "mandatory": true}, - {"id": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:3:1001"]}, - ], - }, -], - - -} \ No newline at end of file