diff --git a/data/conf/samples/ers/cgrates.json b/data/conf/samples/ers/cgrates.json index a8f683499..ff581a0e9 100644 --- a/data/conf/samples/ers/cgrates.json +++ b/data/conf/samples/ers/cgrates.json @@ -181,6 +181,34 @@ {"tag": "ExtraInfo2", "field_id": "ExtraInfo2", "type": "*constant", "value": "ExtraInfo2", "mandatory": true}, ], }, + { + "id": "file_reader_with_filters", + "run_delay": -1, + "type": "*file_csv", + "flags": ["*dryrun"], + "field_separator": ";", + "source_path": "/tmp/ers_with_filters/in", + "processed_path": "/tmp/ers_with_filters/out", + "flags": ["*cdrs","*log"], + "filters":["*string:~*req.3:1002"], + "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value + {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, + {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, + {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, + {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, + {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, + {"tag": "Source", "field_id": "Source", "type": "*composed", "value": "ers_csv", "mandatory": true}, + {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, + {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, + {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, + {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, + {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, + {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, + {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, + {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, + ], + }, ], }, diff --git a/ers/filecsv_it_test.go b/ers/filecsv_it_test.go index 1d1a3331d..438ae55c5 100644 --- a/ers/filecsv_it_test.go +++ b/ers/filecsv_it_test.go @@ -54,6 +54,10 @@ accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;" fileContent3 = `cgrates.org,*voice,SessionFromCsv,*prepaid,1001,ANY2CNT,1002,2018-01-07 17:00:00 +0000 UTC,2018-01-07 17:00:10 +0000 UTC,5m ` + fileContentForFilter = `accid21;*prepaid;itsyscom.com;1002;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 +accid22;*postpaid;itsyscom.com;1002;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 +accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;"";val_extra1` + csvTests = []func(t *testing.T){ testCsvITCreateCdrDirs, testCsvITInitConfig, @@ -69,7 +73,10 @@ accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;" testCsvITTerminateSession, testCsvITProcessCDR, testCsvITAnalyseCDRs, + testCsvITProcessFilteredCDR, + testCsvITAnalyzeFilteredCDR, testCsvITProcessedFiles, + testCsvITCleanupFiles, testCsvITKillEngine, } ) @@ -106,7 +113,7 @@ func testCsvITCreateCdrDirs(t *testing.T) { for _, dir := range []string{"/tmp/ers/in", "/tmp/ers/out", "/tmp/ers2/in", "/tmp/ers2/out", "/tmp/init_session/in", "/tmp/init_session/out", "/tmp/terminate_session/in", "/tmp/terminate_session/out", "/tmp/cdrs/in", - "/tmp/cdrs/out"} { + "/tmp/cdrs/out", "/tmp/ers_with_filters/in", "/tmp/ers_with_filters/out"} { if err := os.RemoveAll(dir); err != nil { t.Fatal("Error removing folder: ", dir, err) } @@ -266,6 +273,34 @@ func testCsvITAnalyseCDRs(t *testing.T) { } } +func testCsvITProcessFilteredCDR(t *testing.T) { + fileName := "file1.csv" + tmpFilePath := path.Join("/tmp", fileName) + if err := ioutil.WriteFile(tmpFilePath, []byte(fileContentForFilter), 0644); err != nil { + t.Fatal(err.Error()) + } + if err := os.Rename(tmpFilePath, path.Join("/tmp/ers_with_filters/in", fileName)); err != nil { + t.Fatal("Error moving file to processing directory: ", err) + } +} + +func testCsvITAnalyzeFilteredCDR(t *testing.T) { + time.Sleep(500 * time.Millisecond) + + var cdrs []*engine.CDR + args := utils.RPCCDRsFilter{NotRunIDs: []string{"CustomerCharges", "SupplierCharges"}, + Sources: []string{"ers_csv"}} + if err := csvRPC.Call(utils.CDRsV1GetCDRs, args, &cdrs); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(cdrs) != 2 { + t.Error("Unexpected number of CDRs returned: ", utils.ToJSON(cdrs)) + } else if cdrs[0].Account != "1002" || cdrs[1].Account != "1002" { + t.Errorf("Expecting: 1002, received: <%s> , <%s>", cdrs[0].Account, cdrs[1].Account) + } else if cdrs[0].Tenant != "itsyscom.com" || cdrs[1].Tenant != "itsyscom.com" { + t.Errorf("Expecting: itsyscom.com, received: <%s> , <%s>", cdrs[0].Tenant, cdrs[1].Tenant) + } +} + func testCsvITProcessedFiles(t *testing.T) { time.Sleep(time.Duration(1 * time.Second)) if outContent1, err := ioutil.ReadFile("/tmp/ers/out/file1.csv"); err != nil { @@ -283,6 +318,22 @@ func testCsvITProcessedFiles(t *testing.T) { } else if fileContent3 != string(outContent3) { t.Errorf("Expecting: %q, received: %q", fileContent3, string(outContent3)) } + if outContent4, err := ioutil.ReadFile("/tmp/ers_with_filters/out/file1.csv"); err != nil { + t.Error(err) + } else if fileContentForFilter != string(outContent4) { + t.Errorf("Expecting: %q, received: %q", fileContentForFilter, string(outContent4)) + } +} + +func testCsvITCleanupFiles(t *testing.T) { + for _, dir := range []string{"/tmp/ers", + "/tmp/ers2", "/tmp/init_session", + "/tmp/terminate_session", "/tmp/cdrs", + "/tmp/ers_with_filters"} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal("Error removing folder: ", dir, err) + } + } } func testCsvITKillEngine(t *testing.T) {