From 22169a17250e3e26e70aee213941d1eea24d8cf9 Mon Sep 17 00:00:00 2001 From: TeoV Date: Mon, 18 May 2020 10:40:03 +0300 Subject: [PATCH] Finish integration test for *file_csv EventExporter --- config/config_test.go | 224 +++++++++++++++++++++++++++-- config/eescfg.go | 5 + config/eescfg_test.go | 178 ++++++++++++++++++++--- data/conf/samples/ees/cgrates.json | 2 +- ees/filecsv_it_test.go | 28 +++- 5 files changed, 411 insertions(+), 26 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 94611a9b9..659090028 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -1881,7 +1881,7 @@ func TestCgrCdfEventExporter(t *testing.T) { Filters: []string{}, AttributeSIDs: []string{}, Flags: utils.FlagsWithParams{}, - ContentFields: []*FCTemplate{ + contentFields: []*FCTemplate{ { Tag: utils.CGRID, Path: "*exp.CGRID", @@ -1982,13 +1982,117 @@ func TestCgrCdfEventExporter(t *testing.T) { RoundingDecimals: utils.IntPointer(4), }, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + Fields: []*FCTemplate{ + { + Tag: utils.CGRID, + Path: "*exp.CGRID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.CGRID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RunID, + Path: "*exp.RunID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RunID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.ToR, + Path: "*exp.ToR", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.ToR", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.OriginID, + Path: "*exp.OriginID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.OriginID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RequestType, + Path: "*exp.RequestType", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RequestType", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Tenant, + Path: "*exp.Tenant", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Tenant", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Category, + Path: "*exp.Category", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Category", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Account, + Path: "*exp.Account", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Account", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Subject, + Path: "*exp.Subject", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Subject", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Destination, + Path: "*exp.Destination", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Destination", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.SetupTime, + Path: "*exp.SetupTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.SetupTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.AnswerTime, + Path: "*exp.AnswerTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.AnswerTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.Usage, + Path: "*exp.Usage", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Usage", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Cost, + Path: "*exp.Cost", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Cost", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + RoundingDecimals: utils.IntPointer(4), + }, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, }, }, } for _, profile := range eCfg.Exporters { - for _, v := range profile.ContentFields { + for _, v := range profile.Fields { + v.ComputePath() + } + for _, v := range profile.contentFields { v.ComputePath() } } @@ -2057,7 +2161,7 @@ func TestCgrCfgEventExporterDefault(t *testing.T) { Timezone: utils.EmptyString, Filters: nil, Flags: utils.FlagsWithParams{}, - ContentFields: []*FCTemplate{ + contentFields: []*FCTemplate{ { Tag: utils.CGRID, Path: "*exp.CGRID", @@ -2158,10 +2262,114 @@ func TestCgrCfgEventExporterDefault(t *testing.T) { RoundingDecimals: utils.IntPointer(4), }, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + Fields: []*FCTemplate{ + { + Tag: utils.CGRID, + Path: "*exp.CGRID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.CGRID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RunID, + Path: "*exp.RunID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RunID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.ToR, + Path: "*exp.ToR", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.ToR", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.OriginID, + Path: "*exp.OriginID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.OriginID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RequestType, + Path: "*exp.RequestType", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RequestType", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Tenant, + Path: "*exp.Tenant", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Tenant", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Category, + Path: "*exp.Category", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Category", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Account, + Path: "*exp.Account", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Account", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Subject, + Path: "*exp.Subject", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Subject", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Destination, + Path: "*exp.Destination", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Destination", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.SetupTime, + Path: "*exp.SetupTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.SetupTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.AnswerTime, + Path: "*exp.AnswerTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.AnswerTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.Usage, + Path: "*exp.Usage", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Usage", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Cost, + Path: "*exp.Cost", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Cost", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + RoundingDecimals: utils.IntPointer(4), + }, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, } - for _, v := range eCfg.ContentFields { + for _, v := range eCfg.Fields { + v.ComputePath() + } + for _, v := range eCfg.contentFields { v.ComputePath() } if !reflect.DeepEqual(cgrCfg.dfltEvExp, eCfg) { diff --git a/config/eescfg.go b/config/eescfg.go index 72f4fc84c..3191f7983 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -252,6 +252,11 @@ func (eeC *EventExporterCfg) Clone() (cln *EventExporterCfg) { cln.Synchronous = eeC.Synchronous cln.Attempts = eeC.Attempts cln.FieldSep = eeC.FieldSep + + cln.Fields = make([]*FCTemplate, len(eeC.Fields)) + for idx, fld := range eeC.Fields { + cln.Fields[idx] = fld.Clone() + } cln.headerFields = make([]*FCTemplate, len(eeC.headerFields)) for idx, fld := range eeC.headerFields { cln.headerFields[idx] = fld.Clone() diff --git a/config/eescfg_test.go b/config/eescfg_test.go index 7da998aed..5f82bf5e2 100644 --- a/config/eescfg_test.go +++ b/config/eescfg_test.go @@ -33,7 +33,7 @@ func TestEventExporterClone(t *testing.T) { FieldSep: ",", Filters: []string{"Filter1", "Filter2"}, Tenant: NewRSRParsersMustCompile("cgrates.org", true, utils.INFIELD_SEP), - ContentFields: []*FCTemplate{ + contentFields: []*FCTemplate{ { Tag: "ToR", Path: "*exp.ToR", @@ -49,10 +49,29 @@ func TestEventExporterClone(t *testing.T) { Mandatory: true, }, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + Fields: []*FCTemplate{ + { + Tag: "ToR", + Path: "*exp.ToR", + Type: "*composed", + Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), + Mandatory: true, + }, + { + Tag: "RandomField", + Path: "*exp.RandomField", + Type: "*composed", + Value: NewRSRParsersMustCompile("Test", true, utils.INFIELD_SEP), + Mandatory: true, + }, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, } - for _, v := range orig.ContentFields { + for _, v := range orig.Fields { + v.ComputePath() + } + for _, v := range orig.contentFields { v.ComputePath() } cloned := orig.Clone() @@ -65,7 +84,7 @@ func TestEventExporterClone(t *testing.T) { FieldSep: ",", Filters: []string{"Filter1", "Filter2"}, Tenant: NewRSRParsersMustCompile("cgrates.org", true, utils.INFIELD_SEP), - ContentFields: []*FCTemplate{ + Fields: []*FCTemplate{ { Tag: "ToR", Path: "*exp.ToR", @@ -81,14 +100,33 @@ func TestEventExporterClone(t *testing.T) { Mandatory: true, }, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + contentFields: []*FCTemplate{ + { + Tag: "ToR", + Path: "*exp.ToR", + Type: "*composed", + Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), + Mandatory: true, + }, + { + Tag: "RandomField", + Path: "*exp.RandomField", + Type: "*composed", + Value: NewRSRParsersMustCompile("Test", true, utils.INFIELD_SEP), + Mandatory: true, + }, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, } - for _, v := range initialOrig.ContentFields { + for _, v := range initialOrig.Fields { + v.ComputePath() + } + for _, v := range initialOrig.contentFields { v.ComputePath() } orig.Filters = []string{"SingleFilter"} - orig.ContentFields = []*FCTemplate{ + orig.contentFields = []*FCTemplate{ { Tag: "ToR", Path: "*exp.ToR", @@ -125,7 +163,7 @@ func TestEventExporterSameID(t *testing.T) { Filters: []string{}, AttributeSIDs: []string{}, Flags: utils.FlagsWithParams{}, - ContentFields: []*FCTemplate{ + Fields: []*FCTemplate{ { Tag: utils.CGRID, Path: "*exp.CGRID", @@ -226,8 +264,109 @@ func TestEventExporterSameID(t *testing.T) { RoundingDecimals: utils.IntPointer(4), }, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + contentFields: []*FCTemplate{ + { + Tag: utils.CGRID, + Path: "*exp.CGRID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.CGRID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RunID, + Path: "*exp.RunID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RunID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.ToR, + Path: "*exp.ToR", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.ToR", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.OriginID, + Path: "*exp.OriginID", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.OriginID", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.RequestType, + Path: "*exp.RequestType", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.RequestType", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Tenant, + Path: "*exp.Tenant", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Tenant", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Category, + Path: "*exp.Category", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Category", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Account, + Path: "*exp.Account", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Account", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Subject, + Path: "*exp.Subject", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Subject", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Destination, + Path: "*exp.Destination", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Destination", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.SetupTime, + Path: "*exp.SetupTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.SetupTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.AnswerTime, + Path: "*exp.AnswerTime", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.AnswerTime", true, utils.INFIELD_SEP), + Layout: "2006-01-02T15:04:05Z07:00", + }, + { + Tag: utils.Usage, + Path: "*exp.Usage", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Usage", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + }, + { + Tag: utils.Cost, + Path: "*exp.Cost", + Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.Cost", true, utils.INFIELD_SEP), + Layout: time.RFC3339, + RoundingDecimals: utils.IntPointer(4), + }, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, }, { ID: "file_exporter1", @@ -239,17 +378,24 @@ func TestEventExporterSameID(t *testing.T) { ExportPath: "/var/spool/cgrates/ees", Attempts: 1, Flags: utils.FlagsWithParams{}, - ContentFields: []*FCTemplate{ + Fields: []*FCTemplate{ {Tag: "CustomTag2", Path: "*exp.CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", true, utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, }, - HeaderFields: []*FCTemplate{}, - TrailerFields: []*FCTemplate{}, + contentFields: []*FCTemplate{ + {Tag: "CustomTag2", Path: "*exp.CustomPath2", Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("CustomValue2", true, utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + }, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, }, }, } for _, profile := range expectedEEsCfg.Exporters { - for _, v := range profile.ContentFields { + for _, v := range profile.Fields { + v.ComputePath() + } + for _, v := range profile.contentFields { v.ComputePath() } } diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index 436296e3f..51dfec1a9 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -58,7 +58,7 @@ "enabled": true, "attributes_conns":["*internal"], "cache": { - "*file_csv": {"limit": -1, "ttl": "1s", "static_ttl": false}, + "*file_csv": {"limit": -1, "ttl": "500ms", "static_ttl": false}, }, "exporters": [ { diff --git a/ees/filecsv_it_test.go b/ees/filecsv_it_test.go index c30edb963..ad7a00e14 100644 --- a/ees/filecsv_it_test.go +++ b/ees/filecsv_it_test.go @@ -21,9 +21,12 @@ along with this program. If not, see package ees import ( + "io/ioutil" "net/rpc" "os" "path" + "path/filepath" + "strings" "testing" "time" @@ -48,8 +51,9 @@ var ( testCsvStartEngine, testCsvRPCConn, testCsvExportEvent, + testCsvVerifyExports, testCsvStopCgrEngine, - //testCsvCleanDirectory, + testCsvCleanDirectory, } ) @@ -211,6 +215,28 @@ func testCsvExportEvent(t *testing.T) { time.Sleep(1 * time.Second) } +func testCsvVerifyExports(t *testing.T) { + var files []string + err := filepath.Walk("/tmp/testExport/", func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, utils.CSVSuffix) { + files = append(files, path) + } + return nil + }) + if err != nil { + t.Error(err) + } + if len(files) != 1 { + t.Errorf("Expected %+v, received: %+v", 1, len(files)) + } + eCnt := "dbafe9c8614c785a65aabd116dd3959c3c56f7f6,*default,*voice,dsafdsaf,*rated,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10000000000,1.01\nea1f1968cc207859672c332364fc7614c86b04c5,*default,*data,abcdef,*rated,AnotherTenant,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10,0.012\n2478e9f18ebcd3c684f3c14596b8bfeab2b0d6d4,*default,*sms,sdfwer,*rated,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,1,0.15\n" + if outContent1, err := ioutil.ReadFile(files[0]); err != nil { + t.Error(err) + } else if eCnt != string(outContent1) { + t.Errorf("Expecting: \n<%q>, \nreceived: \n<%q>", eCnt, string(outContent1)) + } +} + func testCsvStopCgrEngine(t *testing.T) { if err := engine.KillEngine(100); err != nil { t.Error(err)