From 7fbab0c3a2ce571ec739e0560a7374711ae78c8f Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 4 Sep 2020 11:15:50 +0300 Subject: [PATCH] Consider timezone and tenant from exporter first --- data/conf/samples/ees_internal/cgrates.json | 3 +++ data/conf/samples/ees_mongo/cgrates.json | 3 +++ data/conf/samples/ees_mysql/cgrates.json | 15 +++++++++------ ees/ee.go | 8 +++++++- ees/eereq.go | 12 ++++++++++-- ees/ees.go | 14 ++++++++++---- ees/ees_test.go | 4 ++-- ees/filecsv.go | 19 +++++++++++++++---- ees/filefwv.go | 21 +++++++++++++++++---- ees/httppost.go | 6 +++++- ees/virtualee.go | 9 +++++++-- 11 files changed, 88 insertions(+), 26 deletions(-) diff --git a/data/conf/samples/ees_internal/cgrates.json b/data/conf/samples/ees_internal/cgrates.json index 1d68a5853..e8d8a9516 100644 --- a/data/conf/samples/ees_internal/cgrates.json +++ b/data/conf/samples/ees_internal/cgrates.json @@ -65,6 +65,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -90,6 +91,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV2", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -116,6 +118,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV3", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": false, "attempts": 1, "field_separator": ",", diff --git a/data/conf/samples/ees_mongo/cgrates.json b/data/conf/samples/ees_mongo/cgrates.json index 11727688a..39e90cce6 100644 --- a/data/conf/samples/ees_mongo/cgrates.json +++ b/data/conf/samples/ees_mongo/cgrates.json @@ -69,6 +69,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -94,6 +95,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV2", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -120,6 +122,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV3", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": false, "attempts": 1, "field_separator": ",", diff --git a/data/conf/samples/ees_mysql/cgrates.json b/data/conf/samples/ees_mysql/cgrates.json index 8a42b059e..2abf340b1 100644 --- a/data/conf/samples/ees_mysql/cgrates.json +++ b/data/conf/samples/ees_mysql/cgrates.json @@ -67,6 +67,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -81,8 +82,8 @@ {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, - {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, ], @@ -92,6 +93,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV2", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": true, "attempts": 1, "field_separator": ",", @@ -107,8 +109,8 @@ {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, - {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, ], @@ -118,6 +120,7 @@ "type": "*file_csv", "export_path": "/tmp/testCSV3", "tenant": "cgrates.org", + "timezone": "UTC", "synchronous": false, "attempts": 1, "field_separator": ",", @@ -133,8 +136,8 @@ {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string}" }, - {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string}"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, ], diff --git a/ees/ee.go b/ees/ee.go index 191b3d459..cb7dc5a7f 100644 --- a/ees/ee.go +++ b/ees/ee.go @@ -34,7 +34,13 @@ type EventExporter interface { } // NewEventExporter produces exporters -func NewEventExporter(cgrCfg *config.CGRConfig, cfgIdx int, filterS *engine.FilterS, dc utils.MapStorage) (ee EventExporter, err error) { +func NewEventExporter(cgrCfg *config.CGRConfig, cfgIdx int, filterS *engine.FilterS) (ee EventExporter, err error) { + var dc utils.MapStorage + if dc, err = newEEMetrics(utils.FirstNonEmpty( + cgrCfg.EEsCfg().Exporters[cfgIdx].Timezone, + cgrCfg.GeneralCfg().DefaultTimezone)); err != nil { + return + } switch cgrCfg.EEsCfg().Exporters[cfgIdx].Type { case utils.MetaFileCSV: return NewFileCSVee(cgrCfg, cfgIdx, filterS, dc) diff --git a/ees/eereq.go b/ees/eereq.go index bf320d0c1..e63ecd877 100644 --- a/ees/eereq.go +++ b/ees/eereq.go @@ -33,17 +33,25 @@ import ( // NewEventExporterRequest returns a new EventExporterRequest func NewEventExporterRequest(req utils.DataProvider, dc utils.MapStorage, - tnt, timezone string, filterS *engine.FilterS) (eeR *EventExporterRequest) { + tntTpl config.RSRParsers, + dfltTenant, timezone string, filterS *engine.FilterS) (eeR *EventExporterRequest) { eeR = &EventExporterRequest{ req: req, tmz: timezone, - tnt: tnt, filterS: filterS, cnt: utils.NewOrderedNavigableMap(), hdr: utils.NewOrderedNavigableMap(), trl: utils.NewOrderedNavigableMap(), dc: dc, } + // populate tenant + if tntIf, err := eeR.ParseField( + &config.FCTemplate{Type: utils.META_COMPOSED, + Value: tntTpl}); err == nil && tntIf.(string) != "" { + eeR.tnt = tntIf.(string) + } else { + eeR.tnt = dfltTenant + } return } diff --git a/ees/ees.go b/ees/ees.go index 7986af5a4..9c8f65ae0 100644 --- a/ees/ees.go +++ b/ees/ees.go @@ -198,7 +198,7 @@ func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithIDs, rply *ma } } if !isCached { - if ee, err = NewEventExporter(eeS.cfg, cfgIdx, eeS.filterS, newEEMetrics()); err != nil { + if ee, err = NewEventExporter(eeS.cfg, cfgIdx, eeS.filterS); err != nil { return } if hasCache { @@ -265,13 +265,19 @@ func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithIDs, rply *ma return } -func newEEMetrics() utils.MapStorage { +func newEEMetrics(location string) (utils.MapStorage, error) { + tNow := time.Now() + loc, err := time.LoadLocation(location) + if err != nil { + return nil, err + } return utils.MapStorage{ utils.NumberOfEvents: int64(0), utils.PositiveExports: utils.StringSet{}, utils.NegativeExports: utils.StringSet{}, - utils.TimeNow: time.Now(), - } + utils.TimeNow: time.Date(tNow.Year(), tNow.Month(), tNow.Day(), + tNow.Hour(), tNow.Minute(), tNow.Second(), tNow.Nanosecond(), loc), + }, nil } func updateEEMetrics(dc utils.MapStorage, ev engine.MapEvent, timezone string) { diff --git a/ees/ees_test.go b/ees/ees_test.go index d906bb34a..737e902e0 100644 --- a/ees/ees_test.go +++ b/ees/ees_test.go @@ -28,7 +28,7 @@ import ( ) func TestUpdateEEMetrics(t *testing.T) { - dc := newEEMetrics() + dc, _ := newEEMetrics(utils.EmptyString) tnow := time.Now() ev := engine.MapEvent{ utils.AnswerTime: tnow, @@ -37,7 +37,7 @@ func TestUpdateEEMetrics(t *testing.T) { utils.ToR: utils.VOICE, utils.Usage: time.Second, } - exp := newEEMetrics() + exp, _ := newEEMetrics(utils.EmptyString) exp[utils.FirstEventATime] = tnow exp[utils.LastEventATime] = tnow exp[utils.FirstExpOrderID] = int64(1) diff --git a/ees/filecsv.go b/ees/filecsv.go index 818aedd2f..ee4ed2bf1 100644 --- a/ees/filecsv.go +++ b/ees/filecsv.go @@ -105,7 +105,10 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { for k, v := range cgrEv.Event { req[k] = v } - eeReq := NewEventExporterRequest(req, fCsv.dc, cgrEv.Tenant, fCsv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(req, fCsv.dc, fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Tenant, + fCsv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Timezone, + fCsv.cgrCfg.GeneralCfg().DefaultTimezone), fCsv.filterS) if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ContentFields()); err != nil { @@ -118,7 +121,8 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { } csvRecord = append(csvRecord, strVal) } - updateEEMetrics(fCsv.dc, cgrEv.Event, fCsv.cgrCfg.GeneralCfg().DefaultTimezone) + updateEEMetrics(fCsv.dc, cgrEv.Event, utils.FirstNonEmpty(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Timezone, + fCsv.cgrCfg.GeneralCfg().DefaultTimezone)) fCsv.csvWriter.Write(csvRecord) return } @@ -129,7 +133,10 @@ func (fCsv *FileCSVee) composeHeader() (err error) { return } var csvRecord []string - eeReq := NewEventExporterRequest(nil, fCsv.dc, fCsv.cgrCfg.GeneralCfg().DefaultTenant, fCsv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(nil, fCsv.dc, fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Tenant, + fCsv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Timezone, + fCsv.cgrCfg.GeneralCfg().DefaultTimezone), fCsv.filterS) if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].HeaderFields()); err != nil { return @@ -150,7 +157,11 @@ func (fCsv *FileCSVee) composeTrailer() (err error) { return } var csvRecord []string - eeReq := NewEventExporterRequest(nil, fCsv.dc, fCsv.cgrCfg.GeneralCfg().DefaultTenant, fCsv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(nil, fCsv.dc, + fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Tenant, + fCsv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Timezone, + fCsv.cgrCfg.GeneralCfg().DefaultTimezone), fCsv.filterS) if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].TrailerFields()); err != nil { return diff --git a/ees/filefwv.go b/ees/filefwv.go index af917b769..66fec8b56 100644 --- a/ees/filefwv.go +++ b/ees/filefwv.go @@ -95,7 +95,11 @@ func (fFwv *FileFWVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { for k, v := range cgrEv.Event { req[k] = v } - eeReq := NewEventExporterRequest(req, fFwv.dc, cgrEv.Tenant, fFwv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(req, fFwv.dc, + fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Tenant, + fFwv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Timezone, + fFwv.cgrCfg.GeneralCfg().DefaultTimezone), fFwv.filterS) if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].ContentFields()); err != nil { @@ -108,7 +112,8 @@ func (fFwv *FileFWVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { } records = append(records, strVal) } - updateEEMetrics(fFwv.dc, cgrEv.Event, fFwv.cgrCfg.GeneralCfg().DefaultTimezone) + updateEEMetrics(fFwv.dc, cgrEv.Event, utils.FirstNonEmpty(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Timezone, + fFwv.cgrCfg.GeneralCfg().DefaultTimezone)) for _, record := range append(records, "\n") { if _, err = io.WriteString(fFwv.file, record); err != nil { return @@ -123,7 +128,11 @@ func (fFwv *FileFWVee) composeHeader() (err error) { return } var records []string - eeReq := NewEventExporterRequest(nil, fFwv.dc, fFwv.cgrCfg.GeneralCfg().DefaultTenant, fFwv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(nil, fFwv.dc, + fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Tenant, + fFwv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Timezone, + fFwv.cgrCfg.GeneralCfg().DefaultTimezone), fFwv.filterS) if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].HeaderFields()); err != nil { return @@ -149,7 +158,11 @@ func (fFwv *FileFWVee) composeTrailer() (err error) { return } var records []string - eeReq := NewEventExporterRequest(nil, fFwv.dc, fFwv.cgrCfg.GeneralCfg().DefaultTenant, fFwv.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(nil, fFwv.dc, + fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Tenant, + fFwv.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Timezone, + fFwv.cgrCfg.GeneralCfg().DefaultTimezone), fFwv.filterS) if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].TrailerFields()); err != nil { return diff --git a/ees/httppost.go b/ees/httppost.go index 754b8df90..e1a1b67a8 100644 --- a/ees/httppost.go +++ b/ees/httppost.go @@ -79,7 +79,11 @@ func (httpPost *HTTPPost) ExportEvent(cgrEv *utils.CGREvent) (err error) { for k, v := range cgrEv.Event { req[k] = v } - eeReq := NewEventExporterRequest(req, httpPost.dc, cgrEv.Tenant, httpPost.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(req, httpPost.dc, + httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Tenant, + httpPost.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Timezone, + httpPost.cgrCfg.GeneralCfg().DefaultTimezone), httpPost.filterS) if err = eeReq.SetFields(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].ContentFields()); err != nil { diff --git a/ees/virtualee.go b/ees/virtualee.go index d4d5b22d8..0bb24a0ae 100644 --- a/ees/virtualee.go +++ b/ees/virtualee.go @@ -77,12 +77,17 @@ func (vEe *VirtualEe) ExportEvent(cgrEv *utils.CGREvent) (err error) { for k, v := range cgrEv.Event { req[k] = v } - eeReq := NewEventExporterRequest(req, vEe.dc, cgrEv.Tenant, vEe.cgrCfg.GeneralCfg().DefaultTimezone, + eeReq := NewEventExporterRequest(req, vEe.dc, + vEe.cgrCfg.EEsCfg().Exporters[vEe.cfgIdx].Tenant, + vEe.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(vEe.cgrCfg.EEsCfg().Exporters[vEe.cfgIdx].Timezone, + vEe.cgrCfg.GeneralCfg().DefaultTimezone), vEe.filterS) if err = eeReq.SetFields(vEe.cgrCfg.EEsCfg().Exporters[vEe.cfgIdx].ContentFields()); err != nil { return } - updateEEMetrics(vEe.dc, cgrEv.Event, vEe.cgrCfg.GeneralCfg().DefaultTimezone) + updateEEMetrics(vEe.dc, cgrEv.Event, utils.FirstNonEmpty(vEe.cgrCfg.EEsCfg().Exporters[vEe.cfgIdx].Timezone, + vEe.cgrCfg.GeneralCfg().DefaultTimezone)) return }