From 4e80f5e7038063c1ae5e5295a01acaf2491413fa Mon Sep 17 00:00:00 2001 From: Trial97 Date: Wed, 7 Oct 2020 12:45:28 +0300 Subject: [PATCH] Added more tests for empty fields in ees --- config/eescfg.go | 9 ++++++ config/erscfg_test.go | 12 ++++++++ data/conf/samples/ees/cgrates.json | 37 +++++++++++++---------- ees/elastic_it_test.go | 37 +++++++++++++++++++++++ ees/filecsv.go | 6 +++- ees/httppost.go | 47 +++++++++++++++++------------- ees/httppost_it_test.go | 32 ++++++++++++++++++++ 7 files changed, 142 insertions(+), 38 deletions(-) diff --git a/config/eescfg.go b/config/eescfg.go index 25c447f95..9f24bf2bc 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -31,6 +31,15 @@ type EEsCfg struct { Exporters []*EventExporterCfg } +func (eeS *EEsCfg) GetDefaultExporter() *EventExporterCfg { + for _, es := range eeS.Exporters { + if es.ID == utils.MetaDefault { + return es + } + } + return nil +} + func (eeS *EEsCfg) loadFromJsonCfg(jsnCfg *EEsJsonCfg, msgTemplates map[string][]*FCTemplate, sep string, dfltExpCfg *EventExporterCfg) (err error) { if jsnCfg == nil { return diff --git a/config/erscfg_test.go b/config/erscfg_test.go index 3f06b7af2..5fd049959 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -834,3 +834,15 @@ func TestERsloadFromJsonCfg(t *testing.T) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cgrCfg.ersCfg)) } } + +func TestGetDefaultExporter(t *testing.T) { + ees := new(EEsCfg) + if dft := ees.GetDefaultExporter(); dft != nil { + t.Fatalf("Expeceted no default cfg, received: %s", utils.ToJSON(dft)) + } + if cfgCgr, err := NewDefaultCGRConfig(); err != nil { + t.Error(err) + } else if dft := cfgCgr.EEsCfg().GetDefaultExporter(); dft == nil || dft.ID != utils.MetaDefault { + t.Fatalf("Unexpected default cfg returned: %s", utils.ToJSON(dft)) + } +} diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index ffd49e3d1..46dec19df 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -72,22 +72,7 @@ "attempts": 1, "field_separator": ",", "filters": ["*string:~*req.ExporterUsed:CSVExporter"], - "fields":[ - {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, - {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, - {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, - ], + "fields":[], }, { "id": "CSVExporterComposed", @@ -386,6 +371,26 @@ "attempts": 1, "filters": ["*string:~*req.ExporterUsed:HTTPJsonMapExporterWithNoFields"], "fields":[] + }, + { + "id": "ElasticExporterWithNoFields", + "type": "*elastic", + "export_path": "http://localhost:9200", + "tenant": "cgrates.org", + "attempts": 1, + "filters": ["*string:~*req.ExporterUsed:ElasticExporterWithNoFields"], + "fields":[], + }, + { + "id": "HTTPPostExporterWithNoFields", + "type": "*http_post", + "export_path": "http://127.0.0.1:12080/event_http", + "tenant": "cgrates.org", + "flags": ["*attributes"], + "attribute_context": "customContext", + "attempts": 1, + "filters": ["*string:~*req.ExporterUsed:HTTPPostExporterWithNoFields"], + "fields":[] } ] }, diff --git a/ees/elastic_it_test.go b/ees/elastic_it_test.go index 699bb8624..02bfd54c6 100644 --- a/ees/elastic_it_test.go +++ b/ees/elastic_it_test.go @@ -205,6 +205,25 @@ func testElasticExportEvents(t *testing.T) { }, }, } + + eventSMSNoFields := &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "SMSEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("sms2", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.RunID: utils.MetaDefault, + "ExporterUsed": "ElasticExporterWithNoFields", + }, + }, + } var reply map[string]utils.MapStorage if err := elasticRpc.Call(utils.EventExporterSv1ProcessEvent, eventVoice, &reply); err != nil { t.Error(err) @@ -215,6 +234,9 @@ func testElasticExportEvents(t *testing.T) { if err := elasticRpc.Call(utils.EventExporterSv1ProcessEvent, eventSMS, &reply); err != nil { t.Error(err) } + if err := elasticRpc.Call(utils.EventExporterSv1ProcessEvent, eventSMSNoFields, &reply); err != nil { + t.Error(err) + } } func testElasticVerifyExports(t *testing.T) { @@ -301,6 +323,21 @@ func testElasticVerifyExports(t *testing.T) { if !reflect.DeepEqual(eMp, hit.(map[string]interface{})["_source"]) { t.Errorf("Expected %+v, received: %+v", eMp, hit.(map[string]interface{})["_source"]) } + case utils.Sha1("sms2", time.Unix(1383813745, 0).UTC().String()) + ":*default": + eMp := map[string]interface{}{ + utils.CGRID: utils.Sha1("sms2", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.RunID: utils.MetaDefault, + "ExporterUsed": "ElasticExporterWithNoFields", + } + if !reflect.DeepEqual(eMp, hit.(map[string]interface{})["_source"]) { + t.Errorf("Expected %+v, received: %+v", eMp, hit.(map[string]interface{})["_source"]) + } } } } diff --git a/ees/filecsv.go b/ees/filecsv.go index f45369ea6..1f5470191 100644 --- a/ees/filecsv.go +++ b/ees/filecsv.go @@ -102,6 +102,10 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { fCsv.Unlock() }() fCsv.dc[utils.NumberOfEvents] = fCsv.dc[utils.NumberOfEvents].(int64) + 1 + fields := fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ContentFields() + if len(fields) == 0 { // use the default fields in case of empty fields for exporter + fields = fCsv.cgrCfg.EEsCfg().GetDefaultExporter().ContentFields() + } var csvRecord []string req := utils.MapStorage(cgrEv.Event) @@ -111,7 +115,7 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { fCsv.cgrCfg.GeneralCfg().DefaultTimezone), fCsv.filterS) - if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ContentFields()); err != nil { + if err = eeReq.SetFields(fields); err != nil { return } for el := eeReq.cnt.GetFirstElement(); el != nil; el = el.Next() { diff --git a/ees/httppost.go b/ees/httppost.go index 4a16e83a8..103af2ce2 100644 --- a/ees/httppost.go +++ b/ees/httppost.go @@ -74,30 +74,35 @@ func (httpPost *HTTPPost) ExportEvent(cgrEv *utils.CGREvent) (err error) { httpPost.dc[utils.NumberOfEvents] = httpPost.dc[utils.NumberOfEvents].(int64) + 1 urlVals := url.Values{} - req := utils.MapStorage(cgrEv.Event) - eeReq := NewEventExporterRequest(req, httpPost.dc, - httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Tenant, - httpPost.cgrCfg.GeneralCfg().DefaultTenant, - utils.FirstNonEmpty(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Timezone, - httpPost.cgrCfg.GeneralCfg().DefaultTimezone), - httpPost.filterS) - - if err = eeReq.SetFields(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].ContentFields()); err != nil { - return - } - for el := eeReq.cnt.GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.cnt.Field(el.Value); err != nil { + if len(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].ContentFields()) == 0 { + for k, v := range cgrEv.Event { + urlVals.Set(k, utils.IfaceAsString(v)) + } + } else { + req := utils.MapStorage(cgrEv.Event) + eeReq := NewEventExporterRequest(req, httpPost.dc, + httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Tenant, + httpPost.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].Timezone, + httpPost.cgrCfg.GeneralCfg().DefaultTimezone), + httpPost.filterS) + if err = eeReq.SetFields(httpPost.cgrCfg.EEsCfg().Exporters[httpPost.cfgIdx].ContentFields()); err != nil { return } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - return fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) + for el := eeReq.cnt.GetFirstElement(); el != nil; el = el.Next() { + var nmIt utils.NMInterface + if nmIt, err = eeReq.cnt.Field(el.Value); err != nil { + return + } + itm, isNMItem := nmIt.(*config.NMItem) + if !isNMItem { + return fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) + } + if itm == nil { + continue // all attributes, not writable to diameter packet + } + urlVals.Set(strings.Join(itm.Path, utils.NestingSep), utils.IfaceAsString(itm.Data)) } - if itm == nil { - continue // all attributes, not writable to diameter packet - } - urlVals.Set(strings.Join(itm.Path, utils.NestingSep), utils.IfaceAsString(itm.Data)) } updateEEMetrics(httpPost.dc, cgrEv.Event, httpPost.cgrCfg.GeneralCfg().DefaultTimezone) if err = httpPost.httpPoster.PostValues(urlVals); err != nil && diff --git a/ees/httppost_it_test.go b/ees/httppost_it_test.go index 47772e25f..4cd38b7f9 100644 --- a/ees/httppost_it_test.go +++ b/ees/httppost_it_test.go @@ -200,6 +200,28 @@ func testHTTPExportEvent(t *testing.T) { }, }, } + + eventSMSNoFields := &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "SMSEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("sms2", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + utils.OriginID: "sms2", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.RunID: utils.MetaDefault, + "ExporterUsed": "HTTPPostExporterWithNoFields", + }, + }, + } + var reply map[string]utils.MapStorage if err := httpPostRpc.Call(utils.EventExporterSv1ProcessEvent, eventVoice, &reply); err != nil { t.Error(err) @@ -255,4 +277,14 @@ func testHTTPExportEvent(t *testing.T) { t.Errorf("Expected %+v, received: %+v", strVal, rcv) } } + if err := httpPostRpc.Call(utils.EventExporterSv1ProcessEvent, eventSMSNoFields, &reply); err != nil { + t.Error(err) + } + time.Sleep(10 * time.Millisecond) + // verify HTTPValues for eventSMS + for key, strVal := range eventSMSNoFields.Event { + if rcv := httpValues.Get(key); rcv != strVal { + t.Errorf("Expected %+v, received: %+v", strVal, rcv) + } + } }