diff --git a/data/ansible/calls/asterisk/main.yaml b/data/ansible/calls/asterisk/main.yaml index dceee686b..4a9ea2852 100644 --- a/data/ansible/calls/asterisk/main.yaml +++ b/data/ansible/calls/asterisk/main.yaml @@ -5,7 +5,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/calls/freeswitch/main.yaml b/data/ansible/calls/freeswitch/main.yaml index 11ea78f21..e7b2f499e 100644 --- a/data/ansible/calls/freeswitch/main.yaml +++ b/data/ansible/calls/freeswitch/main.yaml @@ -5,7 +5,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/calls/kamailio/main.yaml b/data/ansible/calls/kamailio/main.yaml index 679cc41a1..c9729c5e9 100644 --- a/data/ansible/calls/kamailio/main.yaml +++ b/data/ansible/calls/kamailio/main.yaml @@ -5,7 +5,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/deb_packages/main.yaml b/data/ansible/deb_packages/main.yaml index 51d3bba9a..372b32654 100644 --- a/data/ansible/deb_packages/main.yaml +++ b/data/ansible/deb_packages/main.yaml @@ -29,7 +29,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/docker/main.yaml b/data/ansible/docker/main.yaml index 1a1b87280..24f9139c6 100644 --- a/data/ansible/docker/main.yaml +++ b/data/ansible/docker/main.yaml @@ -17,7 +17,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/drone/main.yaml b/data/ansible/drone/main.yaml index fa9fea503..6d07135bc 100644 --- a/data/ansible/drone/main.yaml +++ b/data/ansible/drone/main.yaml @@ -17,7 +17,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.15.7' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/integration_tests/main.yaml b/data/ansible/integration_tests/main.yaml index 117a2017b..772d5e26c 100644 --- a/data/ansible/integration_tests/main.yaml +++ b/data/ansible/integration_tests/main.yaml @@ -5,7 +5,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/data/ansible/rpm_packages/main.yaml b/data/ansible/rpm_packages/main.yaml index 5fabccc84..fe6507e76 100644 --- a/data/ansible/rpm_packages/main.yaml +++ b/data/ansible/rpm_packages/main.yaml @@ -16,7 +16,7 @@ ##################### Golang Vars ############################# ############################################################### # Go language SDK version number - golang_version: '1.14' + golang_version: '1.16.1' go_version_target: "go version go{{ golang_version }} linux/amd64" # Mirror to download the Go language SDK redistributable package from golang_mirror: 'https://storage.googleapis.com/golang' diff --git a/ees/elastic.go b/ees/elastic.go index 17b59ae19..cec1b5263 100644 --- a/ees/elastic.go +++ b/ees/elastic.go @@ -153,18 +153,8 @@ func (eEe *ElasticEe) ExportEvent(cgrEv *utils.CGREvent) (err error) { return } for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaExp].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - err = fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - return - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaExp].Field(el.Value) + itm := nmIt.(*config.NMItem) valMp[strings.Join(itm.Path, utils.NestingSep)] = utils.IfaceAsString(itm.Data) } } diff --git a/ees/elastic_test.go b/ees/elastic_test.go index e1d96eefa..924dc67d4 100644 --- a/ees/elastic_test.go +++ b/ees/elastic_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -244,3 +245,67 @@ func TestInitCase10(t *testing.T) { t.Errorf("Expected %+v \n but got %+v", utils.ToJSON(eeExpect), utils.ToJSON(ee.opts.WaitForActiveShards)) } } + +func TestElasticExportEvent(t *testing.T) { + cgrCfg := config.NewDefaultCGRConfig() + cgrEv := new(utils.CGREvent) + newIDb := engine.NewInternalDB(nil, nil, true) + newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) + filterS := engine.NewFilterS(cgrCfg, nil, newDM) + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + eEe, err := NewElasticExporter(cgrCfg, 0, filterS, dc) + if err != nil { + t.Error(err) + } + cgrEv.Event = map[string]interface{}{ + "test1": "value", + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + }, + { + Path: "*exp.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("*req.field2", utils.InfieldSep), + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + errExpect := `unsupported protocol scheme ""` + if err := eEe.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but got %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + if err := eEe.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but got %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := eEe.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + eEe.OnEvicted("test", "test") +} diff --git a/ees/filecsv.go b/ees/filecsv.go index 76dc7a162..4410d8229 100644 --- a/ees/filecsv.go +++ b/ees/filecsv.go @@ -106,6 +106,7 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { var csvRecord []string if len(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ContentFields()) == 0 { + csvRecord = make([]string, 0, len(cgrEv.Event)) for _, val := range cgrEv.Event { csvRecord = append(csvRecord, utils.IfaceAsString(val)) } @@ -124,13 +125,7 @@ func (fCsv *FileCSVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ContentFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaExp].FieldAsString(el.Value.Slice()); err != nil { - return - } - csvRecord = append(csvRecord, strVal) - } + csvRecord = eeReq.OrdNavMP[utils.MetaExp].OrderedFieldsAsStrings() } updateEEMetrics(fCsv.dc, cgrEv.Event, utils.FirstNonEmpty(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Timezone, @@ -143,7 +138,6 @@ func (fCsv *FileCSVee) composeHeader() (err error) { if len(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].HeaderFields()) == 0 { return } - var csvRecord []string oNm := map[string]*utils.OrderedNavigableMap{ utils.MetaHdr: utils.NewOrderedNavigableMap(), } @@ -156,14 +150,7 @@ func (fCsv *FileCSVee) composeHeader() (err error) { if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].HeaderFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaHdr].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaHdr].FieldAsString(el.Value.Slice()); err != nil { - return - } - csvRecord = append(csvRecord, strVal) - } - return fCsv.csvWriter.Write(csvRecord) + return fCsv.csvWriter.Write(eeReq.OrdNavMP[utils.MetaHdr].OrderedFieldsAsStrings()) } // Compose and cache the trailer @@ -171,7 +158,6 @@ func (fCsv *FileCSVee) composeTrailer() (err error) { if len(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].TrailerFields()) == 0 { return } - var csvRecord []string oNm := map[string]*utils.OrderedNavigableMap{ utils.MetaTrl: utils.NewOrderedNavigableMap(), } @@ -184,14 +170,8 @@ func (fCsv *FileCSVee) composeTrailer() (err error) { if err = eeReq.SetFields(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].TrailerFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaTrl].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaTrl].FieldAsString(el.Value.Slice()); err != nil { - return - } - csvRecord = append(csvRecord, strVal) - } - return fCsv.csvWriter.Write(csvRecord) + + return fCsv.csvWriter.Write(eeReq.OrdNavMP[utils.MetaTrl].OrderedFieldsAsStrings()) } func (fCsv *FileCSVee) GetMetrics() utils.MapStorage { diff --git a/ees/filecsv_test.go b/ees/filecsv_test.go index e7e1debfe..a31c78b07 100644 --- a/ees/filecsv_test.go +++ b/ees/filecsv_test.go @@ -219,11 +219,11 @@ func TestFileCsvExportEvent(t *testing.T) { cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Fields = []*config.FCTemplate{ { Path: "*exp.1", Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Value: config.NewRSRParsersMustCompile("~*req.test1", utils.InfieldSep), }, { Path: "*exp.2", Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + Value: config.NewRSRParsersMustCompile("3", utils.InfieldSep), }, } for _, field := range cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Fields { @@ -232,12 +232,18 @@ func TestFileCsvExportEvent(t *testing.T) { if err := fCsv.ExportEvent(cgrEv); err != nil { t.Error(err) } + csvNW.Flush() + expected := "value\n" + if expected != byteBuff.String() { + t.Errorf("Expected %q but received %q", expected, byteBuff.String()) + } + byteBuff.Reset() cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].ComputeFields() if err := fCsv.ExportEvent(cgrEv); err != nil { t.Error(err) } csvNW.Flush() - expected := "value\n\n" + expected = "value,3\n" if expected != byteBuff.String() { t.Errorf("Expected %q but received %q", expected, byteBuff.String()) } @@ -263,5 +269,5 @@ func TestFileCsvExportEvent(t *testing.T) { if err := fCsv.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { t.Errorf("Expected %q but received %q", errExpect, err) } - + fCsv.OnEvicted("test", "test") } diff --git a/ees/filefwv.go b/ees/filefwv.go index 173aa2afa..28aabb5d0 100644 --- a/ees/filefwv.go +++ b/ees/filefwv.go @@ -95,6 +95,7 @@ func (fFwv *FileFWVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { fFwv.dc[utils.NumberOfEvents] = fFwv.dc[utils.NumberOfEvents].(int64) + 1 var records []string if len(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].ContentFields()) == 0 { + records = make([]string, 0, len(cgrEv.Event)) for _, val := range cgrEv.Event { records = append(records, utils.IfaceAsString(val)) } @@ -113,22 +114,17 @@ func (fFwv *FileFWVee) ExportEvent(cgrEv *utils.CGREvent) (err error) { if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].ContentFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaExp].FieldAsString(el.Value.Slice()); err != nil { - return - } - records = append(records, strVal) - } + records = eeReq.OrdNavMP[utils.MetaExp].OrderedFieldsAsStrings() } updateEEMetrics(fFwv.dc, cgrEv.Event, utils.FirstNonEmpty(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Timezone, fFwv.cgrCfg.GeneralCfg().DefaultTimezone)) - for _, record := range append(records, "\n") { + for _, record := range records { if _, err = io.WriteString(fFwv.file, record); err != nil { return } } + _, err = io.WriteString(fFwv.file, "\n") return } @@ -137,7 +133,6 @@ func (fFwv *FileFWVee) composeHeader() (err error) { if len(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].HeaderFields()) == 0 { return } - var records []string oNm := map[string]*utils.OrderedNavigableMap{ utils.MetaHdr: utils.NewOrderedNavigableMap(), } @@ -150,18 +145,12 @@ func (fFwv *FileFWVee) composeHeader() (err error) { if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].HeaderFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaHdr].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaHdr].FieldAsString(el.Value.Slice()); err != nil { - return - } - records = append(records, strVal) - } - for _, record := range append(records, "\n") { + for _, record := range eeReq.OrdNavMP[utils.MetaHdr].OrderedFieldsAsStrings() { if _, err = io.WriteString(fFwv.file, record); err != nil { return } } + _, err = io.WriteString(fFwv.file, "\n") return } @@ -170,7 +159,6 @@ func (fFwv *FileFWVee) composeTrailer() (err error) { if len(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].TrailerFields()) == 0 { return } - var records []string oNm := map[string]*utils.OrderedNavigableMap{ utils.MetaTrl: utils.NewOrderedNavigableMap(), } @@ -183,18 +171,12 @@ func (fFwv *FileFWVee) composeTrailer() (err error) { if err = eeReq.SetFields(fFwv.cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].TrailerFields()); err != nil { return } - for el := eeReq.OrdNavMP[utils.MetaTrl].GetFirstElement(); el != nil; el = el.Next() { - var strVal string - if strVal, err = eeReq.OrdNavMP[utils.MetaTrl].FieldAsString(el.Value.Slice()); err != nil { - return - } - records = append(records, strVal) - } - for _, record := range append(records, "\n") { + for _, record := range eeReq.OrdNavMP[utils.MetaTrl].OrderedFieldsAsStrings() { if _, err = io.WriteString(fFwv.file, record); err != nil { return } } + _, err = io.WriteString(fFwv.file, "\n") return } diff --git a/ees/filefwv_test.go b/ees/filefwv_test.go index 68f2228da..eb7020e38 100644 --- a/ees/filefwv_test.go +++ b/ees/filefwv_test.go @@ -55,7 +55,16 @@ func TestFileFwvGetMetrics(t *testing.T) { } } -func TestFilveFwvComposeHeader(t *testing.T) { +// type MyError struct{} + +// func (m *MyError) Error() string { +// return "ERR" +// } + +// func (nopCloser) WriteString(w io.Writer, s string) error { +// return &MyError{} +// } +func TestFileFwvComposeHeader(t *testing.T) { cgrCfg := config.NewDefaultCGRConfig() newIDb := engine.NewInternalDB(nil, nil, true) newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) @@ -116,6 +125,7 @@ func TestFilveFwvComposeHeader(t *testing.T) { if err := fFwv.composeHeader(); err == nil || err.Error() != errExpect { t.Errorf("Expected %q but received %q", errExpect, err) } + } func TestFileFwvComposeTrailer(t *testing.T) { @@ -210,11 +220,11 @@ func TestFileFwvExportEvent(t *testing.T) { cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Fields = []*config.FCTemplate{ { Path: "*exp.1", Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Value: config.NewRSRParsersMustCompile("~*req.test1", utils.InfieldSep), }, { Path: "*exp.2", Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + Value: config.NewRSRParsersMustCompile("3", utils.InfieldSep), }, } for _, field := range cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].Fields { @@ -223,12 +233,18 @@ func TestFileFwvExportEvent(t *testing.T) { if err := fFwv.ExportEvent(cgrEv); err != nil { t.Error(err) } + csvNW.Flush() + expected := "value\n" + if expected != byteBuff.String() { + t.Errorf("Expected %q but received %q", expected, byteBuff.String()) + } cgrCfg.EEsCfg().Exporters[fFwv.cfgIdx].ComputeFields() + byteBuff.Reset() if err := fFwv.ExportEvent(cgrEv); err != nil { t.Error(err) } csvNW.Flush() - expected := "value\n\n" + expected = "value3\n" if expected != byteBuff.String() { t.Errorf("Expected %q but received %q", expected, byteBuff.String()) } diff --git a/ees/httpjsonmap.go b/ees/httpjsonmap.go index cef04bf6a..9c5f353a0 100644 --- a/ees/httpjsonmap.go +++ b/ees/httpjsonmap.go @@ -20,7 +20,6 @@ package ees import ( "encoding/json" - "fmt" "net/http" "strings" "sync" @@ -100,18 +99,8 @@ func (httpEE *HTTPjsonMapEE) ExportEvent(cgrEv *utils.CGREvent) (err error) { return } for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaExp].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - err = fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - return - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaExp].Field(el.Value) + itm := nmIt.(*config.NMItem) valMp[strings.Join(itm.Path, utils.NestingSep)] = utils.IfaceAsString(itm.Data) } if hdr, err = httpEE.composeHeader(); err != nil { @@ -157,18 +146,8 @@ func (httpEE *HTTPjsonMapEE) composeHeader() (hdr http.Header, err error) { return } for el := eeReq.OrdNavMP[utils.MetaHdr].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaHdr].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - err = fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - return - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaHdr].Field(el.Value) //Safe to ignore error, since the path always exists + itm := nmIt.(*config.NMItem) //We only store nmItems in the map hdr.Set(strings.Join(itm.Path, utils.NestingSep), utils.IfaceAsString(itm.Data)) } return diff --git a/ees/httpjsonmap_test.go b/ees/httpjsonmap_test.go new file mode 100644 index 000000000..25f8e87a9 --- /dev/null +++ b/ees/httpjsonmap_test.go @@ -0,0 +1,202 @@ +/* +Real-time Online/Offline Charging System (OerS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package ees + +import ( + "reflect" + "testing" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +func TestHttpJsonMapID(t *testing.T) { + httpEE := &HTTPjsonMapEE{ + id: "3", + } + if rcv := httpEE.ID(); !reflect.DeepEqual(rcv, "3") { + t.Errorf("Expected %+v but got %+v", "3", rcv) + } +} + +func TestHttpJsonMapGetMetrics(t *testing.T) { + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + httpEE := &HTTPjsonMapEE{ + dc: dc, + } + + if rcv := httpEE.GetMetrics(); !reflect.DeepEqual(rcv, httpEE.dc) { + t.Errorf("Expected %+v \n but got %+v", utils.ToJSON(rcv), utils.ToJSON(httpEE.dc)) + } +} + +func TestHttpJsonMapExportEvent(t *testing.T) { + cgrCfg := config.NewDefaultCGRConfig() + cgrCfg.EEsCfg().Exporters[0].Type = utils.MetaSQSjsonMap + cgrEv := new(utils.CGREvent) + newIDb := engine.NewInternalDB(nil, nil, true) + newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) + filterS := engine.NewFilterS(cgrCfg, nil, newDM) + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + + httpEE, err := NewHTTPjsonMapEE(cgrCfg, 0, filterS, dc) + if err != nil { + t.Error(err) + } + cgrEv.Event = map[string]interface{}{ + "test": "string", + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + }, + { + Path: "*exp.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("*req.field2", utils.InfieldSep), + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + errExpect := `Post "/var/spool/cgrates/ees": unsupported protocol scheme ""` + if err := httpEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + if err := httpEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := httpEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + }, + { + Path: "*exp.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + }, + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := httpEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + httpEE.OnEvicted("test", "test") +} + +func TestHttpJsonMapComposeHeader(t *testing.T) { + cgrCfg := config.NewDefaultCGRConfig() + cgrCfg.EEsCfg().Exporters[0].Type = utils.MetaHTTPjson + newIDb := engine.NewInternalDB(nil, nil, true) + newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) + filterS := engine.NewFilterS(cgrCfg, nil, newDM) + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + httpEE, err := NewHTTPjsonMapEE(cgrCfg, 0, filterS, dc) + if err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + }, + { + Path: "*hdr.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field2", utils.InfieldSep), + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + if _, err := httpEE.composeHeader(); err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + if _, err := httpEE.composeHeader(); err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect := "inline parse error for string: <*wrong-type>" + if _, err := httpEE.composeHeader(); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } +} diff --git a/ees/httppost.go b/ees/httppost.go index 9ffdf000f..75b4f584c 100644 --- a/ees/httppost.go +++ b/ees/httppost.go @@ -19,7 +19,6 @@ along with this program. If not, see package ees import ( - "fmt" "net/http" "net/url" "strings" @@ -96,17 +95,8 @@ func (httpPost *HTTPPost) ExportEvent(cgrEv *utils.CGREvent) (err error) { return } for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaExp].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - return fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaExp].Field(el.Value) + itm := nmIt.(*config.NMItem) urlVals.Set(strings.Join(itm.Path, utils.NestingSep), utils.IfaceAsString(itm.Data)) } if hdr, err = httpPost.composeHeader(); err != nil { @@ -150,18 +140,8 @@ func (httpPost *HTTPPost) composeHeader() (hdr http.Header, err error) { return } for el := eeReq.OrdNavMP[utils.MetaHdr].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaHdr].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - err = fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - return - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaHdr].Field(el.Value) + itm := nmIt.(*config.NMItem) hdr.Set(strings.Join(itm.Path, utils.NestingSep), utils.IfaceAsString(itm.Data)) } return diff --git a/ees/httppost_test.go b/ees/httppost_test.go index 59541cc5c..da64b71e7 100644 --- a/ees/httppost_test.go +++ b/ees/httppost_test.go @@ -22,6 +22,8 @@ import ( "reflect" "testing" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -50,3 +52,150 @@ func TestHttpPostGetMetrics(t *testing.T) { t.Errorf("Expected %+v \n but got %+v", utils.ToJSON(rcv), utils.ToJSON(httpPost.dc)) } } + +func TestHttpPostExportEvent(t *testing.T) { + cgrCfg := config.NewDefaultCGRConfig() + cgrCfg.EEsCfg().Exporters[0].Type = utils.MetaHTTPPost + cgrEv := new(utils.CGREvent) + newIDb := engine.NewInternalDB(nil, nil, true) + newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) + filterS := engine.NewFilterS(cgrCfg, nil, newDM) + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + httpPost, err := NewHTTPPostEe(cgrCfg, 0, filterS, dc) + if err != nil { + t.Error(err) + } + cgrEv.Event = map[string]interface{}{ + "Test1": 3, + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + }, + { + Path: "*exp.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("*req.field2", utils.InfieldSep), + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + errExpect := `Post "/var/spool/cgrates/ees": unsupported protocol scheme ""` + if err := httpPost.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + if err := httpPost.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := httpPost.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + }, + { + Path: "*exp.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + }, + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := httpPost.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + httpPost.OnEvicted("test", "test") +} + +func TestHttpPostComposeHeader(t *testing.T) { + cgrCfg := config.NewDefaultCGRConfig() + cgrCfg.EEsCfg().Exporters[0].Type = utils.MetaHTTPPost + newIDb := engine.NewInternalDB(nil, nil, true) + newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) + filterS := engine.NewFilterS(cgrCfg, nil, newDM) + dc, err := newEEMetrics(utils.FirstNonEmpty( + "Local", + utils.EmptyString, + )) + if err != nil { + t.Error(err) + } + httpPost, err := NewHTTPPostEe(cgrCfg, 0, filterS, dc) + if err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + }, + { + Path: "*hdr.2", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field2", utils.InfieldSep), + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + if _, err := httpPost.composeHeader(); err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + if _, err := httpPost.composeHeader(); err != nil { + t.Error(err) + } + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*hdr.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect := "inline parse error for string: <*wrong-type>" + if _, err := httpPost.composeHeader(); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } +} diff --git a/ees/posterjsonmap.go b/ees/posterjsonmap.go index 5de61e3f2..26d3c823e 100644 --- a/ees/posterjsonmap.go +++ b/ees/posterjsonmap.go @@ -20,7 +20,6 @@ package ees import ( "encoding/json" - "fmt" "strings" "sync" @@ -111,18 +110,8 @@ func (pstrEE *PosterJSONMapEE) ExportEvent(cgrEv *utils.CGREvent) (err error) { return } for el := eeReq.OrdNavMP[utils.MetaExp].GetFirstElement(); el != nil; el = el.Next() { - var nmIt utils.NMInterface - if nmIt, err = eeReq.OrdNavMP[utils.MetaExp].Field(el.Value); err != nil { - return - } - itm, isNMItem := nmIt.(*config.NMItem) - if !isNMItem { - err = fmt.Errorf("cannot encode reply value: %s, err: not NMItems", utils.ToJSON(el.Value)) - return - } - if itm == nil { - continue // all attributes, not writable to diameter packet - } + nmIt, _ := eeReq.OrdNavMP[utils.MetaExp].Field(el.Value) + itm := nmIt.(*config.NMItem) valMp[strings.Join(itm.Path, utils.NestingSep)] = utils.IfaceAsString(itm.Data) } } diff --git a/ees/posterjsonmap_test.go b/ees/posterjsonmap_test.go index a8d104bd2..02921cbeb 100644 --- a/ees/posterjsonmap_test.go +++ b/ees/posterjsonmap_test.go @@ -171,7 +171,7 @@ func TestPosterJsonMapExportEvent(t *testing.T) { }, { Path: "*exp.2", Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.field2", utils.InfieldSep), + Value: config.NewRSRParsersMustCompile("*req.field2", utils.InfieldSep), }, } for _, field := range cgrCfg.EEsCfg().Exporters[pstrEE.cfgIdx].Fields { @@ -185,5 +185,25 @@ func TestPosterJsonMapExportEvent(t *testing.T) { if err := pstrEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { t.Errorf("Expected %q but received %q", errExpect, err) } - //////// + cgrCfg.EEsCfg().Exporters[0].Fields = []*config.FCTemplate{ + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + { + Path: "*exp.1", Type: utils.MetaVariable, + Value: config.NewRSRParsersMustCompile("~*req.field1", utils.InfieldSep), + Filters: []string{"*wrong-type"}, + }, + } + for _, field := range cgrCfg.EEsCfg().Exporters[0].Fields { + field.ComputePath() + } + cgrCfg.EEsCfg().Exporters[0].ComputeFields() + errExpect = "inline parse error for string: <*wrong-type>" + if err := pstrEE.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but received %q", errExpect, err) + } + pstrEE.OnEvicted("test", "test") } diff --git a/ees/virtualee_test.go b/ees/virtualee_test.go index 50b883ba5..8bd50f993 100644 --- a/ees/virtualee_test.go +++ b/ees/virtualee_test.go @@ -115,4 +115,5 @@ func TestVirtualEeExportEvent(t *testing.T) { if err := vEe.ExportEvent(cgrEv); err == nil || err.Error() != errExpect { t.Errorf("Expected %q but received %q", errExpect, err) } + vEe.OnEvicted("test", "test") } diff --git a/utils/orderednavigablemap.go b/utils/orderednavigablemap.go index 83f26e6bb..87b1ae281 100644 --- a/utils/orderednavigablemap.go +++ b/utils/orderednavigablemap.go @@ -192,3 +192,13 @@ func (onm *OrderedNavigableMap) RemoveAll() { onm.orderIdx = NewPathItemList() onm.orderRef = make(map[string][]*PathItemElement) } + +// OrderedFieldsAsStrings returns the elements as strings in order they were inserted +func (onm *OrderedNavigableMap) OrderedFieldsAsStrings() (flds []string) { + flds = make([]string, 0, onm.Len()) + for el := onm.GetFirstElement(); el != nil; el = el.Next() { + fld, _ := onm.Field(el.Value) + flds = append(flds, IfaceAsString(fld.Interface())) + } + return +} diff --git a/utils/orderednavigablemap_test.go b/utils/orderednavigablemap_test.go index f7f662b46..2348b1c2a 100644 --- a/utils/orderednavigablemap_test.go +++ b/utils/orderednavigablemap_test.go @@ -967,4 +967,9 @@ func TestOrderedNavigableMapOrderedFields(t *testing.T) { if !reflect.DeepEqual(exp, rcv) { t.Errorf("Expected %+v, received %+v", exp, rcv) } + exp2 := []string{"1003", "1004", "1006"} + rcv2 := nm.OrderedFieldsAsStrings() + if !reflect.DeepEqual(exp2, rcv2) { + t.Errorf("Expected %+v, received %+v", exp2, rcv2) + } }