From f7806216259f4e45c41dfc141d6b28e40ebe60fc Mon Sep 17 00:00:00 2001 From: gezimbll Date: Thu, 8 Jun 2023 11:13:31 -0400 Subject: [PATCH] Add check for els nodes in config --- config/config_defaults.go | 4 +++- config/eescfg.go | 12 ++++++++++++ config/libconfig_json.go | 2 ++ data/conf/samples/ees/cgrates.json | 4 +++- ees/elastic.go | 10 ++++++---- go.mod | 3 ++- go.sum | 4 ++++ 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 6d520580b..e93450911 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -506,7 +506,9 @@ const CGRATES_CFG_JSON = ` // "csvFieldSeparator": ",", // separator used when reading the fields - // Elasticsearch options + // Elasticsearch options + "discoverNodesInterval":"", + "discoverNodesOnStart":false // "elsIndex": "", // ElsIndex // "elsIfPrimaryTerm": 0, // ElsIfPrimaryTerm // "elsIfSeqNo": 0, // ElsIfSeqNo diff --git a/config/eescfg.go b/config/eescfg.go index d51a55e0e..d56873ee6 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -157,6 +157,8 @@ type EventExporterOpts struct { CSVFieldSeparator *string ElsIndex *string ElsIfPrimaryTerm *int + DiscoverNodesOnStart *bool + DiscoverNodesInterval *time.Duration ElsIfSeqNo *int ElsOpType *string ElsPipeline *string @@ -250,6 +252,16 @@ func (eeOpts *EventExporterOpts) loadFromJSONCfg(jsnCfg *EventExporterOptsJson) if jsnCfg.CSVFieldSeparator != nil { eeOpts.CSVFieldSeparator = jsnCfg.CSVFieldSeparator } + if jsnCfg.DiscoverNodesOnStart != nil { + eeOpts.DiscoverNodesOnStart = jsnCfg.DiscoverNodesOnStart + } + if jsnCfg.DiscoverNodesInterval != nil { + var discoverNodesInterval time.Duration + if discoverNodesInterval, err = utils.ParseDurationWithSecs(*jsnCfg.DiscoverNodesInterval); err != nil { + return + } + eeOpts.DiscoverNodesInterval = utils.DurationPointer(discoverNodesInterval) + } if jsnCfg.ElsIndex != nil { eeOpts.ElsIndex = jsnCfg.ElsIndex } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index db77ce844..f7c9fd927 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -299,6 +299,8 @@ type EEsJsonCfg struct { type EventExporterOptsJson struct { CSVFieldSeparator *string `json:"csvFieldSeparator"` + DiscoverNodesInterval *string `json:"discoverNodesInterval"` + DiscoverNodesOnStart *bool `json:"discoverNodesOnStart"` ElsIndex *string `json:"elsIndex"` ElsIfPrimaryTerm *int `json:"elsIfPrimaryTerm"` ElsIfSeqNo *int `json:"elsIfSeqNo"` diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index 22a45345c..781598f7b 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -359,10 +359,12 @@ { "id": "ElasticsearchExporterCluster", "type": "*els", - "export_path": "http://192.168.56.64:9200;http://192.168.56.22:9200", + "export_path": "http://192.168.56.22:9200;http://192.168.56.64:9200", "attempts": 1, "opts": { "elsIndex": "cdrs", + "discoverNodesInterval":"30s", + "discoverNodesOnStart":true, //"elsIfPrimaryTerm": 0, //"elsIfSeqNo": 0, "elsOpType": "", diff --git a/ees/elastic.go b/ees/elastic.go index a76a852bd..8203c57de 100644 --- a/ees/elastic.go +++ b/ees/elastic.go @@ -26,11 +26,11 @@ import ( "strings" "sync" - "github.com/elastic/go-elasticsearch/esapi" + "github.com/elastic/go-elasticsearch/v8/esapi" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" - elasticsearch "github.com/elastic/go-elasticsearch" + elasticsearch "github.com/elastic/go-elasticsearch/v8" ) func NewElasticEE(cfg *config.EventExporterCfg, dc *utils.SafeMapStorage) (eEe *ElasticEE, err error) { @@ -92,7 +92,10 @@ func (eEe *ElasticEE) Connect() (err error) { // create the client if eEe.eClnt == nil { eEe.eClnt, err = elasticsearch.NewClient( - elasticsearch.Config{Addresses: strings.Split(eEe.Cfg().ExportPath, utils.InfieldSep)}, + elasticsearch.Config{ + DiscoverNodesInterval: *eEe.Cfg().Opts.DiscoverNodesInterval, + DiscoverNodesOnStart: *eEe.Cfg().Opts.DiscoverNodesOnStart, + Addresses: strings.Split(eEe.Cfg().ExportPath, utils.InfieldSep)}, ) } eEe.Unlock() @@ -118,7 +121,6 @@ func (eEe *ElasticEE) ExportEvent(ev any, key string) (err error) { IfPrimaryTerm: eEe.opts.IfPrimaryTerm, IfSeqNo: eEe.opts.IfSeqNo, OpType: eEe.opts.OpType, - Parent: eEe.opts.Parent, Pipeline: eEe.opts.Pipeline, Routing: eEe.opts.Routing, Timeout: eEe.opts.Timeout, diff --git a/go.mod b/go.mod index 0716ce1fa..2fbfdc7ee 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/cgrates/ugocodec v0.0.0-20201023092048-df93d0123f60 github.com/creack/pty v1.1.11 github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/elastic/go-elasticsearch v0.0.0 + github.com/elastic/go-elasticsearch/v8 v8.8.0 github.com/ericlagergren/decimal v0.0.0-20191206042408-88212e6cfca9 github.com/fiorix/go-diameter/v4 v4.0.2 github.com/fsnotify/fsnotify v1.4.9 @@ -70,6 +70,7 @@ require ( github.com/couchbase/ghistogram v0.1.0 // indirect github.com/couchbase/moss v0.1.0 // indirect github.com/couchbase/vellum v1.0.2 // indirect + github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6 // indirect github.com/frankban/quicktest v1.11.3 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect diff --git a/go.sum b/go.sum index 6d74a6815..69fdf1179 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,12 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6 h1:1+44gxLdKRnR/Bx/iAtr+XqNcE4e0oODa63+FABNANI= +github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= +github.com/elastic/go-elasticsearch/v8 v8.8.0 h1:yNBPlXNo6wstMG7I3KiZPbLFgA82RMryYqkh1xBMV3A= +github.com/elastic/go-elasticsearch/v8 v8.8.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=