diff --git a/config/config.go b/config/config.go index 3b9573579..ddc23b04b 100755 --- a/config/config.go +++ b/config/config.go @@ -153,6 +153,7 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.cacheCfg.Partitions = make(map[string]*CacheParamCfg) cfg.listenCfg = new(ListenCfg) cfg.httpCfg = new(HTTPCfg) + cfg.httpCfg.ClientOpts = make(map[string]interface{}) cfg.filterSCfg = new(FilterSCfg) cfg.ralsCfg = new(RalsCfg) cfg.ralsCfg.MaxComputedUsage = make(map[string]time.Duration) diff --git a/config/config_defaults.go b/config/config_defaults.go index 885588a7c..ef881b876 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -31,7 +31,6 @@ const CGRATES_CFG_JSON = ` "node_id": "", // identifier of this instance in the cluster, if empty it will be autogenerated "logger":"*syslog", // controls the destination of logs <*syslog|*stdout> "log_level": 6, // control the level of messages logged (0-emerg to 7-debug) - "http_skip_tls_verify": false, // if enabled HttpClient will accept any TLS certificate "rounding_decimals": 5, // system level precision for floats "dbdata_encoding": "*msgpack", // encoding used to store object data in strings: <*msgpack|*json> "tpexport_dir": "/var/spool/cgrates/tpe", // path towards export folder for offline TariffPlans @@ -189,6 +188,24 @@ const CGRATES_CFG_JSON = ` "http_cdrs": "/cdr_http", // CDRS relative URL ("" to disable) "use_basic_auth": false, // use basic authentication "auth_users": {}, // basic authentication usernames and base64-encoded passwords (eg: { "username1": "cGFzc3dvcmQ=", "username2": "cGFzc3dvcmQy "}) + "client_opts":{ + "skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate + // the options to configure the http.Transport + "tls_handshake_timeout": "10s", + "disable_keep_alives": false, + "disable_compression": false, + "max_idle_conns": 100, + "max_idle_conns_per_host": 2, + "max_conns_per_host": 0, + "idle_conn_timeout": "90s", + "response_header_timeout": "0", + "expect_continue_timeout": "0", + "force_attempt_http2": true, + // the optins to configure the net.Dialer + "dial_timeout": "30s", + "dial_fallback_delay": "300ms", + "dial_keep_alive": "30s", + }, }, diff --git a/config/config_it_test.go b/config/config_it_test.go index 46a4550f7..6d567a33b 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -96,7 +96,7 @@ func testNewCgrJsonCfgFromHttp(t *testing.T) { } func testNewCGRConfigFromPath(t *testing.T) { - for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5", + for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "ROUND_DEC": "5", "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts", "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} { os.Setenv(key, val) diff --git a/config/config_json_test.go b/config/config_json_test.go index 9f0cf2952..5deb836ac 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -41,7 +41,6 @@ func TestDfGeneralJsonCfg(t *testing.T) { Node_id: utils.StringPointer(""), Logger: utils.StringPointer(utils.MetaSysLog), Log_level: utils.IntPointer(utils.LOGLEVEL_INFO), - Http_skip_tls_verify: utils.BoolPointer(false), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("*msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), @@ -1568,11 +1567,27 @@ func TestDfHttpJsonCfg(t *testing.T) { Http_Cdrs: utils.StringPointer("/cdr_http"), Use_basic_auth: utils.BoolPointer(false), Auth_users: utils.MapStringStringPointer(map[string]string{}), + Client_opts: map[string]interface{}{ + utils.HTTPClientTLSClientConfigCfg: false, + utils.HTTPClientTLSHandshakeTimeoutCfg: "10s", + utils.HTTPClientDisableKeepAlivesCfg: false, + utils.HTTPClientDisableCompressionCfg: false, + utils.HTTPClientMaxIdleConnsCfg: 100., + utils.HTTPClientMaxIdleConnsPerHostCfg: 2., + utils.HTTPClientMaxConnsPerHostCfg: 0., + utils.HTTPClientIdleConnTimeoutCfg: "90s", + utils.HTTPClientResponseHeaderTimeoutCfg: "0", + utils.HTTPClientExpectContinueTimeoutCfg: "0", + utils.HTTPClientForceAttemptHTTP2Cfg: true, + utils.HTTPClientDialTimeoutCfg: "30s", + utils.HTTPClientDialFallbackDelayCfg: "300ms", + utils.HTTPClientDialKeepAliveCfg: "30s", + }, } if cfg, err := dfCgrJSONCfg.HttpJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { - t.Error("Received: ", cfg) + t.Errorf("Expected: %s ,received: %s", utils.ToJSON(eCfg), utils.ToJSON(cfg)) } } diff --git a/config/config_test.go b/config/config_test.go index 1b6c6b757..ab7d020d4 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -229,9 +229,6 @@ func TestCgrCfgLoadJSONDefaults(t *testing.T) { } func TestCgrCfgJSONDefaultsGeneral(t *testing.T) { - if cgrCfg.GeneralCfg().HttpSkipTlsVerify != false { - t.Errorf("Expected: false, received: %+v", cgrCfg.GeneralCfg().HttpSkipTlsVerify) - } if cgrCfg.GeneralCfg().RoundingDecimals != 5 { t.Errorf("Expected: 5, received: %+v", cgrCfg.GeneralCfg().RoundingDecimals) } diff --git a/config/generalcfg.go b/config/generalcfg.go index f6f0e2d3e..79c30e172 100644 --- a/config/generalcfg.go +++ b/config/generalcfg.go @@ -30,7 +30,6 @@ type GeneralCfg struct { NodeID string // Identifier for this engine instance Logger string // dictates the way logs are displayed/stored LogLevel int // system wide log level, nothing higher than this will be logged - HttpSkipTlsVerify bool // If enabled Http Client will accept any TLS certificate RoundingDecimals int // Number of decimals to round end prices at DBDataEncoding string // The encoding used to store object data in strings: TpExportPath string // Path towards export folder for offline Tariff Plans @@ -101,9 +100,6 @@ func (gencfg *GeneralCfg) loadFromJsonCfg(jsnGeneralCfg *GeneralJsonCfg) (err er if jsnGeneralCfg.Rounding_decimals != nil { gencfg.RoundingDecimals = *jsnGeneralCfg.Rounding_decimals } - if jsnGeneralCfg.Http_skip_tls_verify != nil { - gencfg.HttpSkipTlsVerify = *jsnGeneralCfg.Http_skip_tls_verify - } if jsnGeneralCfg.Tpexport_dir != nil { gencfg.TpExportPath = *jsnGeneralCfg.Tpexport_dir } @@ -156,7 +152,6 @@ func (gencfg *GeneralCfg) AsMapInterface() (initialMP map[string]interface{}) { utils.NodeIDCfg: gencfg.NodeID, utils.LoggerCfg: gencfg.Logger, utils.LogLevelCfg: gencfg.LogLevel, - utils.HttpSkipTlsVerifyCfg: gencfg.HttpSkipTlsVerify, utils.RoundingDecimalsCfg: gencfg.RoundingDecimals, utils.DBDataEncodingCfg: utils.Meta + gencfg.DBDataEncoding, utils.TpExportPathCfg: gencfg.TpExportPath, diff --git a/config/generalcfg_test.go b/config/generalcfg_test.go index beee55971..b448de628 100644 --- a/config/generalcfg_test.go +++ b/config/generalcfg_test.go @@ -27,13 +27,12 @@ import ( func TestGeneralCfgloadFromJsonCfg(t *testing.T) { cfgJSON := &GeneralJsonCfg{ - Node_id: utils.StringPointer("randomID"), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(6), - Http_skip_tls_verify: utils.BoolPointer(false), - Rounding_decimals: utils.IntPointer(5), - Dbdata_encoding: utils.StringPointer("msgpack"), - Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), + Node_id: utils.StringPointer("randomID"), + Logger: utils.StringPointer(utils.MetaSysLog), + Log_level: utils.IntPointer(6), + Rounding_decimals: utils.IntPointer(5), + Dbdata_encoding: utils.StringPointer("msgpack"), + Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), Default_request_type: utils.StringPointer(utils.META_RATED), Default_category: utils.StringPointer(utils.CALL), @@ -52,7 +51,6 @@ func TestGeneralCfgloadFromJsonCfg(t *testing.T) { NodeID: "randomID", Logger: utils.MetaSysLog, LogLevel: 6, - HttpSkipTlsVerify: false, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", @@ -129,7 +127,6 @@ func TestGeneralCfgAsMapInterface(t *testing.T) { "node_id": "cgrates", "logger":"*syslog", "log_level": 6, - "http_skip_tls_verify": false, "rounding_decimals": 5, "dbdata_encoding": "*msgpack", "tpexport_dir": "/var/spool/cgrates/tpe", @@ -156,7 +153,6 @@ func TestGeneralCfgAsMapInterface(t *testing.T) { utils.NodeIDCfg: "cgrates", utils.LoggerCfg: "*syslog", utils.LogLevelCfg: 6, - utils.HttpSkipTlsVerifyCfg: false, utils.RoundingDecimalsCfg: 5, utils.DBDataEncodingCfg: "*msgpack", utils.TpExportPathCfg: "/var/spool/cgrates/tpe", @@ -201,7 +197,6 @@ func TestGeneralCfgAsMapInterface1(t *testing.T) { utils.NodeIDCfg: "ENGINE1", utils.LoggerCfg: "*syslog", utils.LogLevelCfg: 6, - utils.HttpSkipTlsVerifyCfg: false, utils.RoundingDecimalsCfg: 5, utils.DBDataEncodingCfg: "*msgpack", utils.TpExportPathCfg: "/var/spool/cgrates/tpe", diff --git a/config/httpcfg.go b/config/httpcfg.go index 1ec229dc4..6da2a086c 100644 --- a/config/httpcfg.go +++ b/config/httpcfg.go @@ -18,9 +18,16 @@ along with this program. If not, see package config -import "github.com/cgrates/cgrates/utils" +import ( + "crypto/tls" + "net" + "net/http" + "time" -// HTTP config section + "github.com/cgrates/cgrates/utils" +) + +// HTTPCfg is the HTTP config section type HTTPCfg struct { HTTPJsonRPCURL string // JSON RPC relative URL ("" to disable) DispatchersRegistrarURL string // dispatcherH registrar service relative URL @@ -29,9 +36,11 @@ type HTTPCfg struct { HTTPCDRsURL string // CDRS relative URL ("" to disable) HTTPUseBasicAuth bool // Use basic auth for HTTP API HTTPAuthUsers map[string]string // Basic auth user:password map (base64 passwords) + ClientOpts map[string]interface{} + transport *http.Transport } -//loadFromJsonCfg loads Database config from JsonCfg +// loadFromJsonCfg loads Database config from JsonCfg func (httpcfg *HTTPCfg) loadFromJsonCfg(jsnHttpCfg *HTTPJsonCfg) (err error) { if jsnHttpCfg == nil { return nil @@ -57,7 +66,12 @@ func (httpcfg *HTTPCfg) loadFromJsonCfg(jsnHttpCfg *HTTPJsonCfg) (err error) { if jsnHttpCfg.Auth_users != nil { httpcfg.HTTPAuthUsers = *jsnHttpCfg.Auth_users } - return nil + if jsnHttpCfg.Client_opts != nil { + for k, v := range jsnHttpCfg.Client_opts { + httpcfg.ClientOpts[k] = v + } + } + return httpcfg.initTransport() } func (httpcfg *HTTPCfg) AsMapInterface() (initialMP map[string]interface{}) { @@ -68,14 +82,123 @@ func (httpcfg *HTTPCfg) AsMapInterface() (initialMP map[string]interface{}) { utils.HTTPFreeswitchCDRsURLCfg: httpcfg.HTTPFreeswitchCDRsURL, utils.HTTPCDRsURLCfg: httpcfg.HTTPCDRsURL, utils.HTTPUseBasicAuthCfg: httpcfg.HTTPUseBasicAuth, - } - - if httpcfg.HTTPAuthUsers != nil { - httpUsers := make(map[string]interface{}, len(httpcfg.HTTPAuthUsers)) - for key, item := range httpcfg.HTTPAuthUsers { - httpUsers[key] = item - } - initialMP[utils.HTTPAuthUsersCfg] = httpUsers + utils.HTTPAuthUsersCfg: httpcfg.HTTPAuthUsers, + utils.HTTPClientOptsCfg: httpcfg.ClientOpts, } return } + +func (httpcfg *HTTPCfg) initTransport() (err error) { + trsp := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + } + dial := &net.Dialer{ + DualStack: true, + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientTLSClientConfigCfg]; has { + var skipTLSVerify bool + if skipTLSVerify, err = utils.IfaceAsBool(val); err != nil { + return + } + trsp.TLSClientConfig = &tls.Config{InsecureSkipVerify: skipTLSVerify} + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientTLSHandshakeTimeoutCfg]; has { + var tlsHndTimeout time.Duration + if tlsHndTimeout, err = utils.IfaceAsDuration(val); err != nil { + return + } + trsp.TLSHandshakeTimeout = tlsHndTimeout + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientDisableKeepAlivesCfg]; has { + var disKeepAlives bool + if disKeepAlives, err = utils.IfaceAsBool(val); err != nil { + return + } + trsp.DisableKeepAlives = disKeepAlives + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientDisableCompressionCfg]; has { + var disCmp bool + if disCmp, err = utils.IfaceAsBool(val); err != nil { + return + } + trsp.DisableCompression = disCmp + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientMaxIdleConnsCfg]; has { + var maxIdleConns int64 + if maxIdleConns, err = utils.IfaceAsTInt64(val); err != nil { + return + } + trsp.MaxIdleConns = int(maxIdleConns) + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientMaxIdleConnsPerHostCfg]; has { + var maxIdleConns int64 + if maxIdleConns, err = utils.IfaceAsTInt64(val); err != nil { + return + } + trsp.MaxIdleConnsPerHost = int(maxIdleConns) + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientMaxConnsPerHostCfg]; has { + var maxConns int64 + if maxConns, err = utils.IfaceAsTInt64(val); err != nil { + return + } + trsp.MaxConnsPerHost = int(maxConns) + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientIdleConnTimeoutCfg]; has { + var idleTimeout time.Duration + if idleTimeout, err = utils.IfaceAsDuration(val); err != nil { + return + } + trsp.IdleConnTimeout = idleTimeout + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientResponseHeaderTimeoutCfg]; has { + var responseTimeout time.Duration + if responseTimeout, err = utils.IfaceAsDuration(val); err != nil { + return + } + trsp.ResponseHeaderTimeout = responseTimeout + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientExpectContinueTimeoutCfg]; has { + var continueTimeout time.Duration + if continueTimeout, err = utils.IfaceAsDuration(val); err != nil { + return + } + trsp.ExpectContinueTimeout = continueTimeout + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientForceAttemptHTTP2Cfg]; has { + var forceHTTP2 bool + if forceHTTP2, err = utils.IfaceAsBool(val); err != nil { + return + } + trsp.ForceAttemptHTTP2 = forceHTTP2 + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientDialTimeoutCfg]; has { + var timeout time.Duration + if timeout, err = utils.IfaceAsDuration(val); err != nil { + return + } + dial.Timeout = timeout + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientDialFallbackDelayCfg]; has { + var fallDelay time.Duration + if fallDelay, err = utils.IfaceAsDuration(val); err != nil { + return + } + dial.FallbackDelay = fallDelay + } + if val, has := httpcfg.ClientOpts[utils.HTTPClientDialKeepAliveCfg]; has { + var keepAlive time.Duration + if keepAlive, err = utils.IfaceAsDuration(val); err != nil { + return + } + dial.KeepAlive = keepAlive + } + trsp.DialContext = dial.DialContext + httpcfg.transport = trsp + return +} + +// GetDefaultHTTPTransort returns the transport initialized when the config was loaded +func (httpcfg *HTTPCfg) GetDefaultHTTPTransort() *http.Transport { + return httpcfg.transport +} diff --git a/config/httpcfg_test.go b/config/httpcfg_test.go index f82bc7adf..1a73e5380 100644 --- a/config/httpcfg_test.go +++ b/config/httpcfg_test.go @@ -18,8 +18,11 @@ along with this program. If not, see package config import ( + "crypto/tls" + "net/http" "reflect" "testing" + "time" "github.com/cgrates/cgrates/utils" ) @@ -42,12 +45,28 @@ func TestHTTPCfgloadFromJsonCfg(t *testing.T) { HTTPCDRsURL: "/cdr_http", HTTPUseBasicAuth: false, HTTPAuthUsers: map[string]string{}, + ClientOpts: map[string]interface{}{ + utils.HTTPClientTLSClientConfigCfg: false, + utils.HTTPClientTLSHandshakeTimeoutCfg: "10s", + utils.HTTPClientDisableKeepAlivesCfg: false, + utils.HTTPClientDisableCompressionCfg: false, + utils.HTTPClientMaxIdleConnsCfg: 100., + utils.HTTPClientMaxIdleConnsPerHostCfg: 2., + utils.HTTPClientMaxConnsPerHostCfg: 0., + utils.HTTPClientIdleConnTimeoutCfg: "90s", + utils.HTTPClientResponseHeaderTimeoutCfg: "0", + utils.HTTPClientExpectContinueTimeoutCfg: "0", + utils.HTTPClientForceAttemptHTTP2Cfg: true, + utils.HTTPClientDialTimeoutCfg: "30s", + utils.HTTPClientDialFallbackDelayCfg: "300ms", + utils.HTTPClientDialKeepAliveCfg: "30s", + }, } if cfgJsn, err := NewDefaultCGRConfig(); err != nil { t.Error(err) } else if err = cfgJsn.httpCfg.loadFromJsonCfg(cfgJSONStr); err != nil { t.Error(err) - } else if !reflect.DeepEqual(expected, cfgJsn.httpCfg) { + } else if cfgJsn.httpCfg.transport = nil; !reflect.DeepEqual(expected, cfgJsn.httpCfg) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected), utils.ToJSON(cfgJsn.httpCfg)) } } @@ -63,12 +82,28 @@ func TestHTTPCfgAsMapInterface(t *testing.T) { utils.HTTPFreeswitchCDRsURLCfg: "/freeswitch_json", utils.HTTPCDRsURLCfg: "/cdr_http", utils.HTTPUseBasicAuthCfg: false, - utils.HTTPAuthUsersCfg: map[string]interface{}{}, + utils.HTTPAuthUsersCfg: map[string]string{}, + utils.HTTPClientOptsCfg: map[string]interface{}{ + utils.HTTPClientTLSClientConfigCfg: false, + utils.HTTPClientTLSHandshakeTimeoutCfg: "10s", + utils.HTTPClientDisableKeepAlivesCfg: false, + utils.HTTPClientDisableCompressionCfg: false, + utils.HTTPClientMaxIdleConnsCfg: 100., + utils.HTTPClientMaxIdleConnsPerHostCfg: 2., + utils.HTTPClientMaxConnsPerHostCfg: 0., + utils.HTTPClientIdleConnTimeoutCfg: "90s", + utils.HTTPClientResponseHeaderTimeoutCfg: "0", + utils.HTTPClientExpectContinueTimeoutCfg: "0", + utils.HTTPClientForceAttemptHTTP2Cfg: true, + utils.HTTPClientDialTimeoutCfg: "30s", + utils.HTTPClientDialFallbackDelayCfg: "300ms", + utils.HTTPClientDialKeepAliveCfg: "30s", + }, } if cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if rcv := cgrCfg.httpCfg.AsMapInterface(); !reflect.DeepEqual(rcv, eMap) { - t.Errorf("Expected %+v, received %+v", eMap, rcv) + t.Errorf("Expected %+v, received %+v", utils.ToJSON(eMap), utils.ToJSON(rcv)) } } @@ -88,10 +123,26 @@ func TestHTTPCfgAsMapInterface1(t *testing.T) { utils.HTTPFreeswitchCDRsURLCfg: "/freeswitch_json", utils.HTTPCDRsURLCfg: "/cdr_http", utils.HTTPUseBasicAuthCfg: true, - utils.HTTPAuthUsersCfg: map[string]interface{}{ + utils.HTTPAuthUsersCfg: map[string]string{ "user1": "authenticated", "user2": "authenticated", }, + utils.HTTPClientOptsCfg: map[string]interface{}{ + utils.HTTPClientTLSClientConfigCfg: false, + utils.HTTPClientTLSHandshakeTimeoutCfg: "10s", + utils.HTTPClientDisableKeepAlivesCfg: false, + utils.HTTPClientDisableCompressionCfg: false, + utils.HTTPClientMaxIdleConnsCfg: 100., + utils.HTTPClientMaxIdleConnsPerHostCfg: 2., + utils.HTTPClientMaxConnsPerHostCfg: 0., + utils.HTTPClientIdleConnTimeoutCfg: "90s", + utils.HTTPClientResponseHeaderTimeoutCfg: "0", + utils.HTTPClientExpectContinueTimeoutCfg: "0", + utils.HTTPClientForceAttemptHTTP2Cfg: true, + utils.HTTPClientDialTimeoutCfg: "30s", + utils.HTTPClientDialFallbackDelayCfg: "300ms", + utils.HTTPClientDialKeepAliveCfg: "30s", + }, } if cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) @@ -99,3 +150,117 @@ func TestHTTPCfgAsMapInterface1(t *testing.T) { t.Errorf("Expected %+v, received %+v", eMap, rcv) } } + +func TestHTTPCfgGetDefaultHTTPTransort(t *testing.T) { + httpCfg := new(HTTPCfg) + if rply := httpCfg.GetDefaultHTTPTransort(); rply != nil { + t.Errorf("Expected %+v, received %+v", nil, rply) + } +} + +func TestHTTPCfgInitTransport(t *testing.T) { + httpCfg := &HTTPCfg{ + ClientOpts: map[string]interface{}{ + utils.HTTPClientTLSClientConfigCfg: false, + utils.HTTPClientTLSHandshakeTimeoutCfg: "10s", + utils.HTTPClientDisableKeepAlivesCfg: false, + utils.HTTPClientDisableCompressionCfg: false, + utils.HTTPClientMaxIdleConnsCfg: 100., + utils.HTTPClientMaxIdleConnsPerHostCfg: 2., + utils.HTTPClientMaxConnsPerHostCfg: 0., + utils.HTTPClientIdleConnTimeoutCfg: "90s", + utils.HTTPClientResponseHeaderTimeoutCfg: "0", + utils.HTTPClientExpectContinueTimeoutCfg: "0", + utils.HTTPClientForceAttemptHTTP2Cfg: true, + utils.HTTPClientDialTimeoutCfg: "30s", + utils.HTTPClientDialFallbackDelayCfg: "300ms", + utils.HTTPClientDialKeepAliveCfg: "30s", + }, + } + // the dial options are not included + checkTransport := func(t1, t2 *http.Transport) bool { + return t1 != nil && t2 != nil && + t1.TLSClientConfig.InsecureSkipVerify == t2.TLSClientConfig.InsecureSkipVerify && + t1.TLSHandshakeTimeout == t2.TLSHandshakeTimeout && + t1.DisableKeepAlives == t2.DisableKeepAlives && + t1.DisableCompression == t2.DisableCompression && + t1.MaxIdleConns == t2.MaxIdleConns && + t1.MaxIdleConnsPerHost == t2.MaxIdleConnsPerHost && + t1.MaxConnsPerHost == t2.MaxConnsPerHost && + t1.IdleConnTimeout == t2.IdleConnTimeout && + t1.ResponseHeaderTimeout == t2.ResponseHeaderTimeout && + t1.ExpectContinueTimeout == t2.ExpectContinueTimeout && + t1.ForceAttemptHTTP2 == t2.ForceAttemptHTTP2 + } + expTransport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, + TLSHandshakeTimeout: 10 * time.Second, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 2, + MaxConnsPerHost: 0, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + } + if err := httpCfg.initTransport(); err != nil { + t.Fatal(err) + } else if !checkTransport(expTransport, httpCfg.GetDefaultHTTPTransort()) { + t.Errorf("Expected %+v, received %+v", expTransport, httpCfg.GetDefaultHTTPTransort()) + } + + httpCfg.ClientOpts[utils.HTTPClientDialKeepAliveCfg] = "30as" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientDialFallbackDelayCfg] = "300ams" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientDialTimeoutCfg] = "30as" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientForceAttemptHTTP2Cfg] = "string" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientExpectContinueTimeoutCfg] = "0a" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientResponseHeaderTimeoutCfg] = "0a" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientIdleConnTimeoutCfg] = "90as" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientMaxConnsPerHostCfg] = "not a number" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientMaxIdleConnsPerHostCfg] = "not a number" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientMaxIdleConnsCfg] = "not a number" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientDisableCompressionCfg] = "string" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientDisableKeepAlivesCfg] = "string" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientTLSHandshakeTimeoutCfg] = "10as" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } + httpCfg.ClientOpts[utils.HTTPClientTLSClientConfigCfg] = "string" + if err := httpCfg.initTransport(); err == nil { + t.Error("Expected error but the transport was builded succesfully") + } +} diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 3202a4521..c0f16b4ea 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -27,7 +27,6 @@ type GeneralJsonCfg struct { Node_id *string Logger *string Log_level *int - Http_skip_tls_verify *bool Rounding_decimals *int Dbdata_encoding *string Tpexport_dir *string @@ -71,6 +70,7 @@ type HTTPJsonCfg struct { Http_Cdrs *string Use_basic_auth *bool Auth_users *map[string]string + Client_opts map[string]interface{} } type TlsJsonCfg struct { diff --git a/config/multifiles_it_test.go b/config/multifiles_it_test.go index 057e22534..bfbd87265 100644 --- a/config/multifiles_it_test.go +++ b/config/multifiles_it_test.go @@ -31,7 +31,7 @@ import ( var mfCgrCfg *CGRConfig func TestMfInitConfig(t *testing.T) { - for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5", + for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "ROUND_DEC": "5", "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts", "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} { os.Setenv(key, val) @@ -56,7 +56,6 @@ func TestMfEnvReaderITRead(t *testing.T) { NodeID: "d80fac5", Logger: "*syslog", LogLevel: 6, - HttpSkipTlsVerify: false, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 26070b940..6a632149c 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -10,7 +10,6 @@ // "node_id": "", // identifier of this instance in the cluster, if empty it will be autogenerated // "logger":"*syslog", // controls the destination of logs <*syslog|*stdout> // "log_level": 6, // control the level of messages logged (0-emerg to 7-debug) -// "http_skip_tls_verify": false, // if enabled HttpClient will accept any TLS certificate // "rounding_decimals": 5, // system level precision for floats // "dbdata_encoding": "*msgpack", // encoding used to store object data in strings: <*msgpack|*json> // "tpexport_dir": "/var/spool/cgrates/tpe", // path towards export folder for offline TariffPlans @@ -165,6 +164,24 @@ // "http_cdrs": "/cdr_http", // CDRS relative URL ("" to disable) // "use_basic_auth": false, // use basic authentication // "auth_users": {}, // basic authentication usernames and base64-encoded passwords (eg: { "username1": "cGFzc3dvcmQ=", "username2": "cGFzc3dvcmQy "}) +// "client_opts":{ +// "skip_tls_verify": false, // if enabled Http Client will accept any TLS certificate +// // the options to configure the http.Transport +// "tls_handshake_timeout": "10s", +// "disable_keep_alives": false, +// "disable_compression": false, +// "max_idle_conns": 100, +// "max_idle_conns_per_host": 2, +// "max_conns_per_host": 0, +// "idle_conn_timeout": "90s", +// "response_header_timeout": "0", +// "expect_continue_timeout": "0", +// "force_attempt_http2": true, +// // the optins to configure the net.Dialer +// "dial_timeout": "30s", +// "dial_fallback_delay": "300ms", +// "dial_keep_alive": "30s", +// }, // }, diff --git a/data/conf/samples/multifiles/d.json b/data/conf/samples/multifiles/d.json index 4cad502dd..45926e0de 100644 --- a/data/conf/samples/multifiles/d.json +++ b/data/conf/samples/multifiles/d.json @@ -7,7 +7,6 @@ "node_id": "d80fac5", // identifier of this instance in the cluster, if empty it will be autogenerated "logger":"*env:LOGGER", // controls the destination of logs <*syslog|*stdout> "log_level": *env:LOG_LEVEL, // control the level of messages logged (0-emerg to 7-debug) - "http_skip_tls_verify": *env:TLS_VERIFY, // if enabled Http Client will accept any TLS certificate "rounding_decimals": *env:ROUND_DEC, // system level precision for floats "dbdata_encoding": "*env:DB_ENCODING", // encoding used to store object data in strings: <*msgpack|*json> "tpexport_dir": "*env:TP_EXPORT_DIR", // path towards export folder for offline Tariff Plans diff --git a/ees/httpjsonmap.go b/ees/httpjsonmap.go index da7c92074..da0829520 100644 --- a/ees/httpjsonmap.go +++ b/ees/httpjsonmap.go @@ -40,7 +40,7 @@ func NewPosterJSONMapEE(cgrCfg *config.CGRConfig, cfgIdx int, filterS *engine.Fi } switch cgrCfg.EEsCfg().Exporters[cfgIdx].Type { case utils.MetaHTTPjsonMap: - pstrJSON.poster, err = engine.NewHTTPPoster(cgrCfg.GeneralCfg().HttpSkipTlsVerify, + pstrJSON.poster, err = engine.NewHTTPPoster(cgrCfg.HTTPCfg().GetDefaultHTTPTransort(), cgrCfg.GeneralCfg().ReplyTimeout, cgrCfg.EEsCfg().Exporters[cfgIdx].ExportPath, utils.PosterTransportContentTypes[cgrCfg.EEsCfg().Exporters[cfgIdx].Type], cgrCfg.EEsCfg().Exporters[cfgIdx].Attempts) case utils.MetaAMQPjsonMap: diff --git a/ees/httppost.go b/ees/httppost.go index c5e93dc97..8312b7cbc 100644 --- a/ees/httppost.go +++ b/ees/httppost.go @@ -33,7 +33,7 @@ func NewHTTPPostEe(cgrCfg *config.CGRConfig, cfgIdx int, filterS *engine.FilterS dc utils.MapStorage) (httpPost *HTTPPost, err error) { httpPost = &HTTPPost{id: cgrCfg.EEsCfg().Exporters[cfgIdx].ID, cgrCfg: cgrCfg, cfgIdx: cfgIdx, filterS: filterS, dc: dc} - httpPost.httpPoster, err = engine.NewHTTPPoster(cgrCfg.GeneralCfg().HttpSkipTlsVerify, + httpPost.httpPoster, err = engine.NewHTTPPoster(cgrCfg.HTTPCfg().GetDefaultHTTPTransort(), cgrCfg.GeneralCfg().ReplyTimeout, cgrCfg.EEsCfg().Exporters[cfgIdx].ExportPath, utils.PosterTransportContentTypes[cgrCfg.EEsCfg().Exporters[cfgIdx].Type], cgrCfg.EEsCfg().Exporters[cfgIdx].Attempts) return diff --git a/engine/action.go b/engine/action.go index be38dea88..7929202b8 100644 --- a/engine/action.go +++ b/engine/action.go @@ -385,7 +385,7 @@ func callURL(ub *Account, a *Action, acs Actions, extraData interface{}) error { if err != nil { return err } - pstr, err := NewHTTPPoster(config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, + pstr, err := NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), config.CgrConfig().GeneralCfg().ReplyTimeout, a.ExtraParameters, utils.CONTENT_JSON, config.CgrConfig().GeneralCfg().PosterAttempts) if err != nil { @@ -405,7 +405,7 @@ func callURLAsync(ub *Account, a *Action, acs Actions, extraData interface{}) er if err != nil { return err } - pstr, err := NewHTTPPoster(config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, + pstr, err := NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), config.CgrConfig().GeneralCfg().ReplyTimeout, a.ExtraParameters, utils.CONTENT_JSON, config.CgrConfig().GeneralCfg().PosterAttempts) if err != nil { @@ -974,7 +974,7 @@ func postEvent(ub *Account, a *Action, acs Actions, extraData interface{}) error if err != nil { return err } - pstr, err := NewHTTPPoster(config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, + pstr, err := NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), config.CgrConfig().GeneralCfg().ReplyTimeout, a.ExtraParameters, utils.CONTENT_JSON, config.CgrConfig().GeneralCfg().PosterAttempts) if err != nil { diff --git a/engine/cdr.go b/engine/cdr.go index 59ca57b57..8b819d0aa 100644 --- a/engine/cdr.go +++ b/engine/cdr.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "math" + "net/http" "strconv" "strings" "time" @@ -333,7 +334,7 @@ func (cdr *CDR) exportFieldValue(cfgCdrFld *config.FCTemplate, filterS *FilterS) return } -func (cdr *CDR) formatField(cfgFld *config.FCTemplate, httpSkipTLSCheck bool, +func (cdr *CDR) formatField(cfgFld *config.FCTemplate, pstrTransport *http.Transport, groupedCDRs []*CDR, filterS *FilterS) (outVal string, err error) { switch cfgFld.Type { case utils.META_FILLER: @@ -363,7 +364,7 @@ func (cdr *CDR) formatField(cfgFld *config.FCTemplate, httpSkipTLSCheck bool, } if len(httpAddr) == 0 { err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type) - } else if outValByte, err = HttpJsonPost(httpAddr, httpSkipTLSCheck, jsn); err == nil { + } else if outValByte, err = HTTPPostJSON(httpAddr, pstrTransport, jsn); err == nil { outVal = string(outValByte) if len(outVal) == 0 && cfgFld.Mandatory { err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag) @@ -390,7 +391,7 @@ func (cdr *CDR) formatField(cfgFld *config.FCTemplate, httpSkipTLSCheck bool, // AsExportRecord is used in place where we need to export the CDR based on an export template // ExportRecord is a []string to keep it compatible with encoding/csv Writer func (cdr *CDR) AsExportRecord(exportFields []*config.FCTemplate, - httpSkipTLSCheck bool, groupedCDRs []*CDR, filterS *FilterS) (expRecord []string, err error) { + pstrTransport *http.Transport, groupedCDRs []*CDR, filterS *FilterS) (expRecord []string, err error) { nM := utils.MapStorage{ utils.MetaReq: cdr.AsMapStringIface(), utils.MetaEC: cdr.CostDetails, @@ -406,7 +407,7 @@ func (cdr *CDR) AsExportRecord(exportFields []*config.FCTemplate, continue } var fmtOut string - if fmtOut, err = cdr.formatField(cfgFld, httpSkipTLSCheck, groupedCDRs, filterS); err != nil { + if fmtOut, err = cdr.formatField(cfgFld, pstrTransport, groupedCDRs, filterS); err != nil { utils.Logger.Warning(fmt.Sprintf(" error: %s exporting field: %s, CDR: %s\n", err.Error(), utils.ToJSON(cfgFld), utils.ToJSON(cdr))) return nil, err diff --git a/engine/cdr_test.go b/engine/cdr_test.go index a94bc3c2a..6f6d3b763 100644 --- a/engine/cdr_test.go +++ b/engine/cdr_test.go @@ -641,7 +641,8 @@ func TestCDRAsExportRecord(t *testing.T) { Value: prsr, Timezone: "UTC", } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != cdr.Destination { t.Errorf("Expecting:\n%s\nReceived:\n%s", cdr.Destination, expRecord) @@ -660,7 +661,8 @@ func TestCDRAsExportRecord(t *testing.T) { MaskLen: 3, } eDst := "+4986517174***" - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != eDst { t.Errorf("Expecting:\n%s\nReceived:\n%s", eDst, expRecord[0]) @@ -673,7 +675,8 @@ func TestCDRAsExportRecord(t *testing.T) { Value: prsr, MaskDestID: "MASKED_DESTINATIONS", } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != "1" { t.Errorf("Expecting:\n%s\nReceived:\n%s", "1", expRecord[0]) @@ -689,7 +692,8 @@ func TestCDRAsExportRecord(t *testing.T) { Filters: []string{"*string:~*req.Tenant:itsyscom.com"}, Timezone: "UTC", } - if rcrd, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { + if rcrd, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { t.Error(err) } else if len(rcrd) != 0 { t.Error("failed using filter") @@ -706,7 +710,8 @@ func TestCDRAsExportRecord(t *testing.T) { Layout: layout, Timezone: "UTC", } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { t.Error(err) } else if expRecord[0] != "2014-06-11 19:19:00" { t.Error("Expecting: 2014-06-11 19:19:00, got: ", expRecord[0]) @@ -722,7 +727,8 @@ func TestCDRAsExportRecord(t *testing.T) { Layout: layout, Timezone: "UTC", } - if rcrd, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { + if rcrd, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { t.Error(err) } else if len(rcrd) != 0 { t.Error("failed using filter") @@ -737,7 +743,8 @@ func TestCDRAsExportRecord(t *testing.T) { Layout: layout, Timezone: "UTC"} // Test time parse error - if _, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err == nil { + if _, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err == nil { t.Error("Should give error here, got none.") } @@ -748,7 +755,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CGRIDFromCostDetails", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != cdr.CostDetails.CGRID { t.Errorf("Expecting:\n%s\nReceived:\n%s", cdr.CostDetails.CGRID, expRecord) @@ -760,7 +768,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CustomAccountID", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != cdr.CostDetails.AccountSummary.ID { t.Errorf("Expecting:\n%s\nReceived:\n%s", cdr.CostDetails.AccountSummary.ID, expRecord) @@ -774,7 +783,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CustomDestinationID", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != expected { t.Errorf("Expecting: <%q>,\n Received: <%q>", expected, expRecord[0]) @@ -788,7 +798,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CustomDestinationID", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != expected { t.Errorf("Expecting: <%q>,\n Received: <%q>", expected, expRecord[0]) @@ -802,7 +813,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CustomDestinationID", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != expected { t.Errorf("Expecting: <%q>,\n Received: <%q>", expected, expRecord[0]) @@ -816,7 +828,8 @@ func TestCDRAsExportRecord(t *testing.T) { Path: "*exp.CustomDestinationID", Value: prsr, } - if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, nil); err != nil { + if expRecord, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, + config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), nil, nil); err != nil { t.Error(err) } else if expRecord[0] != expected { t.Errorf("Expecting: <%q>,\n Received: <%q>", expected, expRecord[0]) diff --git a/engine/libcdre.go b/engine/libcdre.go index 29213c270..4a1ed801c 100644 --- a/engine/libcdre.go +++ b/engine/libcdre.go @@ -157,7 +157,7 @@ func (expEv *ExportEvents) ReplayFailedPosts(attempts int) (failedEvents *Export switch expEv.Format { case utils.MetaHTTPjsonCDR, utils.MetaHTTPjsonMap, utils.MetaHTTPjson, utils.MetaHTTPPost: var pstr *HTTPPoster - pstr, err = NewHTTPPoster(config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, + pstr, err = NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), config.CgrConfig().GeneralCfg().ReplyTimeout, expEv.Path, utils.PosterTransportContentTypes[expEv.Format], config.CgrConfig().GeneralCfg().PosterAttempts) diff --git a/engine/pstr_http.go b/engine/pstr_http.go index 06d34a478..5e5df447d 100644 --- a/engine/pstr_http.go +++ b/engine/pstr_http.go @@ -20,7 +20,6 @@ package engine import ( "bytes" - "crypto/tls" "fmt" "io/ioutil" "net/http" @@ -30,17 +29,9 @@ import ( "github.com/cgrates/cgrates/utils" ) -// keep it global in order to reuse it -var httpPosterTransport *http.Transport - -// HttpJsonPost posts without automatic failover -func HttpJsonPost(url string, skipTLSVerify bool, content []byte) (respBody []byte, err error) { - if httpPosterTransport == nil { - httpPosterTransport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, - } - } - client := &http.Client{Transport: httpPosterTransport} +// HTTPPostJSON posts without automatic failover +func HTTPPostJSON(url string, posterTransport *http.Transport, content []byte) (respBody []byte, err error) { + client := &http.Client{Transport: posterTransport} var resp *http.Response if resp, err = client.Post(url, "application/json", bytes.NewBuffer(content)); err != nil { return @@ -57,18 +48,13 @@ func HttpJsonPost(url string, skipTLSVerify bool, content []byte) (respBody []by } // NewHTTPPoster return a new HTTP poster -func NewHTTPPoster(skipTLSVerify bool, replyTimeout time.Duration, +func NewHTTPPoster(posterTransport *http.Transport, replyTimeout time.Duration, addr, contentType string, attempts int) (httposter *HTTPPoster, err error) { if !utils.SliceHasMember([]string{utils.CONTENT_FORM, utils.CONTENT_JSON, utils.CONTENT_TEXT}, contentType) { return nil, fmt.Errorf("unsupported ContentType: %s", contentType) } - if httpPosterTransport == nil { - httpPosterTransport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, - } - } return &HTTPPoster{ - httpClient: &http.Client{Transport: httpPosterTransport, Timeout: replyTimeout}, + httpClient: &http.Client{Transport: posterTransport, Timeout: replyTimeout}, addr: addr, contentType: contentType, attempts: attempts, diff --git a/engine/suretax.go b/engine/suretax.go index 86668beb1..1600badab 100644 --- a/engine/suretax.go +++ b/engine/suretax.go @@ -20,7 +20,6 @@ package engine import ( "bytes" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -182,12 +181,9 @@ func SureTaxProcessCdr(cdr *CDR) error { return errors.New("Invalid SureTax configuration") } if sureTaxClient == nil { // First time used, init the client here - tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, - }, + sureTaxClient = &http.Client{ + Transport: config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), } - sureTaxClient = &http.Client{Transport: tr} } req, err := NewSureTaxRequest(cdr, stCfg) if err != nil { diff --git a/engine/z_cdr_it_test.go b/engine/z_cdr_it_test.go index 93bf8d4b2..80c0d5ece 100644 --- a/engine/z_cdr_it_test.go +++ b/engine/z_cdr_it_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) @@ -38,7 +39,7 @@ func TestHttpJsonPost(t *testing.T) { Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } jsn, _ := json.Marshal(cdrOut) - if _, err := HttpJsonPost("http://localhost:8000", false, jsn); err == nil { + if _, err := HTTPPostJSON("http://localhost:8000", config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), jsn); err == nil { t.Error(err) } } diff --git a/engine/z_poster_it_test.go b/engine/z_poster_it_test.go index efa99b0b2..c32239830 100644 --- a/engine/z_poster_it_test.go +++ b/engine/z_poster_it_test.go @@ -67,7 +67,7 @@ func TestHttpJsonPoster(t *testing.T) { config.CgrConfig().GeneralCfg().FailedPostsDir = "/tmp" content := &TestContent{Var1: "Val1", Var2: "Val2"} jsn, _ := json.Marshal(content) - pstr, err := NewHTTPPoster(true, time.Duration(2*time.Second), "http://localhost:8080/invalid", utils.CONTENT_JSON, 3) + pstr, err := NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), time.Duration(2*time.Second), "http://localhost:8080/invalid", utils.CONTENT_JSON, 3) if err != nil { t.Error(err) } @@ -100,7 +100,7 @@ func TestHttpBytesPoster(t *testing.T) { content := []byte(`Test Test2 `) - pstr, err := NewHTTPPoster(true, time.Duration(2*time.Second), "http://localhost:8080/invalid", utils.CONTENT_TEXT, 3) + pstr, err := NewHTTPPoster(config.CgrConfig().HTTPCfg().GetDefaultHTTPTransort(), time.Duration(2*time.Second), "http://localhost:8080/invalid", utils.CONTENT_TEXT, 3) if err != nil { t.Error(err) } diff --git a/ers/partial_csv.go b/ers/partial_csv.go index a02811f98..9e0b018f4 100644 --- a/ers/partial_csv.go +++ b/ers/partial_csv.go @@ -313,7 +313,7 @@ func (rdr *PartialCSVFileER) dumpToFile(itmID string, value interface{}) { utils.ERs, utils.ToJSON(origCgrEvs[0].Event), err.Error())) return } - record, err := cdr.AsExportRecord(rdr.Config().CacheDumpFields, false, nil, rdr.fltrS) + record, err := cdr.AsExportRecord(rdr.Config().CacheDumpFields, rdr.cgrCfg.HTTPCfg().GetDefaultHTTPTransort(), nil, rdr.fltrS) if err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> Converting CDR with CGRID: <%s> to record , ignoring due to error: <%s>", @@ -345,7 +345,7 @@ func (rdr *PartialCSVFileER) dumpToFile(itmID string, value interface{}) { utils.ERs, utils.ToJSON(origCgrEv.Event), err.Error())) return } - record, err = cdr.AsExportRecord(rdr.Config().CacheDumpFields, false, nil, rdr.fltrS) + record, err = cdr.AsExportRecord(rdr.Config().CacheDumpFields, rdr.cgrCfg.HTTPCfg().GetDefaultHTTPTransort(), nil, rdr.fltrS) if err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> Converting CDR with CGRID: <%s> to record , ignoring due to error: <%s>", diff --git a/utils/consts.go b/utils/consts.go index f3bae707a..25a18ecb0 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -1871,7 +1871,6 @@ const ( NodeIDCfg = "node_id" LoggerCfg = "logger" LogLevelCfg = "log_level" - HttpSkipTlsVerifyCfg = "http_skip_tls_verify" RoundingDecimalsCfg = "rounding_decimals" DBDataEncodingCfg = "dbdata_encoding" TpExportPathCfg = "tpexport_dir" @@ -1973,7 +1972,23 @@ const ( HTTPCDRsURLCfg = "http_cdrs" HTTPUseBasicAuthCfg = "use_basic_auth" HTTPAuthUsersCfg = "auth_users" + HTTPClientOptsCfg = "client_opts" ConfigsURL = "configs_url" + + HTTPClientTLSClientConfigCfg = "skip_tls_verify" + HTTPClientTLSHandshakeTimeoutCfg = "tls_handshake_timeout" + HTTPClientDisableKeepAlivesCfg = "disable_keep_alives" + HTTPClientDisableCompressionCfg = "disable_compression" + HTTPClientMaxIdleConnsCfg = "max_idle_conns" + HTTPClientMaxIdleConnsPerHostCfg = "max_idle_conns_per_host" + HTTPClientMaxConnsPerHostCfg = "max_conns_per_host" + HTTPClientIdleConnTimeoutCfg = "idle_conn_timeout" + HTTPClientResponseHeaderTimeoutCfg = "response_header_timeout" + HTTPClientExpectContinueTimeoutCfg = "expect_continue_timeout" + HTTPClientForceAttemptHTTP2Cfg = "force_attempt_http2" + HTTPClientDialTimeoutCfg = "dial_timeout" + HTTPClientDialFallbackDelayCfg = "dial_fallback_delay" + HTTPClientDialKeepAliveCfg = "dial_keep_alive" ) // FilterSCfg