From aaea87eb3323c2e51f8a1bdb77e74a4125b5b4a5 Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Mon, 11 Jul 2022 18:40:32 +0300 Subject: [PATCH] Make data and stordb opts into structs --- cmd/cgr-loader/cgr-loader.go | 75 ++++++++----- cmd/cgr-loader/cgr-loader_it_test.go | 37 ++++--- cmd/cgr-loader/flags_test.go | 11 +- cmd/cgr-migrator/cgr-migrator.go | 77 ++++++++----- cmd/cgr-migrator/cgr-migrator_test.go | 17 +-- cmd/cgr-tester/cgr-tester.go | 70 ++++++++---- config/config.go | 8 +- config/config_defaults.go | 16 ++- config/config_json_test.go | 66 ++++++----- config/config_test.go | 29 +++-- config/configsanity.go | 2 +- config/configsanity_test.go | 4 +- config/datadbcfg.go | 145 +++++++++++++++++++++---- config/datadbcfg_test.go | 60 +++++----- config/erscfg.go | 2 +- config/libconfig_json.go | 29 ++++- config/migratorcfg.go | 58 +++++----- config/migratorcfg_test.go | 69 +++++++----- config/stordbcfg.go | 94 ++++++++++++---- config/stordbcfg_test.go | 53 ++++----- engine/storage_mysql.go | 4 +- engine/storage_postgres.go | 4 +- engine/storage_redis.go | 11 +- engine/storage_utils.go | 76 ++----------- engine/z_datamanager_it_test.go | 2 +- engine/z_filterindexer_it_test.go | 2 +- engine/z_onstor_it_test.go | 2 +- engine/z_stordb_it_test.go | 3 +- general_tests/indexes_redis_it_test.go | 4 +- migrator/migrator_utils.go | 4 +- services/datadb.go | 20 ++-- services/datadb_it_test.go | 112 ++++++++++--------- services/stordb.go | 25 +---- utils/consts.go | 9 +- utils/reflect.go | 23 ++++ 35 files changed, 738 insertions(+), 485 deletions(-) diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index c25edd576..448781a33 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -54,15 +54,25 @@ var ( "The DataDb user's password.") dbDataEncoding = cgrLoaderFlags.String(utils.DBDataEncodingCfg, dfltCfg.GeneralCfg().DBDataEncoding, "The encoding used to store object data in strings") - dbRedisSentinel = cgrLoaderFlags.String(utils.RedisSentinelNameCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg]), + dbRedisMaxConns = cgrLoaderFlags.Int(utils.RedisMaxConnsCfg, dfltCfg.DataDbCfg().Opts.RedisMaxConns, + "The connection pool size") + dbRedisConnectAttempts = cgrLoaderFlags.Int(utils.RedisConnectAttemptsCfg, dfltCfg.DataDbCfg().Opts.RedisConnectAttempts, + "The maximum amount of dial attempts") + dbRedisSentinel = cgrLoaderFlags.String(utils.RedisSentinelNameCfg, dfltCfg.DataDbCfg().Opts.RedisSentinel, "The name of redis sentinel") dbRedisCluster = cgrLoaderFlags.Bool(utils.RedisClusterCfg, false, "Is the redis datadb a cluster") - dbRedisClusterSync = cgrLoaderFlags.String(utils.RedisClusterSyncCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg]), + dbRedisClusterSync = cgrLoaderFlags.Duration(utils.RedisClusterSyncCfg, dfltCfg.DataDbCfg().Opts.RedisClusterSync, "The sync interval for the redis cluster") - dbRedisClusterDownDelay = cgrLoaderFlags.String(utils.RedisClusterOnDownDelayCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]), + dbRedisClusterDownDelay = cgrLoaderFlags.Duration(utils.RedisClusterOnDownDelayCfg, dfltCfg.DataDbCfg().Opts.RedisClusterOndownDelay, "The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state") - dbQueryTimeout = cgrLoaderFlags.String(utils.MongoQueryTimeoutCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]), + dbRedisConnectTimeout = cgrLoaderFlags.Duration(utils.RedisConnectTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisConnectTimeout, + "The amount of wait time until timeout for a connection attempt") + dbRedisReadTimeout = cgrLoaderFlags.Duration(utils.RedisReadTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisReadTimeout, + "The amount of wait time until timeout for reading operations") + dbRedisWriteTimeout = cgrLoaderFlags.Duration(utils.RedisWriteTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisWriteTimeout, + "The amount of wait time until timeout for writing operations") + dbQueryTimeout = cgrLoaderFlags.Duration(utils.MongoQueryTimeoutCfg, dfltCfg.DataDbCfg().Opts.MongoQueryTimeout, "The timeout for queries") dbRedisTls = cgrLoaderFlags.Bool(utils.RedisTLS, false, "Enable TLS when connecting to Redis") dbRedisClientCertificate = cgrLoaderFlags.String(utils.RedisClientCertificate, utils.EmptyString, "Path to the client certificate") @@ -147,39 +157,48 @@ func loadConfig() (ldrCfg *config.CGRConfig) { if *dataDBPasswd != dfltCfg.DataDbCfg().Password { ldrCfg.DataDbCfg().Password = *dataDBPasswd } - - if *dbRedisSentinel != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg]) { - ldrCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg] = *dbRedisSentinel + if *dbRedisMaxConns != dfltCfg.DataDbCfg().Opts.RedisMaxConns { + ldrCfg.DataDbCfg().Opts.RedisMaxConns = *dbRedisMaxConns } - - rdsCls, _ := utils.IfaceAsBool(dfltCfg.DataDbCfg().Opts[utils.RedisClusterCfg]) - if *dbRedisCluster != rdsCls { - ldrCfg.DataDbCfg().Opts[utils.RedisClusterCfg] = *dbRedisCluster + if *dbRedisConnectAttempts != dfltCfg.DataDbCfg().Opts.RedisConnectAttempts { + ldrCfg.DataDbCfg().Opts.RedisConnectAttempts = *dbRedisConnectAttempts } - if *dbRedisClusterSync != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg]) { - ldrCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg] = *dbRedisClusterSync + if *dbRedisSentinel != dfltCfg.DataDbCfg().Opts.RedisSentinel { + ldrCfg.DataDbCfg().Opts.RedisSentinel = *dbRedisSentinel } - if *dbRedisClusterDownDelay != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]) { - ldrCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg] = *dbRedisClusterDownDelay + if *dbRedisCluster != dfltCfg.DataDbCfg().Opts.RedisCluster { + ldrCfg.DataDbCfg().Opts.RedisCluster = *dbRedisCluster } - if *dbQueryTimeout != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]) { - ldrCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg] = *dbQueryTimeout + if *dbRedisClusterSync != dfltCfg.DataDbCfg().Opts.RedisClusterSync { + ldrCfg.DataDbCfg().Opts.RedisClusterSync = *dbRedisClusterSync } - - rdsTLS, _ := utils.IfaceAsBool(dfltCfg.DataDbCfg().Opts[utils.RedisTLS]) - if *dbRedisTls != rdsTLS { - ldrCfg.DataDbCfg().Opts[utils.RedisTLS] = *dbRedisTls + if *dbRedisClusterDownDelay != dfltCfg.DataDbCfg().Opts.RedisClusterOndownDelay { + ldrCfg.DataDbCfg().Opts.RedisClusterOndownDelay = *dbRedisClusterDownDelay } - if *dbRedisClientCertificate != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClientCertificate]) { - ldrCfg.DataDbCfg().Opts[utils.RedisClientCertificate] = *dbRedisClientCertificate + if *dbRedisConnectTimeout != dfltCfg.DataDbCfg().Opts.RedisConnectTimeout { + ldrCfg.DataDbCfg().Opts.RedisConnectTimeout = *dbRedisConnectTimeout } - if *dbRedisClientKey != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClientKey]) { - ldrCfg.DataDbCfg().Opts[utils.RedisClientKey] = *dbRedisClientKey + if *dbRedisReadTimeout != dfltCfg.DataDbCfg().Opts.RedisReadTimeout { + ldrCfg.DataDbCfg().Opts.RedisReadTimeout = *dbRedisReadTimeout } - if *dbRedisCACertificate != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisCACertificate]) { - ldrCfg.DataDbCfg().Opts[utils.RedisCACertificate] = *dbRedisCACertificate + if *dbRedisWriteTimeout != dfltCfg.DataDbCfg().Opts.RedisWriteTimeout { + ldrCfg.DataDbCfg().Opts.RedisWriteTimeout = *dbRedisWriteTimeout + } + if *dbQueryTimeout != dfltCfg.DataDbCfg().Opts.MongoQueryTimeout { + ldrCfg.DataDbCfg().Opts.MongoQueryTimeout = *dbQueryTimeout + } + if *dbRedisTls != dfltCfg.DataDbCfg().Opts.RedisTLS { + ldrCfg.DataDbCfg().Opts.RedisTLS = *dbRedisTls + } + if *dbRedisClientCertificate != dfltCfg.DataDbCfg().Opts.RedisClientCertificate { + ldrCfg.DataDbCfg().Opts.RedisClientCertificate = *dbRedisClientCertificate + } + if *dbRedisClientKey != dfltCfg.DataDbCfg().Opts.RedisClientKey { + ldrCfg.DataDbCfg().Opts.RedisClientKey = *dbRedisClientKey + } + if *dbRedisCACertificate != dfltCfg.DataDbCfg().Opts.RedisCACertificate { + ldrCfg.DataDbCfg().Opts.RedisCACertificate = *dbRedisCACertificate } - if *dbDataEncoding != dfltCfg.GeneralCfg().DBDataEncoding { ldrCfg.GeneralCfg().DBDataEncoding = *dbDataEncoding } diff --git a/cmd/cgr-loader/cgr-loader_it_test.go b/cmd/cgr-loader/cgr-loader_it_test.go index 7dba19c5d..17744b4d1 100644 --- a/cmd/cgr-loader/cgr-loader_it_test.go +++ b/cmd/cgr-loader/cgr-loader_it_test.go @@ -32,6 +32,7 @@ import ( "path" "reflect" "testing" + "time" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -56,6 +57,7 @@ func TestLoadConfig(t *testing.T) { *dataDBUser = "cgrates2" *dataDBPasswd = "toor" *dbRedisSentinel = "sentinel1" + *dbRedisConnectTimeout = 5 * time.Second expDBcfg := &config.DataDbCfg{ Type: utils.Redis, Host: "localhost", @@ -63,16 +65,16 @@ func TestLoadConfig(t *testing.T) { Name: "100", User: "cgrates2", Password: "toor", - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel1", - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterCfg: false, - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "sentinel1", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 5 * time.Second, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -93,13 +95,14 @@ func TestLoadConfig(t *testing.T) { Password: "toor", StringIndexedFields: []string{}, PrefixIndexedFields: []string{}, - Opts: map[string]interface{}{ - utils.SQLConnMaxLifetimeCfg: 0., - utils.MongoQueryTimeoutCfg: "10s", - utils.SQLMaxOpenConnsCfg: 100., - utils.SQLMaxIdleConnsCfg: 10., - utils.SSLModeCfg: "disable", - utils.MysqlLocation: "Local", + Opts: &config.StorDBOpts{ + SQLMaxOpenConns: 100, + SQLMaxIdleConns: 10, + SQLConnMaxLifetime: 0, + MongoQueryTimeout: 10 * time.Second, + PgSSLMode: "disable", + MySQLLocation: "Local", + MySQLDSNParams: map[string]string{}, }, } // Loader diff --git a/cmd/cgr-loader/flags_test.go b/cmd/cgr-loader/flags_test.go index 5ae13a20e..168cf0255 100644 --- a/cmd/cgr-loader/flags_test.go +++ b/cmd/cgr-loader/flags_test.go @@ -18,7 +18,10 @@ along with this program. If not, see package main -import "testing" +import ( + "testing" + "time" +) func TestCGRLoaderFlags(t *testing.T) { if err := cgrLoaderFlags.Parse([]string{"-config_path", "/etc/cgrates"}); err != nil { @@ -83,19 +86,19 @@ func TestCGRLoaderFlags(t *testing.T) { if err := cgrLoaderFlags.Parse([]string{"-redisClusterSync", "3s"}); err != nil { t.Error(err) - } else if *dbRedisClusterSync != "3s" { + } else if *dbRedisClusterSync != 3*time.Second { t.Errorf("Expected 3s, received %+v", *dbRedisClusterSync) } if err := cgrLoaderFlags.Parse([]string{"-redisClusterOndownDelay", "0"}); err != nil { t.Error(err) - } else if *dbRedisClusterDownDelay != "0" { + } else if *dbRedisClusterDownDelay != 0 { t.Errorf("Expected 0, received %+v", *dbRedisClusterDownDelay) } if err := cgrLoaderFlags.Parse([]string{"-mongoQueryTimeout", "5s"}); err != nil { t.Error(err) - } else if *dbQueryTimeout != "5s" { + } else if *dbQueryTimeout != 5*time.Second { t.Errorf("Expected 5s, received %+v", *dbQueryTimeout) } diff --git a/cmd/cgr-migrator/cgr-migrator.go b/cmd/cgr-migrator/cgr-migrator.go index 2a5bdac1f..679e145f6 100644 --- a/cmd/cgr-migrator/cgr-migrator.go +++ b/cmd/cgr-migrator/cgr-migrator.go @@ -63,15 +63,25 @@ var ( "the DataDB password") inDBDataEncoding = cgrMigratorFlags.String(utils.DBDataEncodingCfg, dfltCfg.GeneralCfg().DBDataEncoding, "the encoding used to store object Data in strings") - inDataDBRedisSentinel = cgrMigratorFlags.String(utils.RedisSentinelNameCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg]), + dbRedisMaxConns = cgrMigratorFlags.Int(utils.RedisMaxConnsCfg, dfltCfg.DataDbCfg().Opts.RedisMaxConns, + "The connection pool size") + dbRedisConnectAttempts = cgrMigratorFlags.Int(utils.RedisConnectAttemptsCfg, dfltCfg.DataDbCfg().Opts.RedisConnectAttempts, + "The maximum amount of dial attempts") + inDataDBRedisSentinel = cgrMigratorFlags.String(utils.RedisSentinelNameCfg, dfltCfg.DataDbCfg().Opts.RedisSentinel, "the name of redis sentinel") dbRedisCluster = cgrMigratorFlags.Bool(utils.RedisClusterCfg, false, "Is the redis datadb a cluster") - dbRedisClusterSync = cgrMigratorFlags.String(utils.RedisClusterSyncCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg]), + dbRedisClusterSync = cgrMigratorFlags.Duration(utils.RedisClusterSyncCfg, dfltCfg.DataDbCfg().Opts.RedisClusterSync, "The sync interval for the redis cluster") - dbRedisClusterDownDelay = cgrMigratorFlags.String(utils.RedisClusterOnDownDelayCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]), + dbRedisClusterDownDelay = cgrMigratorFlags.Duration(utils.RedisClusterOnDownDelayCfg, dfltCfg.DataDbCfg().Opts.RedisClusterOndownDelay, "The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state") - dbQueryTimeout = cgrMigratorFlags.String(utils.MongoQueryTimeoutCfg, utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]), + dbRedisConnectTimeout = cgrMigratorFlags.Duration(utils.RedisConnectTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisConnectTimeout, + "The amount of wait time until timeout for a connection attempt") + dbRedisReadTimeout = cgrMigratorFlags.Duration(utils.RedisReadTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisReadTimeout, + "The amount of wait time until timeout for reading operations") + dbRedisWriteTimeout = cgrMigratorFlags.Duration(utils.RedisWriteTimeoutCfg, dfltCfg.DataDbCfg().Opts.RedisWriteTimeout, + "The amount of wait time until timeout for writing operations") + dbQueryTimeout = cgrMigratorFlags.Duration(utils.MongoQueryTimeoutCfg, dfltCfg.DataDbCfg().Opts.MongoQueryTimeout, "The timeout for queries") dbRedisTls = cgrMigratorFlags.Bool(utils.RedisTLS, false, "Enable TLS when connecting to Redis") dbRedisClientCertificate = cgrMigratorFlags.String(utils.RedisClientCertificate, utils.EmptyString, "Path to the client certificate") @@ -169,35 +179,48 @@ func main() { if *inDBDataEncoding != dfltCfg.GeneralCfg().DBDataEncoding { mgrCfg.GeneralCfg().DBDataEncoding = *inDBDataEncoding } - if *inDataDBRedisSentinel != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg]) { - mgrCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg] = *inDataDBRedisSentinel + if *dbRedisMaxConns != dfltCfg.DataDbCfg().Opts.RedisMaxConns { + mgrCfg.DataDbCfg().Opts.RedisMaxConns = *dbRedisMaxConns } - rdsCls, _ := utils.IfaceAsBool(dfltCfg.DataDbCfg().Opts[utils.RedisClusterCfg]) - if *dbRedisCluster != rdsCls { - mgrCfg.DataDbCfg().Opts[utils.RedisClusterCfg] = *dbRedisCluster + if *dbRedisConnectAttempts != dfltCfg.DataDbCfg().Opts.RedisConnectAttempts { + mgrCfg.DataDbCfg().Opts.RedisConnectAttempts = *dbRedisConnectAttempts } - if *dbRedisClusterSync != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg]) { - mgrCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg] = *dbRedisClusterSync + if *inDataDBRedisSentinel != dfltCfg.DataDbCfg().Opts.RedisSentinel { + mgrCfg.DataDbCfg().Opts.RedisSentinel = *inDataDBRedisSentinel } - if *dbRedisClusterDownDelay != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]) { - mgrCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg] = *dbRedisClusterDownDelay + if *dbRedisCluster != dfltCfg.DataDbCfg().Opts.RedisCluster { + mgrCfg.DataDbCfg().Opts.RedisCluster = *dbRedisCluster } - if *dbQueryTimeout != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]) { - mgrCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg] = *dbQueryTimeout + if *dbRedisClusterSync != dfltCfg.DataDbCfg().Opts.RedisClusterSync { + mgrCfg.DataDbCfg().Opts.RedisClusterSync = *dbRedisClusterSync + } + if *dbRedisClusterDownDelay != dfltCfg.DataDbCfg().Opts.RedisClusterOndownDelay { + mgrCfg.DataDbCfg().Opts.RedisClusterOndownDelay = *dbRedisClusterDownDelay + } + if *dbRedisConnectTimeout != dfltCfg.DataDbCfg().Opts.RedisConnectTimeout { + mgrCfg.DataDbCfg().Opts.RedisConnectTimeout = *dbRedisConnectTimeout + } + if *dbRedisReadTimeout != dfltCfg.DataDbCfg().Opts.RedisReadTimeout { + mgrCfg.DataDbCfg().Opts.RedisReadTimeout = *dbRedisReadTimeout + } + if *dbRedisWriteTimeout != dfltCfg.DataDbCfg().Opts.RedisWriteTimeout { + mgrCfg.DataDbCfg().Opts.RedisWriteTimeout = *dbRedisWriteTimeout + } + if *dbQueryTimeout != dfltCfg.DataDbCfg().Opts.MongoQueryTimeout { + mgrCfg.DataDbCfg().Opts.MongoQueryTimeout = *dbQueryTimeout } - rdsTLS, _ := utils.IfaceAsBool(dfltCfg.DataDbCfg().Opts[utils.RedisTLS]) - if *dbRedisTls != rdsTLS { - mgrCfg.DataDbCfg().Opts[utils.RedisTLS] = *dbRedisTls + if *dbRedisTls != dfltCfg.DataDbCfg().Opts.RedisTLS { + mgrCfg.DataDbCfg().Opts.RedisTLS = *dbRedisTls } - if *dbRedisClientCertificate != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClientCertificate]) { - mgrCfg.DataDbCfg().Opts[utils.RedisClientCertificate] = *dbRedisClientCertificate + if *dbRedisClientCertificate != dfltCfg.DataDbCfg().Opts.RedisClientCertificate { + mgrCfg.DataDbCfg().Opts.RedisClientCertificate = *dbRedisClientCertificate } - if *dbRedisClientKey != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisClientKey]) { - mgrCfg.DataDbCfg().Opts[utils.RedisClientKey] = *dbRedisClientKey + if *dbRedisClientKey != dfltCfg.DataDbCfg().Opts.RedisClientKey { + mgrCfg.DataDbCfg().Opts.RedisClientKey = *dbRedisClientKey } - if *dbRedisCACertificate != utils.IfaceAsString(dfltCfg.DataDbCfg().Opts[utils.RedisCACertificate]) { - mgrCfg.DataDbCfg().Opts[utils.RedisCACertificate] = *dbRedisCACertificate + if *dbRedisCACertificate != dfltCfg.DataDbCfg().Opts.RedisCACertificate { + mgrCfg.DataDbCfg().Opts.RedisCACertificate = *dbRedisCACertificate } // outDataDB @@ -252,11 +275,11 @@ func main() { mgrCfg.MigratorCgrCfg().OutDataDBEncoding = *outDBDataEncoding } if *outDataDBRedisSentinel == utils.MetaDataDB { - if utils.IfaceAsString(dfltCfg.MigratorCgrCfg().OutDataDBOpts[utils.RedisSentinelNameCfg]) == utils.IfaceAsString(mgrCfg.MigratorCgrCfg().OutDataDBOpts[utils.RedisSentinelNameCfg]) { - mgrCfg.MigratorCgrCfg().OutDataDBOpts[utils.RedisSentinelNameCfg] = dfltCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg] + if dfltCfg.MigratorCgrCfg().OutDataDBOpts.RedisSentinel == mgrCfg.MigratorCgrCfg().OutDataDBOpts.RedisSentinel { + mgrCfg.MigratorCgrCfg().OutDataDBOpts.RedisSentinel = dfltCfg.DataDbCfg().Opts.RedisSentinel } } else { - mgrCfg.MigratorCgrCfg().OutDataDBOpts[utils.RedisSentinelNameCfg] = *outDataDBRedisSentinel + mgrCfg.MigratorCgrCfg().OutDataDBOpts.RedisSentinel = *outDataDBRedisSentinel } sameDataDB = mgrCfg.MigratorCgrCfg().OutDataDBType == mgrCfg.DataDbCfg().Type && diff --git a/cmd/cgr-migrator/cgr-migrator_test.go b/cmd/cgr-migrator/cgr-migrator_test.go index b6f1a3566..392db6b12 100644 --- a/cmd/cgr-migrator/cgr-migrator_test.go +++ b/cmd/cgr-migrator/cgr-migrator_test.go @@ -20,6 +20,7 @@ package main import ( "testing" + "time" ) // if the flag change this should fail @@ -85,19 +86,19 @@ func TestFlags(t *testing.T) { } else if !*dbRedisCluster { t.Errorf("Expected true received:%v ", *dbRedisCluster) } - if err := cgrMigratorFlags.Parse([]string{"-redisClusterSync", "true"}); err != nil { + if err := cgrMigratorFlags.Parse([]string{"-redisClusterSync", "3s"}); err != nil { t.Fatal(err) - } else if *dbRedisClusterSync != "true" { - t.Errorf("Expected true received:%v ", *dbRedisClusterSync) + } else if *dbRedisClusterSync != 3*time.Second { + t.Errorf("Expected 3s received:%v ", *dbRedisClusterSync) } - if err := cgrMigratorFlags.Parse([]string{"-redisClusterOndownDelay", "true"}); err != nil { + if err := cgrMigratorFlags.Parse([]string{"-redisClusterOndownDelay", "3s"}); err != nil { t.Fatal(err) - } else if *dbRedisClusterDownDelay != "true" { - t.Errorf("Expected true received:%v ", *dbRedisClusterDownDelay) + } else if *dbRedisClusterDownDelay != 3*time.Second { + t.Errorf("Expected 3s received:%v ", *dbRedisClusterDownDelay) } - if err := cgrMigratorFlags.Parse([]string{"-mongoQueryTimeout", "true"}); err != nil { + if err := cgrMigratorFlags.Parse([]string{"-mongoQueryTimeout", "3s"}); err != nil { t.Fatal(err) - } else if *dbQueryTimeout != "true" { + } else if *dbQueryTimeout != 3*time.Second { t.Errorf("Expected true received:%v ", *dbQueryTimeout) } if err := cgrMigratorFlags.Parse([]string{"-redisTLS", "true"}); err != nil { diff --git a/cmd/cgr-tester/cgr-tester.go b/cmd/cgr-tester/cgr-tester.go index 3b06ed5dd..036c908d4 100644 --- a/cmd/cgr-tester/cgr-tester.go +++ b/cmd/cgr-tester/cgr-tester.go @@ -46,23 +46,33 @@ var ( cfgPath = cgrTesterFlags.String("config_path", "", "Configuration directory path.") - parallel = cgrTesterFlags.Int("parallel", 0, "run n requests in parallel") - datadbType = cgrTesterFlags.String("datadb_type", cgrConfig.DataDbCfg().Type, "The type of the DataDb database ") - datadbHost = cgrTesterFlags.String("datadb_host", cgrConfig.DataDbCfg().Host, "The DataDb host to connect to.") - datadbPort = cgrTesterFlags.String("datadb_port", cgrConfig.DataDbCfg().Port, "The DataDb port to bind to.") - datadbName = cgrTesterFlags.String("datadb_name", cgrConfig.DataDbCfg().Name, "The name/number of the DataDb to connect to.") - datadbUser = cgrTesterFlags.String("datadb_user", cgrConfig.DataDbCfg().User, "The DataDb user to sign in as.") - datadbPass = cgrTesterFlags.String("datadb_pass", cgrConfig.DataDbCfg().Password, "The DataDb user's password.") - dbdataEncoding = cgrTesterFlags.String("dbdata_encoding", cgrConfig.GeneralCfg().DBDataEncoding, "The encoding used to store object data in strings.") - redisSentinel = cgrTesterFlags.String("redisSentinel", utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisSentinelNameCfg]), "The name of redis sentinel") + parallel = cgrTesterFlags.Int("parallel", 0, "run n requests in parallel") + datadbType = cgrTesterFlags.String("datadb_type", cgrConfig.DataDbCfg().Type, "The type of the DataDb database ") + datadbHost = cgrTesterFlags.String("datadb_host", cgrConfig.DataDbCfg().Host, "The DataDb host to connect to.") + datadbPort = cgrTesterFlags.String("datadb_port", cgrConfig.DataDbCfg().Port, "The DataDb port to bind to.") + datadbName = cgrTesterFlags.String("datadb_name", cgrConfig.DataDbCfg().Name, "The name/number of the DataDb to connect to.") + datadbUser = cgrTesterFlags.String("datadb_user", cgrConfig.DataDbCfg().User, "The DataDb user to sign in as.") + datadbPass = cgrTesterFlags.String("datadb_pass", cgrConfig.DataDbCfg().Password, "The DataDb user's password.") + dbdataEncoding = cgrTesterFlags.String("dbdata_encoding", cgrConfig.GeneralCfg().DBDataEncoding, "The encoding used to store object data in strings.") + dbRedisMaxConns = cgrTesterFlags.Int(utils.RedisMaxConnsCfg, cgrConfig.DataDbCfg().Opts.RedisMaxConns, + "The connection pool size") + dbRedisConnectAttempts = cgrTesterFlags.Int(utils.RedisConnectAttemptsCfg, cgrConfig.DataDbCfg().Opts.RedisConnectAttempts, + "The maximum amount of dial attempts") + redisSentinel = cgrTesterFlags.String("redisSentinel", cgrConfig.DataDbCfg().Opts.RedisSentinel, "The name of redis sentinel") dbRedisCluster = cgrTesterFlags.Bool("redisCluster", false, "Is the redis datadb a cluster") - dbRedisClusterSync = cgrTesterFlags.String("redisClusterSync", utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisClusterSyncCfg]), + dbRedisClusterSync = cgrTesterFlags.Duration("redisClusterSync", cgrConfig.DataDbCfg().Opts.RedisClusterSync, "The sync interval for the redis cluster") - dbRedisClusterDownDelay = cgrTesterFlags.String("redisClusterOndownDelay", utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]), + dbRedisClusterDownDelay = cgrTesterFlags.Duration("redisClusterOndownDelay", cgrConfig.DataDbCfg().Opts.RedisClusterOndownDelay, "The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state") - dbQueryTimeout = cgrTesterFlags.String("mongoQueryTimeout", utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]), + dbQueryTimeout = cgrTesterFlags.Duration("mongoQueryTimeout", cgrConfig.DataDbCfg().Opts.MongoQueryTimeout, "The timeout for queries") + dbRedisConnectTimeout = cgrTesterFlags.Duration(utils.RedisConnectTimeoutCfg, cgrConfig.DataDbCfg().Opts.RedisConnectTimeout, + "The amount of wait time until timeout for a connection attempt") + dbRedisReadTimeout = cgrTesterFlags.Duration(utils.RedisReadTimeoutCfg, cgrConfig.DataDbCfg().Opts.RedisReadTimeout, + "The amount of wait time until timeout for reading operations") + dbRedisWriteTimeout = cgrTesterFlags.Duration(utils.RedisWriteTimeoutCfg, cgrConfig.DataDbCfg().Opts.RedisWriteTimeout, + "The amount of wait time until timeout for writing operations") raterAddress = cgrTesterFlags.String("rater_address", "", "Rater address for remote tests. Empty for internal rater.") tor = cgrTesterFlags.String("tor", utils.MetaVoice, "The type of record to use in queries.") category = cgrTesterFlags.String("category", "call", "The Record category to test.") @@ -200,21 +210,35 @@ func main() { if *dbdataEncoding != "" { tstCfg.GeneralCfg().DBDataEncoding = *dbdataEncoding } - if *redisSentinel != utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisSentinelNameCfg]) { - tstCfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg] = *redisSentinel + if *dbRedisMaxConns != cgrConfig.DataDbCfg().Opts.RedisMaxConns { + tstCfg.DataDbCfg().Opts.RedisMaxConns = *dbRedisMaxConns } - rdsCls, _ := utils.IfaceAsBool(cgrConfig.DataDbCfg().Opts[utils.RedisClusterCfg]) - if *dbRedisCluster != rdsCls { - tstCfg.DataDbCfg().Opts[utils.RedisClusterCfg] = *dbRedisCluster + if *dbRedisConnectAttempts != cgrConfig.DataDbCfg().Opts.RedisConnectAttempts { + tstCfg.DataDbCfg().Opts.RedisConnectAttempts = *dbRedisConnectAttempts } - if *dbRedisClusterSync != utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisClusterSyncCfg]) { - tstCfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg] = *dbRedisClusterSync + if *redisSentinel != cgrConfig.DataDbCfg().Opts.RedisSentinel { + tstCfg.DataDbCfg().Opts.RedisSentinel = *redisSentinel } - if *dbRedisClusterDownDelay != utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]) { - tstCfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg] = *dbRedisClusterDownDelay + if *dbRedisCluster != cgrConfig.DataDbCfg().Opts.RedisCluster { + tstCfg.DataDbCfg().Opts.RedisCluster = *dbRedisCluster } - if *dbQueryTimeout != utils.IfaceAsString(cgrConfig.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]) { - tstCfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg] = *dbQueryTimeout + if *dbRedisClusterSync != cgrConfig.DataDbCfg().Opts.RedisClusterSync { + tstCfg.DataDbCfg().Opts.RedisClusterSync = *dbRedisClusterSync + } + if *dbRedisClusterDownDelay != cgrConfig.DataDbCfg().Opts.RedisClusterOndownDelay { + tstCfg.DataDbCfg().Opts.RedisClusterOndownDelay = *dbRedisClusterDownDelay + } + if *dbRedisConnectTimeout != cgrConfig.DataDbCfg().Opts.RedisConnectTimeout { + tstCfg.DataDbCfg().Opts.RedisConnectTimeout = *dbRedisConnectTimeout + } + if *dbRedisReadTimeout != cgrConfig.DataDbCfg().Opts.RedisReadTimeout { + tstCfg.DataDbCfg().Opts.RedisReadTimeout = *dbRedisReadTimeout + } + if *dbRedisWriteTimeout != cgrConfig.DataDbCfg().Opts.RedisWriteTimeout { + tstCfg.DataDbCfg().Opts.RedisWriteTimeout = *dbRedisWriteTimeout + } + if *dbQueryTimeout != cgrConfig.DataDbCfg().Opts.MongoQueryTimeout { + tstCfg.DataDbCfg().Opts.MongoQueryTimeout = *dbQueryTimeout } if *cpuprofile != "" { diff --git a/config/config.go b/config/config.go index 360419aa8..ac8596d03 100644 --- a/config/config.go +++ b/config/config.go @@ -123,10 +123,10 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { cfg.generalCfg.NodeID = utils.UUIDSha1Prefix() cfg.dataDbCfg = new(DataDbCfg) cfg.dataDbCfg.Items = make(map[string]*ItemOpt) - cfg.dataDbCfg.Opts = make(map[string]interface{}) + cfg.dataDbCfg.Opts = &DataDBOpts{} cfg.storDbCfg = new(StorDbCfg) cfg.storDbCfg.Items = make(map[string]*ItemOpt) - cfg.storDbCfg.Opts = make(map[string]interface{}) + cfg.storDbCfg.Opts = &StorDBOpts{} cfg.tlsCfg = new(TLSCfg) cfg.cacheCfg = new(CacheCfg) cfg.cacheCfg.Partitions = make(map[string]*CacheParamCfg) @@ -165,8 +165,8 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { cfg.registrarCCfg.Dispatchers.Hosts = make(map[string][]*RemoteHost) cfg.loaderCgrCfg = new(LoaderCgrCfg) cfg.migratorCgrCfg = new(MigratorCgrCfg) - cfg.migratorCgrCfg.OutDataDBOpts = make(map[string]interface{}) - cfg.migratorCgrCfg.OutStorDBOpts = make(map[string]interface{}) + cfg.migratorCgrCfg.OutDataDBOpts = &DataDBOpts{} + cfg.migratorCgrCfg.OutStorDBOpts = &StorDBOpts{} cfg.mailerCfg = new(MailerCfg) cfg.loaderCfg = make(LoaderSCfgs, 0) cfg.apier = new(ApierCfg) diff --git a/config/config_defaults.go b/config/config_defaults.go index 70f0c877e..d35394e76 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -134,10 +134,15 @@ const CGRATES_CFG_JSON = ` "*reverse_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, }, "opts":{ + "redisMaxConns": 10, // the connection pool size + "redisConnectAttempts": 20, // the maximum amount of dial attempts "redisSentinel": "", // the name of sentinel when used "redisCluster": false, // if enabled the datadb will try to connect to the redis cluster "redisClusterSync": "5s", // the sync interval for the redis cluster "redisClusterOndownDelay": "0", // the delay before executing the commands if the redis cluster is in the CLUSTERDOWN state + "redisConnectTimeout": "0", // the amount of wait time until timeout for a connection attempt + "redisReadTimeout": "0", // the amount of wait time until timeout for reading operations + "redisWriteTimeout": "0", // the amount of wait time until timeout for writing operations "mongoQueryTimeout":"10s", // timeout for query when mongo is used "redisTLS": false, // if true it will use a tls connection and use the redisClientCertificate, redisClientKey and redisCACertificate for tls connection "redisClientCertificate":"", // path to client certificate @@ -162,7 +167,7 @@ const CGRATES_CFG_JSON = ` "sqlConnMaxLifetime": 0, // maximum amount of time in seconds a connection may be reused (0 for unlimited), not applying for mongo "mysqlDSNParams": {}, // DSN extra paramss "mongoQueryTimeout":"10s", // timeout for query when mongo is used - "postgresSSLMode":"disable", // postgresSSLMode in case of *postgres + "pgSSLMode":"disable", // pgSSLMode in case of *postgres "mysqlLocation": "Local", // the location the time from mysql is retrived }, "items":{ @@ -404,11 +409,11 @@ const CGRATES_CFG_JSON = ` // SQL // "sqlDBName": "cgrates", // the name of the database from were the events are read // "sqlTableName": "cdrs", // the name of the table from were the events are read - // "postgresSSLMode": "disable", // the ssl mode for postgres db + // "pgSSLMode": "disable", // the ssl mode for postgres db // "sqlDBNameProcessed": "", // the name of the database were the events are sent after they are processed // "sqlTableNameProcessed": "", // the name of the table were the events are sent after they are processed - // "postgresSSLModeProcessed": "", // the ssl mode for postgres db + // "pgSSLModeProcessed": "", // the ssl mode for postgres db // SQS and S3 // "awsRegion": "", @@ -1054,10 +1059,15 @@ const CGRATES_CFG_JSON = ` "out_stordb_password": "", "users_filters":[], "out_datadb_opts":{ + "redisMaxConns": 10, // the connection pool size + "redisConnectAttempts": 20, // the maximum amount of dial attempts "redisSentinel": "", "redisCluster": false, "redisClusterSync": "5s", "redisClusterOndownDelay": "0", + "redisConnectTimeout": "0", // the amount of wait time until timeout for a connection attempt + "redisReadTimeout": "0", // the amount of wait time until timeout for reading operations + "redisWriteTimeout": "0", // the amount of wait time until timeout for writing operations "redisTLS": false, // enable TLS when connecting to Redis and use the redisClientCertificate, redisClientKey and redisCACertificate for TLS connection "redisClientCertificate":"", // path to client certificate "redisClientKey":"", // path to client key diff --git a/config/config_json_test.go b/config/config_json_test.go index 53da9df43..043bb9381 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -274,16 +274,21 @@ func TestDfDataDbJsonCfg(t *testing.T) { Replication_filtered: utils.BoolPointer(false), Remote_conn_id: utils.StringPointer(""), Replication_cache: utils.StringPointer(""), - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "", - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterCfg: false, - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &DBOptsJson{ + RedisMaxConns: utils.IntPointer(10), + RedisConnectAttempts: utils.IntPointer(20), + RedisSentinel: utils.StringPointer(utils.EmptyString), + RedisCluster: utils.BoolPointer(false), + RedisClusterSync: utils.StringPointer("5s"), + RedisClusterOndownDelay: utils.StringPointer("0"), + RedisConnectTimeout: utils.StringPointer("0"), + RedisReadTimeout: utils.StringPointer("0"), + RedisWriteTimeout: utils.StringPointer("0"), + MongoQueryTimeout: utils.StringPointer("10s"), + RedisTLS: utils.BoolPointer(false), + RedisClientCertificate: utils.StringPointer(utils.EmptyString), + RedisClientKey: utils.StringPointer(utils.EmptyString), + RedisCACertificate: utils.StringPointer(utils.EmptyString), }, Items: &map[string]*ItemOptJson{ utils.MetaAccounts: { @@ -541,14 +546,14 @@ func TestDfStorDBJsonCfg(t *testing.T) { Db_password: utils.StringPointer(""), String_indexed_fields: &[]string{}, Prefix_indexed_fields: &[]string{}, - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.SQLMaxOpenConnsCfg: 100., - utils.SQLMaxIdleConnsCfg: 10., - utils.SQLConnMaxLifetimeCfg: 0., - utils.MYSQLDSNParams: make(map[string]interface{}), - utils.SSLModeCfg: utils.PostgressSSLModeDisable, - utils.MysqlLocation: "Local", + Opts: &DBOptsJson{ + SQLMaxOpenConns: utils.IntPointer(100), + SQLMaxIdleConns: utils.IntPointer(10), + MongoQueryTimeout: utils.StringPointer("10s"), + SQLConnMaxLifetime: utils.StringPointer("0"), + MySQLDSNParams: make(map[string]string), + PgSSLMode: utils.StringPointer(utils.PostgressSSLModeDisable), + MySQLLocation: utils.StringPointer("Local"), }, Items: &map[string]*ItemOptJson{ utils.CacheTBLTPTimings: { @@ -1872,16 +1877,21 @@ func TestDfMigratorCfg(t *testing.T) { Out_storDB_user: utils.StringPointer("cgrates"), Out_storDB_password: utils.StringPointer(""), Users_filters: &[]string{}, - Out_storDB_opts: make(map[string]interface{}), - Out_dataDB_opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "", - utils.RedisClusterCfg: false, - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Out_storDB_opts: &DBOptsJson{}, + Out_dataDB_opts: &DBOptsJson{ + RedisMaxConns: utils.IntPointer(10), + RedisConnectAttempts: utils.IntPointer(20), + RedisSentinel: utils.StringPointer(utils.EmptyString), + RedisCluster: utils.BoolPointer(false), + RedisClusterSync: utils.StringPointer("5s"), + RedisClusterOndownDelay: utils.StringPointer("0"), + RedisConnectTimeout: utils.StringPointer("0"), + RedisReadTimeout: utils.StringPointer("0"), + RedisWriteTimeout: utils.StringPointer("0"), + RedisTLS: utils.BoolPointer(false), + RedisClientCertificate: utils.StringPointer(utils.EmptyString), + RedisClientKey: utils.StringPointer(utils.EmptyString), + RedisCACertificate: utils.StringPointer(utils.EmptyString), }, } dfCgrJSONCfg, err := NewCgrJsonCfgFromBytes([]byte(CGRATES_CFG_JSON)) diff --git a/config/config_test.go b/config/config_test.go index 1603a5cd9..d683fb4d1 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -376,11 +376,11 @@ func TestCgrCfgJSONDefaultsStorDB(t *testing.T) { if cgrCfg.StorDbCfg().Password != "" { t.Errorf("Expecting: , received: %+v", cgrCfg.StorDbCfg().Password) } - if cgrCfg.StorDbCfg().Opts[utils.SQLMaxOpenConnsCfg] != 100. { - t.Errorf("Expecting: 100 , received: %+v", cgrCfg.StorDbCfg().Opts[utils.SQLMaxOpenConnsCfg]) + if cgrCfg.StorDbCfg().Opts.SQLMaxOpenConns != 100 { + t.Errorf("Expecting: 100 , received: %+v", cgrCfg.StorDbCfg().Opts.SQLMaxOpenConns) } - if cgrCfg.StorDbCfg().Opts[utils.SQLMaxIdleConnsCfg] != 10. { - t.Errorf("Expecting: 10 , received: %+v", cgrCfg.StorDbCfg().Opts[utils.SQLMaxIdleConnsCfg]) + if cgrCfg.StorDbCfg().Opts.SQLMaxIdleConns != 10 { + t.Errorf("Expecting: 10 , received: %+v", cgrCfg.StorDbCfg().Opts.SQLMaxIdleConns) } if !reflect.DeepEqual(cgrCfg.StorDbCfg().StringIndexedFields, []string{}) { t.Errorf("Expecting: %+v , received: %+v", []string{}, cgrCfg.StorDbCfg().StringIndexedFields) @@ -3290,17 +3290,16 @@ func TestCgrMigratorCfgDefault(t *testing.T) { OutStorDBName: "cgrates", OutStorDBUser: "cgrates", OutStorDBPassword: "", - OutDataDBOpts: map[string]interface{}{ - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + OutDataDBOpts: &DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: utils.EmptyString, + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisTLS: false, }, - OutStorDBOpts: make(map[string]interface{}), + OutStorDBOpts: &StorDBOpts{}, } if !reflect.DeepEqual(cgrCfg.MigratorCgrCfg(), eMgrCfg) { t.Errorf("expected: %+v, received: %+v", utils.ToJSON(eMgrCfg), utils.ToJSON(cgrCfg.MigratorCgrCfg())) @@ -3633,7 +3632,7 @@ func TestV1GetConfigStorDB(t *testing.T) { utils.SQLConnMaxLifetimeCfg: 0., utils.MYSQLDSNParams: make(map[string]interface{}), utils.MongoQueryTimeoutCfg: "10s", - utils.SSLModeCfg: "disable", + utils.PgSSLModeCfg: "disable", utils.MysqlLocation: "Local", }, utils.ItemsCfg: map[string]interface{}{}, diff --git a/config/configsanity.go b/config/configsanity.go index 5365ded19..42e9669ed 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -894,7 +894,7 @@ func (cfg *CGRConfig) checkConfigSanity() error { if cfg.storDbCfg.Type == utils.Postgres { if !utils.IsSliceMember([]string{utils.PostgressSSLModeDisable, utils.PostgressSSLModeAllow, utils.PostgressSSLModePrefer, utils.PostgressSSLModeRequire, utils.PostgressSSLModeVerifyCa, - utils.PostgressSSLModeVerifyFull}, utils.IfaceAsString(cfg.storDbCfg.Opts[utils.SSLModeCfg])) { + utils.PostgressSSLModeVerifyFull}, cfg.storDbCfg.Opts.PgSSLMode) { return fmt.Errorf("<%s> unsupported sslmode for storDB", utils.StorDB) } } diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 90620a4ba..b530a301a 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1673,8 +1673,8 @@ func TestConfigSanityStorDB(t *testing.T) { cfg = NewDefaultCGRConfig() cfg.storDbCfg = &StorDbCfg{ Type: utils.Postgres, - Opts: map[string]interface{}{ - utils.SSLModeCfg: "wrongSSLMode", + Opts: &StorDBOpts{ + PgSSLMode: "wrongSSLMode", }, } expected := " unsupported sslmode for storDB" diff --git a/config/datadbcfg.go b/config/datadbcfg.go index 723acf5a9..c7dac23be 100644 --- a/config/datadbcfg.go +++ b/config/datadbcfg.go @@ -27,6 +27,23 @@ import ( "github.com/cgrates/cgrates/utils" ) +type DataDBOpts struct { + RedisMaxConns int + RedisConnectAttempts int + RedisSentinel string + RedisCluster bool + RedisClusterSync time.Duration + RedisClusterOndownDelay time.Duration + RedisConnectTimeout time.Duration + RedisReadTimeout time.Duration + RedisWriteTimeout time.Duration + MongoQueryTimeout time.Duration + RedisTLS bool + RedisClientCertificate string + RedisClientKey string + RedisCACertificate string +} + // DataDbCfg Database config type DataDbCfg struct { Type string @@ -41,13 +58,74 @@ type DataDbCfg struct { RplFiltered bool RplCache string Items map[string]*ItemOpt - Opts map[string]interface{} + Opts *DataDBOpts +} + +func (dbOpts *DataDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) { + if jsnCfg == nil { + return + } + if jsnCfg.RedisMaxConns != nil { + dbOpts.RedisMaxConns = *jsnCfg.RedisMaxConns + } + if jsnCfg.RedisConnectAttempts != nil { + dbOpts.RedisConnectAttempts = *jsnCfg.RedisConnectAttempts + } + if jsnCfg.RedisSentinel != nil { + dbOpts.RedisSentinel = *jsnCfg.RedisSentinel + } + if jsnCfg.RedisCluster != nil { + dbOpts.RedisCluster = *jsnCfg.RedisCluster + } + if jsnCfg.RedisClusterSync != nil { + if dbOpts.RedisClusterSync, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisClusterSync); err != nil { + return + } + } + if jsnCfg.RedisClusterOndownDelay != nil { + if dbOpts.RedisClusterOndownDelay, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisClusterOndownDelay); err != nil { + return + } + } + if jsnCfg.RedisConnectTimeout != nil { + if dbOpts.RedisConnectTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisConnectTimeout); err != nil { + return + } + } + if jsnCfg.RedisReadTimeout != nil { + if dbOpts.RedisReadTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisReadTimeout); err != nil { + return + } + } + if jsnCfg.RedisWriteTimeout != nil { + if dbOpts.RedisWriteTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisWriteTimeout); err != nil { + return + } + } + if jsnCfg.MongoQueryTimeout != nil { + if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil { + return + } + } + if jsnCfg.RedisTLS != nil { + dbOpts.RedisTLS = *jsnCfg.RedisTLS + } + if jsnCfg.RedisClientCertificate != nil { + dbOpts.RedisClientCertificate = *jsnCfg.RedisClientCertificate + } + if jsnCfg.RedisClientKey != nil { + dbOpts.RedisClientKey = *jsnCfg.RedisClientKey + } + if jsnCfg.RedisCACertificate != nil { + dbOpts.RedisCACertificate = *jsnCfg.RedisCACertificate + } + return } // loadFromJSONCfg loads Database config from JsonCfg func (dbcfg *DataDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { if jsnDbCfg == nil { - return nil + return } if jsnDbCfg.Db_type != nil { dbcfg.Type = strings.TrimPrefix(*jsnDbCfg.Db_type, "*") @@ -103,11 +181,6 @@ func (dbcfg *DataDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.Items[kJsn] = val } } - if jsnDbCfg.Opts != nil { - for k, v := range jsnDbCfg.Opts { - dbcfg.Opts[k] = v - } - } if jsnDbCfg.Replication_filtered != nil { dbcfg.RplFiltered = *jsnDbCfg.Replication_filtered } @@ -117,9 +190,31 @@ func (dbcfg *DataDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { if jsnDbCfg.Replication_cache != nil { dbcfg.RplCache = *jsnDbCfg.Replication_cache } + if jsnDbCfg.Opts != nil { + err = dbcfg.Opts.loadFromJSONCfg(jsnDbCfg.Opts) + } return } +func (dbOpts *DataDBOpts) Clone() *DataDBOpts { + return &DataDBOpts{ + RedisMaxConns: dbOpts.RedisMaxConns, + RedisConnectAttempts: dbOpts.RedisConnectAttempts, + RedisSentinel: dbOpts.RedisSentinel, + RedisCluster: dbOpts.RedisCluster, + RedisClusterSync: dbOpts.RedisClusterSync, + RedisClusterOndownDelay: dbOpts.RedisClusterOndownDelay, + RedisConnectTimeout: dbOpts.RedisConnectTimeout, + RedisReadTimeout: dbOpts.RedisReadTimeout, + RedisWriteTimeout: dbOpts.RedisWriteTimeout, + MongoQueryTimeout: dbOpts.MongoQueryTimeout, + RedisTLS: dbOpts.RedisTLS, + RedisClientCertificate: dbOpts.RedisClientCertificate, + RedisClientKey: dbOpts.RedisClientKey, + RedisCACertificate: dbOpts.RedisCACertificate, + } +} + // Clone returns the cloned object func (dbcfg *DataDbCfg) Clone() (cln *DataDbCfg) { cln = &DataDbCfg{ @@ -133,15 +228,11 @@ func (dbcfg *DataDbCfg) Clone() (cln *DataDbCfg) { RplCache: dbcfg.RplCache, RmtConnID: dbcfg.RmtConnID, Items: make(map[string]*ItemOpt), - Opts: make(map[string]interface{}), + Opts: dbcfg.Opts.Clone(), } for k, itm := range dbcfg.Items { cln.Items[k] = itm.Clone() } - for k, v := range dbcfg.Opts { - cln.Opts[k] = v - } - if dbcfg.RmtConns != nil { cln.RmtConns = make([]string, len(dbcfg.RmtConns)) for i, conn := range dbcfg.RmtConns { @@ -158,8 +249,24 @@ func (dbcfg *DataDbCfg) Clone() (cln *DataDbCfg) { } // AsMapInterface returns the config as a map[string]interface{} -func (dbcfg *DataDbCfg) AsMapInterface() (initialMP map[string]interface{}) { - initialMP = map[string]interface{}{ +func (dbcfg *DataDbCfg) AsMapInterface() (mp map[string]interface{}) { + opts := map[string]interface{}{ + utils.RedisMaxConnsCfg: dbcfg.Opts.RedisMaxConns, + utils.RedisConnectAttemptsCfg: dbcfg.Opts.RedisConnectAttempts, + utils.RedisSentinelNameCfg: dbcfg.Opts.RedisSentinel, + utils.RedisClusterCfg: dbcfg.Opts.RedisCluster, + utils.RedisClusterSyncCfg: dbcfg.Opts.RedisClusterSync.String(), + utils.RedisClusterOnDownDelayCfg: dbcfg.Opts.RedisClusterOndownDelay.String(), + utils.RedisConnectTimeoutCfg: dbcfg.Opts.RedisConnectTimeout.String(), + utils.RedisReadTimeoutCfg: dbcfg.Opts.RedisReadTimeout.String(), + utils.RedisWriteTimeoutCfg: dbcfg.Opts.RedisWriteTimeout.String(), + utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(), + utils.RedisTLS: dbcfg.Opts.RedisTLS, + utils.RedisClientCertificate: dbcfg.Opts.RedisClientCertificate, + utils.RedisClientKey: dbcfg.Opts.RedisClientKey, + utils.RedisCACertificate: dbcfg.Opts.RedisCACertificate, + } + mp = map[string]interface{}{ utils.DataDbTypeCfg: utils.Meta + dbcfg.Type, utils.DataDbHostCfg: dbcfg.Host, utils.DataDbNameCfg: dbcfg.Name, @@ -170,21 +277,17 @@ func (dbcfg *DataDbCfg) AsMapInterface() (initialMP map[string]interface{}) { utils.ReplicationConnsCfg: dbcfg.RplConns, utils.ReplicationFilteredCfg: dbcfg.RplFiltered, utils.ReplicationCache: dbcfg.RplCache, + utils.OptsCfg: opts, } - opts := make(map[string]interface{}) - for k, v := range dbcfg.Opts { - opts[k] = v - } - initialMP[utils.OptsCfg] = opts if dbcfg.Items != nil { items := make(map[string]interface{}) for key, item := range dbcfg.Items { items[key] = item.AsMapInterface() } - initialMP[utils.ItemsCfg] = items + mp[utils.ItemsCfg] = items } if dbcfg.Port != "" { - initialMP[utils.DataDbPortCfg], _ = strconv.Atoi(dbcfg.Port) + mp[utils.DataDbPortCfg], _ = strconv.Atoi(dbcfg.Port) } return } diff --git a/config/datadbcfg_test.go b/config/datadbcfg_test.go index c6d4f2fee..f8dad6897 100644 --- a/config/datadbcfg_test.go +++ b/config/datadbcfg_test.go @@ -46,8 +46,8 @@ func TestDataDbCfgloadFromJsonCfg(t *testing.T) { Replicate: utils.BoolPointer(false), }, }, - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DBOptsJson{ + RedisSentinel: utils.StringPointer("sentinel"), }, } expected := &DataDbCfg{ @@ -74,8 +74,8 @@ func TestDataDbCfgloadFromJsonCfg(t *testing.T) { Replicate: false, }, }, - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, } jsnCfg := NewDefaultCGRConfig() @@ -87,9 +87,9 @@ func TestDataDbCfgloadFromJsonCfg(t *testing.T) { if !reflect.DeepEqual(expected.Items[utils.MetaAccounts], jsnCfg.dataDbCfg.Items[utils.MetaAccounts]) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected.Items[utils.MetaAccounts]), utils.ToJSON(jsnCfg.dataDbCfg.Items[utils.MetaAccounts])) - } else if !reflect.DeepEqual(expected.Opts[utils.RedisSentinelNameCfg], jsnCfg.dataDbCfg.Opts[utils.RedisSentinelNameCfg]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected.Opts[utils.RedisSentinelNameCfg]), - utils.ToJSON(jsnCfg.dataDbCfg.Opts[utils.RedisSentinelNameCfg])) + } else if !reflect.DeepEqual(expected.Opts.RedisSentinel, jsnCfg.dataDbCfg.Opts.RedisSentinel) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected.Opts.RedisSentinel), + utils.ToJSON(jsnCfg.dataDbCfg.Opts.RedisSentinel)) } else if !reflect.DeepEqual(expected.RplConns, jsnCfg.dataDbCfg.RplConns) { t.Errorf("Expected %+v \n, received %+v", expected.RplConns, jsnCfg.dataDbCfg.RplConns) } @@ -139,7 +139,7 @@ func TestItemCfgloadFromJson(t *testing.T) { func TestDataDbCfgloadFromJsonCfgPort(t *testing.T) { var dbcfg DataDbCfg - dbcfg.Opts = make(map[string]interface{}) + dbcfg.Opts = &DataDBOpts{} cfgJSONStr := `{ "data_db": { "db_type": "mongo", @@ -147,7 +147,7 @@ func TestDataDbCfgloadFromJsonCfgPort(t *testing.T) { }` expected := DataDbCfg{ Type: "mongo", - Opts: make(map[string]interface{}), + Opts: &DataDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -167,7 +167,7 @@ func TestDataDbCfgloadFromJsonCfgPort(t *testing.T) { expected = DataDbCfg{ Type: "mongo", Port: "27017", - Opts: make(map[string]interface{}), + Opts: &DataDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -187,7 +187,7 @@ func TestDataDbCfgloadFromJsonCfgPort(t *testing.T) { expected = DataDbCfg{ Type: "internal", Port: "internal", - Opts: make(map[string]interface{}), + Opts: &DataDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -217,7 +217,7 @@ func TestDataDBRemoteReplication(t *testing.T) { } }` - dbcfg.Opts = make(map[string]interface{}) + dbcfg.Opts = &DataDBOpts{} expected = DataDbCfg{ Type: "redis", Host: "127.0.0.1", @@ -226,8 +226,8 @@ func TestDataDBRemoteReplication(t *testing.T) { User: "cgrates", Password: "password", RmtConns: []string{"Conn1"}, - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { @@ -255,8 +255,8 @@ func TestDataDBRemoteReplication(t *testing.T) { Name: "10", User: "cgrates", Password: "password", - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, RmtConns: []string{"Conn1"}, RplConns: []string{"Conn2"}, @@ -286,8 +286,8 @@ func TestDataDBRemoteReplication(t *testing.T) { Name: "10", User: "cgrates", Password: "password", - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, RmtConns: []string{"Conn1", "Conn2", "Conn3"}, RplConns: []string{"Conn4", "Conn5"}, @@ -347,12 +347,12 @@ func TestDataDbCfgloadFromJsonCfgItems(t *testing.T) { Limit: -1, }, }, - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, } dbcfg.Items = make(map[string]*ItemOpt) - dbcfg.Opts = make(map[string]interface{}) + dbcfg.Opts = &DataDBOpts{} if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) } else if jsnDataDbCfg, err := jsnCfg.DbJsonCfg(DATADB_JSN); err != nil { @@ -390,8 +390,8 @@ func TestDataDbCfgloadFromJsonCfgItems(t *testing.T) { Name: "10", User: "cgrates", Password: "password", - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, RmtConns: []string{"Conn1"}, Items: map[string]*ItemOpt{ @@ -446,8 +446,8 @@ func TestDataDbCfgloadFromJsonCfgItems(t *testing.T) { Name: "10", User: "cgrates", Password: "password", - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DataDBOpts{ + RedisSentinel: "sentinel", }, RmtConns: []string{"Conn1"}, Items: map[string]*ItemOpt{ @@ -549,8 +549,8 @@ func TestCloneDataDB(t *testing.T) { Replicate: utils.BoolPointer(false), }, }, - Opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: "sentinel", + Opts: &DBOptsJson{ + RedisSentinel: utils.StringPointer("sentinel"), }, } jsnCfg := NewDefaultCGRConfig() @@ -561,9 +561,9 @@ func TestCloneDataDB(t *testing.T) { if !reflect.DeepEqual(rcv.Items[utils.MetaAccounts], jsnCfg.dataDbCfg.Items[utils.MetaAccounts]) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(rcv.Items[utils.MetaAccounts]), utils.ToJSON(jsnCfg.dataDbCfg.Items[utils.MetaAccounts])) - } else if !reflect.DeepEqual(rcv.Opts[utils.RedisSentinelNameCfg], jsnCfg.dataDbCfg.Opts[utils.RedisSentinelNameCfg]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(rcv.Opts[utils.RedisSentinelNameCfg]), - utils.ToJSON(jsnCfg.dataDbCfg.Opts[utils.RedisSentinelNameCfg])) + } else if !reflect.DeepEqual(rcv.Opts.RedisSentinel, jsnCfg.dataDbCfg.Opts.RedisSentinel) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(rcv.Opts.RedisSentinel), + utils.ToJSON(jsnCfg.dataDbCfg.Opts.RedisSentinel)) } else if !reflect.DeepEqual(rcv.RplConns, jsnCfg.dataDbCfg.RplConns) { t.Errorf("Expected %+v \n, received %+v", rcv.RplConns, jsnCfg.dataDbCfg.RplConns) } diff --git a/config/erscfg.go b/config/erscfg.go index e47baeecf..e9eb35b38 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -788,7 +788,7 @@ func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string opts[utils.SQLTableNameOpt] = *er.Opts.SQLTableName } if er.Opts.SSLMode != nil { - opts[utils.SSLModeCfg] = *er.Opts.SSLMode + opts[utils.PgSSLModeCfg] = *er.Opts.SSLMode } if er.Opts.SQLDBNameProcessed != nil { opts[utils.SQLDBNameProcessedCfg] = *er.Opts.SQLDBNameProcessed diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 963ebb8cc..ada033a25 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -99,6 +99,29 @@ type TlsJsonCfg struct { Ca_certificate *string } +type DBOptsJson struct { + RedisMaxConns *int `json:"redisMaxConns"` + RedisConnectAttempts *int `json:"redisConnectAttempts"` + RedisSentinel *string `json:"redisSentinel"` + RedisCluster *bool `json:"redisCluster"` + RedisClusterSync *string `json:"redisClusterSync"` + RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"` + RedisConnectTimeout *string `json:"redisConnectTimeout"` + RedisReadTimeout *string `json:"redisReadTimeout"` + RedisWriteTimeout *string `json:"redisWriteTimeout"` + MongoQueryTimeout *string `json:"mongoQueryTimeout"` + RedisTLS *bool `json:"redisTLS"` + RedisClientCertificate *string `json:"redisClientCertificate"` + RedisClientKey *string `json:"redisClientKey"` + RedisCACertificate *string `json:"redisCACertificate"` + SQLMaxOpenConns *int `json:"sqlMaxOpenConns"` + SQLMaxIdleConns *int `json:"sqlMaxIdleConns"` + SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"` + MySQLDSNParams map[string]string `json:"mysqlDSNParams"` + PgSSLMode *string `json:"pgSSLMode"` + MySQLLocation *string `json:"mysqlLocation"` +} + // Database config type DbJsonCfg struct { Db_type *string @@ -115,7 +138,7 @@ type DbJsonCfg struct { Replication_filtered *bool Replication_cache *string Items *map[string]*ItemOptJson - Opts map[string]interface{} + Opts *DBOptsJson } type ItemOptJson struct { @@ -753,8 +776,8 @@ type MigratorCfgJson struct { Out_storDB_user *string Out_storDB_password *string Users_filters *[]string - Out_dataDB_opts map[string]interface{} - Out_storDB_opts map[string]interface{} + Out_dataDB_opts *DBOptsJson + Out_storDB_opts *DBOptsJson } type FcTemplateJsonCfg struct { diff --git a/config/migratorcfg.go b/config/migratorcfg.go index 4d7266c3d..b226c52ea 100644 --- a/config/migratorcfg.go +++ b/config/migratorcfg.go @@ -40,8 +40,8 @@ type MigratorCgrCfg struct { OutStorDBUser string OutStorDBPassword string UsersFilters []string - OutDataDBOpts map[string]interface{} - OutStorDBOpts map[string]interface{} + OutDataDBOpts *DataDBOpts + OutStorDBOpts *StorDBOpts } func (mg *MigratorCgrCfg) loadFromJSONCfg(jsnCfg *MigratorCfgJson) (err error) { @@ -93,33 +93,36 @@ func (mg *MigratorCgrCfg) loadFromJSONCfg(jsnCfg *MigratorCfgJson) (err error) { mg.UsersFilters[i] = v } } - if jsnCfg.Out_dataDB_opts != nil { - for k, v := range jsnCfg.Out_dataDB_opts { - mg.OutDataDBOpts[k] = v - } + err = mg.OutDataDBOpts.loadFromJSONCfg(jsnCfg.Out_dataDB_opts) } if jsnCfg.Out_storDB_opts != nil { - for k, v := range jsnCfg.Out_storDB_opts { - mg.OutStorDBOpts[k] = v - } + err = mg.OutStorDBOpts.loadFromJSONCfg(jsnCfg.Out_storDB_opts) } return nil } // AsMapInterface returns the config as a map[string]interface{} func (mg *MigratorCgrCfg) AsMapInterface() (initialMP map[string]interface{}) { - fltrs := []string{} - if mg.UsersFilters != nil { - fltrs = mg.UsersFilters + outDataDBOpts := map[string]interface{}{ + utils.RedisSentinelNameCfg: mg.OutDataDBOpts.RedisSentinel, + utils.RedisClusterCfg: mg.OutDataDBOpts.RedisCluster, + utils.RedisClusterSyncCfg: mg.OutDataDBOpts.RedisClusterSync.String(), + utils.RedisClusterOnDownDelayCfg: mg.OutDataDBOpts.RedisClusterOndownDelay.String(), + utils.MongoQueryTimeoutCfg: mg.OutDataDBOpts.MongoQueryTimeout.String(), + utils.RedisTLS: mg.OutDataDBOpts.RedisTLS, + utils.RedisClientCertificate: mg.OutDataDBOpts.RedisClientCertificate, + utils.RedisClientKey: mg.OutDataDBOpts.RedisClientKey, + utils.RedisCACertificate: mg.OutDataDBOpts.RedisCACertificate, } - outDataDBOpts := make(map[string]interface{}) - for k, v := range mg.OutDataDBOpts { - outDataDBOpts[k] = v - } - outStorDBOpts := make(map[string]interface{}) - for k, v := range mg.OutStorDBOpts { - outStorDBOpts[k] = v + outStorDBOpts := map[string]interface{}{ + utils.SQLMaxOpenConnsCfg: mg.OutStorDBOpts.SQLMaxOpenConns, + utils.SQLMaxIdleConnsCfg: mg.OutStorDBOpts.SQLMaxIdleConns, + utils.SQLConnMaxLifetime: mg.OutStorDBOpts.SQLConnMaxLifetime.String(), + utils.MongoQueryTimeoutCfg: mg.OutStorDBOpts.MongoQueryTimeout.String(), + utils.PgSSLModeCfg: mg.OutStorDBOpts.PgSSLMode, + utils.MysqlLocation: mg.OutStorDBOpts.MySQLLocation, + utils.MYSQLDSNParams: mg.OutStorDBOpts.MySQLDSNParams, } return map[string]interface{}{ utils.OutDataDBTypeCfg: mg.OutDataDBType, @@ -137,7 +140,7 @@ func (mg *MigratorCgrCfg) AsMapInterface() (initialMP map[string]interface{}) { utils.OutStorDBPasswordCfg: mg.OutStorDBPassword, utils.OutDataDBOptsCfg: outDataDBOpts, utils.OutStorDBOptsCfg: outStorDBOpts, - utils.UsersFiltersCfg: fltrs, + utils.UsersFiltersCfg: utils.CloneStringSlice(mg.UsersFilters), } } @@ -157,20 +160,11 @@ func (mg MigratorCgrCfg) Clone() (cln *MigratorCgrCfg) { OutStorDBName: mg.OutStorDBName, OutStorDBUser: mg.OutStorDBUser, OutStorDBPassword: mg.OutStorDBPassword, - OutDataDBOpts: make(map[string]interface{}), - OutStorDBOpts: make(map[string]interface{}), + OutDataDBOpts: mg.OutDataDBOpts.Clone(), + OutStorDBOpts: mg.OutStorDBOpts.Clone(), } if mg.UsersFilters != nil { - cln.UsersFilters = make([]string, len(mg.UsersFilters)) - for i, f := range mg.UsersFilters { - cln.UsersFilters[i] = f - } - } - for k, v := range mg.OutDataDBOpts { - cln.OutDataDBOpts[k] = v - } - for k, v := range mg.OutStorDBOpts { - cln.OutStorDBOpts[k] = v + cln.UsersFilters = utils.CloneStringSlice(mg.UsersFilters) } return } diff --git a/config/migratorcfg_test.go b/config/migratorcfg_test.go index 8d0d7f5f6..cf3327aa7 100644 --- a/config/migratorcfg_test.go +++ b/config/migratorcfg_test.go @@ -20,6 +20,7 @@ package config import ( "reflect" "testing" + "time" "github.com/cgrates/cgrates/utils" ) @@ -39,12 +40,12 @@ func TestMigratorCgrCfgloadFromJsonCfg(t *testing.T) { Out_storDB_name: utils.StringPointer(utils.CGRateSLwr), Out_storDB_user: utils.StringPointer(utils.CGRateSLwr), Out_storDB_password: utils.StringPointer(utils.EmptyString), - Out_dataDB_opts: map[string]interface{}{ - utils.RedisClusterCfg: true, - utils.RedisClusterSyncCfg: "10s", + Out_dataDB_opts: &DBOptsJson{ + RedisCluster: utils.BoolPointer(true), + RedisClusterSync: utils.StringPointer("10s"), }, - Out_storDB_opts: map[string]interface{}{ - utils.RedisSentinelNameCfg: utils.EmptyString, + Out_storDB_opts: &DBOptsJson{ + SQLMaxOpenConns: utils.IntPointer(100), }, } expected := &MigratorCgrCfg{ @@ -61,18 +62,20 @@ func TestMigratorCgrCfgloadFromJsonCfg(t *testing.T) { OutStorDBName: utils.CGRateSLwr, OutStorDBUser: utils.CGRateSLwr, OutStorDBPassword: utils.EmptyString, - OutDataDBOpts: map[string]interface{}{ - utils.RedisSentinelNameCfg: utils.EmptyString, - utils.RedisClusterCfg: true, - utils.RedisClusterSyncCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + OutDataDBOpts: &DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: utils.EmptyString, + RedisCluster: true, + RedisClusterSync: 10 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + RedisTLS: false, }, - OutStorDBOpts: map[string]interface{}{ - utils.RedisSentinelNameCfg: utils.EmptyString, + OutStorDBOpts: &StorDBOpts{ + SQLMaxOpenConns: 100, }, } cfg := NewDefaultCGRConfig() @@ -202,10 +205,15 @@ func TestMigratorCgrCfgAsMapInterface2(t *testing.T) { utils.UsersFiltersCfg: []string{}, utils.OutStorDBOptsCfg: map[string]interface{}{}, utils.OutDataDBOptsCfg: map[string]interface{}{ + utils.RedisMaxConnsCfg: 10., + utils.RedisConnectAttemptsCfg: 20., utils.RedisSentinelNameCfg: "", utils.RedisClusterCfg: false, utils.RedisClusterSyncCfg: "5s", utils.RedisClusterOnDownDelayCfg: "0", + utils.RedisConnectTimeoutCfg: "0", + utils.RedisReadTimeoutCfg: "0", + utils.RedisWriteTimeoutCfg: "0", utils.RedisTLS: false, utils.RedisClientCertificate: "", utils.RedisClientKey: "", @@ -236,18 +244,21 @@ func TestMigratorCgrCfgClone(t *testing.T) { OutStorDBUser: utils.CGRateSLwr, OutStorDBPassword: utils.EmptyString, UsersFilters: []string{utils.AccountField}, - OutDataDBOpts: map[string]interface{}{ - utils.RedisSentinelNameCfg: utils.EmptyString, - utils.RedisClusterCfg: true, - utils.RedisClusterSyncCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + OutDataDBOpts: &DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: utils.EmptyString, + RedisCluster: true, + RedisClusterSync: 10 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, - OutStorDBOpts: map[string]interface{}{ - utils.RedisSentinelNameCfg: utils.EmptyString, + OutStorDBOpts: &StorDBOpts{ + PgSSLMode: "disable", }, } rcv := sa.Clone() @@ -257,10 +268,10 @@ func TestMigratorCgrCfgClone(t *testing.T) { if rcv.UsersFilters[0] = ""; sa.UsersFilters[0] != utils.AccountField { t.Errorf("Expected clone to not modify the cloned") } - if rcv.OutDataDBOpts[utils.RedisSentinelNameCfg] = "1"; sa.OutDataDBOpts[utils.RedisSentinelNameCfg] != "" { + if rcv.OutDataDBOpts.RedisSentinel = "1"; sa.OutDataDBOpts.RedisSentinel != "" { t.Errorf("Expected clone to not modify the cloned") } - if rcv.OutStorDBOpts[utils.RedisSentinelNameCfg] = "1"; sa.OutStorDBOpts[utils.RedisSentinelNameCfg] != "" { + if rcv.OutStorDBOpts.PgSSLMode = "1"; sa.OutStorDBOpts.PgSSLMode != "" { t.Errorf("Expected clone to not modify the cloned") } } diff --git a/config/stordbcfg.go b/config/stordbcfg.go index d8e2d0b0b..99537313d 100644 --- a/config/stordbcfg.go +++ b/config/stordbcfg.go @@ -22,10 +22,21 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/cgrates/cgrates/utils" ) +type StorDBOpts struct { + SQLMaxOpenConns int + SQLMaxIdleConns int + SQLConnMaxLifetime time.Duration + MongoQueryTimeout time.Duration + PgSSLMode string + MySQLLocation string + MySQLDSNParams map[string]string +} + // StorDbCfg StroreDb config type StorDbCfg struct { Type string // Should reflect the database type used to store logs @@ -39,7 +50,40 @@ type StorDbCfg struct { RmtConns []string // Remote DataDB connIDs RplConns []string // Replication connIDs Items map[string]*ItemOpt - Opts map[string]interface{} + Opts *StorDBOpts +} + +func (dbOpts *StorDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) { + if jsnCfg == nil { + return + } + if jsnCfg.SQLMaxOpenConns != nil { + dbOpts.SQLMaxOpenConns = *jsnCfg.SQLMaxOpenConns + } + if jsnCfg.SQLMaxIdleConns != nil { + dbOpts.SQLMaxIdleConns = *jsnCfg.SQLMaxIdleConns + } + if jsnCfg.SQLConnMaxLifetime != nil { + if dbOpts.SQLConnMaxLifetime, err = utils.ParseDurationWithNanosecs(*jsnCfg.SQLConnMaxLifetime); err != nil { + return + } + } + if jsnCfg.MySQLDSNParams != nil { + dbOpts.MySQLDSNParams = make(map[string]string) + dbOpts.MySQLDSNParams = jsnCfg.MySQLDSNParams + } + if jsnCfg.MongoQueryTimeout != nil { + if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil { + return + } + } + if jsnCfg.PgSSLMode != nil { + dbOpts.PgSSLMode = *jsnCfg.PgSSLMode + } + if jsnCfg.MySQLLocation != nil { + dbOpts.MySQLLocation = *jsnCfg.MySQLLocation + } + return } // loadFromJSONCfg loads StoreDb config from JsonCfg @@ -93,11 +137,6 @@ func (dbcfg *StorDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.RplConns[i] = item } } - if jsnDbCfg.Opts != nil { - for k, v := range jsnDbCfg.Opts { - dbcfg.Opts[k] = v - } - } if jsnDbCfg.Items != nil { for kJsn, vJsn := range *jsnDbCfg.Items { val := new(ItemOpt) @@ -107,9 +146,24 @@ func (dbcfg *StorDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.Items[kJsn] = val } } + if jsnDbCfg.Opts != nil { + err = dbcfg.Opts.loadFromJSONCfg(jsnDbCfg.Opts) + } return nil } +func (dbOpts *StorDBOpts) Clone() *StorDBOpts { + return &StorDBOpts{ + SQLMaxOpenConns: dbOpts.SQLMaxOpenConns, + SQLMaxIdleConns: dbOpts.SQLMaxIdleConns, + SQLConnMaxLifetime: dbOpts.SQLConnMaxLifetime, + MySQLDSNParams: dbOpts.MySQLDSNParams, + MongoQueryTimeout: dbOpts.MongoQueryTimeout, + PgSSLMode: dbOpts.PgSSLMode, + MySQLLocation: dbOpts.MySQLLocation, + } +} + // Clone returns the cloned object func (dbcfg *StorDbCfg) Clone() (cln *StorDbCfg) { cln = &StorDbCfg{ @@ -121,14 +175,11 @@ func (dbcfg *StorDbCfg) Clone() (cln *StorDbCfg) { Password: dbcfg.Password, Items: make(map[string]*ItemOpt), - Opts: make(map[string]interface{}), + Opts: dbcfg.Opts.Clone(), } for key, item := range dbcfg.Items { cln.Items[key] = item.Clone() } - for key, val := range dbcfg.Opts { - cln.Opts[key] = val - } if dbcfg.StringIndexedFields != nil { cln.StringIndexedFields = make([]string, len(dbcfg.StringIndexedFields)) for i, idx := range dbcfg.StringIndexedFields { @@ -157,8 +208,17 @@ func (dbcfg *StorDbCfg) Clone() (cln *StorDbCfg) { } // AsMapInterface returns the config as a map[string]interface{} -func (dbcfg *StorDbCfg) AsMapInterface() (initialMP map[string]interface{}) { - initialMP = map[string]interface{}{ +func (dbcfg *StorDbCfg) AsMapInterface() (mp map[string]interface{}) { + opts := map[string]interface{}{ + utils.SQLMaxOpenConnsCfg: dbcfg.Opts.SQLMaxOpenConns, + utils.SQLMaxIdleConnsCfg: dbcfg.Opts.SQLMaxIdleConns, + utils.SQLConnMaxLifetime: dbcfg.Opts.SQLConnMaxLifetime.String(), + utils.MYSQLDSNParams: dbcfg.Opts.MySQLDSNParams, + utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(), + utils.PgSSLModeCfg: dbcfg.Opts.PgSSLMode, + utils.MysqlLocation: dbcfg.Opts.MySQLLocation, + } + mp = map[string]interface{}{ utils.DataDbTypeCfg: utils.Meta + dbcfg.Type, utils.DataDbHostCfg: dbcfg.Host, utils.DataDbNameCfg: dbcfg.Name, @@ -168,22 +228,18 @@ func (dbcfg *StorDbCfg) AsMapInterface() (initialMP map[string]interface{}) { utils.PrefixIndexedFieldsCfg: dbcfg.PrefixIndexedFields, utils.RemoteConnsCfg: dbcfg.RmtConns, utils.ReplicationConnsCfg: dbcfg.RplConns, + utils.OptsCfg: opts, } - opts := make(map[string]interface{}) - for k, v := range dbcfg.Opts { - opts[k] = v - } - initialMP[utils.OptsCfg] = opts if dbcfg.Items != nil { items := make(map[string]interface{}) for key, item := range dbcfg.Items { items[key] = item.AsMapInterface() } - initialMP[utils.ItemsCfg] = items + mp[utils.ItemsCfg] = items } if dbcfg.Port != utils.EmptyString { dbPort, _ := strconv.Atoi(dbcfg.Port) - initialMP[utils.DataDbPortCfg] = dbPort + mp[utils.DataDbPortCfg] = dbPort } return } diff --git a/config/stordbcfg_test.go b/config/stordbcfg_test.go index 599fa0598..c5013c0ab 100644 --- a/config/stordbcfg_test.go +++ b/config/stordbcfg_test.go @@ -20,6 +20,7 @@ package config import ( "reflect" "testing" + "time" "github.com/cgrates/cgrates/utils" ) @@ -46,12 +47,12 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) { Replicate: utils.BoolPointer(false), }, }, - Opts: map[string]interface{}{ - utils.SQLMaxOpenConnsCfg: 100., - utils.SQLMaxIdleConnsCfg: 10., - utils.SQLConnMaxLifetimeCfg: 0., - utils.MYSQLDSNParams: make(map[string]string), - utils.MysqlLocation: "UTC", + Opts: &DBOptsJson{ + SQLMaxOpenConns: utils.IntPointer(100), + SQLMaxIdleConns: utils.IntPointer(10), + SQLConnMaxLifetime: utils.StringPointer("0"), + MySQLDSNParams: make(map[string]string), + MySQLLocation: utils.StringPointer("UTC"), }, } expected := &StorDbCfg{ @@ -75,14 +76,14 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) { Replicate: false, }, }, - Opts: map[string]interface{}{ - utils.SQLMaxOpenConnsCfg: 100., - utils.SQLMaxIdleConnsCfg: 10., - utils.SQLConnMaxLifetimeCfg: 0., - utils.MYSQLDSNParams: make(map[string]string), - utils.MongoQueryTimeoutCfg: "10s", - utils.SSLModeCfg: "disable", - utils.MysqlLocation: "UTC", + Opts: &StorDBOpts{ + SQLMaxOpenConns: 100, + SQLMaxIdleConns: 10, + SQLConnMaxLifetime: 0, + MongoQueryTimeout: 10 * time.Second, + PgSSLMode: "disable", + MySQLLocation: "UTC", + MySQLDSNParams: make(map[string]string), }, } jsonCfg := NewDefaultCGRConfig() @@ -137,10 +138,10 @@ func TestStoreDbCfgloadFromJsonCfgPort(t *testing.T) { "db_type": "mongo", } }` - dbcfg.Opts = make(map[string]interface{}) + dbcfg.Opts = &StorDBOpts{} expected := StorDbCfg{ Type: "mongo", - Opts: make(map[string]interface{}), + Opts: &StorDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -160,7 +161,7 @@ func TestStoreDbCfgloadFromJsonCfgPort(t *testing.T) { expected = StorDbCfg{ Type: "mongo", Port: "27017", - Opts: make(map[string]interface{}), + Opts: &StorDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -180,7 +181,7 @@ func TestStoreDbCfgloadFromJsonCfgPort(t *testing.T) { expected = StorDbCfg{ Type: "internal", Port: "internal", - Opts: make(map[string]interface{}), + Opts: &StorDBOpts{}, } if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { t.Error(err) @@ -239,7 +240,7 @@ func TestStorDbCfgAsMapInterface(t *testing.T) { utils.SQLConnMaxLifetimeCfg: 0., utils.MYSQLDSNParams: make(map[string]interface{}), utils.MongoQueryTimeoutCfg: "10s", - utils.SSLModeCfg: "disable", + utils.PgSSLModeCfg: "disable", utils.MysqlLocation: "UTC", }, utils.ItemsCfg: map[string]interface{}{ @@ -289,12 +290,12 @@ func TestStorDbCfgClone(t *testing.T) { Replicate: false, }, }, - Opts: map[string]interface{}{ - utils.SQLMaxOpenConnsCfg: 100., - utils.SQLMaxIdleConnsCfg: 10., - utils.SQLConnMaxLifetimeCfg: 0., - utils.MongoQueryTimeoutCfg: "10s", - utils.SSLModeCfg: "disable", + Opts: &StorDBOpts{ + SQLMaxOpenConns: 100, + SQLMaxIdleConns: 10, + SQLConnMaxLifetime: 0, + MySQLDSNParams: make(map[string]string), + MySQLLocation: "UTC", }, } rcv := ban.Clone() @@ -318,7 +319,7 @@ func TestStorDbCfgClone(t *testing.T) { if rcv.Items[utils.MetaCDRs].Remote = false; !ban.Items[utils.MetaCDRs].Remote { t.Errorf("Expected clone to not modify the cloned") } - if rcv.Opts[utils.SSLModeCfg] = ""; ban.Opts[utils.SSLModeCfg] != "disable" { + if rcv.Opts.PgSSLMode = ""; ban.Opts.PgSSLMode != "disable" { t.Errorf("Expected clone to not modify the cloned") } diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 4b5a02a05..dae6b5e4a 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -32,7 +32,7 @@ type MySQLStorage struct { } func NewMySQLStorage(host, port, name, user, password string, - maxConn, maxIdleConn, connMaxLifetime int, location string, dsnParams map[string]string) (*SQLStorage, error) { + maxConn, maxIdleConn int, connMaxLifetime time.Duration, location string, dsnParams map[string]string) (*SQLStorage, error) { connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=%s&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", user, password, host, port, name, location) db, err := gorm.Open(mysql.Open(connectString+AppendToMysqlDSNOpts(dsnParams)), &gorm.Config{AllowGlobalUpdate: true}) @@ -50,7 +50,7 @@ func NewMySQLStorage(host, port, name, user, password string, } mySQLStorage.Db.SetMaxIdleConns(maxIdleConn) mySQLStorage.Db.SetMaxOpenConns(maxConn) - mySQLStorage.Db.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) + mySQLStorage.Db.SetConnMaxLifetime(connMaxLifetime) //db.LogMode(true) mySQLStorage.db = db return &SQLStorage{ diff --git a/engine/storage_postgres.go b/engine/storage_postgres.go index 97714e78c..5a72706cf 100644 --- a/engine/storage_postgres.go +++ b/engine/storage_postgres.go @@ -28,7 +28,7 @@ import ( ) // NewPostgresStorage returns the posgres storDB -func NewPostgresStorage(host, port, name, user, password, sslmode string, maxConn, maxIdleConn, connMaxLifetime int) (*SQLStorage, error) { +func NewPostgresStorage(host, port, name, user, password, sslmode string, maxConn, maxIdleConn int, connMaxLifetime time.Duration) (*SQLStorage, error) { connectString := fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=%s", host, port, name, user, password, sslmode) db, err := gorm.Open(postgres.Open(connectString), &gorm.Config{AllowGlobalUpdate: true}) if err != nil { @@ -43,7 +43,7 @@ func NewPostgresStorage(host, port, name, user, password, sslmode string, maxCon } postgressStorage.Db.SetMaxIdleConns(maxIdleConn) postgressStorage.Db.SetMaxOpenConns(maxConn) - postgressStorage.Db.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) + postgressStorage.Db.SetConnMaxLifetime(connMaxLifetime) //db.LogMode(true) postgressStorage.db = db return &SQLStorage{ diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 0801dba26..2437b5b49 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -73,14 +73,14 @@ const ( func NewRedisStorage(address string, db int, user, pass, mrshlerStr string, maxConns, attempts int, sentinelName string, isCluster bool, clusterSync, - clusterOnDownDelay time.Duration, tlsConn bool, - tlsClientCert, tlsClientKey, tlsCACert string) (_ *RedisStorage, err error) { + clusterOnDownDelay time.Duration, connTimeout, readTimeout, writeTimeout time.Duration, + tlsConn bool, tlsClientCert, tlsClientKey, tlsCACert string) (_ *RedisStorage, err error) { var ms Marshaler if ms, err = NewMarshaler(mrshlerStr); err != nil { return } - dialOpts := make([]radix.DialOpt, 1, 3) + dialOpts := make([]radix.DialOpt, 1, 6) dialOpts[0] = radix.DialSelectDB(db) if pass != utils.EmptyString { if user == utils.EmptyString { @@ -119,6 +119,11 @@ func NewRedisStorage(address string, db int, user, pass, mrshlerStr string, })) } + dialOpts = append(dialOpts, + radix.DialReadTimeout(readTimeout), + radix.DialWriteTimeout(writeTimeout), + radix.DialConnectTimeout(connTimeout)) + var client radix.Client if client, err = newRedisClient(address, sentinelName, isCluster, clusterSync, clusterOnDownDelay, diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 220a665be..b4ec76e8a 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -32,7 +32,7 @@ import ( // NewDataDBConn creates a DataDB connection func NewDataDBConn(dbType, host, port, name, user, - pass, marshaler string, opts map[string]interface{}, + pass, marshaler string, opts *config.DataDBOpts, itmsCfg map[string]*config.ItemOpt) (d DataDB, err error) { switch dbType { case utils.Redis: @@ -45,33 +45,12 @@ func NewDataDBConn(dbType, host, port, name, user, if port != "" && !strings.Contains(host, ":") { host += ":" + port } - var isCluster bool - if isCluster, err = utils.IfaceAsBool(opts[utils.RedisClusterCfg]); err != nil { - return - } - var clusterSync, clusterOnDownDelay time.Duration - if clusterSync, err = utils.IfaceAsDuration(opts[utils.RedisClusterSyncCfg]); err != nil { - return - } - if clusterOnDownDelay, err = utils.IfaceAsDuration(opts[utils.RedisClusterOnDownDelayCfg]); err != nil { - return - } - var hasTlsConn bool - if hasTlsConn, err = utils.IfaceAsBool(opts[utils.RedisTLS]); err != nil { - return - } - d, err = NewRedisStorage(host, dbNo, user, pass, marshaler, - utils.RedisMaxConns, utils.RedisMaxAttempts, utils.IfaceAsString(opts[utils.RedisSentinelNameCfg]), - isCluster, clusterSync, clusterOnDownDelay, hasTlsConn, - utils.IfaceAsString(opts[utils.RedisClientCertificate]), - utils.IfaceAsString(opts[utils.RedisClientKey]), - utils.IfaceAsString(opts[utils.RedisCACertificate])) + d, err = NewRedisStorage(host, dbNo, user, pass, marshaler, opts.RedisMaxConns, opts.RedisConnectAttempts, + opts.RedisSentinel, opts.RedisCluster, opts.RedisClusterSync, opts.RedisClusterOndownDelay, + opts.RedisConnectTimeout, opts.RedisReadTimeout, opts.RedisWriteTimeout, opts.RedisTLS, + opts.RedisClientCertificate, opts.RedisClientKey, opts.RedisCACertificate) case utils.Mongo: - var ttl time.Duration - if ttl, err = utils.IfaceAsDuration(opts[utils.MongoQueryTimeoutCfg]); err != nil { - return - } - d, err = NewMongoStorage(host, port, name, user, pass, marshaler, utils.DataDB, nil, ttl) + d, err = NewMongoStorage(host, port, name, user, pass, marshaler, utils.DataDB, nil, opts.MongoQueryTimeout) case utils.Internal: d = NewInternalDB(nil, nil, true, itmsCfg) default: @@ -83,47 +62,16 @@ func NewDataDBConn(dbType, host, port, name, user, // NewStorDBConn returns a StorDB(implements Storage interface) based on dbType func NewStorDBConn(dbType, host, port, name, user, pass, marshaler string, stringIndexedFields, prefixIndexedFields []string, - opts map[string]interface{}, itmsCfg map[string]*config.ItemOpt) (db StorDB, err error) { + opts *config.StorDBOpts, itmsCfg map[string]*config.ItemOpt) (db StorDB, err error) { switch dbType { case utils.Mongo: - var ttl time.Duration - if ttl, err = utils.IfaceAsDuration(opts[utils.MongoQueryTimeoutCfg]); err != nil { - return nil, err - } - db, err = NewMongoStorage(host, port, name, user, pass, marshaler, utils.StorDB, stringIndexedFields, ttl) + db, err = NewMongoStorage(host, port, name, user, pass, marshaler, utils.StorDB, stringIndexedFields, opts.MongoQueryTimeout) case utils.Postgres: - var maxConn, maxIdleConn, connMaxLifetime int64 - if maxConn, err = utils.IfaceAsTInt64(opts[utils.SQLMaxOpenConnsCfg]); err != nil { - return - } - if maxIdleConn, err = utils.IfaceAsTInt64(opts[utils.SQLMaxIdleConnsCfg]); err != nil { - return - } - if connMaxLifetime, err = utils.IfaceAsTInt64(opts[utils.SQLConnMaxLifetimeCfg]); err != nil { - return - } - db, err = NewPostgresStorage(host, port, name, user, pass, utils.IfaceAsString(opts[utils.SSLModeCfg]), - int(maxConn), int(maxIdleConn), int(connMaxLifetime)) + db, err = NewPostgresStorage(host, port, name, user, pass, opts.PgSSLMode, + opts.SQLMaxOpenConns, opts.SQLMaxIdleConns, opts.SQLConnMaxLifetime) case utils.MySQL: - var maxConn, maxIdleConn, connMaxLifetime int64 - - if maxConn, err = utils.IfaceAsTInt64(opts[utils.SQLMaxOpenConnsCfg]); err != nil { - return - } - if maxIdleConn, err = utils.IfaceAsTInt64(opts[utils.SQLMaxIdleConnsCfg]); err != nil { - return - } - if connMaxLifetime, err = utils.IfaceAsTInt64(opts[utils.SQLConnMaxLifetimeCfg]); err != nil { - return - } - optsDB := make(map[string]string) - if optsNew, has := opts[utils.MYSQLDSNParams]; has { - for key, val := range optsNew.(map[string]interface{}) { - optsDB[key] = utils.IfaceAsString(val) - } - } - db, err = NewMySQLStorage(host, port, name, user, pass, int(maxConn), int(maxIdleConn), - int(connMaxLifetime), utils.IfaceAsString(opts[utils.MysqlLocation]), optsDB) + db, err = NewMySQLStorage(host, port, name, user, pass, opts.SQLMaxOpenConns, opts.SQLMaxIdleConns, + opts.SQLConnMaxLifetime, opts.MySQLLocation, opts.MySQLDSNParams) case utils.Internal: db = NewInternalDB(stringIndexedFields, prefixIndexedFields, false, itmsCfg) default: diff --git a/engine/z_datamanager_it_test.go b/engine/z_datamanager_it_test.go index 67944393b..ce02846f8 100644 --- a/engine/z_datamanager_it_test.go +++ b/engine/z_datamanager_it_test.go @@ -53,7 +53,7 @@ func TestDMitinitDB(t *testing.T) { dataDB, err = NewRedisStorage( fmt.Sprintf("%s:%s", cfg.DataDbCfg().Host, cfg.DataDbCfg().Port), 4, cfg.DataDbCfg().User, cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, - utils.RedisMaxConns, utils.RedisMaxAttempts, "", false, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) + 10, 20, "", false, 0, 0, 0, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } diff --git a/engine/z_filterindexer_it_test.go b/engine/z_filterindexer_it_test.go index 74e36072d..11c2cd589 100644 --- a/engine/z_filterindexer_it_test.go +++ b/engine/z_filterindexer_it_test.go @@ -99,7 +99,7 @@ func TestFilterIndexerIT(t *testing.T) { redisDB, err := NewRedisStorage( fmt.Sprintf("%s:%s", cfg.DataDbCfg().Host, cfg.DataDbCfg().Port), 4, cfg.DataDbCfg().User, cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, - utils.RedisMaxConns, utils.RedisMaxAttempts, "", false, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) + 10, 20, "", false, 0, 0, 0, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } diff --git a/engine/z_onstor_it_test.go b/engine/z_onstor_it_test.go index b2d8dc68f..2f813e538 100644 --- a/engine/z_onstor_it_test.go +++ b/engine/z_onstor_it_test.go @@ -93,7 +93,7 @@ func TestOnStorIT(t *testing.T) { rdsITdb, err = NewRedisStorage( fmt.Sprintf("%s:%s", cfg.DataDbCfg().Host, cfg.DataDbCfg().Port), 4, cfg.DataDbCfg().User, cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, - utils.RedisMaxConns, utils.RedisMaxAttempts, "", false, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) + 10, 20, "", false, 0, 0, 0, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) if err != nil { t.Fatal("Could not connect to Redis", err.Error()) } diff --git a/engine/z_stordb_it_test.go b/engine/z_stordb_it_test.go index 0adbff3b6..d0f441e25 100644 --- a/engine/z_stordb_it_test.go +++ b/engine/z_stordb_it_test.go @@ -107,8 +107,7 @@ func TestStorDBit(t *testing.T) { if storDB, err = NewPostgresStorage(storDBCfg.StorDbCfg().Host, storDBCfg.StorDbCfg().Port, storDBCfg.StorDbCfg().Name, storDBCfg.StorDbCfg().User, storDBCfg.StorDbCfg().Password, - utils.IfaceAsString(storDBCfg.StorDbCfg().Opts[utils.SSLModeCfg]), - 100, 10, 0); err != nil { + storDBCfg.StorDbCfg().Opts.PgSSLMode, 100, 10, 0); err != nil { t.Fatal(err) } storDB.(*SQLStorage).db.Config.Logger = logger.Default.LogMode(logger.Silent) diff --git a/general_tests/indexes_redis_it_test.go b/general_tests/indexes_redis_it_test.go index b521b96f5..5f0e3d85c 100644 --- a/general_tests/indexes_redis_it_test.go +++ b/general_tests/indexes_redis_it_test.go @@ -33,8 +33,8 @@ import ( func TestIndexesRedis(t *testing.T) { cfg := config.NewDefaultCGRConfig() db, err := engine.NewRedisStorage(cfg.DataDbCfg().Host+":"+cfg.DataDbCfg().Port, 10, cfg.DataDbCfg().User, - cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, utils.RedisMaxConns, utils.RedisMaxAttempts, - utils.EmptyString, false, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) + cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, 10, 20, + utils.EmptyString, false, 0, 0, 0, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString) if err != nil { t.Fatal(err) } diff --git a/migrator/migrator_utils.go b/migrator/migrator_utils.go index d24c43e25..033062464 100644 --- a/migrator/migrator_utils.go +++ b/migrator/migrator_utils.go @@ -33,7 +33,7 @@ var ( func NewMigratorDataDB(db_type, host, port, name, user, pass, marshaler string, cacheCfg *config.CacheCfg, - opts map[string]interface{}, itmsCfg map[string]*config.ItemOpt) (db MigratorDataDB, err error) { + opts *config.DataDBOpts, itmsCfg map[string]*config.ItemOpt) (db MigratorDataDB, err error) { var dbCon engine.DataDB if dbCon, err = engine.NewDataDBConn(db_type, host, port, name, user, pass, marshaler, opts, nil); err != nil { @@ -56,7 +56,7 @@ func NewMigratorDataDB(db_type, host, port, name, user, pass, func NewMigratorStorDB(db_type, host, port, name, user, pass, marshaler string, stringIndexedFields, prefixIndexedFields []string, - opts map[string]interface{}, itmsCfg map[string]*config.ItemOpt) (db MigratorStorDB, err error) { + opts *config.StorDBOpts, itmsCfg map[string]*config.ItemOpt) (db MigratorStorDB, err error) { var storDb engine.StorDB if storDb, err = engine.NewStorDBConn(db_type, host, port, name, user, pass, marshaler, stringIndexedFields, prefixIndexedFields, opts, itmsCfg); err != nil { diff --git a/services/datadb.go b/services/datadb.go index d41c4a6c9..d7d8cbbdc 100644 --- a/services/datadb.go +++ b/services/datadb.go @@ -21,7 +21,6 @@ package services import ( "fmt" "sync" - "time" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -95,16 +94,12 @@ func (db *DataDBService) Reload() (err error) { return } if db.cfg.DataDbCfg().Type == utils.Mongo { - var ttl time.Duration - if ttl, err = utils.IfaceAsDuration(db.cfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg]); err != nil { - return - } mgo, canCast := db.dm.DataDB().(*engine.MongoStorage) if !canCast { return fmt.Errorf("can't conver DataDB of type %s to MongoStorage", db.cfg.DataDbCfg().Type) } - mgo.SetTTL(ttl) + mgo.SetTTL(db.cfg.DataDbCfg().Opts.MongoQueryTimeout) } return } @@ -171,10 +166,15 @@ func (db *DataDBService) needsConnectionReload() bool { } } return db.oldDBCfg.Type == utils.Redis && - (db.oldDBCfg.Opts[utils.RedisSentinelNameCfg] != db.cfg.DataDbCfg().Opts[utils.RedisSentinelNameCfg] || - db.oldDBCfg.Opts[utils.RedisClusterCfg] != db.cfg.DataDbCfg().Opts[utils.RedisClusterCfg] || - db.oldDBCfg.Opts[utils.RedisClusterSyncCfg] != db.cfg.DataDbCfg().Opts[utils.RedisClusterSyncCfg] || - db.oldDBCfg.Opts[utils.RedisClusterOnDownDelayCfg] != db.cfg.DataDbCfg().Opts[utils.RedisClusterOnDownDelayCfg]) + (db.oldDBCfg.Opts.RedisMaxConns != db.cfg.DataDbCfg().Opts.RedisMaxConns || + db.oldDBCfg.Opts.RedisConnectAttempts != db.cfg.DataDbCfg().Opts.RedisConnectAttempts || + db.oldDBCfg.Opts.RedisSentinel != db.cfg.DataDbCfg().Opts.RedisSentinel || + db.oldDBCfg.Opts.RedisCluster != db.cfg.DataDbCfg().Opts.RedisCluster || + db.oldDBCfg.Opts.RedisClusterSync != db.cfg.DataDbCfg().Opts.RedisClusterSync || + db.oldDBCfg.Opts.RedisClusterOndownDelay != db.cfg.DataDbCfg().Opts.RedisClusterOndownDelay || + db.oldDBCfg.Opts.RedisConnectTimeout != db.cfg.DataDbCfg().Opts.RedisConnectTimeout || + db.oldDBCfg.Opts.RedisReadTimeout != db.cfg.DataDbCfg().Opts.RedisReadTimeout || + db.oldDBCfg.Opts.RedisWriteTimeout != db.cfg.DataDbCfg().Opts.RedisWriteTimeout) } // GetDMChan returns the DataManager chanel diff --git a/services/datadb_it_test.go b/services/datadb_it_test.go index 44a8ad9ee..5ed8608f8 100644 --- a/services/datadb_it_test.go +++ b/services/datadb_it_test.go @@ -88,16 +88,18 @@ func TestDataDBReload(t *testing.T) { Port: "27017", Name: "10", User: "cgrates", - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -214,16 +216,18 @@ func TestDataDBReloadBadType(t *testing.T) { Port: "27017", Name: "10", User: "cgrates", - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -334,16 +338,18 @@ func TestDataDBReloadErrorMarsheler(t *testing.T) { Port: "27017", Name: "10", User: "cgrates", - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -549,16 +555,18 @@ func TestDataDBReloadCastError(t *testing.T) { Port: "27017", Name: "10", User: "cgrates", - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -706,16 +714,18 @@ func TestDataDBReloadIfaceAsDurationError(t *testing.T) { Port: "27017", Name: "10", User: "cgrates", - Opts: map[string]interface{}{ - utils.MongoQueryTimeoutCfg: "10s", - utils.RedisClusterOnDownDelayCfg: "0", - utils.RedisClusterSyncCfg: "5s", - utils.RedisClusterCfg: false, - utils.RedisSentinelNameCfg: "", - utils.RedisTLS: false, - utils.RedisClientCertificate: "", - utils.RedisClientKey: "", - utils.RedisCACertificate: "", + Opts: &config.DataDBOpts{ + RedisMaxConns: 10, + RedisConnectAttempts: 20, + RedisSentinel: "", + RedisCluster: false, + RedisClusterSync: 5 * time.Second, + RedisClusterOndownDelay: 0, + RedisConnectTimeout: 0, + RedisReadTimeout: 0, + RedisWriteTimeout: 0, + MongoQueryTimeout: 10 * time.Second, + RedisTLS: false, }, RmtConns: []string{}, RplConns: []string{}, @@ -794,7 +804,7 @@ func TestDataDBReloadIfaceAsDurationError(t *testing.T) { Remote: false}, }, } - cfg.DataDbCfg().Opts[utils.MongoQueryTimeoutCfg] = true + cfg.DataDbCfg().Opts.MongoQueryTimeout = true db.dm = nil err = db.Reload() if err == nil || err.Error() != "cannot convert field: true to time.Duration" { diff --git a/services/stordb.go b/services/stordb.go index 0b057f412..c2e8f0672 100644 --- a/services/stordb.go +++ b/services/stordb.go @@ -21,7 +21,6 @@ package services import ( "fmt" "sync" - "time" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -101,11 +100,7 @@ func (db *StorDBService) Reload() (err error) { return fmt.Errorf("can't conver StorDB of type %s to MongoStorage", db.cfg.StorDbCfg().Type) } - var ttl time.Duration - if ttl, err = utils.IfaceAsDuration(db.cfg.StorDbCfg().Opts[utils.MongoQueryTimeoutCfg]); err != nil { - return - } - mgo.SetTTL(ttl) + mgo.SetTTL(db.cfg.StorDbCfg().Opts.MongoQueryTimeout) } else if db.cfg.StorDbCfg().Type == utils.Postgres || db.cfg.StorDbCfg().Type == utils.MySQL { msql, canCast := db.db.(*engine.SQLStorage) @@ -113,19 +108,9 @@ func (db *StorDBService) Reload() (err error) { return fmt.Errorf("can't conver StorDB of type %s to SQLStorage", db.cfg.StorDbCfg().Type) } - var maxConn, maxIdleConn, connMaxLifetime int64 - if maxConn, err = utils.IfaceAsTInt64(db.cfg.StorDbCfg().Opts[utils.SQLMaxOpenConnsCfg]); err != nil { - return - } - if maxIdleConn, err = utils.IfaceAsTInt64(db.cfg.StorDbCfg().Opts[utils.SQLMaxIdleConnsCfg]); err != nil { - return - } - if connMaxLifetime, err = utils.IfaceAsTInt64(db.cfg.StorDbCfg().Opts[utils.SQLConnMaxLifetimeCfg]); err != nil { - return - } - msql.Db.SetMaxOpenConns(int(maxConn)) - msql.Db.SetMaxIdleConns(int(maxIdleConn)) - msql.Db.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) + msql.Db.SetMaxOpenConns(db.cfg.StorDbCfg().Opts.SQLMaxOpenConns) + msql.Db.SetMaxIdleConns(db.cfg.StorDbCfg().Opts.SQLMaxIdleConns) + msql.Db.SetConnMaxLifetime(db.cfg.StorDbCfg().Opts.SQLConnMaxLifetime) } else if db.cfg.StorDbCfg().Type == utils.Internal { idb, canCast := db.db.(*engine.InternalDB) if !canCast { @@ -203,5 +188,5 @@ func (db *StorDBService) needsConnectionReload() bool { return true } return db.cfg.StorDbCfg().Type == utils.Postgres && - utils.IfaceAsString(db.oldDBCfg.Opts[utils.SSLModeCfg]) != utils.IfaceAsString(db.cfg.StorDbCfg().Opts[utils.SSLModeCfg]) + db.oldDBCfg.Opts.PgSSLMode != db.cfg.StorDbCfg().Opts.PgSSLMode } diff --git a/utils/consts.go b/utils/consts.go index b7e027558..1285ec6fd 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -154,8 +154,6 @@ const ( CGRateS = "CGRateS" Version = "v0.11.0~dev" DiameterFirmwareRevision = 918 - RedisMaxConns = 10 - RedisMaxAttempts = 20 CGRateSLwr = "cgrates" Postgres = "postgres" MySQL = "mysql" @@ -1967,7 +1965,7 @@ const ( PrefixIndexedFieldsCfg = "prefix_indexed_fields" SuffixIndexedFieldsCfg = "suffix_indexed_fields" MongoQueryTimeoutCfg = "mongoQueryTimeout" - SSLModeCfg = "postgresSSLMode" + PgSSLModeCfg = "pgSSLMode" ItemsCfg = "items" OptsCfg = "opts" Tenants = "tenants" @@ -1983,10 +1981,15 @@ const ( DataDbNameCfg = "db_name" DataDbUserCfg = "db_user" DataDbPassCfg = "db_password" + RedisMaxConnsCfg = "redisMaxConns" + RedisConnectAttemptsCfg = "redisConnectAttempts" RedisSentinelNameCfg = "redisSentinel" RedisClusterCfg = "redisCluster" RedisClusterSyncCfg = "redisClusterSync" RedisClusterOnDownDelayCfg = "redisClusterOndownDelay" + RedisConnectTimeoutCfg = "redisConnectTimeout" + RedisReadTimeoutCfg = "redisReadTimeout" + RedisWriteTimeoutCfg = "redisWriteTimeout" RedisTLS = "redisTLS" RedisClientCertificate = "redisClientCertificate" RedisClientKey = "redisClientKey" diff --git a/utils/reflect.go b/utils/reflect.go index 9100d3127..5dcec2bd5 100644 --- a/utils/reflect.go +++ b/utils/reflect.go @@ -219,6 +219,29 @@ func IfaceAsDuration(itm interface{}) (d time.Duration, err error) { return } +// IfaceAsTInt converts interface to type int +func IfaceAsTInt(itm interface{}) (i int, err error) { + switch it := itm.(type) { + case int: + return it, nil + case time.Duration: + return int(it.Nanoseconds()), nil + case int32: + return int(it), nil + case int64: + return int(it), nil + case float32: + return int(it), nil + case float64: + return int(it), nil + case string: + return strconv.Atoi(it) + default: + err = fmt.Errorf("cannot convert field<%T>: %+v to int", it, it) + } + return +} + func IfaceAsInt64(itm interface{}) (i int64, err error) { switch it := itm.(type) { case int: