From 9f053c04e8f1ce662802bd4aa0a27f702df4bf6b Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Fri, 8 Oct 2021 09:52:34 +0300 Subject: [PATCH] Add stordb opts to struct and adjust word order in SQLConnMaxLifetime constant --- config/config.go | 2 +- config/config_defaults.go | 2 +- config/datadbcfg.go | 23 ++++--- config/stordbcfg.go | 106 ++++++++++++++++++++++++----- data/conf/cgrates/cgrates.json | 2 +- data/conf/samples/ees/cgrates.json | 4 +- ees/sql.go | 2 +- ees/sql_it_test.go | 4 +- utils/consts.go | 2 +- 9 files changed, 113 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index 51fe881c8..91fc27992 100644 --- a/config/config.go +++ b/config/config.go @@ -102,7 +102,7 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { }, storDbCfg: &StorDbCfg{ Items: make(map[string]*ItemOpt), - Opts: make(map[string]interface{}), + Opts: &StorDBOpts{}, }, tlsCfg: new(TLSCfg), cacheCfg: &CacheCfg{Partitions: make(map[string]*CacheParamCfg)}, diff --git a/config/config_defaults.go b/config/config_defaults.go index e8a6cc735..4bab9247f 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -493,7 +493,7 @@ const CGRATES_CFG_JSON = ` // SQL // "sqlMaxIdleConns": 0, // SQLMaxIdleConns // "sqlMaxOpenConns": 0, // SQLMaxOpenConns - // "sqlMaxConnLifetime": 0, // SQLMaxConnLifetime + // "sqlConnMaxLifetime": 0, // SQLMaxConnLifetime // "sqlTableName":"cdrs", // the name of the table from where the events are exported diff --git a/config/datadbcfg.go b/config/datadbcfg.go index c2701ccd8..04ea43ab5 100644 --- a/config/datadbcfg.go +++ b/config/datadbcfg.go @@ -383,15 +383,20 @@ func diffMapItemOptJson(d map[string]*ItemOptJson, v1, v2 map[string]*ItemOpt) m } type DataDBOptsJson struct { - RedisSentinel *string `json:"*redisSentinel"` - RedisCluster *bool `json:"*redisCluster"` - RedisClusterSync *string `json:"*redisClusterSync"` - RedisClusterOndownDelay *string `json:"*redisClusterOndownDelay"` - MongoQueryTimeout *string `json:"*mongoQueryTimeout"` - RedisTLS *bool `json:"*redisTLS"` - RedisClientCertificate *string `json:"*redisClientCertificate"` - RedisClientKey *string `json:"*redisClientKey"` - RedisCACertificate *string `json:"*redisCACertificate"` + RedisSentinel *string `json:"redisSentinel"` + RedisCluster *bool `json:"redisCluster"` + RedisClusterSync *string `json:"redisClusterSync"` + RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"` + MongoQueryTimeout *string `json:"mongoQueryTimeout"` + RedisTLS *bool `json:"redisTLS"` + RedisClientCertificate *string `json:"redisClientCertificate"` + RedisClientKey *string `json:"redisClientKey"` + RedisCACertificate *string `json:"redisCACertificate"` + SQLMaxOpenConns *int `json:"sqlMaxOpenConns"` + SQLMaxIdleConns *int `json:"sqlMaxIdleConns"` + SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"` + SSLMode *string `json:"mongoQueryTimeout"` + MySQLLocation *string `json:"mysqlLocation"` } // Database config diff --git a/config/stordbcfg.go b/config/stordbcfg.go index dd0fb99ef..3bd66b79d 100644 --- a/config/stordbcfg.go +++ b/config/stordbcfg.go @@ -22,11 +22,21 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/utils" ) +type StorDBOpts struct { + SQLMaxOpenConns int + SQLMaxIdleConns int + SQLConnMaxLifetime time.Duration + MongoQueryTimeout time.Duration + SSLMode string + MySQLLocation string +} + // StorDbCfg StroreDb config type StorDbCfg struct { Type string // Should reflect the database type used to store logs @@ -40,7 +50,7 @@ type StorDbCfg struct { RmtConns []string // Remote DataDB connIDs RplConns []string // Replication connIDs Items map[string]*ItemOpt - Opts map[string]interface{} + Opts *StorDBOpts } // loadStorDBCfg loads the StorDB section of the configuration @@ -52,6 +62,35 @@ func (dbcfg *StorDbCfg) Load(ctx *context.Context, jsnCfg ConfigDB, _ *CGRConfig return dbcfg.loadFromJSONCfg(jsnDataDbCfg) } +func (dbOpts *StorDBOpts) loadFromJSONCfg(jsnCfg *DataDBOptsJson) (err error) { + if jsnCfg == nil { + return + } + if jsnCfg.SQLMaxOpenConns != nil { + dbOpts.SQLMaxOpenConns = *jsnCfg.SQLMaxOpenConns + } + if jsnCfg.SQLMaxIdleConns != nil { + dbOpts.SQLMaxIdleConns = *jsnCfg.SQLMaxIdleConns + } + if jsnCfg.SQLConnMaxLifetime != nil { + if dbOpts.SQLConnMaxLifetime, err = utils.ParseDurationWithNanosecs(*jsnCfg.SQLConnMaxLifetime); err != nil { + return + } + } + if jsnCfg.MongoQueryTimeout != nil { + if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil { + return + } + } + if jsnCfg.SSLMode != nil { + dbOpts.SSLMode = *jsnCfg.SSLMode + } + if jsnCfg.MySQLLocation != nil { + dbOpts.MySQLLocation = *jsnCfg.MySQLLocation + } + return +} + // loadFromJSONCfg loads StoreDb config from JsonCfg func (dbcfg *StorDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { if jsnDbCfg == nil { @@ -103,11 +142,6 @@ func (dbcfg *StorDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.RplConns[i] = item } } - if jsnDbCfg.Opts != nil { - for k, v := range jsnDbCfg.Opts { - dbcfg.Opts[k] = v - } - } if jsnDbCfg.Items != nil { for kJsn, vJsn := range jsnDbCfg.Items { val := new(ItemOpt) @@ -115,12 +149,26 @@ func (dbcfg *StorDbCfg) loadFromJSONCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.Items[kJsn] = val } } + if jsnDbCfg.Opts != nil { + err = dbcfg.Opts.loadFromJSONCfg(jsnDbCfg.Opts) + } return nil } func (StorDbCfg) SName() string { return StorDBJSON } func (dbcfg StorDbCfg) CloneSection() Section { return dbcfg.Clone() } +func (dbOpts *StorDBOpts) Clone() *StorDBOpts { + return &StorDBOpts{ + SQLMaxOpenConns: dbOpts.SQLMaxOpenConns, + SQLMaxIdleConns: dbOpts.SQLMaxIdleConns, + SQLConnMaxLifetime: dbOpts.SQLConnMaxLifetime, + MongoQueryTimeout: dbOpts.MongoQueryTimeout, + SSLMode: dbOpts.SSLMode, + MySQLLocation: dbOpts.MySQLLocation, + } +} + // Clone returns the cloned object func (dbcfg StorDbCfg) Clone() (cln *StorDbCfg) { cln = &StorDbCfg{ @@ -132,14 +180,11 @@ func (dbcfg StorDbCfg) Clone() (cln *StorDbCfg) { Password: dbcfg.Password, Items: make(map[string]*ItemOpt), - Opts: make(map[string]interface{}), + Opts: dbcfg.Opts.Clone(), } for key, item := range dbcfg.Items { cln.Items[key] = item.Clone() } - for key, val := range dbcfg.Opts { - cln.Opts[key] = val - } if dbcfg.StringIndexedFields != nil { cln.StringIndexedFields = utils.CloneStringSlice(dbcfg.StringIndexedFields) } @@ -157,6 +202,14 @@ func (dbcfg StorDbCfg) Clone() (cln *StorDbCfg) { // AsMapInterface returns the config as a map[string]interface{} func (dbcfg StorDbCfg) AsMapInterface(string) interface{} { + opts := map[string]interface{}{ + utils.SQLMaxOpenConnsCfg: dbcfg.Opts.SQLMaxOpenConns, + utils.SQLMaxIdleConnsCfg: dbcfg.Opts.SQLMaxIdleConns, + utils.SQLConnMaxLifetime: dbcfg.Opts.SQLConnMaxLifetime.String(), + utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(), + utils.SSLModeCfg: dbcfg.Opts.SSLMode, + utils.MysqlLocation: dbcfg.Opts.MySQLLocation, + } mp := map[string]interface{}{ utils.DataDbTypeCfg: utils.Meta + dbcfg.Type, utils.DataDbHostCfg: dbcfg.Host, @@ -167,12 +220,8 @@ func (dbcfg StorDbCfg) AsMapInterface(string) interface{} { utils.PrefixIndexedFieldsCfg: dbcfg.PrefixIndexedFields, utils.RemoteConnsCfg: dbcfg.RmtConns, utils.ReplicationConnsCfg: dbcfg.RplConns, + utils.OptsCfg: opts, } - opts := make(map[string]interface{}) - for k, v := range dbcfg.Opts { - opts[k] = v - } - mp[utils.OptsCfg] = opts if dbcfg.Items != nil { items := make(map[string]interface{}) for key, item := range dbcfg.Items { @@ -187,6 +236,31 @@ func (dbcfg StorDbCfg) AsMapInterface(string) interface{} { return mp } +func diffStorDBOptsJsonCfg(d *DataDBOptsJson, v1, v2 *StorDBOpts) *DataDBOptsJson { + if d == nil { + d = new(DataDBOptsJson) + } + if v1.SQLMaxOpenConns != v2.SQLMaxOpenConns { + d.SQLMaxOpenConns = utils.IntPointer(v2.SQLMaxOpenConns) + } + if v1.SQLMaxIdleConns != v2.SQLMaxIdleConns { + d.SQLMaxIdleConns = utils.IntPointer(v2.SQLMaxIdleConns) + } + if v1.SQLConnMaxLifetime != v2.SQLConnMaxLifetime { + d.SQLConnMaxLifetime = utils.StringPointer(v2.SQLConnMaxLifetime.String()) + } + if v1.MongoQueryTimeout != v2.MongoQueryTimeout { + d.MongoQueryTimeout = utils.StringPointer(v2.MongoQueryTimeout.String()) + } + if v1.SSLMode != v2.SSLMode { + d.SSLMode = utils.StringPointer(v2.SSLMode) + } + if v1.MySQLLocation != v2.MySQLLocation { + d.MySQLLocation = utils.StringPointer(v2.MySQLLocation) + } + return d +} + func diffStorDBDbJsonCfg(d *DbJsonCfg, v1, v2 *StorDbCfg) *DbJsonCfg { if d == nil { d = new(DbJsonCfg) @@ -226,7 +300,7 @@ func diffStorDBDbJsonCfg(d *DbJsonCfg, v1, v2 *StorDbCfg) *DbJsonCfg { } d.Items = diffMapItemOptJson(d.Items, v1.Items, v2.Items) - d.Opts = diffMap(d.Opts, v1.Opts, v2.Opts) + d.Opts = diffStorDBOptsJsonCfg(d.Opts, v1.Opts, v2.Opts) return d } diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index bdb09e27c..d5122b160 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -449,7 +449,7 @@ // // SQL // // "sqlMaxIdleConns": 0, // SQLMaxIdleConns // // "sqlMaxOpenConns": 0, // SQLMaxOpenConns -// // "sqlMaxConnLifetime": 0, // SQLMaxConnLifetime +// // "sqlConnMaxLifetime": 0, // SQLMaxConnLifetime // // "sqlTableName":"cdrs", // the name of the table from where the events are exported diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index e559a1315..ab374b612 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -385,7 +385,7 @@ "sslMode": "disable", "sqlMaxIdleConns": "10", "sqlMaxOpenConns": "100", - "sqlMaxConnLifetime": "0", + "sqlConnMaxLifetime": "0", }, "fields":[ // in case that the path is *exp.*row user must complete all the fields one to one with his sql schema in the correct order {"tag": "CGRID", "path": "*exp.*row", "type": "*group", "value": "~*req.CGRID"}, @@ -405,7 +405,7 @@ "sslMode": "disable", "sqlMaxIdleConns": "10", "sqlMaxOpenConns": "100", - "sqlMaxConnLifetime": "0", + "sqlConnMaxLifetime": "0", }, "fields":[ // the path constains *exp.columnName {"tag": "CGRID", "path": "*exp.cgrid", "type": "*variable", "value": "~*req.CGRID"}, diff --git a/ees/sql.go b/ees/sql.go index a1c4d5cc1..bed594abb 100644 --- a/ees/sql.go +++ b/ees/sql.go @@ -121,7 +121,7 @@ func openDB(dialect gorm.Dialector, opts map[string]interface{}) (db *gorm.DB, s } sqlDB.SetMaxOpenConns(int(val)) } - if iface, has := opts[utils.SQLMaxConnLifetime]; has { + if iface, has := opts[utils.SQLConnMaxLifetime]; has { val, err := utils.IfaceAsDuration(iface) if err != nil { return nil, nil, err diff --git a/ees/sql_it_test.go b/ees/sql_it_test.go index f00b5f9a3..9a2841cd3 100644 --- a/ees/sql_it_test.go +++ b/ees/sql_it_test.go @@ -270,7 +270,7 @@ func TestOpenDB2Err(t *testing.T) { func TestOpenDB3(t *testing.T) { dialect := mysql.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", "cgrates", "CGRateS.org", "127.0.0.1", "3306", "cgrates")) - _, _, err := openDB(dialect, map[string]interface{}{utils.SQLMaxConnLifetime: 2}) + _, _, err := openDB(dialect, map[string]interface{}{utils.SQLConnMaxLifetime: 2}) if err != nil { t.Error(err) } @@ -279,7 +279,7 @@ func TestOpenDB3(t *testing.T) { func TestOpenDB3Err(t *testing.T) { dialect := mysql.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", "cgrates", "CGRateS.org", "127.0.0.1", "3306", "cgrates")) - _, _, err := openDB(dialect, map[string]interface{}{utils.SQLMaxConnLifetime: "test"}) + _, _, err := openDB(dialect, map[string]interface{}{utils.SQLConnMaxLifetime: "test"}) errExpect := "time: invalid duration \"test\"" if err == nil || err.Error() != errExpect { t.Errorf("Expected %v but received %v", errExpect, err) diff --git a/utils/consts.go b/utils/consts.go index 5c8c1a106..2450ce18d 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -2359,7 +2359,7 @@ const ( SQLTableNameOpt = "sqlTableName" SQLMaxOpenConns = "sqlMaxOpenConns" - SQLMaxConnLifetime = "sqlMaxConnLifetime" + SQLConnMaxLifetime = "sqlConnMaxLifetime" // fileCSV CSVRowLengthOpt = "csvRowLength"