From 5912e6aacf899a9ad2eef7eba518b43e37dc44f2 Mon Sep 17 00:00:00 2001 From: porosnicuadrian Date: Tue, 11 Jan 2022 10:00:36 +0200 Subject: [PATCH] Added DSNopts for stor_db --- config/config_defaults.go | 1 + config/config_json_test.go | 1 + config/config_test.go | 3 ++- config/datadbcfg.go | 29 +++++++++++++++-------------- config/stordbcfg.go | 11 +++++++++++ config/stordbcfg_test.go | 15 +++++++++------ data/conf/samples/ees/cgrates.json | 1 - ees/sql.go | 15 ++------------- engine/storage_mysql.go | 16 +++++++++++++--- engine/storage_utils.go | 2 +- engine/z_stordb_it_test.go | 2 +- 11 files changed, 56 insertions(+), 40 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 57295664a..521c411a4 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -162,6 +162,7 @@ const CGRATES_CFG_JSON = ` "sqlMaxOpenConns": 100, // maximum database connections opened, not applying for mongo "sqlMaxIdleConns": 10, // maximum database connections idle, not applying for mongo "sqlConnMaxLifetime": "0", // maximum amount of time a connection may be reused (0 for unlimited), not applying for mongo + "sqlDSNParams":{}, // DSN params for opening db "mongoQueryTimeout":"10s", // timeout for query when mongo is used "sslMode":"disable", // sslMode in case of *postgres "mysqlLocation": "Local", // the location the time from mysql is retrived diff --git a/config/config_json_test.go b/config/config_json_test.go index addbfca7f..11760b0c2 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -525,6 +525,7 @@ func TestDfStorDBJsonCfg(t *testing.T) { SQLMaxOpenConns: utils.IntPointer(100), SQLMaxIdleConns: utils.IntPointer(10), SQLConnMaxLifetime: utils.StringPointer("0"), + SQLDSNParams: make(map[string]string), SSLMode: utils.StringPointer(utils.PostgressSSLModeDisable), MySQLLocation: utils.StringPointer("Local"), }, diff --git a/config/config_test.go b/config/config_test.go index ded234b07..eb41ddd7f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3928,6 +3928,7 @@ func TestV1GetConfigStorDB(t *testing.T) { utils.SQLMaxOpenConnsCfg: 100, utils.SQLMaxIdleConnsCfg: 10, utils.SQLConnMaxLifetimeCfg: "0s", + utils.SQLDSNParams: make(map[string]string), utils.MongoQueryTimeoutCfg: "10s", utils.SSLModeCfg: "disable", utils.MysqlLocation: "Local", @@ -5097,7 +5098,7 @@ func TestV1GetConfigAsJSONDataDB(t *testing.T) { func TestV1GetConfigAsJSONStorDB(t *testing.T) { var reply string - expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` + expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":"0s","sqlDSNParams":{},"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` cfgCgr := NewDefaultCGRConfig() if err := cfgCgr.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Sections: []string{StorDBJSON}}, &reply); err != nil { t.Error(err) diff --git a/config/datadbcfg.go b/config/datadbcfg.go index 1b69cdc07..12c197e14 100644 --- a/config/datadbcfg.go +++ b/config/datadbcfg.go @@ -431,20 +431,21 @@ func diffMapItemOptJson(d map[string]*ItemOptJson, v1, v2 map[string]*ItemOpt) m } type DBOptsJson struct { - RedisSentinel *string `json:"redisSentinel"` - RedisCluster *bool `json:"redisCluster"` - RedisClusterSync *string `json:"redisClusterSync"` - RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"` - MongoQueryTimeout *string `json:"mongoQueryTimeout"` - RedisTLS *bool `json:"redisTLS"` - RedisClientCertificate *string `json:"redisClientCertificate"` - RedisClientKey *string `json:"redisClientKey"` - RedisCACertificate *string `json:"redisCACertificate"` - SQLMaxOpenConns *int `json:"sqlMaxOpenConns"` - SQLMaxIdleConns *int `json:"sqlMaxIdleConns"` - SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"` - SSLMode *string `json:"sslMode"` - MySQLLocation *string `json:"mysqlLocation"` + RedisSentinel *string `json:"redisSentinel"` + RedisCluster *bool `json:"redisCluster"` + RedisClusterSync *string `json:"redisClusterSync"` + RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"` + MongoQueryTimeout *string `json:"mongoQueryTimeout"` + RedisTLS *bool `json:"redisTLS"` + RedisClientCertificate *string `json:"redisClientCertificate"` + RedisClientKey *string `json:"redisClientKey"` + RedisCACertificate *string `json:"redisCACertificate"` + SQLMaxOpenConns *int `json:"sqlMaxOpenConns"` + SQLMaxIdleConns *int `json:"sqlMaxIdleConns"` + SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"` + SQLDSNParams map[string]string `json:"sqlDSNParams"` + SSLMode *string `json:"sslMode"` + MySQLLocation *string `json:"mysqlLocation"` } // Database config diff --git a/config/stordbcfg.go b/config/stordbcfg.go index d9d86ac0c..da968ba36 100644 --- a/config/stordbcfg.go +++ b/config/stordbcfg.go @@ -20,6 +20,7 @@ package config import ( "fmt" + "reflect" "strconv" "strings" "time" @@ -32,6 +33,7 @@ type StorDBOpts struct { SQLMaxOpenConns int SQLMaxIdleConns int SQLConnMaxLifetime time.Duration + SQLDSNParams map[string]string MongoQueryTimeout time.Duration SSLMode string MySQLLocation string @@ -77,6 +79,10 @@ func (dbOpts *StorDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) { return } } + if jsnCfg.SQLDSNParams != nil { + dbOpts.SQLDSNParams = make(map[string]string) + dbOpts.SQLDSNParams = jsnCfg.SQLDSNParams + } if jsnCfg.MongoQueryTimeout != nil { if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil { return @@ -165,6 +171,7 @@ func (dbOpts *StorDBOpts) Clone() *StorDBOpts { SQLMaxOpenConns: dbOpts.SQLMaxOpenConns, SQLMaxIdleConns: dbOpts.SQLMaxIdleConns, SQLConnMaxLifetime: dbOpts.SQLConnMaxLifetime, + SQLDSNParams: dbOpts.SQLDSNParams, MongoQueryTimeout: dbOpts.MongoQueryTimeout, SSLMode: dbOpts.SSLMode, MySQLLocation: dbOpts.MySQLLocation, @@ -208,6 +215,7 @@ func (dbcfg StorDbCfg) AsMapInterface(string) interface{} { utils.SQLMaxOpenConnsCfg: dbcfg.Opts.SQLMaxOpenConns, utils.SQLMaxIdleConnsCfg: dbcfg.Opts.SQLMaxIdleConns, utils.SQLConnMaxLifetime: dbcfg.Opts.SQLConnMaxLifetime.String(), + utils.SQLDSNParams: dbcfg.Opts.SQLDSNParams, utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(), utils.SSLModeCfg: dbcfg.Opts.SSLMode, utils.MysqlLocation: dbcfg.Opts.MySQLLocation, @@ -251,6 +259,9 @@ func diffStorDBOptsJsonCfg(d *DBOptsJson, v1, v2 *StorDBOpts) *DBOptsJson { if v1.SQLConnMaxLifetime != v2.SQLConnMaxLifetime { d.SQLConnMaxLifetime = utils.StringPointer(v2.SQLConnMaxLifetime.String()) } + if !reflect.DeepEqual(v1.SQLDSNParams, v2.SQLDSNParams) { + d.SQLDSNParams = v2.SQLDSNParams + } if v1.MongoQueryTimeout != v2.MongoQueryTimeout { d.MongoQueryTimeout = utils.StringPointer(v2.MongoQueryTimeout.String()) } diff --git a/config/stordbcfg_test.go b/config/stordbcfg_test.go index 017aca330..3908c78b5 100644 --- a/config/stordbcfg_test.go +++ b/config/stordbcfg_test.go @@ -48,6 +48,7 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) { SQLMaxOpenConns: utils.IntPointer(100), SQLMaxIdleConns: utils.IntPointer(10), SQLConnMaxLifetime: utils.StringPointer("0"), + SQLDSNParams: make(map[string]string), MySQLLocation: utils.StringPointer("UTC"), }, } @@ -71,6 +72,7 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) { Opts: &StorDBOpts{ SQLMaxOpenConns: 100, SQLMaxIdleConns: 10, + SQLDSNParams: make(map[string]string), MongoQueryTimeout: 10 * time.Second, SSLMode: "disable", MySQLLocation: "UTC", @@ -258,6 +260,7 @@ func TestStorDbCfgAsMapInterface(t *testing.T) { utils.SQLMaxOpenConnsCfg: 100, utils.SQLMaxIdleConnsCfg: 10, utils.SQLConnMaxLifetimeCfg: "0s", + utils.SQLDSNParams: make(map[string]string), utils.MongoQueryTimeoutCfg: "10s", utils.SSLModeCfg: "disable", utils.MysqlLocation: "UTC", @@ -273,16 +276,16 @@ func TestStorDbCfgAsMapInterface(t *testing.T) { rcv := cfgCgr.storDbCfg.AsMapInterface("").(map[string]interface{}) if !reflect.DeepEqual(eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg], rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]) { - t.Errorf("Expected %+v, received %+v", eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg], - rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]) + t.Errorf("Expected %+v, received %+v", utils.ToJSON(eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]), + utils.ToJSON(rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg])) } else if !reflect.DeepEqual(eMap[utils.OptsCfg], rcv[utils.OptsCfg]) { - t.Errorf("Expected %+v \n, received %+v", eMap[utils.OptsCfg], rcv[utils.OptsCfg]) + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.OptsCfg]), utils.ToJSON(rcv[utils.OptsCfg])) } else if !reflect.DeepEqual(eMap[utils.PrefixIndexedFieldsCfg], rcv[utils.PrefixIndexedFieldsCfg]) { - t.Errorf("Expected %+v \n, received %+v", eMap[utils.PrefixIndexedFieldsCfg], rcv[utils.PrefixIndexedFieldsCfg]) + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.PrefixIndexedFieldsCfg]), utils.ToJSON(rcv[utils.PrefixIndexedFieldsCfg])) } else if !reflect.DeepEqual(eMap[utils.RemoteConnsCfg], rcv[utils.RemoteConnsCfg]) { - t.Errorf("Expected %+v \n, received %+v", eMap[utils.RemoteConnsCfg], rcv[utils.RemoteConnsCfg]) + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.RemoteConnsCfg]), utils.ToJSON(rcv[utils.RemoteConnsCfg])) } else if !reflect.DeepEqual(eMap[utils.ReplicationConnsCfg], rcv[utils.ReplicationConnsCfg]) { - t.Errorf("Expected %+v \n, received %+v", eMap[utils.ReplicationConnsCfg], rcv[utils.ReplicationConnsCfg]) + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.ReplicationConnsCfg]), utils.ToJSON(rcv[utils.ReplicationConnsCfg])) } } } diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index 4dbc821a8..3d607f3d0 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -461,7 +461,6 @@ "sqlMaxIdleConns": 10, "sqlMaxOpenConns": 100, "sqlConnMaxLifetime": "0", - }, "fields":[ // the path constains *exp.columnName {"tag": "CGRID", "path": "*exp.cgrid", "type": "*variable", "value": "~*req.CGRID"}, diff --git a/ees/sql.go b/ees/sql.go index ebf0a8cb5..bccbf5820 100644 --- a/ees/sql.go +++ b/ees/sql.go @@ -31,6 +31,7 @@ import ( "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -90,7 +91,7 @@ func (sqlEe *SQLEe) initDialector() (err error) { switch u.Scheme { case utils.MySQL: sqlEe.dialect = mysql.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", - u.User.Username(), password, u.Hostname(), u.Port(), dbname) + appendToMysqlDSNOpts(sqlEe.Cfg().Opts)) + u.User.Username(), password, u.Hostname(), u.Port(), dbname) + engine.AppendToMysqlDSNOpts(sqlEe.Cfg().Opts.SQLDSNParams)) case utils.Postgres: sqlEe.dialect = postgres.Open(fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=%s", u.Hostname(), u.Port(), dbname, u.User.Username(), password, ssl)) default: @@ -99,18 +100,6 @@ func (sqlEe *SQLEe) initDialector() (err error) { return } -func appendToMysqlDSNOpts(opts *config.EventExporterOpts) string { - if opts.SQLDSNParams != nil { - var dsn string - for key, val := range opts.SQLDSNParams { - dsn = dsn + "&" + key + "=" + val - } - utils.Logger.Debug("dsn: " + dsn) - return dsn - } - return utils.EmptyString -} - func openDB(dialect gorm.Dialector, opts *config.EventExporterOpts) (db *gorm.DB, sqlDB *sql.DB, err error) { if db, err = gorm.Open(dialect, &gorm.Config{AllowGlobalUpdate: true}); err != nil { return diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index a038b255d..7f796f670 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -32,14 +32,13 @@ type MySQLStorage struct { } func NewMySQLStorage(host, port, name, user, password string, - maxConn, maxIdleConn int, connMaxLifetime time.Duration, location string) (*SQLStorage, error) { + maxConn, maxIdleConn int, connMaxLifetime time.Duration, location string, dsnParams map[string]string) (*SQLStorage, error) { connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=%s&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", user, password, host, port, name, location) - db, err := gorm.Open(mysql.Open(connectString), &gorm.Config{AllowGlobalUpdate: true}) + db, err := gorm.Open(mysql.Open(connectString+AppendToMysqlDSNOpts(dsnParams)), &gorm.Config{AllowGlobalUpdate: true}) if err != nil { return nil, err } - mySQLStorage := new(MySQLStorage) if mySQLStorage.DB, err = db.DB(); err != nil { return nil, err @@ -60,6 +59,17 @@ func NewMySQLStorage(host, port, name, user, password string, }, nil } +func AppendToMysqlDSNOpts(opts map[string]string) string { + if opts != nil { + var dsn string + for key, val := range opts { + dsn = dsn + "&" + key + "=" + val + } + return dsn + } + return utils.EmptyString +} + // SetVersions will set a slice of versions, updating existing func (msqlS *MySQLStorage) SetVersions(vrs Versions, overwrite bool) (err error) { tx := msqlS.db.Begin() diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 6f7826495..e958e6981 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -69,7 +69,7 @@ func NewStorDBConn(dbType, host, port, name, user, pass, marshaler string, opts.SQLMaxOpenConns, opts.SQLMaxIdleConns, opts.SQLConnMaxLifetime) case utils.MySQL: db, err = NewMySQLStorage(host, port, name, user, pass, opts.SQLMaxOpenConns, opts.SQLMaxIdleConns, - opts.SQLConnMaxLifetime, opts.MySQLLocation) + opts.SQLConnMaxLifetime, opts.MySQLLocation, opts.SQLDSNParams) case utils.Internal: db = NewInternalDB(stringIndexedFields, prefixIndexedFields, itmsCfg) default: diff --git a/engine/z_stordb_it_test.go b/engine/z_stordb_it_test.go index f1eba6e3c..964854308 100644 --- a/engine/z_stordb_it_test.go +++ b/engine/z_stordb_it_test.go @@ -74,7 +74,7 @@ func TestStorDBit(t *testing.T) { if storDB, err = NewMySQLStorage(storCfg.StorDbCfg().Host, storCfg.StorDbCfg().Port, storCfg.StorDbCfg().Name, storCfg.StorDbCfg().User, storCfg.StorDbCfg().Password, - 100, 10, 0, "UTC"); err != nil { + 100, 10, 0, "UTC", storCfg.StorDbCfg().Opts.SQLDSNParams); err != nil { t.Fatal(err) } storDB.(*SQLStorage).db.Config.Logger = logger.Default.LogMode(logger.Silent)