uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Added DSNopts for stor_db

Browse Source
This commit is contained in:
porosnicuadrian
2022-01-11 10:00:36 +02:00
committed by Dan Christian Bogos
parent b3879194c1
commit 5912e6aacf
11 changed files with 56 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config/config_defaults.go
View File

@@ -162,6 +162,7 @@ const CGRATES_CFG_JSON = `
"sqlMaxOpenConns": 100, // maximum database connections opened, not applying for mongo
"sqlMaxIdleConns": 10, // maximum database connections idle, not applying for mongo
"sqlConnMaxLifetime": "0", // maximum amount of time a connection may be reused (0 for unlimited), not applying for mongo
"sqlDSNParams":{}, // DSN params for opening db
"mongoQueryTimeout":"10s", // timeout for query when mongo is used
"sslMode":"disable", // sslMode in case of *postgres
"mysqlLocation": "Local", // the location the time from mysql is retrived

1
config/config_json_test.go
View File

@@ -525,6 +525,7 @@ func TestDfStorDBJsonCfg(t *testing.T) {
SQLMaxOpenConns: utils.IntPointer(100),
SQLMaxIdleConns: utils.IntPointer(10),
SQLConnMaxLifetime: utils.StringPointer("0"),
SQLDSNParams: make(map[string]string),
SSLMode: utils.StringPointer(utils.PostgressSSLModeDisable),
MySQLLocation: utils.StringPointer("Local"),
},

3
config/config_test.go
View File

@@ -3928,6 +3928,7 @@ func TestV1GetConfigStorDB(t *testing.T) {
utils.SQLMaxOpenConnsCfg: 100,
utils.SQLMaxIdleConnsCfg: 10,
utils.SQLConnMaxLifetimeCfg: "0s",
utils.SQLDSNParams: make(map[string]string),
utils.MongoQueryTimeoutCfg: "10s",
utils.SSLModeCfg: "disable",
utils.MysqlLocation: "Local",
@@ -5097,7 +5098,7 @@ func TestV1GetConfigAsJSONDataDB(t *testing.T) {
func TestV1GetConfigAsJSONStorDB(t *testing.T) {
var reply string
expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}`
expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":"0s","sqlDSNParams":{},"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}`
cfgCgr := NewDefaultCGRConfig()
if err := cfgCgr.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Sections: []string{StorDBJSON}}, &reply); err != nil {
t.Error(err)

29
config/datadbcfg.go
View File

@@ -431,20 +431,21 @@ func diffMapItemOptJson(d map[string]*ItemOptJson, v1, v2 map[string]*ItemOpt) m
}
type DBOptsJson struct {
RedisSentinel *string `json:"redisSentinel"`
RedisCluster *bool `json:"redisCluster"`
RedisClusterSync *string `json:"redisClusterSync"`
RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"`
MongoQueryTimeout *string `json:"mongoQueryTimeout"`
RedisTLS *bool `json:"redisTLS"`
RedisClientCertificate *string `json:"redisClientCertificate"`
RedisClientKey *string `json:"redisClientKey"`
RedisCACertificate *string `json:"redisCACertificate"`
SQLMaxOpenConns *int `json:"sqlMaxOpenConns"`
SQLMaxIdleConns *int `json:"sqlMaxIdleConns"`
SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"`
SSLMode *string `json:"sslMode"`
MySQLLocation *string `json:"mysqlLocation"`
RedisSentinel *string `json:"redisSentinel"`
RedisCluster *bool `json:"redisCluster"`
RedisClusterSync *string `json:"redisClusterSync"`
RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"`
MongoQueryTimeout *string `json:"mongoQueryTimeout"`
RedisTLS *bool `json:"redisTLS"`
RedisClientCertificate *string `json:"redisClientCertificate"`
RedisClientKey *string `json:"redisClientKey"`
RedisCACertificate *string `json:"redisCACertificate"`
SQLMaxOpenConns *int `json:"sqlMaxOpenConns"`
SQLMaxIdleConns *int `json:"sqlMaxIdleConns"`
SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"`
SQLDSNParams map[string]string `json:"sqlDSNParams"`
SSLMode *string `json:"sslMode"`
MySQLLocation *string `json:"mysqlLocation"`
}
// Database config

11
config/stordbcfg.go
View File

@@ -20,6 +20,7 @@ package config
import (
"fmt"
"reflect"
"strconv"
"strings"
"time"
@@ -32,6 +33,7 @@ type StorDBOpts struct {
SQLMaxOpenConns int
SQLMaxIdleConns int
SQLConnMaxLifetime time.Duration
SQLDSNParams map[string]string
MongoQueryTimeout time.Duration
SSLMode string
MySQLLocation string
@@ -77,6 +79,10 @@ func (dbOpts *StorDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) {
return
}
}
if jsnCfg.SQLDSNParams != nil {
dbOpts.SQLDSNParams = make(map[string]string)
dbOpts.SQLDSNParams = jsnCfg.SQLDSNParams
}
if jsnCfg.MongoQueryTimeout != nil {
if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil {
return
@@ -165,6 +171,7 @@ func (dbOpts *StorDBOpts) Clone() *StorDBOpts {
SQLMaxOpenConns: dbOpts.SQLMaxOpenConns,
SQLMaxIdleConns: dbOpts.SQLMaxIdleConns,
SQLConnMaxLifetime: dbOpts.SQLConnMaxLifetime,
SQLDSNParams: dbOpts.SQLDSNParams,
MongoQueryTimeout: dbOpts.MongoQueryTimeout,
SSLMode: dbOpts.SSLMode,
MySQLLocation: dbOpts.MySQLLocation,
@@ -208,6 +215,7 @@ func (dbcfg StorDbCfg) AsMapInterface(string) interface{} {
utils.SQLMaxOpenConnsCfg: dbcfg.Opts.SQLMaxOpenConns,
utils.SQLMaxIdleConnsCfg: dbcfg.Opts.SQLMaxIdleConns,
utils.SQLConnMaxLifetime: dbcfg.Opts.SQLConnMaxLifetime.String(),
utils.SQLDSNParams: dbcfg.Opts.SQLDSNParams,
utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(),
utils.SSLModeCfg: dbcfg.Opts.SSLMode,
utils.MysqlLocation: dbcfg.Opts.MySQLLocation,
@@ -251,6 +259,9 @@ func diffStorDBOptsJsonCfg(d *DBOptsJson, v1, v2 *StorDBOpts) *DBOptsJson {
if v1.SQLConnMaxLifetime != v2.SQLConnMaxLifetime {
d.SQLConnMaxLifetime = utils.StringPointer(v2.SQLConnMaxLifetime.String())
}
if !reflect.DeepEqual(v1.SQLDSNParams, v2.SQLDSNParams) {
d.SQLDSNParams = v2.SQLDSNParams
}
if v1.MongoQueryTimeout != v2.MongoQueryTimeout {
d.MongoQueryTimeout = utils.StringPointer(v2.MongoQueryTimeout.String())
}

15
config/stordbcfg_test.go
View File

@@ -48,6 +48,7 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) {
SQLMaxOpenConns: utils.IntPointer(100),
SQLMaxIdleConns: utils.IntPointer(10),
SQLConnMaxLifetime: utils.StringPointer("0"),
SQLDSNParams: make(map[string]string),
MySQLLocation: utils.StringPointer("UTC"),
},
}
@@ -71,6 +72,7 @@ func TestStoreDbCfgloadFromJsonCfgCase1(t *testing.T) {
Opts: &StorDBOpts{
SQLMaxOpenConns: 100,
SQLMaxIdleConns: 10,
SQLDSNParams: make(map[string]string),
MongoQueryTimeout: 10 * time.Second,
SSLMode: "disable",
MySQLLocation: "UTC",
@@ -258,6 +260,7 @@ func TestStorDbCfgAsMapInterface(t *testing.T) {
utils.SQLMaxOpenConnsCfg: 100,
utils.SQLMaxIdleConnsCfg: 10,
utils.SQLConnMaxLifetimeCfg: "0s",
utils.SQLDSNParams: make(map[string]string),
utils.MongoQueryTimeoutCfg: "10s",
utils.SSLModeCfg: "disable",
utils.MysqlLocation: "UTC",
@@ -273,16 +276,16 @@ func TestStorDbCfgAsMapInterface(t *testing.T) {
rcv := cfgCgr.storDbCfg.AsMapInterface("").(map[string]interface{})
if !reflect.DeepEqual(eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg],
rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]) {
t.Errorf("Expected %+v, received %+v", eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg],
rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg])
t.Errorf("Expected %+v, received %+v", utils.ToJSON(eMap[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]),
utils.ToJSON(rcv[utils.ItemsCfg].(map[string]interface{})[utils.SessionSConnsCfg]))
} else if !reflect.DeepEqual(eMap[utils.OptsCfg], rcv[utils.OptsCfg]) {
t.Errorf("Expected %+v \n, received %+v", eMap[utils.OptsCfg], rcv[utils.OptsCfg])
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.OptsCfg]), utils.ToJSON(rcv[utils.OptsCfg]))
} else if !reflect.DeepEqual(eMap[utils.PrefixIndexedFieldsCfg], rcv[utils.PrefixIndexedFieldsCfg]) {
t.Errorf("Expected %+v \n, received %+v", eMap[utils.PrefixIndexedFieldsCfg], rcv[utils.PrefixIndexedFieldsCfg])
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.PrefixIndexedFieldsCfg]), utils.ToJSON(rcv[utils.PrefixIndexedFieldsCfg]))
} else if !reflect.DeepEqual(eMap[utils.RemoteConnsCfg], rcv[utils.RemoteConnsCfg]) {
t.Errorf("Expected %+v \n, received %+v", eMap[utils.RemoteConnsCfg], rcv[utils.RemoteConnsCfg])
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.RemoteConnsCfg]), utils.ToJSON(rcv[utils.RemoteConnsCfg]))
} else if !reflect.DeepEqual(eMap[utils.ReplicationConnsCfg], rcv[utils.ReplicationConnsCfg]) {
t.Errorf("Expected %+v \n, received %+v", eMap[utils.ReplicationConnsCfg], rcv[utils.ReplicationConnsCfg])
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.ReplicationConnsCfg]), utils.ToJSON(rcv[utils.ReplicationConnsCfg]))
}
}
}

1
data/conf/samples/ees/cgrates.json
View File

@@ -461,7 +461,6 @@
"sqlMaxIdleConns": 10,
"sqlMaxOpenConns": 100,
"sqlConnMaxLifetime": "0",
},
"fields":[ // the path constains *exp.columnName
{"tag": "CGRID", "path": "*exp.cgrid", "type": "*variable", "value": "~*req.CGRID"},

15
ees/sql.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
@@ -90,7 +91,7 @@ func (sqlEe *SQLEe) initDialector() (err error) {
switch u.Scheme {
case utils.MySQL:
sqlEe.dialect = mysql.Open(fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'",
u.User.Username(), password, u.Hostname(), u.Port(), dbname) + appendToMysqlDSNOpts(sqlEe.Cfg().Opts))
u.User.Username(), password, u.Hostname(), u.Port(), dbname) + engine.AppendToMysqlDSNOpts(sqlEe.Cfg().Opts.SQLDSNParams))
case utils.Postgres:
sqlEe.dialect = postgres.Open(fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=%s", u.Hostname(), u.Port(), dbname, u.User.Username(), password, ssl))
default:
@@ -99,18 +100,6 @@ func (sqlEe *SQLEe) initDialector() (err error) {
return
}
func appendToMysqlDSNOpts(opts *config.EventExporterOpts) string {
if opts.SQLDSNParams != nil {
var dsn string
for key, val := range opts.SQLDSNParams {
dsn = dsn + "&" + key + "=" + val
}
utils.Logger.Debug("dsn: " + dsn)
return dsn
}
return utils.EmptyString
}
func openDB(dialect gorm.Dialector, opts *config.EventExporterOpts) (db *gorm.DB, sqlDB *sql.DB, err error) {
if db, err = gorm.Open(dialect, &gorm.Config{AllowGlobalUpdate: true}); err != nil {
return

16
engine/storage_mysql.go
View File

@@ -32,14 +32,13 @@ type MySQLStorage struct {
}
func NewMySQLStorage(host, port, name, user, password string,
maxConn, maxIdleConn int, connMaxLifetime time.Duration, location string) (*SQLStorage, error) {
maxConn, maxIdleConn int, connMaxLifetime time.Duration, location string, dsnParams map[string]string) (*SQLStorage, error) {
connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=%s&parseTime=true&sql_mode='ALLOW_INVALID_DATES'",
user, password, host, port, name, location)
db, err := gorm.Open(mysql.Open(connectString), &gorm.Config{AllowGlobalUpdate: true})
db, err := gorm.Open(mysql.Open(connectString+AppendToMysqlDSNOpts(dsnParams)), &gorm.Config{AllowGlobalUpdate: true})
if err != nil {
return nil, err
}
mySQLStorage := new(MySQLStorage)
if mySQLStorage.DB, err = db.DB(); err != nil {
return nil, err
@@ -60,6 +59,17 @@ func NewMySQLStorage(host, port, name, user, password string,
}, nil
}
func AppendToMysqlDSNOpts(opts map[string]string) string {
if opts != nil {
var dsn string
for key, val := range opts {
dsn = dsn + "&" + key + "=" + val
}
return dsn
}
return utils.EmptyString
}
// SetVersions will set a slice of versions, updating existing
func (msqlS *MySQLStorage) SetVersions(vrs Versions, overwrite bool) (err error) {
tx := msqlS.db.Begin()

2
engine/storage_utils.go
View File

@@ -69,7 +69,7 @@ func NewStorDBConn(dbType, host, port, name, user, pass, marshaler string,
opts.SQLMaxOpenConns, opts.SQLMaxIdleConns, opts.SQLConnMaxLifetime)
case utils.MySQL:
db, err = NewMySQLStorage(host, port, name, user, pass, opts.SQLMaxOpenConns, opts.SQLMaxIdleConns,
opts.SQLConnMaxLifetime, opts.MySQLLocation)
opts.SQLConnMaxLifetime, opts.MySQLLocation, opts.SQLDSNParams)
case utils.Internal:
db = NewInternalDB(stringIndexedFields, prefixIndexedFields, itmsCfg)
default:

2
engine/z_stordb_it_test.go
View File

@@ -74,7 +74,7 @@ func TestStorDBit(t *testing.T) {
if storDB, err = NewMySQLStorage(storCfg.StorDbCfg().Host,
storCfg.StorDbCfg().Port, storCfg.StorDbCfg().Name,
storCfg.StorDbCfg().User, storCfg.StorDbCfg().Password,
100, 10, 0, "UTC"); err != nil {
100, 10, 0, "UTC", storCfg.StorDbCfg().Opts.SQLDSNParams); err != nil {
t.Fatal(err)
}
storDB.(*SQLStorage).db.Config.Logger = logger.Default.LogMode(logger.Silent)