From 37569193423f2001c649389e47faedc4296a95e1 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 3 Oct 2014 16:30:50 +0200 Subject: [PATCH] Adding maximumConnections and maximumIdleConnections on database side --- apier/v1/apier_local_test.go | 2 +- cdrc/cdrc_local_test.go | 2 +- cmd/cgr-engine/cgr-engine.go | 2 +- cmd/cgr-loader/cgr-loader.go | 6 ++++-- config/config.go | 10 ++++++++++ config/config_test.go | 4 ++++ config/test_data.txt | 2 ++ data/conf/cgrates.cfg | 2 ++ engine/loader_local_test.go | 2 +- engine/mediator_local_test.go | 3 ++- engine/storage_mysql.go | 6 +++--- engine/storage_sql_local_test.go | 3 ++- engine/storage_utils.go | 12 ++++++------ general_tests/multiplecdrc_local_test.go | 2 +- general_tests/tutorial_local_test.go | 3 ++- 15 files changed, 42 insertions(+), 19 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 1acaf4ef8..2e4a746f2 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -92,7 +92,7 @@ func TestCreateTables(t *testing.T) { t.Fatal("Unsupported storDbType") } var mysql *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil { + if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { t.Fatal("Error on opening database connection: ", err) } else { mysql = d.(*engine.MySQLStorage) diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index 42787c726..e32c853df 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -110,7 +110,7 @@ func TestEmptyTables(t *testing.T) { t.Fatal("Unsupported storDbType") } var mysql *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil { + if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { t.Fatal("Error on opening database connection: ", err) } else { mysql = d.(*engine.MySQLStorage) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 9e7627983..da112849c 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -364,7 +364,7 @@ func main() { logDb = ratingDb.(engine.LogStorage) } else { logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, - cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding) + cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) if err != nil { // Cannot configure logger database, show stopper engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err)) return diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index c0d6f19b4..a81c1d4db 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -90,9 +90,11 @@ func main() { ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name, *ratingdb_user, *ratingdb_pass, *dbdata_encoding) accountDb, errAccDb = engine.ConfigureAccountingStorage(*accountdb_type, *accountdb_host, *accountdb_port, *accountdb_name, *accountdb_user, *accountdb_pass, *dbdata_encoding) - storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) + storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding, + cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns) } else if *toStorDb { // Import from csv files to storDb - storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) + storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding, + cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns) } else { // Default load from csv files to dataDb ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name, *ratingdb_user, *ratingdb_pass, *dbdata_encoding) diff --git a/config/config.go b/config/config.go index ae7cc9196..a36be6149 100644 --- a/config/config.go +++ b/config/config.go @@ -73,6 +73,8 @@ type CGRConfig struct { StorDBName string // The name of the database to connect to. StorDBUser string // The user to sign in as. StorDBPass string // The user's password. + StorDBMaxOpenConns int // Maximum database connections opened + StorDBMaxIdleConns int // Maximum idle connections to keep opened DBDataEncoding string // The encoding used to store object data in strings: RPCJSONListen string // RPC JSON listening address RPCGOBListen string // RPC GOB listening address @@ -155,6 +157,8 @@ func (self *CGRConfig) setDefaults() error { self.StorDBName = "cgrates" self.StorDBUser = "cgrates" self.StorDBPass = "CGRateS.org" + self.StorDBMaxOpenConns = 100 + self.StorDBMaxIdleConns = 10 self.DBDataEncoding = utils.MSGPACK self.RPCJSONListen = "127.0.0.1:2012" self.RPCGOBListen = "127.0.0.1:2013" @@ -346,6 +350,12 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("global", "stordb_passwd"); hasOpt { cfg.StorDBPass, _ = c.GetString("global", "stordb_passwd") } + if hasOpt = c.HasOption("global", "stordb_max_open_conns"); hasOpt { + cfg.StorDBMaxOpenConns, _ = c.GetInt("global", "stordb_max_open_conns") + } + if hasOpt = c.HasOption("global", "stordb_max_idle_conns"); hasOpt { + cfg.StorDBMaxIdleConns, _ = c.GetInt("global", "stordb_max_idle_conns") + } if hasOpt = c.HasOption("global", "dbdata_encoding"); hasOpt { cfg.DBDataEncoding, _ = c.GetString("global", "dbdata_encoding") } diff --git a/config/config_test.go b/config/config_test.go index e7c88484d..c1f79923a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -64,6 +64,8 @@ func TestDefaults(t *testing.T) { eCfg.StorDBName = "cgrates" eCfg.StorDBUser = "cgrates" eCfg.StorDBPass = "CGRateS.org" + eCfg.StorDBMaxOpenConns = 100 + eCfg.StorDBMaxIdleConns = 10 eCfg.DBDataEncoding = utils.MSGPACK eCfg.RPCJSONListen = "127.0.0.1:2012" eCfg.RPCGOBListen = "127.0.0.1:2013" @@ -187,6 +189,8 @@ func TestConfigFromFile(t *testing.T) { eCfg.StorDBName = "test" eCfg.StorDBUser = "test" eCfg.StorDBPass = "test" + eCfg.StorDBMaxOpenConns = 99 + eCfg.StorDBMaxIdleConns = 99 eCfg.DBDataEncoding = "test" eCfg.RPCJSONListen = "test" eCfg.RPCGOBListen = "test" diff --git a/config/test_data.txt b/config/test_data.txt index 23e6d26c1..3a99cbcf2 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -20,6 +20,8 @@ stordb_port = test # The port to reach the logdb. stordb_name = test # The name of the log database to connect to. stordb_user = test # Username to use when connecting to logdb. stordb_passwd = test # Password to use when connecting to logdb. +stordb_max_open_conns = 99 # Maximum database connections opened +stordb_max_idle_conns = 99 # Maximum database connections idle dbdata_encoding = test # The encoding used to scategorye object data in strings: rpc_json_listen = test # RPC JSON listening address rpc_gob_listen = test # RPC GOB listening address diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index b36f9d99b..91e439aa3 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -23,6 +23,8 @@ # stordb_name = cgrates # The name of the log database to connect to. # stordb_user = cgrates # Username to use when connecting to stordb. # stordb_passwd = CGRateS.org # Password to use when connecting to stordb. +# stordb_max_open_conns = 0 # Maximum database connections opened +# stordb_max_idle_conns = -1 # Maximum database connections idle # dbdata_encoding = msgpack # The encoding used to store object data in strings: # rpc_json_listen = 127.0.0.1:2012 # RPC JSON listening address # rpc_gob_listen = 127.0.0.1:2013 # RPC GOB listening address diff --git a/engine/loader_local_test.go b/engine/loader_local_test.go index 4d46d0a8b..848621ab1 100644 --- a/engine/loader_local_test.go +++ b/engine/loader_local_test.go @@ -95,7 +95,7 @@ func TestCreateStorTpTables(t *testing.T) { return } var db *MySQLStorage - if d, err := NewMySQLStorage(lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass); err != nil { + if d, err := NewMySQLStorage(lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass, lCfg.StorDBMaxOpenConns, lCfg.StorDBMaxIdleConns); err != nil { t.Error("Error on opening database connection: ", err) return } else { diff --git a/engine/mediator_local_test.go b/engine/mediator_local_test.go index a35481487..376fc7512 100644 --- a/engine/mediator_local_test.go +++ b/engine/mediator_local_test.go @@ -85,7 +85,8 @@ func TestInitStorDb(t *testing.T) { } var mysql *MySQLStorage var err error - if cdrStor, err = ConfigureCdrStorage(cgrCfg.StorDBType, cgrCfg.StorDBHost, cgrCfg.StorDBPort, cgrCfg.StorDBName, cgrCfg.StorDBUser, cgrCfg.StorDBPass); err != nil { + if cdrStor, err = ConfigureCdrStorage(cgrCfg.StorDBType, cgrCfg.StorDBHost, cgrCfg.StorDBPort, cgrCfg.StorDBName, cgrCfg.StorDBUser, cgrCfg.StorDBPass, + cgrCfg.StorDBMaxOpenConns, cgrCfg.StorDBMaxIdleConns); err != nil { t.Fatal("Error on opening database connection: ", err) } else { mysql = cdrStor.(*MySQLStorage) diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 9463b71a1..4f0d50e2d 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -29,7 +29,7 @@ type MySQLStorage struct { *SQLStorage } -func NewMySQLStorage(host, port, name, user, password string) (Storage, error) { +func NewMySQLStorage(host, port, name, user, password string, maxConn, maxIdleConn int) (Storage, error) { connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true", user, password, host, port, name) db, err := gorm.Open("mysql", connectString) if err != nil { @@ -39,8 +39,8 @@ func NewMySQLStorage(host, port, name, user, password string) (Storage, error) { if err != nil { return nil, err } - db.DB().SetMaxIdleConns(10) - db.DB().SetMaxOpenConns(100) + db.DB().SetMaxIdleConns(maxIdleConn) + db.DB().SetMaxOpenConns(maxConn) //db.LogMode(true) return &MySQLStorage{&SQLStorage{Db: db.DB(), db: db}}, nil diff --git a/engine/storage_sql_local_test.go b/engine/storage_sql_local_test.go index caad42bcc..eb532ec0a 100644 --- a/engine/storage_sql_local_test.go +++ b/engine/storage_sql_local_test.go @@ -49,7 +49,8 @@ func TestCreateTables(t *testing.T) { return } cgrConfig, _ := config.NewDefaultCGRConfig() - if d, err := NewMySQLStorage(cgrConfig.StorDBHost, cgrConfig.StorDBPort, cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass); err != nil { + if d, err := NewMySQLStorage(cgrConfig.StorDBHost, cgrConfig.StorDBPort, cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass, + cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns); err != nil { t.Error("Error on opening database connection: ", err) return } else { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 49d1a5ca4..7d09672f0 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -86,7 +86,7 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler return db, nil } -func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string) (db LogStorage, err error) { +func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LogStorage, err error) { var d Storage switch db_type { /* @@ -107,7 +107,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string d, err = NewPostgresStorage(host, port, name, user, pass) */ case utils.MYSQL: - d, err = NewMySQLStorage(host, port, name, user, pass) + d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) default: err = errors.New("unknown db") } @@ -117,7 +117,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string return d.(LogStorage), nil } -func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string) (db LoadStorage, err error) { +func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LoadStorage, err error) { var d Storage switch db_type { /* @@ -126,7 +126,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin db = d.(LoadStorage) */ case utils.MYSQL: - d, err = NewMySQLStorage(host, port, name, user, pass) + d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) db = d.(LoadStorage) default: err = errors.New("unknown db") @@ -137,7 +137,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin return db, nil } -func ConfigureCdrStorage(db_type, host, port, name, user, pass string) (db CdrStorage, err error) { +func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int) (db CdrStorage, err error) { var d Storage switch db_type { /* @@ -146,7 +146,7 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string) (db CdrSt db = d.(CdrStorage) */ case utils.MYSQL: - d, err = NewMySQLStorage(host, port, name, user, pass) + d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) db = d.(CdrStorage) default: err = errors.New("unknown db") diff --git a/general_tests/multiplecdrc_local_test.go b/general_tests/multiplecdrc_local_test.go index b1466ea7f..f365e7ab4 100644 --- a/general_tests/multiplecdrc_local_test.go +++ b/general_tests/multiplecdrc_local_test.go @@ -76,7 +76,7 @@ func TestEmptyTables(t *testing.T) { t.Fatal("Unsupported storDbType") } var mysql *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil { + if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { t.Fatal("Error on opening database connection: ", err) } else { mysql = d.(*engine.MySQLStorage) diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 1d71c849a..b2db0abc4 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -54,7 +54,8 @@ func TestTutLclResetDb(t *testing.T) { if !*testLocal { return } - if db, err := engine.NewMySQLStorage(tutCfg.StorDBHost, tutCfg.StorDBPort, tutCfg.StorDBName, tutCfg.StorDBUser, tutCfg.StorDBPass); err != nil { + if db, err := engine.NewMySQLStorage(tutCfg.StorDBHost, tutCfg.StorDBPort, tutCfg.StorDBName, tutCfg.StorDBUser, tutCfg.StorDBPass, + tutCfg.StorDBMaxOpenConns, tutCfg.StorDBMaxIdleConns); err != nil { t.Error(err) } else if errFlush := db.Flush(); errFlush != nil { t.Error(err)