uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

MongoDb configurable indexes in cgrates.json, fixes #340

Browse Source
This commit is contained in:
DanB
2016-01-01 16:22:47 +01:00
parent 2e5305725c
commit 140e6f2da8
12 changed files with 56 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
apier/v2/cdrs_mongo_local_test.go
View File

@@ -68,7 +68,8 @@ func TestV2CdrsMongoInjectUnratedCdr(t *testing.T) {
if !*testLocal {
return
}
mongoDb, err := engine.NewMongoStorage(cdrsMongoCfg.StorDBHost, cdrsMongoCfg.StorDBPort, cdrsMongoCfg.StorDBName, cdrsMongoCfg.StorDBUser, cdrsMongoCfg.StorDBPass)
mongoDb, err := engine.NewMongoStorage(cdrsMongoCfg.StorDBHost, cdrsMongoCfg.StorDBPort, cdrsMongoCfg.StorDBName,
cdrsMongoCfg.StorDBUser, cdrsMongoCfg.StorDBPass, cdrsMongoCfg.StorDBCDRSIndexes)
if err != nil {
t.Error("Error on opening database connection: ", err)
return

2
cmd/cgr-engine/cgr-engine.go
View File

@@ -644,7 +644,7 @@ func main() {
}
if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary
logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes)
if err != nil { // Cannot configure logger database, show stopper
utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
return

4
cmd/cgr-loader/cgr-loader.go
View File

@@ -157,10 +157,10 @@ func main() {
*tpdb_user, *tpdb_pass, *dbdata_encoding)
accountDb, errAccDb = engine.ConfigureAccountingStorage(*datadb_type, *datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass, *dbdata_encoding)
storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns, cgrConfig.StorDBCDRSIndexes)
} else if *toStorDb { // Import from csv files to storDb
storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns, cgrConfig.StorDBCDRSIndexes)
} else { // Default load from csv files to dataDb
ratingDb, errRatingDb = engine.ConfigureRatingStorage(*tpdb_type, *tpdb_host, *tpdb_port, *tpdb_name,
*tpdb_user, *tpdb_pass, *dbdata_encoding)

32
config/config.go
View File

@@ -174,20 +174,21 @@ type CGRConfig struct {
TpDbUser string // The user to sign in as.
TpDbPass string // The user's password.
DataDbType string
DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets.
DataDbPort string // The port to bind to.
DataDbName string // The name of the database to connect to.
DataDbUser string // The user to sign in as.
DataDbPass string // The user's password.
LoadHistorySize int // Maximum number of records to archive in load history
StorDBType string // Should reflect the database type used to store logs
StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets.
StorDBPort string // Th e port to bind to.
StorDBName string // The name of the database to connect to.
StorDBUser string // The user to sign in as.
StorDBPass string // The user's password.
StorDBMaxOpenConns int // Maximum database connections opened
StorDBMaxIdleConns int // Maximum idle connections to keep opened
DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets.
DataDbPort string // The port to bind to.
DataDbName string // The name of the database to connect to.
DataDbUser string // The user to sign in as.
DataDbPass string // The user's password.
LoadHistorySize int // Maximum number of records to archive in load history
StorDBType string // Should reflect the database type used to store logs
StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets.
StorDBPort string // Th e port to bind to.
StorDBName string // The name of the database to connect to.
StorDBUser string // The user to sign in as.
StorDBPass string // The user's password.
StorDBMaxOpenConns int // Maximum database connections opened
StorDBMaxIdleConns int // Maximum idle connections to keep opened
StorDBCDRSIndexes []string
DBDataEncoding string // The encoding used to store object data in strings: <msgpack|json>
RPCJSONListen string // RPC JSON listening address
RPCGOBListen string // RPC GOB listening address
@@ -577,6 +578,9 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error {
if jsnStorDbCfg.Max_idle_conns != nil {
self.StorDBMaxIdleConns = *jsnStorDbCfg.Max_idle_conns
}
if jsnStorDbCfg.Cdrs_indexes != nil {
self.StorDBCDRSIndexes = *jsnStorDbCfg.Cdrs_indexes
}
}
if jsnGeneralCfg != nil {

7
config/config_defaults.go
View File

@@ -53,7 +53,7 @@ const CGRATES_CFG_JSON = `
"tariffplan_db": { // database used to store active tariff plan configuration
"db_type": "redis", // tariffplan_db type: <redis>
"db_type": "redis", // tariffplan_db type: <redis|mongo>
"db_host": "127.0.0.1", // tariffplan_db host address
"db_port": 6379, // port to reach the tariffplan_db
"db_name": "10", // tariffplan_db name to connect to
@@ -63,7 +63,7 @@ const CGRATES_CFG_JSON = `
"data_db": { // database used to store runtime data (eg: accounts, cdr stats)
"db_type": "redis", // data_db type: <redis>
"db_type": "redis", // data_db type: <redis|mongo>
"db_host": "127.0.0.1", // data_db host address
"db_port": 6379, // data_db port to reach the database
"db_name": "11", // data_db database name to connect to
@@ -74,7 +74,7 @@ const CGRATES_CFG_JSON = `
"stor_db": { // database used to store offline tariff plans and CDRs
"db_type": "mysql", // stor database type to use: <mysql|postgres>
"db_type": "mysql", // stor database type to use: <mongo|mysql|postgres>
"db_host": "127.0.0.1", // the host to connect to
"db_port": 3306, // the port to reach the stordb
"db_name": "cgrates", // stor database name
@@ -82,6 +82,7 @@ const CGRATES_CFG_JSON = `
"db_passwd": "CGRateS.org", // password to use when connecting to stordb
"max_open_conns": 100, // maximum database connections opened
"max_idle_conns": 10, // maximum database connections idle
"cdrs_indexes": [], // indexes on cdrs table to speed up queries, used only in case of mongo
},

1
config/config_json_test.go
View File

@@ -109,6 +109,7 @@ func TestDfDbJsonCfg(t *testing.T) {
Db_passwd: utils.StringPointer("CGRateS.org"),
Max_open_conns: utils.IntPointer(100),
Max_idle_conns: utils.IntPointer(10),
Cdrs_indexes: utils.StringSlicePointer([]string{}),
}
if cfg, err := dfCgrJsonCfg.DbJsonCfg(STORDB_JSN); err != nil {
t.Error(err)

1
config/libconfig_json.go
View File

@@ -54,6 +54,7 @@ type DbJsonCfg struct {
Max_open_conns *int // Used only in case of storDb
Max_idle_conns *int
Load_history_size *int // Used in case of dataDb to limit the length of the loads history
Cdrs_indexes *[]string
}
// Balancer config section

4
engine/calldesc.go
View File

@@ -47,11 +47,11 @@ func init() {
ratingStorage, _ = NewMapStorage()
accountingStorage, _ = NewMapStorage()
case "mongo":
ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "")
ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil)
if err != nil {
log.Fatal(err)
}
accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "")
accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil)
if err != nil {
log.Fatal(err)
}

2
engine/libtest.go
View File

@@ -53,7 +53,7 @@ func InitDataDb(cfg *config.CGRConfig) error {
func InitStorDb(cfg *config.CGRConfig) error {
storDb, err := ConfigureLoadStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes)
if err != nil {
return err
}

6
engine/storage_cdrs_it_test.go
View File

@@ -95,7 +95,7 @@ func testSetCDR(cfg *config.CGRConfig) error {
return err
}
cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes)
if err != nil {
return err
}
@@ -205,7 +205,7 @@ func testSMCosts(cfg *config.CGRConfig) error {
return err
}
cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes)
if err != nil {
return err
}
@@ -238,7 +238,7 @@ func testGetCDRs(cfg *config.CGRConfig) error {
return err
}
cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass,
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes)
if err != nil {
return err
}

18
engine/storage_mongo_datadb.go
View File

@@ -87,7 +87,7 @@ type MongoStorage struct {
ms Marshaler
}
func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) {
func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string) (*MongoStorage, error) {
address := fmt.Sprintf("%s:%s", host, port)
if user != "" && pass != "" {
address = fmt.Sprintf("%s:%s@%s", user, pass, address)
@@ -222,13 +222,21 @@ func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) {
Background: false,
Sparse: false,
}
collections = []string{utils.TBL_CDRS}
for _, col := range collections {
if err = ndb.C(col).EnsureIndex(index); err != nil {
if err = ndb.C(utils.TBL_CDRS).EnsureIndex(index); err != nil {
return nil, err
}
for _, idxKey := range cdrsIndexes {
index = mgo.Index{
Key: []string{idxKey},
Unique: false,
DropDups: false,
Background: false,
Sparse: false,
}
if err = ndb.C(utils.TBL_CDRS).EnsureIndex(index); err != nil {
return nil, err
}
}
return &MongoStorage{db: ndb, session: session, ms: NewCodecMsgpackMarshaler()}, err
}

16
engine/storage_utils.go
View File

@@ -42,7 +42,7 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler str
}
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass)
d, err = NewMongoStorage(host, port, name, user, pass, nil)
db = d.(RatingStorage)
default:
err = errors.New("unknown db")
@@ -68,7 +68,7 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler
}
d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass)
d, err = NewMongoStorage(host, port, name, user, pass, nil)
db = d.(AccountingStorage)
default:
err = errors.New("unknown db")
@@ -79,7 +79,7 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler
return d, nil
}
func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LogStorage, err error) {
func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int, cdrsIndexes []string) (db LogStorage, err error) {
var d LogStorage
switch db_type {
/*
@@ -96,7 +96,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string
d, err = NewRedisStorage(host, db_nb, pass, marshaler)
*/
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass)
d, err = NewMongoStorage(host, port, name, user, pass, nil)
case utils.POSTGRES:
d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MYSQL:
@@ -110,7 +110,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string
return d, nil
}
func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LoadStorage, err error) {
func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int, cdrsIndexes []string) (db LoadStorage, err error) {
var d LoadStorage
switch db_type {
case utils.POSTGRES:
@@ -118,7 +118,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin
case utils.MYSQL:
d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass)
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes)
default:
err = errors.New("unknown db")
}
@@ -128,7 +128,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin
return d, nil
}
func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int) (db CdrStorage, err error) {
func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int, cdrsIndexes []string) (db CdrStorage, err error) {
var d CdrStorage
switch db_type {
case utils.POSTGRES:
@@ -136,7 +136,7 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn,
case utils.MYSQL:
d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn)
case utils.MONGO:
d, err = NewMongoStorage(host, port, name, user, pass)
d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes)
default:
err = errors.New("unknown db")
}