diff --git a/apier/v2/cdrs_mongo_local_test.go b/apier/v2/cdrs_mongo_local_test.go index a0e248015..14e0d2e9f 100644 --- a/apier/v2/cdrs_mongo_local_test.go +++ b/apier/v2/cdrs_mongo_local_test.go @@ -68,7 +68,8 @@ func TestV2CdrsMongoInjectUnratedCdr(t *testing.T) { if !*testLocal { return } - mongoDb, err := engine.NewMongoStorage(cdrsMongoCfg.StorDBHost, cdrsMongoCfg.StorDBPort, cdrsMongoCfg.StorDBName, cdrsMongoCfg.StorDBUser, cdrsMongoCfg.StorDBPass) + mongoDb, err := engine.NewMongoStorage(cdrsMongoCfg.StorDBHost, cdrsMongoCfg.StorDBPort, cdrsMongoCfg.StorDBName, + cdrsMongoCfg.StorDBUser, cdrsMongoCfg.StorDBPass, cdrsMongoCfg.StorDBCDRSIndexes) if err != nil { t.Error("Error on opening database connection: ", err) return diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 452b8cda5..352bd9ba0 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -644,7 +644,7 @@ func main() { } if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, - cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { // Cannot configure logger database, show stopper utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err)) return diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index db41efc3d..c027b2b00 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -157,10 +157,10 @@ func main() { *tpdb_user, *tpdb_pass, *dbdata_encoding) accountDb, errAccDb = engine.ConfigureAccountingStorage(*datadb_type, *datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass, *dbdata_encoding) storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding, - cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns) + cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns, cgrConfig.StorDBCDRSIndexes) } else if *toStorDb { // Import from csv files to storDb storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding, - cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns) + cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns, cgrConfig.StorDBCDRSIndexes) } else { // Default load from csv files to dataDb ratingDb, errRatingDb = engine.ConfigureRatingStorage(*tpdb_type, *tpdb_host, *tpdb_port, *tpdb_name, *tpdb_user, *tpdb_pass, *dbdata_encoding) diff --git a/config/config.go b/config/config.go index 0be3b935d..e8dcfc582 100644 --- a/config/config.go +++ b/config/config.go @@ -174,20 +174,21 @@ type CGRConfig struct { TpDbUser string // The user to sign in as. TpDbPass string // The user's password. DataDbType string - DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - DataDbPort string // The port to bind to. - DataDbName string // The name of the database to connect to. - DataDbUser string // The user to sign in as. - DataDbPass string // The user's password. - LoadHistorySize int // Maximum number of records to archive in load history - StorDBType string // Should reflect the database type used to store logs - StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - StorDBPort string // Th e port to bind to. - StorDBName string // The name of the database to connect to. - StorDBUser string // The user to sign in as. - StorDBPass string // The user's password. - StorDBMaxOpenConns int // Maximum database connections opened - StorDBMaxIdleConns int // Maximum idle connections to keep opened + DataDbHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + DataDbPort string // The port to bind to. + DataDbName string // The name of the database to connect to. + DataDbUser string // The user to sign in as. + DataDbPass string // The user's password. + LoadHistorySize int // Maximum number of records to archive in load history + StorDBType string // Should reflect the database type used to store logs + StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + StorDBPort string // Th e port to bind to. + StorDBName string // The name of the database to connect to. + StorDBUser string // The user to sign in as. + StorDBPass string // The user's password. + StorDBMaxOpenConns int // Maximum database connections opened + StorDBMaxIdleConns int // Maximum idle connections to keep opened + StorDBCDRSIndexes []string DBDataEncoding string // The encoding used to store object data in strings: RPCJSONListen string // RPC JSON listening address RPCGOBListen string // RPC GOB listening address @@ -577,6 +578,9 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) error { if jsnStorDbCfg.Max_idle_conns != nil { self.StorDBMaxIdleConns = *jsnStorDbCfg.Max_idle_conns } + if jsnStorDbCfg.Cdrs_indexes != nil { + self.StorDBCDRSIndexes = *jsnStorDbCfg.Cdrs_indexes + } } if jsnGeneralCfg != nil { diff --git a/config/config_defaults.go b/config/config_defaults.go index 845ad4141..7d5edc4f3 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -53,7 +53,7 @@ const CGRATES_CFG_JSON = ` "tariffplan_db": { // database used to store active tariff plan configuration - "db_type": "redis", // tariffplan_db type: + "db_type": "redis", // tariffplan_db type: "db_host": "127.0.0.1", // tariffplan_db host address "db_port": 6379, // port to reach the tariffplan_db "db_name": "10", // tariffplan_db name to connect to @@ -63,7 +63,7 @@ const CGRATES_CFG_JSON = ` "data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: + "db_type": "redis", // data_db type: "db_host": "127.0.0.1", // data_db host address "db_port": 6379, // data_db port to reach the database "db_name": "11", // data_db database name to connect to @@ -74,7 +74,7 @@ const CGRATES_CFG_JSON = ` "stor_db": { // database used to store offline tariff plans and CDRs - "db_type": "mysql", // stor database type to use: + "db_type": "mysql", // stor database type to use: "db_host": "127.0.0.1", // the host to connect to "db_port": 3306, // the port to reach the stordb "db_name": "cgrates", // stor database name @@ -82,6 +82,7 @@ const CGRATES_CFG_JSON = ` "db_passwd": "CGRateS.org", // password to use when connecting to stordb "max_open_conns": 100, // maximum database connections opened "max_idle_conns": 10, // maximum database connections idle + "cdrs_indexes": [], // indexes on cdrs table to speed up queries, used only in case of mongo }, diff --git a/config/config_json_test.go b/config/config_json_test.go index 535e0752f..4b485a23b 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -109,6 +109,7 @@ func TestDfDbJsonCfg(t *testing.T) { Db_passwd: utils.StringPointer("CGRateS.org"), Max_open_conns: utils.IntPointer(100), Max_idle_conns: utils.IntPointer(10), + Cdrs_indexes: utils.StringSlicePointer([]string{}), } if cfg, err := dfCgrJsonCfg.DbJsonCfg(STORDB_JSN); err != nil { t.Error(err) diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 975bcd5ea..7dda0b6ec 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -54,6 +54,7 @@ type DbJsonCfg struct { Max_open_conns *int // Used only in case of storDb Max_idle_conns *int Load_history_size *int // Used in case of dataDb to limit the length of the loads history + Cdrs_indexes *[]string } // Balancer config section diff --git a/engine/calldesc.go b/engine/calldesc.go index 96dacd589..6fc95807e 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -47,11 +47,11 @@ func init() { ratingStorage, _ = NewMapStorage() accountingStorage, _ = NewMapStorage() case "mongo": - ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "") + ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "", nil) if err != nil { log.Fatal(err) } - accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "") + accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "", nil) if err != nil { log.Fatal(err) } diff --git a/engine/libtest.go b/engine/libtest.go index 69a8f48b7..99877e0be 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -53,7 +53,7 @@ func InitDataDb(cfg *config.CGRConfig) error { func InitStorDb(cfg *config.CGRConfig) error { storDb, err := ConfigureLoadStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, - cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { return err } diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index 5f886aa4e..96c695731 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -95,7 +95,7 @@ func testSetCDR(cfg *config.CGRConfig) error { return err } cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, - cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { return err } @@ -205,7 +205,7 @@ func testSMCosts(cfg *config.CGRConfig) error { return err } cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, - cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { return err } @@ -238,7 +238,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { return err } cdrStorage, err := ConfigureCdrStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, - cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns, cfg.StorDBCDRSIndexes) if err != nil { return err } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 53273c18d..b77313b59 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -87,7 +87,7 @@ type MongoStorage struct { ms Marshaler } -func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) { +func NewMongoStorage(host, port, db, user, pass string, cdrsIndexes []string) (*MongoStorage, error) { address := fmt.Sprintf("%s:%s", host, port) if user != "" && pass != "" { address = fmt.Sprintf("%s:%s@%s", user, pass, address) @@ -222,13 +222,21 @@ func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) { Background: false, Sparse: false, } - collections = []string{utils.TBL_CDRS} - for _, col := range collections { - if err = ndb.C(col).EnsureIndex(index); err != nil { + if err = ndb.C(utils.TBL_CDRS).EnsureIndex(index); err != nil { + return nil, err + } + for _, idxKey := range cdrsIndexes { + index = mgo.Index{ + Key: []string{idxKey}, + Unique: false, + DropDups: false, + Background: false, + Sparse: false, + } + if err = ndb.C(utils.TBL_CDRS).EnsureIndex(index); err != nil { return nil, err } } - return &MongoStorage{db: ndb, session: session, ms: NewCodecMsgpackMarshaler()}, err } diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 5d550cee9..ba093af58 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -42,7 +42,7 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler str } d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) + d, err = NewMongoStorage(host, port, name, user, pass, nil) db = d.(RatingStorage) default: err = errors.New("unknown db") @@ -68,7 +68,7 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler } d, err = NewRedisStorage(host, db_nb, pass, marshaler, utils.REDIS_MAX_CONNS) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) + d, err = NewMongoStorage(host, port, name, user, pass, nil) db = d.(AccountingStorage) default: err = errors.New("unknown db") @@ -79,7 +79,7 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler return d, nil } -func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LogStorage, err error) { +func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int, cdrsIndexes []string) (db LogStorage, err error) { var d LogStorage switch db_type { /* @@ -96,7 +96,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string d, err = NewRedisStorage(host, db_nb, pass, marshaler) */ case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) + d, err = NewMongoStorage(host, port, name, user, pass, nil) case utils.POSTGRES: d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MYSQL: @@ -110,7 +110,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string return d, nil } -func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LoadStorage, err error) { +func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int, cdrsIndexes []string) (db LoadStorage, err error) { var d LoadStorage switch db_type { case utils.POSTGRES: @@ -118,7 +118,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) + d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes) default: err = errors.New("unknown db") } @@ -128,7 +128,7 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin return d, nil } -func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int) (db CdrStorage, err error) { +func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int, cdrsIndexes []string) (db CdrStorage, err error) { var d CdrStorage switch db_type { case utils.POSTGRES: @@ -136,7 +136,7 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) + d, err = NewMongoStorage(host, port, name, user, pass, cdrsIndexes) default: err = errors.New("unknown db") }