diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 97a58fc0c..4006e438c 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -312,7 +312,7 @@ func main() { var logDb engine.LogStorage var loadDb engine.LoadStorage var cdrDb engine.CdrStorage - dataDb, err = engine.ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, cfg.DataDBName, cfg.DataDBUser, cfg.DataDBPass) + dataDb, err = engine.ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, cfg.DataDBName, cfg.DataDBUser, cfg.DataDBPass, cfg.DBDataEncoding) if err != nil { // Cannot configure getter database, show stopper engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err)) return @@ -322,7 +322,7 @@ func main() { if cfg.StorDBType == SAME { logDb = dataDb.(engine.LogStorage) } else { - logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass) + logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding) if err != nil { // Cannot configure logger database, show stopper engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err)) return diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 0f3446647..311c199ec 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -47,6 +47,8 @@ var ( stor_db_user = flag.String("stordb_user", cgrConfig.StorDBUser, "The storDb user to sign in as.") stor_db_pass = flag.String("stordb_passwd", cgrConfig.StorDBPass, "The storDb user's password.") + dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings") + flush = flag.Bool("flush", false, "Flush the database before importing") tpid = flag.String("tpid", "", "The tariff plan id from the database") dataPath = flag.String("path", ".", "The path containing the data files") @@ -70,12 +72,12 @@ func main() { var storDb engine.LoadStorage // Init necessary db connections if *fromStorDb { - dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) - storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) + dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding) + storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) } else if *toStorDb { // Import from csv files to storDb - storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass) + storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding) } else { // Default load from csv files to dataDb - dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass) + dataDb, errDataDb = engine.ConfigureDataStorage(*data_db_type, *data_db_host, *data_db_port, *data_db_name, *data_db_user, *data_db_pass, *dbdata_encoding) } // Defer databases opened to be closed when we are done for _, db := range []engine.Storage{dataDb, storDb} { diff --git a/config/config.go b/config/config.go index 6576ef347..3a2bea688 100644 --- a/config/config.go +++ b/config/config.go @@ -45,12 +45,14 @@ type CGRConfig struct { DataDBName string // The name of the database to connect to. DataDBUser string // The user to sign in as. DataDBPass string // The user's password. + DataDBEncoding string // The encoding used to store objects in string keys StorDBType string // Should reflect the database type used to store logs StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. StorDBPort string // The port to bind to. StorDBName string // The name of the database to connect to. StorDBUser string // The user to sign in as. StorDBPass string // The user's password. + DBDataEncoding string // The encoding used to store object data in strings: RPCEncoding string // RPC encoding used on APIs: . DefaultReqType string // Use this request type if not defined on top DefaultTOR string // set default type of record @@ -113,6 +115,7 @@ func (self *CGRConfig) setDefaults() error { self.StorDBName = "cgrates" self.StorDBUser = "cgrates" self.StorDBPass = "CGRateS.org" + self.DBDataEncoding = utils.MSGPACK self.RPCEncoding = JSON self.DefaultReqType = utils.RATED self.DefaultTOR = "0" @@ -228,6 +231,9 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("global", "stordb_passwd"); hasOpt { cfg.StorDBPass, _ = c.GetString("global", "stordb_passwd") } + if hasOpt = c.HasOption("global", "dbdata_encoding"); hasOpt { + cfg.DBDataEncoding, _ = c.GetString("global", "dbdata_encoding") + } if hasOpt = c.HasOption("global", "rpc_encoding"); hasOpt { cfg.RPCEncoding, _ = c.GetString("global", "rpc_encoding") } diff --git a/config/config_test.go b/config/config_test.go index 04e8325c6..58266f88b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -46,6 +46,7 @@ func TestDefaults(t *testing.T) { eCfg.StorDBName = "cgrates" eCfg.StorDBUser = "cgrates" eCfg.StorDBPass = "CGRateS.org" + eCfg.DBDataEncoding = utils.MSGPACK eCfg.RPCEncoding = JSON eCfg.DefaultReqType = utils.RATED eCfg.DefaultTOR = "0" @@ -141,6 +142,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.StorDBName = "test" eCfg.StorDBUser = "test" eCfg.StorDBPass = "test" + eCfg.DBDataEncoding = "test" eCfg.RPCEncoding = "test" eCfg.DefaultReqType = "test" eCfg.DefaultTOR = "test" diff --git a/config/test_data.txt b/config/test_data.txt index 132b16506..006d72f5b 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -14,6 +14,7 @@ stordb_port = test # The port to reach the logdb. stordb_name = test # The name of the log database to connect to. stordb_user = test # Username to use when connecting to logdb. stordb_passwd = test # Password to use when connecting to logdb. +dbdata_encoding = test # The encoding used to store object data in strings: rpc_encoding = test # RPC encoding used on APIs: . default_reqtype = test # Default request type to consider when missing from requests: <""|prepaid|postpaid|pseudoprepaid|rated>. default_tor = test # Default Type of Record to consider when missing from requests. diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 8a29c8369..b8d1c0de0 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -17,6 +17,7 @@ # stordb_name = cgrates # The name of the log database to connect to. # stordb_user = cgrates # Username to use when connecting to stordb. # stordb_passwd = CGRateS.org # Password to use when connecting to stordb. +# dbdata_encoding = msgpack # The encoding used to store object data in strings: # rpc_encoding = json # RPC encoding used on APIs: . # default_reqtype = rated # Default request type to consider when missing from requests: <""|prepaid|postpaid|pseudoprepaid|rated>. # default_tor = 0 # Default Type of Record to consider when missing from requests. diff --git a/engine/storage_redis.go b/engine/storage_redis.go index d0d0ac95c..baaa20349 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -34,7 +34,7 @@ type RedisStorage struct { ms Marshaler } -func NewRedisStorage(address string, db int, pass string) (Storage, error) { +func NewRedisStorage(address string, db int, pass, mrshlerStr string) (Storage, error) { addrSplit := strings.Split(address, ":") host := addrSplit[0] port := 6379 @@ -56,7 +56,15 @@ func NewRedisStorage(address string, db int, pass string) (Storage, error) { return nil, err } } - return &RedisStorage{db: ndb, dbNb: db, ms: NewCodecMsgpackMarshaler()}, nil + var mrshler Marshaler + if mrshlerStr == utils.MSGPACK { + mrshler = NewCodecMsgpackMarshaler() + } else if mrshlerStr == utils.JSON { + mrshler = new(JSONMarshaler) + } else { + return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr) + } + return &RedisStorage{db: ndb, dbNb: db, ms: mrshler}, nil } func (rs *RedisStorage) Close() { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 7436118f7..4619ed30a 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -26,7 +26,7 @@ import ( // Various helpers to deal with database -func ConfigureDataStorage(db_type, host, port, name, user, pass string) (db DataStorage, err error) { +func ConfigureDataStorage(db_type, host, port, name, user, pass, marshaler string) (db DataStorage, err error) { var d Storage switch db_type { case utils.REDIS: @@ -39,7 +39,7 @@ func ConfigureDataStorage(db_type, host, port, name, user, pass string) (db Data if port != "" { host += ":" + port } - d, err = NewRedisStorage(host, db_nb, pass) + d, err = NewRedisStorage(host, db_nb, pass, marshaler) db = d.(DataStorage) case utils.MONGO: d, err = NewMongoStorage(host, port, name, user, pass) @@ -53,7 +53,7 @@ func ConfigureDataStorage(db_type, host, port, name, user, pass string) (db Data return db, nil } -func ConfigureLogStorage(db_type, host, port, name, user, pass string) (db LogStorage, err error) { +func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string) (db LogStorage, err error) { var d Storage switch db_type { case utils.REDIS: @@ -66,7 +66,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass string) (db LogSt if port != "" { host += ":" + port } - d, err = NewRedisStorage(host, db_nb, pass) + d, err = NewRedisStorage(host, db_nb, pass, marshaler) db = d.(LogStorage) case utils.MONGO: d, err = NewMongoStorage(host, port, name, user, pass) @@ -86,7 +86,7 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass string) (db LogSt return db, nil } -func ConfigureLoadStorage(db_type, host, port, name, user, pass string) (db LoadStorage, err error) { +func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string) (db LoadStorage, err error) { var d Storage switch db_type { case utils.POSTGRES: diff --git a/utils/consts.go b/utils/consts.go index 12098d4ea..fa1800b8c 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -55,4 +55,6 @@ const ( ROUNDING_MIDDLE = "*middle" ROUNDING_DOWN = "*down" COMMENT_CHAR = '#' + JSON = "json" + MSGPACK = "msgpack" )