uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Add redisPoolPipelineLimit configuration field for datadb

Browse Source 
Sets the maximum number of commands that can be pipelined before flushing.
0 means no limit and pipelines will only be limited by the time window.
This commit is contained in:
ionutboangiu
2024-07-04 19:38:25 +03:00
committed by Dan Christian Bogos
parent 8ed26c769e
commit eb3bd5cc1d
20 changed files with 116 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
config/config_defaults.go
View File

@@ -143,16 +143,17 @@ const CGRATES_CFG_JSON = `
"redisCluster": false, // if enabled the datadb will try to connect to the redis cluster
"redisClusterSync": "5s", // the sync interval for the redis cluster
"redisClusterOndownDelay": "0", // the delay before executing the commands if the redis cluster is in the CLUSTERDOWN state
"redisPoolPipelineWindow": "150µs", // duration after which internal pipelines are flushed (0 disables implicit pipelining)
"redisConnectTimeout": "0", // the amount of wait time until timeout for a connection attempt
"redisReadTimeout": "0", // the amount of wait time until timeout for reading operations
"redisWriteTimeout": "0", // the amount of wait time until timeout for writing operations
"mongoQueryTimeout":"10s", // timeout for query when mongo is used
"mongoConnScheme": "mongodb", // scheme for MongoDB connection <mongodb|mongodb+srv>
"redisPoolPipelineWindow": "150µs", // duration after which internal pipelines are flushed (0 disables implicit pipelining)
"redisPoolPipelineLimit": 0, // maximum number of commands that can be pipelined before flushing (0 means no limit)
"redisTLS": false, // if true it will use a tls connection and use the redisClientCertificate, redisClientKey and redisCACertificate for tls connection
"redisClientCertificate":"", // path to client certificate
"redisClientKey":"", // path to client key
"redisCACertificate":"", // path to CA certificate (populate for self-signed certificate otherwise let it empty)
"mongoQueryTimeout":"10s", // timeout for query when mongo is used
"mongoConnScheme": "mongodb" // scheme for MongoDB connection <mongodb|mongodb+srv>
}
},
@@ -1118,15 +1119,16 @@ const CGRATES_CFG_JSON = `
"redisCluster": false,
"redisClusterSync": "5s",
"redisClusterOndownDelay": "0",
"redisPoolPipelineWindow": "150µs", // duration after which internal pipelines are flushed (0 disables implicit pipelining)
"redisConnectTimeout": "0", // the amount of wait time until timeout for a connection attempt
"redisReadTimeout": "0", // the amount of wait time until timeout for reading operations
"redisWriteTimeout": "0", // the amount of wait time until timeout for writing operations
"redisPoolPipelineWindow": "150µs", // duration after which internal pipelines are flushed (0 disables implicit pipelining)
"redisPoolPipelineLimit": 0, // maximum number of commands that can be pipelined before flushing (0 means no limit)
"redisTLS": false, // enable TLS when connecting to Redis and use the redisClientCertificate, redisClientKey and redisCACertificate for TLS connection
"redisClientCertificate":"", // path to client certificate
"redisClientKey":"", // path to client key
"redisCACertificate":"", // path to CA certificate (populate for self-signed certificate otherwise let it empty)
"mongoConnScheme": "mongodb", // scheme for MongoDB connection <mongodb|mongodb+srv>
"mongoConnScheme": "mongodb" // scheme for MongoDB connection <mongodb|mongodb+srv>
},
"out_stordb_opts":{
"mongoConnScheme": "mongodb" // scheme for MongoDB connection <mongodb|mongodb+srv>

2
config/config_json_test.go
View File

@@ -293,6 +293,7 @@ func TestDfDataDbJsonCfg(t *testing.T) {
RedisClusterSync: utils.StringPointer("5s"),
RedisClusterOndownDelay: utils.StringPointer("0"),
RedisPoolPipelineWindow: utils.StringPointer("150µs"),
RedisPoolPipelineLimit: utils.IntPointer(0),
RedisConnectTimeout: utils.StringPointer("0"),
RedisReadTimeout: utils.StringPointer("0"),
RedisWriteTimeout: utils.StringPointer("0"),
@@ -1942,6 +1943,7 @@ func TestDfMigratorCfg(t *testing.T) {
RedisClusterSync: utils.StringPointer("5s"),
RedisClusterOndownDelay: utils.StringPointer("0"),
RedisPoolPipelineWindow: utils.StringPointer("150µs"),
RedisPoolPipelineLimit: utils.IntPointer(0),
RedisConnectTimeout: utils.StringPointer("0"),
RedisReadTimeout: utils.StringPointer("0"),
RedisWriteTimeout: utils.StringPointer("0"),

7
config/config_test.go
View File

File diff suppressed because one or more lines are too long

38
config/datadbcfg.go
View File

@@ -34,16 +34,17 @@ type DataDBOpts struct {
RedisCluster bool
RedisClusterSync time.Duration
RedisClusterOndownDelay time.Duration
RedisPoolPipelineWindow time.Duration
RedisConnectTimeout time.Duration
RedisReadTimeout time.Duration
RedisWriteTimeout time.Duration
MongoQueryTimeout time.Duration
MongoConnScheme string
RedisPoolPipelineWindow time.Duration
RedisPoolPipelineLimit int
RedisTLS bool
RedisClientCertificate string
RedisClientKey string
RedisCACertificate string
MongoQueryTimeout time.Duration
MongoConnScheme string
}
// DataDbCfg Database config
@@ -89,11 +90,6 @@ func (dbOpts *DataDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) {
return
}
}
if jsnCfg.RedisPoolPipelineWindow != nil {
if dbOpts.RedisPoolPipelineWindow, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisPoolPipelineWindow); err != nil {
return
}
}
if jsnCfg.RedisConnectTimeout != nil {
if dbOpts.RedisConnectTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisConnectTimeout); err != nil {
return
@@ -109,13 +105,13 @@ func (dbOpts *DataDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) {
return
}
}
if jsnCfg.MongoQueryTimeout != nil {
if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil {
if jsnCfg.RedisPoolPipelineWindow != nil {
if dbOpts.RedisPoolPipelineWindow, err = utils.ParseDurationWithNanosecs(*jsnCfg.RedisPoolPipelineWindow); err != nil {
return
}
}
if jsnCfg.MongoConnScheme != nil {
dbOpts.MongoConnScheme = *jsnCfg.MongoConnScheme
if jsnCfg.RedisPoolPipelineLimit != nil {
dbOpts.RedisPoolPipelineLimit = *jsnCfg.RedisPoolPipelineLimit
}
if jsnCfg.RedisTLS != nil {
dbOpts.RedisTLS = *jsnCfg.RedisTLS
@@ -129,6 +125,14 @@ func (dbOpts *DataDBOpts) loadFromJSONCfg(jsnCfg *DBOptsJson) (err error) {
if jsnCfg.RedisCACertificate != nil {
dbOpts.RedisCACertificate = *jsnCfg.RedisCACertificate
}
if jsnCfg.MongoQueryTimeout != nil {
if dbOpts.MongoQueryTimeout, err = utils.ParseDurationWithNanosecs(*jsnCfg.MongoQueryTimeout); err != nil {
return
}
}
if jsnCfg.MongoConnScheme != nil {
dbOpts.MongoConnScheme = *jsnCfg.MongoConnScheme
}
return
}
@@ -218,10 +222,11 @@ func (dbOpts *DataDBOpts) Clone() *DataDBOpts {
RedisCluster: dbOpts.RedisCluster,
RedisClusterSync: dbOpts.RedisClusterSync,
RedisClusterOndownDelay: dbOpts.RedisClusterOndownDelay,
RedisPoolPipelineWindow: dbOpts.RedisPoolPipelineWindow,
RedisConnectTimeout: dbOpts.RedisConnectTimeout,
RedisReadTimeout: dbOpts.RedisReadTimeout,
RedisWriteTimeout: dbOpts.RedisWriteTimeout,
RedisPoolPipelineWindow: dbOpts.RedisPoolPipelineWindow,
RedisPoolPipelineLimit: dbOpts.RedisPoolPipelineLimit,
RedisTLS: dbOpts.RedisTLS,
RedisClientCertificate: dbOpts.RedisClientCertificate,
RedisClientKey: dbOpts.RedisClientKey,
@@ -269,16 +274,17 @@ func (dbcfg *DataDbCfg) AsMapInterface() (mp map[string]any) {
utils.RedisClusterCfg: dbcfg.Opts.RedisCluster,
utils.RedisClusterSyncCfg: dbcfg.Opts.RedisClusterSync.String(),
utils.RedisClusterOnDownDelayCfg: dbcfg.Opts.RedisClusterOndownDelay.String(),
utils.RedisPoolPipelineWindowCfg: dbcfg.Opts.RedisPoolPipelineWindow.String(),
utils.RedisConnectTimeoutCfg: dbcfg.Opts.RedisConnectTimeout.String(),
utils.RedisReadTimeoutCfg: dbcfg.Opts.RedisReadTimeout.String(),
utils.RedisWriteTimeoutCfg: dbcfg.Opts.RedisWriteTimeout.String(),
utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(),
utils.MongoConnSchemeCfg: dbcfg.Opts.MongoConnScheme,
utils.RedisPoolPipelineWindowCfg: dbcfg.Opts.RedisPoolPipelineWindow.String(),
utils.RedisPoolPipelineLimitCfg: dbcfg.Opts.RedisPoolPipelineLimit,
utils.RedisTLS: dbcfg.Opts.RedisTLS,
utils.RedisClientCertificate: dbcfg.Opts.RedisClientCertificate,
utils.RedisClientKey: dbcfg.Opts.RedisClientKey,
utils.RedisCACertificate: dbcfg.Opts.RedisCACertificate,
utils.MongoQueryTimeoutCfg: dbcfg.Opts.MongoQueryTimeout.String(),
utils.MongoConnSchemeCfg: dbcfg.Opts.MongoConnScheme,
}
mp = map[string]any{
utils.DataDbTypeCfg: dbcfg.Type,

7
config/libconfig_json.go
View File

@@ -107,16 +107,17 @@ type DBOptsJson struct {
RedisCluster *bool `json:"redisCluster"`
RedisClusterSync *string `json:"redisClusterSync"`
RedisClusterOndownDelay *string `json:"redisClusterOndownDelay"`
RedisPoolPipelineWindow *string `json:"redisPoolPipelineWindow"`
RedisConnectTimeout *string `json:"redisConnectTimeout"`
RedisReadTimeout *string `json:"redisReadTimeout"`
RedisWriteTimeout *string `json:"redisWriteTimeout"`
MongoQueryTimeout *string `json:"mongoQueryTimeout"`
MongoConnScheme *string `json:"mongoConnScheme"`
RedisPoolPipelineWindow *string `json:"redisPoolPipelineWindow"`
RedisPoolPipelineLimit *int `json:"redisPoolPipelineLimit"`
RedisTLS *bool `json:"redisTLS"`
RedisClientCertificate *string `json:"redisClientCertificate"`
RedisClientKey *string `json:"redisClientKey"`
RedisCACertificate *string `json:"redisCACertificate"`
MongoQueryTimeout *string `json:"mongoQueryTimeout"`
MongoConnScheme *string `json:"mongoConnScheme"`
SQLMaxOpenConns *int `json:"sqlMaxOpenConns"`
SQLMaxIdleConns *int `json:"sqlMaxIdleConns"`
SQLConnMaxLifetime *string `json:"sqlConnMaxLifetime"`

7
config/migratorcfg.go
View File

@@ -119,16 +119,17 @@ func (mg *MigratorCgrCfg) AsMapInterface() (initialMP map[string]any) {
utils.RedisClusterCfg: mg.OutDataDBOpts.RedisCluster,
utils.RedisClusterSyncCfg: mg.OutDataDBOpts.RedisClusterSync.String(),
utils.RedisClusterOnDownDelayCfg: mg.OutDataDBOpts.RedisClusterOndownDelay.String(),
utils.RedisPoolPipelineWindowCfg: mg.OutDataDBOpts.RedisPoolPipelineWindow.String(),
utils.RedisConnectTimeoutCfg: mg.OutDataDBOpts.RedisConnectTimeout.String(),
utils.RedisReadTimeoutCfg: mg.OutDataDBOpts.RedisReadTimeout.String(),
utils.RedisWriteTimeoutCfg: mg.OutDataDBOpts.RedisWriteTimeout.String(),
utils.MongoQueryTimeoutCfg: mg.OutDataDBOpts.MongoQueryTimeout.String(),
utils.MongoConnSchemeCfg: mg.OutDataDBOpts.MongoConnScheme,
utils.RedisPoolPipelineWindowCfg: mg.OutDataDBOpts.RedisPoolPipelineWindow.String(),
utils.RedisPoolPipelineLimitCfg: mg.OutDataDBOpts.RedisPoolPipelineLimit,
utils.RedisTLS: mg.OutDataDBOpts.RedisTLS,
utils.RedisClientCertificate: mg.OutDataDBOpts.RedisClientCertificate,
utils.RedisClientKey: mg.OutDataDBOpts.RedisClientKey,
utils.RedisCACertificate: mg.OutDataDBOpts.RedisCACertificate,
utils.MongoQueryTimeoutCfg: mg.OutDataDBOpts.MongoQueryTimeout.String(),
utils.MongoConnSchemeCfg: mg.OutDataDBOpts.MongoConnScheme,
}
outStorDBOpts := map[string]any{
utils.SQLMaxOpenConnsCfg: mg.OutStorDBOpts.SQLMaxOpenConns,

7
config/migratorcfg_test.go
View File

@@ -154,6 +154,7 @@ func TestMigratorCgrCfgAsMapInterface(t *testing.T) {
utils.RedisClusterSyncCfg: "2s",
utils.RedisClusterOnDownDelayCfg: "1ns",
utils.RedisPoolPipelineWindowCfg: "150µs",
utils.RedisPoolPipelineLimitCfg: 0,
utils.RedisConnectTimeoutCfg: "0s",
utils.RedisReadTimeoutCfg: "3s",
utils.RedisWriteTimeoutCfg: "3s",
@@ -177,6 +178,8 @@ func TestMigratorCgrCfgAsMapInterface1(t *testing.T) {
"out_datadb_opts": {
"redisSentinel": "out_datadb_redis_sentinel",
"redisConnectTimeout": "5s",
"redisPoolPipelineWindow": "1ms",
"redisPoolPipelineLimit": 3
},
},
}`
@@ -214,7 +217,8 @@ func TestMigratorCgrCfgAsMapInterface1(t *testing.T) {
utils.RedisClusterCfg: false,
utils.RedisClusterSyncCfg: "5s",
utils.RedisClusterOnDownDelayCfg: "0s",
utils.RedisPoolPipelineWindowCfg: "150µs",
utils.RedisPoolPipelineWindowCfg: "1ms",
utils.RedisPoolPipelineLimitCfg: 3,
utils.RedisConnectTimeoutCfg: "5s",
utils.RedisReadTimeoutCfg: "0s",
utils.RedisWriteTimeoutCfg: "0s",
@@ -270,6 +274,7 @@ func TestMigratorCgrCfgAsMapInterface2(t *testing.T) {
utils.RedisClusterSyncCfg: "5s",
utils.RedisClusterOnDownDelayCfg: "0s",
utils.RedisPoolPipelineWindowCfg: "150µs",
utils.RedisPoolPipelineLimitCfg: 0,
utils.RedisConnectTimeoutCfg: "0s",
utils.RedisReadTimeoutCfg: "0s",
utils.RedisWriteTimeoutCfg: "0s",