From 1657f015fcd8efc3ed8a036b5f07ae3ec65e275e Mon Sep 17 00:00:00 2001 From: arberkatellari Date: Thu, 30 Oct 2025 09:16:08 +0200 Subject: [PATCH] make accounts storable in mysql --- apis/accounts_it_test.go | 6 +- config/config_defaults.go | 4 +- .../acc_generaltest_mysql_acc/cgrates.json | 147 ++++++++++++++++++ data/conf/samples/mysql_acc/cgrates.json | 141 +++++++++++++++++ data/storage/mysql/create_accounts_tables.sql | 11 +- data/storage/mysql/setup_cgr_db.sh | 4 +- .../postgres/create_accounts_tables.sql | 6 +- engine/filters.go | 11 +- engine/storage_mysql.go | 21 +-- engine/storage_postgres.go | 45 ------ engine/storage_sql.go | 56 ++++++- engine/storage_utils.go | 3 +- general_tests/accounts_it_test.go | 4 + 13 files changed, 376 insertions(+), 83 deletions(-) create mode 100644 data/conf/samples/acc_generaltest_mysql_acc/cgrates.json create mode 100644 data/conf/samples/mysql_acc/cgrates.json diff --git a/apis/accounts_it_test.go b/apis/accounts_it_test.go index 24151a5f2..8bd05bd4d 100644 --- a/apis/accounts_it_test.go +++ b/apis/accounts_it_test.go @@ -85,6 +85,10 @@ func TestAccSIT(t *testing.T) { case utils.MetaMongo: accPrfConfigDIR = "tutmongo" case utils.MetaMySQL: + accPrfConfigDIR = "mysql_acc" + for _, stest := range sTestsAccPrf { + t.Run(accPrfConfigDIR, stest) + } accPrfConfigDIR = "tutmysql" case utils.MetaPostgres: accPrfConfigDIR = "tutpostgres" @@ -1507,7 +1511,7 @@ func testAccRefundCharges(t *testing.T) { }, &result); err != nil { t.Error(err) } else { - if *utils.DBType == utils.MetaPostgres { + if *utils.DBType == utils.MetaPostgres || accPrfConfigDIR == "mysql_acc" { acc.Account.Balances["CB"].Units = utils.NewDecimalFromFloat64(50) } if !reflect.DeepEqual(result, acc.Account) { diff --git a/config/config_defaults.go b/config/config_defaults.go index 23c58f003..51ef8129b 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -125,8 +125,10 @@ const CGRATES_CFG_JSON = ` }, }, "items":{ - // compatible db types: <*internal|*redis|*mongo> + // compatible with all db types "*accounts": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, + + // compatible db types: <*internal|*redis|*mongo> "*actions": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*resource_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*resources": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, diff --git a/data/conf/samples/acc_generaltest_mysql_acc/cgrates.json b/data/conf/samples/acc_generaltest_mysql_acc/cgrates.json new file mode 100644 index 000000000..538a37686 --- /dev/null +++ b/data/conf/samples/acc_generaltest_mysql_acc/cgrates.json @@ -0,0 +1,147 @@ +{ +// CGRateS Configuration file +// + + +"general": { + "reply_timeout": "50s", +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + +"db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_conns": { + "*default": { + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to + }, + "StorDB": { // The id of the DB connection + "db_type": "mysql", // db type: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 3306, // db port to reach the database + "db_name": "cgrates", // db database name to connect to + "db_user": "cgrates", // username to use when connecting to the database + "db_password": "CGRateS.org" // password to use when connecting to the database + }, + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*accounts": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + + } +}, + +"loaders": [ + { + "id": "*default", + "enabled": true, + "tenant": "cgrates.org", + "lockfile_path": ".cgr.lck", + "tp_in_dir": "/usr/share/cgrates/tariffplans/testit", + "tp_out_dir": "", + }, +], + +"cdrs": { + "enabled": true, + "chargers_conns":["*internal"], +}, + + +"attributes": { + "enabled": true, + "stats_conns": ["*localhost"], + "resources_conns": ["*localhost"], + "accounts_conns": ["*localhost"] +}, + + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"], +}, + + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"routes": { + "enabled": true, + "prefix_indexed_fields":["*req.Destination"], + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "rates_conns": ["*internal"], +}, + + +"sessions": { + "enabled": true, + "routes_conns": ["*internal"], + "resources_conns": ["*internal"], + "attributes_conns": ["*internal"], + "rates_conns": ["*internal"], + "cdrs_conns": ["*internal"], + "chargers_conns": ["*internal"], +}, + + +"migrator":{ + "users_filters":["Account"], +}, + + +"admins": { + "enabled": true, +}, + + +"rates": { + "enabled": true +}, + + +"actions": { + "enabled": true, + "accounts_conns": ["*localhost"] +}, + + +"accounts": { + "enabled": true +}, + + +"filters": { + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "accounts_conns": ["*internal"], +}, + + +} diff --git a/data/conf/samples/mysql_acc/cgrates.json b/data/conf/samples/mysql_acc/cgrates.json new file mode 100644 index 000000000..ded6ab1b7 --- /dev/null +++ b/data/conf/samples/mysql_acc/cgrates.json @@ -0,0 +1,141 @@ +{ +// CGRateS Configuration file +// + + +"general": { + "reply_timeout": "50s", +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + +"db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_conns": { + "*default": { // The id of the DB connection + "db_type": "redis", // db type: + "db_host": "127.0.0.1", + "db_port": 6379, // db port to reach the database + "db_name": "10", // db database name to connect to + "db_user": "cgrates", + }, + "StorDB": { // The id of the DB connection + "db_type": "mysql", // db type: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 3306, // db port to reach the database + "db_name": "cgrates", // db database name to connect to + "db_user": "cgrates", // username to use when connecting to the database + "db_password": "CGRateS.org" // password to use when connecting to the database + }, + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*accounts": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"cdrs": { + "enabled": true, + "chargers_conns":["*internal"], +}, + + +"attributes": { + "enabled": true, + "stats_conns": ["*localhost"], + "resources_conns": ["*localhost"], + "accounts_conns": ["*localhost"] +}, + +"chargers": { + "enabled": true, + "attributes_conns": ["*internal"], +}, + + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"], +}, + + +"thresholds": { + "enabled": true, + "store_interval": "1s", +}, + + +"routes": { + "enabled": true, + "prefix_indexed_fields":["*req.Destination"], + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "rates_conns": ["*internal"], +}, + + +"sessions": { + "enabled": true, + "routes_conns": ["*internal"], + "resources_conns": ["*internal"], + "attributes_conns": ["*internal"], + "rates_conns": ["*internal"], + "cdrs_conns": ["*internal"], + "chargers_conns": ["*internal"], +}, + + +"migrator":{ + + "users_filters":["Account"], +}, + + +"admins": { + "enabled": true, + "scheduler_conns": ["*internal"], +}, + + +"rates": { + "enabled": true +}, + + +"actions": { + "enabled": true, + "accounts_conns": ["*localhost"] +}, + + +"accounts": { + "enabled": true +}, + + +"filters": { + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "accounts_conns": ["*internal"], +}, + +"tpes": { + "enabled": true +}, + +} diff --git a/data/storage/mysql/create_accounts_tables.sql b/data/storage/mysql/create_accounts_tables.sql index 9c2d79736..c7ffde89f 100644 --- a/data/storage/mysql/create_accounts_tables.sql +++ b/data/storage/mysql/create_accounts_tables.sql @@ -2,4 +2,13 @@ -- Table structure for table `accounts` -- --- placeholder file \ No newline at end of file +DROP TABLE IF EXISTS accounts; +CREATE TABLE accounts ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tenant` VARCHAR(40) NOT NULL, + `id` VARCHAR(64) NOT NULL, + `account` JSON NOT NULL, + PRIMARY KEY (`pk`), + UNIQUE KEY unique_tenant_id (`tenant`, `id`) +); +CREATE UNIQUE INDEX accounts_idx ON accounts (`id`); \ No newline at end of file diff --git a/data/storage/mysql/setup_cgr_db.sh b/data/storage/mysql/setup_cgr_db.sh index bfc68ad4f..042bdb932 100755 --- a/data/storage/mysql/setup_cgr_db.sh +++ b/data/storage/mysql/setup_cgr_db.sh @@ -16,12 +16,14 @@ DIR="$(dirname "$(readlink -f "$0")")" mysql -u $1 -p$2 -h $host < "$DIR"/create_db_with_users.sql cu=$? +mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_accounts_tables.sql +acct=$? mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_cdrs_tables.sql cdrt=$? mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_tariffplan_tables.sql tpt=$? -if [ $cu = 0 ] && [ $cdrt = 0 ] && [ $tpt = 0 ]; then +if [ $cu = 0 ] && [ $acct = 0 ] && [ $cdrt = 0 ] && [ $tpt = 0 ]; then echo "\n\t+++ CGR-DB successfully set-up! +++\n" exit 0 fi diff --git a/data/storage/postgres/create_accounts_tables.sql b/data/storage/postgres/create_accounts_tables.sql index b65f6a776..1c9fe3d9f 100644 --- a/data/storage/postgres/create_accounts_tables.sql +++ b/data/storage/postgres/create_accounts_tables.sql @@ -6,8 +6,8 @@ DROP TABLE IF EXISTS accounts; CREATE TABLE accounts ( pk SERIAL PRIMARY KEY, tenant VARCHAR(40) NOT NULL, - id varchar(64) NOT NULL, - account jsonb NOT NULL, + id VARCHAR(64) NOT NULL, + account JSONB NOT NULL, UNIQUE (tenant, id) ); -CREATE UNIQUE INDEX accounts_unique ON accounts ("id"); +CREATE UNIQUE INDEX accounts_idx ON accounts ("id"); diff --git a/engine/filters.go b/engine/filters.go index b6130be14..e99a7aad9 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -1060,23 +1060,24 @@ func (fltr *FilterRule) FilterToSQLQuery() (conditions []string) { if len(fltr.Values) == 0 { switch fltr.Type { case utils.MetaExists, utils.MetaNotExists: - if not { + if not { // not existing means Column IS NULL if firstItem == utils.EmptyString { - conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", restOfItems)) + conditions = append(conditions, fmt.Sprintf("%s IS NULL", restOfItems)) return } - queryPart := fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", firstItem, restOfItems) + queryPart := fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", firstItem, restOfItems) if strings.HasPrefix(restOfItems, `"*`) { queryPart = fmt.Sprintf("JSON_UNQUOTE(%s)", queryPart) } conditions = append(conditions, queryPart) return } + // existing means Column IS NOT NULL if firstItem == utils.EmptyString { - conditions = append(conditions, fmt.Sprintf("%s IS NULL", restOfItems)) + conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", restOfItems)) return } - queryPart := fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", firstItem, restOfItems) + queryPart := fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", firstItem, restOfItems) if strings.HasPrefix(restOfItems, `"*`) { queryPart = fmt.Sprintf("JSON_UNQUOTE(%s)", queryPart) } diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 0792658d8..14b4a8b29 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/utils" "gorm.io/driver/mysql" "gorm.io/gorm" @@ -33,8 +32,9 @@ type MySQLStorage struct { SQLStorage } -func NewMySQLStorage(host, port, name, user, password string, - maxConn, maxIdleConn, logLevel int, connMaxLifetime time.Duration, location string, dsnParams map[string]string) (*SQLStorage, error) { +func NewMySQLStorage(host, port, name, user, password string, maxConn, maxIdleConn, + logLevel int, connMaxLifetime time.Duration, location string, + dsnParams map[string]string) (*SQLStorage, error) { connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=%s&parseTime=true&sql_mode='ALLOW_INVALID_DATES'", user, password, host, port, name, location) db, err := gorm.Open(mysql.Open(connectString+AppendToMysqlDSNOpts(dsnParams)), &gorm.Config{AllowGlobalUpdate: true, Logger: logger.Default.LogMode(logger.LogLevel(logLevel))}) @@ -92,21 +92,6 @@ func (msqlS *MySQLStorage) SetVersions(vrs Versions, overwrite bool) (err error) return } -// DataDB method not implemented yet -func (msqlS *MySQLStorage) GetAccountDrv(ctx *context.Context, tenant, id string) (ap *utils.Account, err error) { - return nil, utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (msqlS *MySQLStorage) SetAccountDrv(ctx *context.Context, ap *utils.Account) (err error) { - return utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (msqlS *MySQLStorage) RemoveAccountDrv(ctx *context.Context, tenant, id string) (err error) { - return utils.ErrNotImplemented -} - func (msqlS *MySQLStorage) extraFieldsExistsQry(field string) string { return fmt.Sprintf(" extra_fields LIKE '%%\"%s\":%%'", field) } diff --git a/engine/storage_postgres.go b/engine/storage_postgres.go index 05b8957b6..45e23dc6f 100644 --- a/engine/storage_postgres.go +++ b/engine/storage_postgres.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/utils" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -102,50 +101,6 @@ func (poS *PostgresStorage) SetVersions(vrs Versions, overwrite bool) (err error return } -func (poS *PostgresStorage) GetAccountDrv(_ *context.Context, tenant, id string) (ap *utils.Account, err error) { - var result []*AccountJSONMdl - if err = poS.db.Model(&AccountJSONMdl{}).Where(&AccountJSONMdl{Tenant: tenant, - ID: id}).Find(&result).Error; err != nil { - return - } - if len(result) == 0 { - return nil, utils.ErrNotFound - } - return utils.MapStringInterfaceToAccount(result[0].Account) -} - -func (poS *PostgresStorage) SetAccountDrv(_ *context.Context, ap *utils.Account) (err error) { - tx := poS.db.Begin() - mdl := &AccountJSONMdl{ - Tenant: ap.Tenant, - ID: ap.ID, - Account: ap.AsMapStringInterface(), - } - if err = tx.Model(&AccountJSONMdl{}).Where( - AccountJSONMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete( - AccountJSONMdl{Account: mdl.Account}).Error; err != nil { - tx.Rollback() - return - } - if err = tx.Save(mdl).Error; err != nil { - tx.Rollback() - return - } - tx.Commit() - return -} - -func (poS *PostgresStorage) RemoveAccountDrv(_ *context.Context, tenant, id string) (err error) { - tx := poS.db.Begin() - if err = tx.Model(&AccountJSONMdl{}).Where(&AccountJSONMdl{Tenant: tenant, ID: id}). - Delete(&AccountJSONMdl{}).Error; err != nil { - tx.Rollback() - return - } - tx.Commit() - return -} - func (poS *PostgresStorage) extraFieldsExistsQry(field string) string { return fmt.Sprintf(" extra_fields ?'%s'", field) } diff --git a/engine/storage_sql.go b/engine/storage_sql.go index dc0431ff6..4873a4095 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -74,13 +74,6 @@ func (sqls *SQLStorage) SelectDatabase(dbName string) (err error) { return } -// returns all keys in table matching the prefix -func (sqls *SQLStorage) getAllIndexKeys(_ *context.Context, table, prefix string) (ids []string, err error) { - err = sqls.db.Table(table).Select("id").Where("id LIKE ?", prefix+"%"). - Pluck("id", &ids).Error - return -} - // returns all keys in table matching the Tenant and ID func (sqls *SQLStorage) getAllKeysMatchingTenantID(_ *context.Context, table string, tntID *utils.TenantID) (ids []string, err error) { matchingTntID := []utils.TenantID{} @@ -387,6 +380,55 @@ func (sqls *SQLStorage) RemoveCDRs(ctx *context.Context, qryFltr []*Filter) (err return } +// GetAccountDrv will get the account from the DB matching the tenant and id provided. +// Decimal fields ending in `.0` will be read as whole numbers but still in decimal type. +// (50.0 -> 50) +func (sqls *SQLStorage) GetAccountDrv(ctx *context.Context, tenant, id string) (ap *utils.Account, err error) { + var result []*AccountJSONMdl + if err = sqls.db.Model(&AccountJSONMdl{}).Where(&AccountJSONMdl{Tenant: tenant, + ID: id}).Find(&result).Error; err != nil { + return + } + if len(result) == 0 { + return nil, utils.ErrNotFound + } + return utils.MapStringInterfaceToAccount(result[0].Account) +} + +// SetAccountDrv will set in DB the provided Account +func (sqls *SQLStorage) SetAccountDrv(ctx *context.Context, ap *utils.Account) (err error) { + tx := sqls.db.Begin() + mdl := &AccountJSONMdl{ + Tenant: ap.Tenant, + ID: ap.ID, + Account: ap.AsMapStringInterface(), + } + if err = tx.Model(&AccountJSONMdl{}).Where( + AccountJSONMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete( + AccountJSONMdl{Account: mdl.Account}).Error; err != nil { + tx.Rollback() + return + } + if err = tx.Save(mdl).Error; err != nil { + tx.Rollback() + return + } + tx.Commit() + return +} + +// RemoveAccountDrv will remove from DB the account matching the tenamt and id provided +func (sqls *SQLStorage) RemoveAccountDrv(ctx *context.Context, tenant, id string) (err error) { + tx := sqls.db.Begin() + if err = tx.Model(&AccountJSONMdl{}).Where(&AccountJSONMdl{Tenant: tenant, ID: id}). + Delete(&AccountJSONMdl{}).Error; err != nil { + tx.Rollback() + return + } + tx.Commit() + return +} + // AddLoadHistory DataDB method not implemented yet func (sqls *SQLStorage) AddLoadHistory(ldInst *utils.LoadInstance, loadHistSize int, transactionID string) error { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 6aaf68401..0172d5570 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -64,7 +64,8 @@ func NewDataDBConn(dbType, host, port, name, user, opts.SQLMaxIdleConns, opts.SQLLogLevel, opts.SQLConnMaxLifetime, opts.MySQLLocation, opts.SQLDSNParams) case utils.MetaInternal: - d, err = NewInternalDB(stringIndexedFields, prefixIndexedFields, opts.ToTransCacheOpts(), itmsCfg) + d, err = NewInternalDB(stringIndexedFields, prefixIndexedFields, + opts.ToTransCacheOpts(), itmsCfg) default: err = fmt.Errorf("unsupported db_type <%s>", dbType) } diff --git a/general_tests/accounts_it_test.go b/general_tests/accounts_it_test.go index ebf4865f1..de7e27647 100644 --- a/general_tests/accounts_it_test.go +++ b/general_tests/accounts_it_test.go @@ -66,6 +66,10 @@ func TestAccIT(t *testing.T) { case utils.MetaInternal: accConfDIR = "acc_generaltest_internal" case utils.MetaMySQL: + accConfDIR = "acc_generaltest_mysql_acc" + for _, stest := range sTestsAcc { + t.Run(accConfDIR, stest) + } accConfDIR = "acc_generaltest_mysql" case utils.MetaMongo: accConfDIR = "acc_generaltest_mongo"