Make IPProfiles and IPAllocations storable in MySQL and Postgres

This commit is contained in:
arberkatellari
2025-10-31 16:02:34 +02:00
committed by Dan Christian Bogos
parent 1657f015fc
commit 40a8e6ae31
14 changed files with 363 additions and 70 deletions

View File

@@ -127,13 +127,13 @@ const CGRATES_CFG_JSON = `
"items":{
// compatible with all db types
"*accounts": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*ip_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*ip_allocations": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
// compatible db types: <*internal|*redis|*mongo>
"*actions": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*resource_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*resources": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*ip_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*ip_allocations": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*statqueue_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},
"*threshold_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"},

View File

@@ -1060,7 +1060,8 @@ func (cfg *CGRConfig) checkConfigSanity() error {
dataDBTypes := []string{utils.MetaInternal, utils.MetaRedis, utils.MetaMongo,
utils.Internal, utils.Redis, utils.Mongo}
if item != utils.MetaAccounts {
if item != utils.MetaAccounts && item != utils.MetaIPProfiles &&
item != utils.MetaIPAllocations {
if item == utils.MetaCDRs {
if !slices.Contains(storDBTypes, cfg.dbCfg.DBConns[val.DBConn].Type) {
return fmt.Errorf("<%s> db item can only be of types <%v>, got <%s>", item,

View File

@@ -1,14 +0,0 @@
--
-- Table structure for table `accounts`
--
DROP TABLE IF EXISTS accounts;
CREATE TABLE accounts (
`pk` int(11) NOT NULL AUTO_INCREMENT,
`tenant` VARCHAR(40) NOT NULL,
`id` VARCHAR(64) NOT NULL,
`account` JSON NOT NULL,
PRIMARY KEY (`pk`),
UNIQUE KEY unique_tenant_id (`tenant`, `id`)
);
CREATE UNIQUE INDEX accounts_idx ON accounts (`id`);

View File

@@ -0,0 +1,36 @@
--
-- Table structure for table `accounts`
--
DROP TABLE IF EXISTS accounts;
CREATE TABLE accounts (
`pk` int(11) NOT NULL AUTO_INCREMENT,
`tenant` VARCHAR(40) NOT NULL,
`id` VARCHAR(64) NOT NULL,
`account` JSON NOT NULL,
PRIMARY KEY (`pk`),
UNIQUE KEY unique_tenant_id (`tenant`, `id`)
);
CREATE UNIQUE INDEX accounts_idx ON accounts (`id`);
DROP TABLE IF EXISTS ip_profiles;
CREATE TABLE ip_profiles (
`pk` int(11) NOT NULL AUTO_INCREMENT,
`tenant` VARCHAR(40) NOT NULL,
`id` VARCHAR(64) NOT NULL,
`ip_profile` JSON NOT NULL,
PRIMARY KEY (`pk`),
UNIQUE KEY unique_tenant_id (`tenant`, `id`)
);
CREATE UNIQUE INDEX ip_profiles_idx ON ip_profiles (`id`);
DROP TABLE IF EXISTS ip_allocations;
CREATE TABLE ip_allocations (
`pk` int(11) NOT NULL AUTO_INCREMENT,
`tenant` VARCHAR(40) NOT NULL,
`id` VARCHAR(64) NOT NULL,
`ip_allocation` JSON NOT NULL,
PRIMARY KEY (`pk`),
UNIQUE KEY unique_tenant_id (`tenant`, `id`)
);
CREATE UNIQUE INDEX ip_allocations_idx ON ip_allocations (`id`);

View File

@@ -16,7 +16,7 @@ DIR="$(dirname "$(readlink -f "$0")")"
mysql -u $1 -p$2 -h $host < "$DIR"/create_db_with_users.sql
cu=$?
mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_accounts_tables.sql
mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_db_tables.sql
acct=$?
mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_cdrs_tables.sql
cdrt=$?

View File

@@ -1,13 +0,0 @@
--
-- Table structure for table `accounts`
--
DROP TABLE IF EXISTS accounts;
CREATE TABLE accounts (
pk SERIAL PRIMARY KEY,
tenant VARCHAR(40) NOT NULL,
id VARCHAR(64) NOT NULL,
account JSONB NOT NULL,
UNIQUE (tenant, id)
);
CREATE UNIQUE INDEX accounts_idx ON accounts ("id");

View File

@@ -0,0 +1,34 @@
--
-- Table structure for table `accounts`
--
DROP TABLE IF EXISTS accounts;
CREATE TABLE accounts (
pk SERIAL PRIMARY KEY,
tenant VARCHAR(40) NOT NULL,
id VARCHAR(64) NOT NULL,
account JSONB NOT NULL,
UNIQUE (tenant, id)
);
CREATE UNIQUE INDEX accounts_idx ON accounts ("id");
DROP TABLE IF EXISTS ip_profiles;
CREATE TABLE ip_profiles (
pk SERIAL PRIMARY KEY,
tenant VARCHAR(40) NOT NULL,
id VARCHAR(64) NOT NULL,
ip_profile JSONB NOT NULL,
UNIQUE (tenant, id)
);
CREATE UNIQUE INDEX ip_profiles_idx ON ip_profiles ("id");
DROP TABLE IF EXISTS ip_allocations;
CREATE TABLE ip_allocations (
pk SERIAL PRIMARY KEY,
tenant VARCHAR(40) NOT NULL,
id VARCHAR(64) NOT NULL,
ip_allocation JSONB NOT NULL,
UNIQUE (tenant, id)
);
CREATE UNIQUE INDEX ip_allocations_idx ON ip_allocations ("id");

View File

@@ -17,7 +17,7 @@ DIR="$(dirname "$(readlink -f "$0")")"
export PGPASSWORD="CGRateS.org"
psql -U $user -h $host -d cgrates -f "$DIR"/create_accounts_tables.sql
psql -U $user -h $host -d cgrates -f "$DIR"/create_db_tables.sql
acct=$?
psql -U $user -h $host -d cgrates -f "$DIR"/create_cdrs_tables.sql
cdrt=$?

View File

@@ -393,3 +393,25 @@ type AccountJSONMdl struct {
func (AccountJSONMdl) TableName() string {
return utils.TBLAccounts
}
type IPProfileMdl struct {
PK uint `gorm:"primary_key"`
Tenant string `index:"0" re:".*"`
ID string `index:"1" re:".*"`
IPProfile utils.JSONB `gorm:"type:jsonb" index:"2" re:".*"`
}
func (IPProfileMdl) TableName() string {
return utils.TBLIPProfiles
}
type IPAllocationMdl struct {
PK uint `gorm:"primary_key"`
Tenant string `index:"0" re:".*"`
ID string `index:"1" re:".*"`
IPAllocation utils.JSONB `gorm:"type:jsonb" index:"2" re:".*"`
}
func (IPAllocationMdl) TableName() string {
return utils.TBLIPAllocations
}

View File

@@ -77,8 +77,10 @@ func (sqls *SQLStorage) SelectDatabase(dbName string) (err error) {
// returns all keys in table matching the Tenant and ID
func (sqls *SQLStorage) getAllKeysMatchingTenantID(_ *context.Context, table string, tntID *utils.TenantID) (ids []string, err error) {
matchingTntID := []utils.TenantID{}
err = sqls.db.Table(table).Select("tenant, id").Where("tenant = ? AND id LIKE ?", tntID.Tenant, tntID.ID+"%").
Find(&matchingTntID).Error
if err = sqls.db.Table(table).Select("tenant, id").Where("tenant = ? AND id LIKE ?", tntID.Tenant, tntID.ID+"%").
Find(&matchingTntID).Error; err != nil {
return nil, err
}
ids = make([]string, len(matchingTntID))
for i, result := range matchingTntID {
ids[i] = utils.ConcatenatedKey(result.Tenant, result.ID)
@@ -98,6 +100,10 @@ func (sqls *SQLStorage) GetKeysForPrefix(ctx *context.Context, prefix string) (k
switch category {
case utils.AccountPrefix:
keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLAccounts, tntID)
case utils.IPProfilesPrefix:
keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLIPProfiles, tntID)
case utils.IPAllocationsPrefix:
keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLIPAllocations, tntID)
default:
err = fmt.Errorf("unsupported prefix in GetKeysForPrefix: %q", prefix)
}
@@ -429,6 +435,94 @@ func (sqls *SQLStorage) RemoveAccountDrv(ctx *context.Context, tenant, id string
return
}
func (sqls *SQLStorage) GetIPProfileDrv(ctx *context.Context, tenant, id string) (*utils.IPProfile, error) {
var result []*IPProfileMdl
if err := sqls.db.Model(&IPProfileMdl{}).Where(&IPProfileMdl{Tenant: tenant,
ID: id}).Find(&result).Error; err != nil {
return nil, err
}
if len(result) == 0 {
return nil, utils.ErrNotFound
}
return utils.MapStringInterfaceToIPProfile(result[0].IPProfile)
}
func (sqls *SQLStorage) SetIPProfileDrv(ctx *context.Context, ipp *utils.IPProfile) error {
tx := sqls.db.Begin()
mdl := &IPProfileMdl{
Tenant: ipp.Tenant,
ID: ipp.ID,
IPProfile: ipp.AsMapStringInterface(),
}
if err := tx.Model(&IPProfileMdl{}).Where(
IPProfileMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete(
IPProfileMdl{IPProfile: mdl.IPProfile}).Error; err != nil {
tx.Rollback()
return err
}
if err := tx.Save(mdl).Error; err != nil {
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func (sqls *SQLStorage) RemoveIPProfileDrv(ctx *context.Context, tenant, id string) error {
tx := sqls.db.Begin()
if err := tx.Model(&IPProfileMdl{}).Where(&IPProfileMdl{Tenant: tenant, ID: id}).
Delete(&IPProfileMdl{}).Error; err != nil {
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func (sqls *SQLStorage) GetIPAllocationsDrv(ctx *context.Context, tenant, id string) (*utils.IPAllocations, error) {
var result []*IPAllocationMdl
if err := sqls.db.Model(&IPAllocationMdl{}).Where(&IPAllocationMdl{Tenant: tenant,
ID: id}).Find(&result).Error; err != nil {
return nil, err
}
if len(result) == 0 {
return nil, utils.ErrNotFound
}
return utils.MapStringInterfaceToIPAllocations(result[0].IPAllocation)
}
func (sqls *SQLStorage) SetIPAllocationsDrv(ctx *context.Context, ip *utils.IPAllocations) error {
tx := sqls.db.Begin()
mdl := &IPAllocationMdl{
Tenant: ip.Tenant,
ID: ip.ID,
IPAllocation: ip.AsMapStringInterface(),
}
if err := tx.Model(&IPAllocationMdl{}).Where(
IPAllocationMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete(
IPAllocationMdl{IPAllocation: mdl.IPAllocation}).Error; err != nil {
tx.Rollback()
return err
}
if err := tx.Save(mdl).Error; err != nil {
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func (sqls *SQLStorage) RemoveIPAllocationsDrv(ctx *context.Context, tenant, id string) error {
tx := sqls.db.Begin()
if err := tx.Model(&IPAllocationMdl{}).Where(&IPAllocationMdl{Tenant: tenant, ID: id}).
Delete(&IPAllocationMdl{}).Error; err != nil {
tx.Rollback()
return err
}
tx.Commit()
return nil
}
// AddLoadHistory DataDB method not implemented yet
func (sqls *SQLStorage) AddLoadHistory(ldInst *utils.LoadInstance,
loadHistSize int, transactionID string) error {
@@ -501,36 +595,6 @@ func (sqls *SQLStorage) RemoveResourceDrv(ctx *context.Context, tenant, id strin
return utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) GetIPProfileDrv(ctx *context.Context, tenant, id string) (*utils.IPProfile, error) {
return nil, utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) SetIPProfileDrv(ctx *context.Context, ipp *utils.IPProfile) error {
return utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) RemoveIPProfileDrv(ctx *context.Context, tenant, id string) error {
return utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) GetIPAllocationsDrv(ctx *context.Context, tenant, id string) (*utils.IPAllocations, error) {
return nil, utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) SetIPAllocationsDrv(ctx *context.Context, ip *utils.IPAllocations) error {
return utils.ErrNotImplemented
}
// DataDB method not implemented yet
func (sqls *SQLStorage) RemoveIPAllocationsDrv(ctx *context.Context, tenant, id string) error {
return utils.ErrNotImplemented
}
// GetStatQueueProfileDrv DataDB method not implemented yet
func (sqls *SQLStorage) GetStatQueueProfileDrv(ctx *context.Context, tenant string, id string) (sq *StatQueueProfile, err error) {
return nil, utils.ErrNotImplemented

View File

@@ -21,6 +21,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>
package ips
import (
"fmt"
"net/netip"
"reflect"
"testing"
@@ -115,22 +116,35 @@ var (
// TODO: move anything sessions related to sessions once ips implementation
// is complete.
func TestIPsIT(t *testing.T) {
var items string
var dbCfg engine.DBCfg
switch *utils.DBType {
case utils.MetaInternal:
dbCfg = engine.InternalDBCfg
case utils.MetaMySQL:
dbCfg = engine.MySQLDBCfg
items = `"items": {
"*ip_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"},
"*ip_allocations": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}
},`
case utils.MetaMongo:
dbCfg = engine.MongoDBCfg
case utils.MetaPostgres:
t.SkipNow()
dbCfg = engine.PostgresDBCfg
items = `"items": {
"*ip_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"},
"*ip_allocations": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}
},`
default:
t.Fatal("unsupported dbtype value")
}
ng := engine.TestEngine{
ConfigJSON: `{
ConfigJSON: fmt.Sprintf(`{
"logger": {
"level": 7
},
"sessions": {
"enabled": true,
"ips_conns": ["*localhost"],
@@ -187,8 +201,9 @@ func TestIPsIT(t *testing.T) {
"db_conns": {
"*default": {
"db_type": "*internal"
}
},
},
%s
"opts":{
"internalDBRewriteInterval": "0s",
"internalDBDumpInterval": "0s"
@@ -197,7 +212,7 @@ func TestIPsIT(t *testing.T) {
"admins": {
"enabled": true
}
}`,
}`, items),
TpFiles: map[string]string{
utils.IPsCsv: `
#Tenant[0],ID[1],FilterIDs[2],Weights[3],TTL[4],Stored[5],PoolID[6],PoolFilterIDs[7],PoolType[8],PoolRange[9],PoolStrategy[10],PoolMessage[11],PoolWeights[12],PoolBlockers[13]
@@ -209,11 +224,12 @@ cgrates.org,IPs2,*string:~*req.Account:1002,;20,2s,false,POOL1,*string:~*req.Des
},
DBCfg: dbCfg,
Encoding: *utils.Encoding,
// LogBuffer: new(bytes.Buffer),
// LogBuffer: new(bytes.Buffer),
// GracefulShutdown: true,
}
// t.Cleanup(func() { fmt.Println(ng.LogBuffer) })
client, _ := ng.Run(t)
time.Sleep(50 * time.Millisecond) // wait for all services
t.Run("admins apis", func(t *testing.T) {
var reply string

View File

@@ -371,6 +371,7 @@ func (acc *Account) CacheClone() any {
return acc.Clone()
}
// AsMapStringInterface converts Account struct to map[string]any
func (acc *Account) AsMapStringInterface() map[string]any {
if acc == nil {
return nil

View File

@@ -286,7 +286,7 @@ const (
LoadIDPrefix = "lid_"
LoadInstKey = "load_history"
CreateCDRsTablesSQL = "create_cdrs_tables.sql"
CreateAccountsTablesSQL = "create_accounts_tables.sql"
CreateAccountsTablesSQL = "create_db_tables.sql"
CreateTariffPlanTablesSQL = "create_tariffplan_tables.sql"
TestSQL = "TEST_SQL"
MetaAsc = "*asc"
@@ -530,6 +530,8 @@ const (
AllocationMessage = "AllocationMessage"
AddressPool = "AddressPool"
Pools = "Pools"
Allocations = "Allocations"
TTLIndex = "TTLIndex"
Allocation = "Allocation"
Range = "Range"
Stored = "Stored"
@@ -1908,6 +1910,8 @@ const (
TBLTPChargers = "tp_chargers"
TBLVersions = "versions"
TBLAccounts = "accounts"
TBLIPProfiles = "ip_profiles"
TBLIPAllocations = "ip_allocations"
OldSMCosts = "sm_costs"
TBLTPDispatchers = "tp_dispatcher_profiles"
TBLTPDispatcherHosts = "tp_dispatcher_hosts"

View File

@@ -629,6 +629,87 @@ func (a *IPAllocations) Config() *IPProfile {
return a.prfl
}
// AsMapStringInterface converts IPProfile struct to map[string]any
func (p *IPProfile) AsMapStringInterface() map[string]any {
if p == nil {
return nil
}
return map[string]any{
Tenant: p.Tenant,
ID: p.ID,
FilterIDs: p.FilterIDs,
Weights: p.Weights,
TTL: p.TTL,
Stored: p.Stored,
Pools: p.Pools,
}
}
// MapStringInterfaceToIPProfile converts map[string]any to IPProfile struct
func MapStringInterfaceToIPProfile(m map[string]any) (*IPProfile, error) {
ipp := &IPProfile{}
if v, ok := m[Tenant].(string); ok {
ipp.Tenant = v
}
if v, ok := m[ID].(string); ok {
ipp.ID = v
}
ipp.FilterIDs = InterfaceToStringSlice(m[FilterIDs])
ipp.Weights = InterfaceToDynamicWeights(m[Weights])
if v, ok := m[TTL].(string); ok {
if dur, err := time.ParseDuration(v); err == nil {
ipp.TTL = dur
}
} else if v, ok := m[TTL].(float64); ok { // for -1 cases
ipp.TTL = time.Duration(v)
}
if v, ok := m[Stored].(bool); ok {
ipp.Stored = v
}
ipp.Pools = InterfaceToPools(m[Pools])
return ipp, nil
}
// InterfaceToPools converts any to []*IPPool
func InterfaceToPools(v any) []*IPPool {
if v == nil {
return nil
}
if pools, ok := v.([]any); ok {
ipPools := make([]*IPPool, 0, len(pools))
for _, p := range pools {
pm, ok := p.(map[string]any)
if !ok {
break
}
pool := &IPPool{}
if v, ok := pm[ID].(string); ok {
pool.ID = v
}
pool.FilterIDs = InterfaceToStringSlice(pm[FilterIDs])
if v, ok := pm[Type].(string); ok {
pool.Type = v
}
if v, ok := pm[Range].(string); ok {
pool.Range = v
}
if v, ok := pm[Strategy].(string); ok {
pool.Strategy = v
}
if v, ok := pm[Message].(string); ok {
pool.Message = v
}
pool.Weights = InterfaceToDynamicWeights(pm[Weights])
pool.Blockers = InterfaceToDynamicBlockers(pm[Blockers])
ipPools = append(ipPools, pool)
}
return ipPools
}
return nil
}
// TenantID returns the unique ID in a multi-tenant environment
func (a *IPAllocations) TenantID() string {
return ConcatenatedKey(a.Tenant, a.ID)
@@ -670,3 +751,64 @@ func (a *IPAllocations) Clone() *IPAllocations {
func IPAllocationsLockKey(tnt, id string) string {
return ConcatenatedKey(CacheIPAllocations, tnt, id)
}
// AsMapStringInterface converts IPProfile struct to map[string]any
func (p *IPAllocations) AsMapStringInterface() map[string]any {
if p == nil {
return nil
}
return map[string]any{
Tenant: p.Tenant,
ID: p.ID,
Allocations: p.Allocations,
TTLIndex: p.TTLIndex,
}
}
// MapStringInterfaceToIPAllocations converts map[string]any to IPAllocations struct
func MapStringInterfaceToIPAllocations(m map[string]any) (*IPAllocations, error) {
ipa := &IPAllocations{}
if v, ok := m[Tenant].(string); ok {
ipa.Tenant = v
}
if v, ok := m[ID].(string); ok {
ipa.ID = v
}
ipa.Allocations = InterfaceToAllocations(m[Allocations])
ipa.TTLIndex = InterfaceToStringSlice(m[TTLIndex])
return ipa, nil
}
// InterfaceToAllocations converts any to map[string]*PoolAllocation
func InterfaceToAllocations(v any) map[string]*PoolAllocation {
if v == nil {
return nil
}
if allocs, ok := v.(map[string]any); ok {
ipAllocs := make(map[string]*PoolAllocation)
for allocID, val := range allocs {
allocMap, ok := val.(map[string]any)
if !ok {
break
}
allocation := &PoolAllocation{}
if v, ok := allocMap[PoolID].(string); ok {
allocation.PoolID = v
}
if v, ok := allocMap[Address].(string); ok {
if addr, err := netip.ParseAddr(v); err == nil {
allocation.Address = addr
}
}
if v, ok := allocMap[Time].(string); ok {
if t, err := time.Parse(time.RFC3339, v); err == nil {
allocation.Time = t
}
}
ipAllocs[allocID] = allocation
}
return ipAllocs
}
return nil
}