From c51d3f27ded2128c2735fd2c2c35367de86953ba Mon Sep 17 00:00:00 2001 From: arberkatellari Date: Wed, 12 Nov 2025 14:25:02 +0200 Subject: [PATCH] Make Filters and RouteProfiles storable in MySQL and Postgres --- apis/filters_it_test.go | 10 +- config/config_defaults.go | 4 +- config/configsanity.go | 2 + data/conf/samples/filters_mysql/cgrates.json | 3 +- .../samples/filters_postgres/cgrates.json | 114 +++++++++++++ data/conf/samples/filters_redis/cgrates.json | 113 +++++++++++++ .../samples/routes_cases_mysql/cgrates.json | 3 +- .../routes_cases_postgres/cgrates.json | 3 +- .../routes_generaltests_mysql/cgrates.json | 3 +- .../routes_generaltests_postgres/cgrates.json | 90 ++++++++++ .../routes_generaltests_redis/cgrates.json | 89 ++++++++++ data/conf/samples/routes_mysql/cgrates.json | 3 +- .../conf/samples/routes_postgres/cgrates.json | 50 ++++++ data/conf/samples/routes_redis/cgrates.json | 49 ++++++ data/storage/mysql/create_db_tables.sql | 24 ++- data/storage/postgres/create_db_tables.sql | 22 +++ engine/filters.go | 63 +++++++ engine/models.go | 22 +++ engine/storage_sql.go | 159 ++++++++++++++---- general_tests/dynamic_thresholds_it_test.go | 72 ++++++++ general_tests/filters_it_test.go | 10 +- general_tests/route_it_test.go | 4 +- routes/routes_it_test.go | 4 +- utils/consts.go | 2 + utils/resources.go | 2 +- utils/routes.go | 81 +++++++++ 26 files changed, 944 insertions(+), 57 deletions(-) create mode 100644 data/conf/samples/filters_postgres/cgrates.json create mode 100644 data/conf/samples/filters_redis/cgrates.json create mode 100644 data/conf/samples/routes_generaltests_postgres/cgrates.json create mode 100644 data/conf/samples/routes_generaltests_redis/cgrates.json create mode 100644 data/conf/samples/routes_postgres/cgrates.json create mode 100644 data/conf/samples/routes_redis/cgrates.json diff --git a/apis/filters_it_test.go b/apis/filters_it_test.go index 3ab0093ae..420cfdd45 100644 --- a/apis/filters_it_test.go +++ b/apis/filters_it_test.go @@ -75,15 +75,15 @@ var ( func TestFiltersIT(t *testing.T) { switch *utils.DBType { case utils.MetaInternal: - fltrConfigDIR = "tutinternal" + fltrConfigDIR = "filters_internal" case utils.MetaMongo: - fltrConfigDIR = "tutmongo" + fltrConfigDIR = "filters_mongo" case utils.MetaRedis: - t.SkipNow() + fltrConfigDIR = "filters_redis" case utils.MetaMySQL: - fltrConfigDIR = "tutmysql" + fltrConfigDIR = "filters_mysql" case utils.MetaPostgres: - t.SkipNow() + fltrConfigDIR = "filters_postgres" default: t.Fatal("Unknown Database type") } diff --git a/config/config_defaults.go b/config/config_defaults.go index 3b39a4bfb..d8b2f4c1d 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -178,11 +178,11 @@ const CGRATES_CFG_JSON = ` "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*threshold_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*thresholds": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, + "*filters": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, // compatible db types: <*internal|*redis|*mongo> "*actions": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, - "*filters": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, - "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*rate_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*load_ids": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "*default"}, "*resource_filter_indexes" : {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate": false, "dbConn": "*default"}, diff --git a/config/configsanity.go b/config/configsanity.go index afb2f426a..4f7f8d4dc 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -1039,6 +1039,8 @@ func (cfg *CGRConfig) checkConfigSanity() error { utils.MetaStatQueues, utils.MetaThresholdProfiles, utils.MetaThresholds, + utils.MetaFilters, + utils.MetaRouteProfiles, } for _, dbcfg := range cfg.dbCfg.DBConns { if dbcfg.Type == utils.MetaInternal { diff --git a/data/conf/samples/filters_mysql/cgrates.json b/data/conf/samples/filters_mysql/cgrates.json index 49b3b2c05..0d3f57339 100644 --- a/data/conf/samples/filters_mysql/cgrates.json +++ b/data/conf/samples/filters_mysql/cgrates.json @@ -34,7 +34,8 @@ } }, "items": { - "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*filters": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} } }, diff --git a/data/conf/samples/filters_postgres/cgrates.json b/data/conf/samples/filters_postgres/cgrates.json new file mode 100644 index 000000000..7c29756f6 --- /dev/null +++ b/data/conf/samples/filters_postgres/cgrates.json @@ -0,0 +1,114 @@ +{ + +"general": { + "node_id": "CGRFilterS" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "postgres", + "db_host": "127.0.0.1", + "db_port": 5432, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*filters": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"rates": { + "enabled": true, + "thresholds_conns": ["*localhost"] +}, + +"filters": { + "stats_conns": ["*localhost"], + "resources_conns": ["*localhost"], + "accounts_conns": ["*localhost"] +}, + +"accounts": { + "enabled": true +}, + + +"resources": { + "enabled": true, + "indexed_selects":false, + "store_interval": "1s" +}, + + +"stats": { + "enabled": true, + "indexed_selects":false, + "store_interval": "1s" +}, + + +"thresholds": { + "enabled": true, + "indexed_selects":false, + "actions_conns": ["*localhost"], + "store_interval": "1s" +}, + +"actions": { + "enabled": true +}, + + +"attributes": { + "enabled": true, + "prefix_indexed_fields": ["*req.CustomField"] +}, + + +"chargers": { + "enabled": true, + "suffix_indexed_fields": ["*req.Subject"], + "attributes_conns": ["*internal"] +}, + + +"admins": { + "enabled": true, + "caches_conns":["*localhost"], +}, + +"loaders": [ + { + "id": "*default", + "enabled": true, + "tenant": "cgrates.org", + "lockfile_path": ".cgr.lck", + "tp_in_dir": "/usr/share/cgrates/tariffplans/testit", + "tp_out_dir": "" + } +] + + +} diff --git a/data/conf/samples/filters_redis/cgrates.json b/data/conf/samples/filters_redis/cgrates.json new file mode 100644 index 000000000..49b3b2c05 --- /dev/null +++ b/data/conf/samples/filters_redis/cgrates.json @@ -0,0 +1,113 @@ +{ + +"general": { + "node_id": "CGRFilterS" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "mysql", + "db_host": "127.0.0.1", + "db_port": 3306, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"rates": { + "enabled": true, + "thresholds_conns": ["*localhost"] +}, + +"filters": { + "stats_conns": ["*localhost"], + "resources_conns": ["*localhost"], + "accounts_conns": ["*localhost"] +}, + +"accounts": { + "enabled": true +}, + + +"resources": { + "enabled": true, + "indexed_selects":false, + "store_interval": "1s" +}, + + +"stats": { + "enabled": true, + "indexed_selects":false, + "store_interval": "1s" +}, + + +"thresholds": { + "enabled": true, + "indexed_selects":false, + "actions_conns": ["*localhost"], + "store_interval": "1s" +}, + +"actions": { + "enabled": true +}, + + +"attributes": { + "enabled": true, + "prefix_indexed_fields": ["*req.CustomField"] +}, + + +"chargers": { + "enabled": true, + "suffix_indexed_fields": ["*req.Subject"], + "attributes_conns": ["*internal"] +}, + + +"admins": { + "enabled": true, + "caches_conns":["*localhost"], +}, + +"loaders": [ + { + "id": "*default", + "enabled": true, + "tenant": "cgrates.org", + "lockfile_path": ".cgr.lck", + "tp_in_dir": "/usr/share/cgrates/tariffplans/testit", + "tp_out_dir": "" + } +] + + +} diff --git a/data/conf/samples/routes_cases_mysql/cgrates.json b/data/conf/samples/routes_cases_mysql/cgrates.json index d92d2f12b..6765b9aed 100644 --- a/data/conf/samples/routes_cases_mysql/cgrates.json +++ b/data/conf/samples/routes_cases_mysql/cgrates.json @@ -28,7 +28,8 @@ "items": { "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, "*statqueue_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, - "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} } }, diff --git a/data/conf/samples/routes_cases_postgres/cgrates.json b/data/conf/samples/routes_cases_postgres/cgrates.json index a7b7f17aa..4444079f9 100644 --- a/data/conf/samples/routes_cases_postgres/cgrates.json +++ b/data/conf/samples/routes_cases_postgres/cgrates.json @@ -28,7 +28,8 @@ "items": { "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, "*statqueue_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, - "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} } }, diff --git a/data/conf/samples/routes_generaltests_mysql/cgrates.json b/data/conf/samples/routes_generaltests_mysql/cgrates.json index fb0d502cf..cc4d542f3 100644 --- a/data/conf/samples/routes_generaltests_mysql/cgrates.json +++ b/data/conf/samples/routes_generaltests_mysql/cgrates.json @@ -32,7 +32,8 @@ } }, "items": { - "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} } }, diff --git a/data/conf/samples/routes_generaltests_postgres/cgrates.json b/data/conf/samples/routes_generaltests_postgres/cgrates.json new file mode 100644 index 000000000..4131bf3df --- /dev/null +++ b/data/conf/samples/routes_generaltests_postgres/cgrates.json @@ -0,0 +1,90 @@ +{ +"general": { + "reply_timeout": "50s" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "postgres", + "db_host": "127.0.0.1", + "db_port": 5432, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + +"thresholds": { + "enabled": true, + "store_interval": "1s" +}, + +"routes": { + "enabled": true, + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "rates_conns": ["*internal"], + "accounts_conns": ["*internal"] +}, + +"admins": { + "enabled": true +}, + +"rates": { + "enabled": true +}, + +"accounts": { + "enabled": true +}, + +"loaders": [ + { + "id": "*default", + "enabled": true, + "tenant": "cgrates.org", + "lockfile_path": ".cgr.lck", + "tp_in_dir": "/usr/share/cgrates/tariffplans/testit", + "tp_out_dir": "" + } +] + +} + \ No newline at end of file diff --git a/data/conf/samples/routes_generaltests_redis/cgrates.json b/data/conf/samples/routes_generaltests_redis/cgrates.json new file mode 100644 index 000000000..fb0d502cf --- /dev/null +++ b/data/conf/samples/routes_generaltests_redis/cgrates.json @@ -0,0 +1,89 @@ +{ +"general": { + "reply_timeout": "50s" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "mysql", + "db_host": "127.0.0.1", + "db_port": 3306, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"resources": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + + +"stats": { + "enabled": true, + "store_interval": "1s", + "thresholds_conns": ["*internal"] +}, + +"thresholds": { + "enabled": true, + "store_interval": "1s" +}, + +"routes": { + "enabled": true, + "stats_conns": ["*internal"], + "resources_conns": ["*internal"], + "rates_conns": ["*internal"], + "accounts_conns": ["*internal"] +}, + +"admins": { + "enabled": true +}, + +"rates": { + "enabled": true +}, + +"accounts": { + "enabled": true +}, + +"loaders": [ + { + "id": "*default", + "enabled": true, + "tenant": "cgrates.org", + "lockfile_path": ".cgr.lck", + "tp_in_dir": "/usr/share/cgrates/tariffplans/testit", + "tp_out_dir": "" + } +] + +} + \ No newline at end of file diff --git a/data/conf/samples/routes_mysql/cgrates.json b/data/conf/samples/routes_mysql/cgrates.json index b1a61b9c4..02ee27670 100644 --- a/data/conf/samples/routes_mysql/cgrates.json +++ b/data/conf/samples/routes_mysql/cgrates.json @@ -33,7 +33,8 @@ } }, "items": { - "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} } }, diff --git a/data/conf/samples/routes_postgres/cgrates.json b/data/conf/samples/routes_postgres/cgrates.json new file mode 100644 index 000000000..4693f7bcf --- /dev/null +++ b/data/conf/samples/routes_postgres/cgrates.json @@ -0,0 +1,50 @@ +{ + +"general": { + "reply_timeout": "50s" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "postgres", + "db_host": "127.0.0.1", + "db_port": 5432, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"}, + "*route_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"routes": { + "enabled": true +}, + +"admins": { + "enabled": true, + "scheduler_conns": ["*internal"] +} + +} \ No newline at end of file diff --git a/data/conf/samples/routes_redis/cgrates.json b/data/conf/samples/routes_redis/cgrates.json new file mode 100644 index 000000000..b1a61b9c4 --- /dev/null +++ b/data/conf/samples/routes_redis/cgrates.json @@ -0,0 +1,49 @@ +{ + +"general": { + "reply_timeout": "50s" +}, + +"logger": { + "level": 7 +}, + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080" +}, + +"db": { + "db_conns": { + "*default": { + "db_type": "redis", + "db_host": "127.0.0.1", + "db_port": 6379, + "db_name": "10", + "db_user": "cgrates" + }, + "StorDB": { + "db_type": "mysql", + "db_host": "127.0.0.1", + "db_port": 3306, + "db_name": "cgrates", + "db_user": "cgrates", + "db_password": "CGRateS.org" + } + }, + "items": { + "*cdrs": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false, "dbConn": "StorDB"} + } +}, + +"routes": { + "enabled": true +}, + +"admins": { + "enabled": true, + "scheduler_conns": ["*internal"] +} + +} \ No newline at end of file diff --git a/data/storage/mysql/create_db_tables.sql b/data/storage/mysql/create_db_tables.sql index 8d81d4035..b0b1b67bb 100644 --- a/data/storage/mysql/create_db_tables.sql +++ b/data/storage/mysql/create_db_tables.sql @@ -132,4 +132,26 @@ CREATE TABLE thresholds ( PRIMARY KEY (`pk`), UNIQUE KEY unique_tenant_id (`tenant`, `id`) ); -CREATE UNIQUE INDEX thresholds_idx ON thresholds (`id`); \ No newline at end of file +CREATE UNIQUE INDEX thresholds_idx ON thresholds (`id`); + +DROP TABLE IF EXISTS filters; +CREATE TABLE filters ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tenant` VARCHAR(40) NOT NULL, + `id` VARCHAR(64) NOT NULL, + `filter` JSON NOT NULL, + PRIMARY KEY (`pk`), + UNIQUE KEY unique_tenant_id (`tenant`, `id`) +); +CREATE INDEX filters_idx ON filters (`id`); + +DROP TABLE IF EXISTS route_profiles; +CREATE TABLE route_profiles ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tenant` VARCHAR(40) NOT NULL, + `id` VARCHAR(64) NOT NULL, + `route_profile` JSON NOT NULL, + PRIMARY KEY (`pk`), + UNIQUE KEY unique_tenant_id (`tenant`, `id`) +); +CREATE UNIQUE INDEX route_profiles_idx ON route_profiles (`id`); \ No newline at end of file diff --git a/data/storage/postgres/create_db_tables.sql b/data/storage/postgres/create_db_tables.sql index 93d3458ce..a94ed2e1b 100644 --- a/data/storage/postgres/create_db_tables.sql +++ b/data/storage/postgres/create_db_tables.sql @@ -130,3 +130,25 @@ CREATE TABLE thresholds ( UNIQUE (tenant, id) ); CREATE UNIQUE INDEX thresholds_idx ON thresholds ("id"); + + +DROP TABLE IF EXISTS filters; +CREATE TABLE filters ( + pk SERIAL PRIMARY KEY, + tenant VARCHAR(40) NOT NULL, + id VARCHAR(64) NOT NULL, + filter JSONB NOT NULL, + UNIQUE (tenant, id) +); +CREATE INDEX filters_idx ON filters ("id"); + + +DROP TABLE IF EXISTS route_profiles; +CREATE TABLE route_profiles ( + pk SERIAL PRIMARY KEY, + tenant VARCHAR(40) NOT NULL, + id VARCHAR(64) NOT NULL, + route_profile JSONB NOT NULL, + UNIQUE (tenant, id) +); +CREATE UNIQUE INDEX route_profiles_idx ON route_profiles ("id"); diff --git a/engine/filters.go b/engine/filters.go index e99a7aad9..83f43234e 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -985,6 +985,69 @@ func (fltr *Filter) FieldAsInterface(fldPath []string) (_ any, err error) { return fltr.Rules[idx].FieldAsInterface(fldPath[1:]) } +// AsMapStringInterface converts Filter struct to map[string]any +func (fltr *Filter) AsMapStringInterface() map[string]any { + if fltr == nil { + return nil + } + return map[string]any{ + utils.Tenant: fltr.Tenant, + utils.ID: fltr.ID, + utils.Rules: fltr.Rules, + } +} + +// MapStringInterfaceToFilter converts map[string]any to Filter struct +func MapStringInterfaceToFilter(m map[string]any) (*Filter, error) { + fltr := &Filter{} + + if v, ok := m[utils.Tenant].(string); ok { + fltr.Tenant = v + } + if v, ok := m[utils.ID].(string); ok { + fltr.ID = v + } + fltr.Rules = InterfaceToRules(m[utils.Rules]) + return fltr, nil +} + +// InterfaceToRules converts interface to []*FilterRule +func InterfaceToRules(v any) []*FilterRule { + if v == nil { + return nil + } + if val, ok := v.([]any); !ok { + return nil + } else { + result := make([]*FilterRule, 0, len(val)) + for _, item := range val { + if filters, ok := item.(map[string]any); ok { + result = append(result, MapStringInterfaceToFilterRule(filters)) + } + } + return result + } +} + +// MapStringInterfaceToFilterRule converts map[string]any to *FilterRule +func MapStringInterfaceToFilterRule(m map[string]any) *FilterRule { + if m == nil { + return nil + } + rule := &FilterRule{} + + if v, ok := m[utils.Type].(string); ok { + rule.Type = v + } + + if v, ok := m[utils.Element].(string); ok { + rule.Element = v + } + + rule.Values = utils.InterfaceToStringSlice(m[utils.Values]) + return rule +} + func (fltr *FilterRule) String() string { return utils.ToJSON(fltr) } func (fltr *FilterRule) FieldAsString(fldPath []string) (_ string, err error) { var val any diff --git a/engine/models.go b/engine/models.go index 2f05c702c..72315038d 100644 --- a/engine/models.go +++ b/engine/models.go @@ -514,3 +514,25 @@ type ThresholdJSONMdl struct { func (ThresholdJSONMdl) TableName() string { return utils.TBLThresholds } + +type FilterJSONMdl struct { + PK uint `gorm:"primary_key"` + Tenant string `index:"0" re:".*"` + ID string `index:"1" re:".*"` + Filter utils.JSONB `gorm:"type:jsonb" index:"2" re:".*"` +} + +func (FilterJSONMdl) TableName() string { + return utils.TBLFilters +} + +type RouteProfileMdl struct { + PK uint `gorm:"primary_key"` + Tenant string `index:"0" re:".*"` + ID string `index:"1" re:".*"` + RouteProfile utils.JSONB `gorm:"type:jsonb" index:"2" re:".*"` +} + +func (RouteProfileMdl) TableName() string { + return utils.TBLRouteProfiles +} diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 543aec309..cc9573917 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -123,6 +123,10 @@ func (sqls *SQLStorage) GetKeysForPrefix(ctx *context.Context, prefix string) (k keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLThresholdProfiles, tntID) case utils.ThresholdPrefix: keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLThresholds, tntID) + case utils.FilterPrefix: + keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLFilters, tntID) + case utils.RouteProfilePrefix: + keys, err = sqls.getAllKeysMatchingTenantID(ctx, utils.TBLRouteProfiles, tntID) default: err = fmt.Errorf("unsupported prefix in GetKeysForPrefix: %q", prefix) } @@ -948,6 +952,126 @@ func (sqls *SQLStorage) RemoveThresholdDrv(ctx *context.Context, tenant, id stri return } +func (sqls *SQLStorage) GetFilterDrv(ctx *context.Context, tenant, id string) (f *Filter, err error) { + var result []*FilterJSONMdl + if err = sqls.db.Model(&FilterJSONMdl{}).Where(&FilterJSONMdl{Tenant: tenant, + ID: id}).Find(&result).Error; err != nil { + return nil, err + } + if len(result) == 0 { + return nil, utils.ErrNotFound + } + return MapStringInterfaceToFilter(result[0].Filter) +} + +func (sqls *SQLStorage) SetFilterDrv(ctx *context.Context, f *Filter) (err error) { + tx := sqls.db.Begin() + mdl := &FilterJSONMdl{ + Tenant: f.Tenant, + ID: f.ID, + Filter: f.AsMapStringInterface(), + } + if err = tx.Model(&FilterJSONMdl{}).Where( + FilterJSONMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete( + FilterJSONMdl{}).Error; err != nil { + tx.Rollback() + return + } + if err = tx.Save(mdl).Error; err != nil { + tx.Rollback() + return + } + tx.Commit() + return +} + +func (sqls *SQLStorage) RemoveFilterDrv(ctx *context.Context, tenant, id string) (err error) { + tx := sqls.db.Begin() + if err = tx.Model(&FilterJSONMdl{}).Where(&FilterJSONMdl{Tenant: tenant, ID: id}). + Delete(&FilterJSONMdl{}).Error; err != nil { + tx.Rollback() + return err + } + tx.Commit() + return +} + +func (sqls *SQLStorage) GetRouteProfileDrv(ctx *context.Context, tenant, id string) (rp *utils.RouteProfile, err error) { + var result []*RouteProfileMdl + if err = sqls.db.Model(&RouteProfileMdl{}).Where(&RouteProfileMdl{Tenant: tenant, + ID: id}).Find(&result).Error; err != nil { + return nil, err + } + if len(result) == 0 { + return nil, utils.ErrNotFound + } + return utils.MapStringInterfaceToRouteProfile(result[0].RouteProfile), nil +} + +func (sqls *SQLStorage) SetRouteProfileDrv(ctx *context.Context, rp *utils.RouteProfile) (err error) { + tx := sqls.db.Begin() + mdl := &RouteProfileMdl{ + Tenant: rp.Tenant, + ID: rp.ID, + RouteProfile: rp.AsMapStringInterface(), + } + if err = tx.Model(&RouteProfileMdl{}).Where( + RouteProfileMdl{Tenant: mdl.Tenant, ID: mdl.ID}).Delete( + RouteProfileMdl{}).Error; err != nil { + tx.Rollback() + return + } + if err = tx.Save(mdl).Error; err != nil { + tx.Rollback() + return + } + tx.Commit() + return +} + +func (sqls *SQLStorage) RemoveRouteProfileDrv(ctx *context.Context, tenant, id string) (err error) { + tx := sqls.db.Begin() + if err = tx.Model(&RouteProfileMdl{}).Where(&RouteProfileMdl{Tenant: tenant, ID: id}). + Delete(&RouteProfileMdl{}).Error; err != nil { + tx.Rollback() + return err + } + tx.Commit() + return +} + +// Used to check if specific subject is stored using prefix key attached to entity +func (sqls *SQLStorage) HasDataDrv(ctx *context.Context, category, subject, tenant string) (has bool, err error) { + var categoryModelMap = map[string]any{ + utils.AccountPrefix: &AccountJSONMdl{}, + utils.ActionProfilePrefix: &ActionProfileJSONMdl{}, + utils.ResourcesPrefix: &ResourceJSONMdl{}, + utils.ResourceProfilesPrefix: &ResourceProfileMdl{}, + utils.IPAllocationsPrefix: &IPAllocationMdl{}, + utils.IPProfilesPrefix: &IPProfileMdl{}, + utils.StatQueuePrefix: &StatQueueMdl{}, + utils.StatQueueProfilePrefix: &StatQueueProfileMdl{}, + utils.ThresholdPrefix: &ThresholdJSONMdl{}, + utils.ThresholdProfilePrefix: &ThresholdProfileMdl{}, + utils.FilterPrefix: &FilterJSONMdl{}, + utils.RouteProfilePrefix: &RouteProfileMdl{}, + utils.AttributeProfilePrefix: &AttributeProfileMdl{}, + utils.ChargerProfilePrefix: &ChargerProfileMdl{}, + // utils.TrendPrefix: &TrendJSONMdl{}, + // utils.TrendProfilePrefix: &TrendProfileMdl{}, + // utils.RateProfilePrefix: &RateProfileJSONMdl{}, + } + model, ok := categoryModelMap[category] + if !ok { + return false, fmt.Errorf("unsupported category in HasDataDrv: %s", category) + } + var count int64 // if it finds 1, return + err = sqls.db.Model(model).Where("tenant = ? AND id = ?", tenant, subject). + Limit(1).Count(&count).Error + + return count > 0, err +} + // AddLoadHistory DataDB method not implemented yet func (sqls *SQLStorage) AddLoadHistory(ldInst *utils.LoadInstance, loadHistSize int, transactionID string) error { @@ -979,11 +1103,6 @@ func (sqls *SQLStorage) RewriteDataDB() (err error) { return utils.ErrNotImplemented } -// DataDB method not implemented yet -func (sqls *SQLStorage) HasDataDrv(ctx *context.Context, category, subject, tenant string) (exists bool, err error) { - return false, utils.ErrNotImplemented -} - // DataDB method not implemented yet func (sqls *SQLStorage) GetLoadHistory(limit int, skipCache bool, transactionID string) (loadInsts []*utils.LoadInstance, err error) { @@ -1050,36 +1169,6 @@ func (sqls *SQLStorage) RemoveRankingDrv(ctx *context.Context, tenant, id string return utils.ErrNotImplemented } -// DataDB method not implemented yet -func (sqls *SQLStorage) GetFilterDrv(ctx *context.Context, tenant, id string) (r *Filter, err error) { - return nil, utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (sqls *SQLStorage) SetFilterDrv(ctx *context.Context, r *Filter) (err error) { - return utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (sqls *SQLStorage) RemoveFilterDrv(ctx *context.Context, tenant, id string) (err error) { - return utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (sqls *SQLStorage) GetRouteProfileDrv(ctx *context.Context, tenant, id string) (r *utils.RouteProfile, err error) { - return nil, utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (sqls *SQLStorage) SetRouteProfileDrv(ctx *context.Context, r *utils.RouteProfile) (err error) { - return utils.ErrNotImplemented -} - -// DataDB method not implemented yet -func (sqls *SQLStorage) RemoveRouteProfileDrv(ctx *context.Context, tenant, id string) (err error) { - return utils.ErrNotImplemented -} - // GetStorageType returns the storage type that is being used func (sqls *SQLStorage) GetStorageType() string { return utils.MetaMySQL diff --git a/general_tests/dynamic_thresholds_it_test.go b/general_tests/dynamic_thresholds_it_test.go index 4985435b0..d192b522a 100644 --- a/general_tests/dynamic_thresholds_it_test.go +++ b/general_tests/dynamic_thresholds_it_test.go @@ -66,6 +66,42 @@ func TestDynThdIT(t *testing.T) { Limit: utils.IntPointer(-1), DbConn: utils.StringPointer(utils.StorDB), }, + utils.MetaStatQueueProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaStatQueues: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaAttributeProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaResourceProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaResources: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaFilters: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaRouteProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaIPProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaIPAllocations: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, }, }} case utils.MetaMongo: @@ -95,6 +131,42 @@ func TestDynThdIT(t *testing.T) { Limit: utils.IntPointer(-1), DbConn: utils.StringPointer(utils.StorDB), }, + utils.MetaStatQueueProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaStatQueues: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaAttributeProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaResourceProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaResources: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaFilters: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaRouteProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaIPProfiles: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, + utils.MetaIPAllocations: { + Limit: utils.IntPointer(-1), + DbConn: utils.StringPointer(utils.StorDB), + }, }, }} default: diff --git a/general_tests/filters_it_test.go b/general_tests/filters_it_test.go index e752f612c..9c38be848 100644 --- a/general_tests/filters_it_test.go +++ b/general_tests/filters_it_test.go @@ -72,13 +72,13 @@ func TestFltrIT(t *testing.T) { case utils.MetaInternal: fltrConfDIR = "filters_internal" case utils.MetaRedis: - t.SkipNow() + fltrConfDIR = "filters_redis" case utils.MetaMySQL: fltrConfDIR = "filters_mysql" case utils.MetaMongo: fltrConfDIR = "filters_mongo" case utils.MetaPostgres: - t.SkipNow() + fltrConfDIR = "filters_postgres" default: t.Fatal("Unknown Database type") } @@ -119,8 +119,10 @@ func testV1FltrRpcConn(t *testing.T) { func testV1FltrLoadTarrifPlans(t *testing.T) { caching := utils.MetaReload - if fltrCfg.DbCfg().DBConns[utils.MetaDefault].Type == utils.MetaInternal { - caching = utils.MetaNone + for _, dbc := range fltrCfg.DbCfg().DBConns { + if dbc.Type == utils.MetaInternal { + caching = utils.MetaNone + } } var reply string if err := fltrRpc.Call(context.Background(), utils.LoaderSv1Run, diff --git a/general_tests/route_it_test.go b/general_tests/route_it_test.go index a87ef7178..271eccaf3 100644 --- a/general_tests/route_it_test.go +++ b/general_tests/route_it_test.go @@ -71,13 +71,13 @@ func TestRouteSV1IT(t *testing.T) { case utils.MetaInternal: splSv1ConfDIR = "routes_generaltests_internal" case utils.MetaRedis: - t.SkipNow() + splSv1ConfDIR = "routes_generaltests_redis" case utils.MetaMySQL: splSv1ConfDIR = "routes_generaltests_mysql" case utils.MetaMongo: splSv1ConfDIR = "routes_generaltests_mongo" case utils.MetaPostgres: - t.SkipNow() + splSv1ConfDIR = "routes_generaltests_postgres" default: t.Fatal("Unknown Database type") } diff --git a/routes/routes_it_test.go b/routes/routes_it_test.go index b6e7dd30d..49a013197 100644 --- a/routes/routes_it_test.go +++ b/routes/routes_it_test.go @@ -84,11 +84,11 @@ func TestRoutesIT(t *testing.T) { case utils.MetaMongo: roConfigDIR = "routes_mongo" case utils.MetaRedis: - t.SkipNow() + roConfigDIR = "routes_redis" case utils.MetaMySQL: roConfigDIR = "routes_mysql" case utils.MetaPostgres: - t.SkipNow() + roConfigDIR = "routes_postgres" default: t.Fatal("Unknown Database type") } diff --git a/utils/consts.go b/utils/consts.go index 652589e9c..434802b94 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -1924,6 +1924,8 @@ const ( TBLStatQueues = "stat_queues" TBLThresholdProfiles = "threshold_profiles" TBLThresholds = "thresholds" + TBLFilters = "filters" + TBLRouteProfiles = "route_profiles" OldSMCosts = "sm_costs" TBLTPDispatchers = "tp_dispatcher_profiles" TBLTPDispatcherHosts = "tp_dispatcher_hosts" diff --git a/utils/resources.go b/utils/resources.go index 940ec06ed..43a49bc1b 100644 --- a/utils/resources.go +++ b/utils/resources.go @@ -380,7 +380,7 @@ func (rp *ResourceProfile) AsMapStringInterface() map[string]any { Tenant: rp.Tenant, ID: rp.ID, FilterIDs: rp.FilterIDs, - UsageTTL: rp.FilterIDs, + UsageTTL: rp.UsageTTL, Limit: rp.Limit, AllocationMessage: rp.AllocationMessage, Blocker: rp.Blocker, diff --git a/utils/routes.go b/utils/routes.go index b000685a4..6cad7225c 100644 --- a/utils/routes.go +++ b/utils/routes.go @@ -316,6 +316,87 @@ func (rp *RouteProfile) FieldAsInterface(fldPath []string) (_ any, err error) { return rp.Routes[*idx].FieldAsInterface(fldPath[1:]) } +// AsMapStringInterface converts RouteProfile struct to map[string]any +func (rp *RouteProfile) AsMapStringInterface() map[string]any { + if rp == nil { + return nil + } + return map[string]any{ + Tenant: rp.Tenant, + ID: rp.ID, + FilterIDs: rp.FilterIDs, + Weights: rp.Weights, + Blockers: rp.Blockers, + Sorting: rp.Sorting, + SortingParameters: rp.SortingParameters, + Routes: rp.Routes, + } +} + +// MapStringInterfaceToRouteProfile converts map[string]any to RouteProfile struct +func MapStringInterfaceToRouteProfile(m map[string]any) *RouteProfile { + rp := &RouteProfile{} + + if v, ok := m[Tenant].(string); ok { + rp.Tenant = v + } + if v, ok := m[ID].(string); ok { + rp.ID = v + } + rp.FilterIDs = InterfaceToStringSlice(m[FilterIDs]) + rp.Weights = InterfaceToDynamicWeights(m[Weights]) + rp.Blockers = InterfaceToDynamicBlockers(m[Blockers]) + if v, ok := m[Sorting].(string); ok { + rp.Sorting = v + } + rp.SortingParameters = InterfaceToStringSlice(m[SortingParameters]) + rp.Routes = InterfaceToRoutes(m[Routes]) + return rp +} + +// InterfaceToRoutes converts interface to []*Route +func InterfaceToRoutes(v any) []*Route { + if v == nil { + return nil + } + switch val := v.(type) { + case []*Route: + return val + case []any: + var result []*Route + for _, item := range val { + if m, ok := item.(map[string]any); ok { + route := MapStringInterfaceToRoute(m) + result = append(result, route) + } else if r, ok := item.(*Route); ok { + result = append(result, r) + } + } + return result + default: + return nil + } +} + +// MapStringInterfaceToRoute converts map[string]any to Route struct +func MapStringInterfaceToRoute(m map[string]any) *Route { + r := &Route{} + if v, ok := m[ID].(string); ok { + r.ID = v + } + r.FilterIDs = InterfaceToStringSlice(m[FilterIDs]) + r.AccountIDs = InterfaceToStringSlice(m[AccountIDs]) + r.RateProfileIDs = InterfaceToStringSlice(m[RateProfileIDs]) + r.ResourceIDs = InterfaceToStringSlice(m[ResourceIDs]) + r.StatIDs = InterfaceToStringSlice(m[StatIDs]) + r.Weights = InterfaceToDynamicWeights(m[Weights]) + r.Blockers = InterfaceToDynamicBlockers(m[Blockers]) + if v, ok := m[RouteParameters].(string); ok { + r.RouteParameters = v + } + return r +} + // Route defines a single route within a RouteProfile. type Route struct { ID string // RouteID