uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to ERS to update or move ERS SQL events to a new table & add *export flag for ERS readers

Browse Source
This commit is contained in:
arberkatellari
2024-12-10 20:05:23 +02:00
committed by Dan Christian Bogos
parent a2b86e1a02
commit d35b14b6b9
30 changed files with 1898 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
engine/filters.go
View File

@@ -795,3 +795,150 @@ func (fltr *FilterRule) passHttp(dDP utils.DataProvider) (bool, error) {
func (fltr *FilterRule) ElementItems() []string {
return strings.Split(fltr.Element, utils.NestingSep)
}
// Creates mysql conditions used in WHERE statement out of filters
func (fltr *FilterRule) FilterToSQLQuery() (conditions []string) {
var firstItem string // Excluding ~*req, hold the first item of an element, left empty if no more than 1 item in element. e.g. "cost_details" out of ~*req.cost_details.Charges[0].RatingID or "" out of ~*req.answer_time
var restOfItems string // Excluding ~*req, hold the rest of the items past the first one. If only 1 item in all element, holds that item. e.g. "Charges[0].RatingID" out of ~*req.cost_details.Charges[0].RatingID or "answer_time" out of ~*req.answer_time
not := strings.HasPrefix(fltr.Type, utils.MetaNot)
elementItems := fltr.ElementItems()[1:] // exclude first item: ~*req
if len(elementItems) > 1 {
firstItem = elementItems[0]
restOfItems = strings.Join(elementItems[1:], utils.NestingSep)
} else {
restOfItems = elementItems[0]
}
// here are for the filters that their values are empty: *exists, *notexists, *empty, *notempty..
if len(fltr.Values) == 0 {
switch fltr.Type {
case utils.MetaExists, utils.MetaNotExists:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", restOfItems))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", firstItem, restOfItems))
return
}
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s IS NULL", restOfItems))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", firstItem, restOfItems))
case utils.MetaEmpty, utils.MetaNotEmpty:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s != ''", restOfItems))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != ''", firstItem, restOfItems))
return
}
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s == ''", restOfItems))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') == ''", firstItem, restOfItems))
}
return
}
// here are for the filters that can have more than one value: *string, *prefix, *suffix ..
for _, value := range fltr.Values {
switch value { // in case we have boolean values, it should be queried over 1 or 0
case "true":
value = "1"
case "false":
value = "0"
}
var singleCond string
switch fltr.Type {
case utils.MetaString, utils.MetaNotString, utils.MetaEqual, utils.MetaNotEqual:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s != '%s'", restOfItems, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != '%s'",
firstItem, restOfItems, value))
continue
}
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s = '%s'", restOfItems, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') = '%s'", firstItem, restOfItems, value)
}
case utils.MetaLessThan, utils.MetaLessOrEqual, utils.MetaGreaterThan, utils.MetaGreaterOrEqual:
parsedValAny := utils.StringToInterface(value)
if fltr.Type == utils.MetaGreaterOrEqual {
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s >= '%v'", restOfItems, parsedValAny)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') >= '%v'", firstItem, restOfItems, parsedValAny)
}
} else if fltr.Type == utils.MetaGreaterThan {
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s > '%v'", restOfItems, parsedValAny)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') > '%v'", firstItem, restOfItems, parsedValAny)
}
} else if fltr.Type == utils.MetaLessOrEqual {
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s <= '%v'", restOfItems, parsedValAny)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') <= '%v'", firstItem, restOfItems, parsedValAny)
}
} else if fltr.Type == utils.MetaLessThan {
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s < '%v'", restOfItems, parsedValAny)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') < '%v'", firstItem, restOfItems, parsedValAny)
}
}
case utils.MetaPrefix, utils.MetaNotPrefix:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%s%%'", restOfItems, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%s%%'", firstItem, restOfItems, value))
continue
}
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s LIKE '%s%%'", restOfItems, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%s%%'", firstItem, restOfItems, value)
}
case utils.MetaSuffix, utils.MetaNotSuffix:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%%%s'", restOfItems, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%%%s'", firstItem, restOfItems, value))
continue
}
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s LIKE '%%%s'", restOfItems, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%%%s'", firstItem, restOfItems, value)
}
case utils.MetaRegex, utils.MetaNotRegex:
if not {
if firstItem == utils.EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT REGEXP '%s'", restOfItems, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT REGEXP '%s'", firstItem, restOfItems, value))
continue
}
if firstItem == utils.EmptyString {
singleCond = fmt.Sprintf("%s REGEXP '%s'", restOfItems, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') REGEXP '%s'", firstItem, restOfItems, value)
}
}
conditions = append(conditions, singleCond)
}
return
}

291
engine/filters_test.go
View File

@@ -3115,3 +3115,294 @@ func TestNewFilterRule(t *testing.T) {
}
})
}
func TestFilterToSQLQuery(t *testing.T) {
tests := []struct {
name string
fltrRule FilterRule
expected []string
}{
{"MetaEqual with values", FilterRule{Type: utils.MetaEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"RatingID2"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') = 'RatingID2'"}},
{"MetaExists with no values", FilterRule{Type: utils.MetaExists, Element: "~*req.answer_time", Values: nil}, []string{"answer_time IS NULL"}},
{"MetaExists with JSON field", FilterRule{Type: utils.MetaExists, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NULL"}},
{"MetaNotExists with no values", FilterRule{Type: utils.MetaNotExists, Element: "~*req.answer_time", Values: nil}, []string{"answer_time IS NOT NULL"}},
{"MetaNotExists with JSON field", FilterRule{Type: utils.MetaNotExists, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NOT NULL"}},
{"MetaString with values", FilterRule{Type: utils.MetaString, Element: "~*req.answer_time", Values: []string{"value1", "value2"}}, []string{"answer_time = 'value1'", "answer_time = 'value2'"}},
{"MetaNotString with values", FilterRule{Type: utils.MetaNotString, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"value1"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != 'value1'"}},
{"MetaEmpty with no values", FilterRule{Type: utils.MetaEmpty, Element: "~*req.answer_time", Values: nil}, []string{"answer_time == ''"}},
{"MetaEmpty with JSON field", FilterRule{Type: utils.MetaEmpty, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') == ''"}},
{"MetaNotEmpty with no values", FilterRule{Type: utils.MetaNotEmpty, Element: "~*req.answer_time", Values: nil}, []string{"answer_time != ''"}},
{"MetaNotEmpty with JSON field", FilterRule{Type: utils.MetaNotEmpty, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != ''"}},
{"MetaGreaterOrEqual with values", FilterRule{Type: utils.MetaGreaterOrEqual, Element: "~*req.answer_time", Values: []string{"10"}}, []string{"answer_time >= '10'"}},
{"MetaGreaterThan with values", FilterRule{Type: utils.MetaGreaterThan, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"20"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') > '20'"}},
{"MetaLessThan with values", FilterRule{Type: utils.MetaLessThan, Element: "~*req.answer_time", Values: []string{"5"}}, []string{"answer_time < '5'"}},
{"MetaLessOrEqual with values", FilterRule{Type: utils.MetaLessOrEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"15"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') <= '15'"}},
{"MetaPrefix with values", FilterRule{Type: utils.MetaPrefix, Element: "~*req.answer_time", Values: []string{"pre"}}, []string{"answer_time LIKE 'pre%'"}},
{"MetaNotPrefix with values", FilterRule{Type: utils.MetaNotPrefix, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"pre"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE 'pre%'"}},
{"MetaSuffix with values", FilterRule{Type: utils.MetaSuffix, Element: "~*req.answer_time", Values: []string{"suf"}}, []string{"answer_time LIKE '%suf'"}},
{"MetaNotSuffix with values", FilterRule{Type: utils.MetaNotSuffix, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"suf"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE '%suf'"}},
{"MetaGreaterOrEqual with JSON field", FilterRule{Type: utils.MetaGreaterOrEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"100"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') >= '100'"}},
{"MetaRegex with values", FilterRule{Type: utils.MetaRegex, Element: "~*req.answer_time", Values: []string{"pattern1", "pattern2"}}, []string{"answer_time REGEXP 'pattern1'", "answer_time REGEXP 'pattern2'"}},
{"MetaNotRegex with values", FilterRule{Type: utils.MetaNotRegex, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"pattern"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT REGEXP 'pattern'"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.fltrRule.FilterToSQLQuery()
if len(got) != len(tt.expected) {
t.Errorf("expected %v, got %v", tt.expected, got)
return
}
for i, cond := range got {
if cond != tt.expected[i] {
t.Errorf("expected %v, got %v", tt.expected[i], cond)
}
}
})
}
}
func TestFilterToSQLQueryValidations(t *testing.T) {
tests := []struct {
name string
fltrRule FilterRule
expected []string
}{
{
name: "Boolean true value",
fltrRule: FilterRule{
Type: utils.MetaString,
Element: "~*req.active",
Values: []string{"true"},
},
expected: []string{"active = '1'"},
},
{
name: "Boolean false value",
fltrRule: FilterRule{
Type: utils.MetaString,
Element: "~*req.active",
Values: []string{"false"},
},
expected: []string{"active = '0'"},
},
{
name: "Greater than or equal with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaGreaterOrEqual,
Element: "~*req.score",
Values: []string{"10"},
},
expected: []string{"score >= '10'"},
},
{
name: "Greater than with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaGreaterThan,
Element: "~*req.score",
Values: []string{"20"},
},
expected: []string{"score > '20'"},
},
{
name: "Less than or equal with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaLessOrEqual,
Element: "~*req.score",
Values: []string{"30"},
},
expected: []string{"score <= '30'"},
},
{
name: "Less than with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaLessThan,
Element: "~*req.score",
Values: []string{"40"},
},
expected: []string{"score < '40'"},
},
{
name: "Prefix NOT LIKE with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaNotPrefix,
Element: "~*req.name",
Values: []string{"prefix"},
},
expected: []string{"name NOT LIKE 'prefix%'"},
},
{
name: "Prefix LIKE with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaPrefix,
Element: "~*req.data.name",
Values: []string{"prefix"},
},
expected: []string{"JSON_VALUE(data, '$.name') LIKE 'prefix%'"},
},
{
name: "Suffix NOT LIKE with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaNotSuffix,
Element: "~*req.name",
Values: []string{"suffix"},
},
expected: []string{"name NOT LIKE '%suffix'"},
},
{
name: "Suffix LIKE with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaSuffix,
Element: "~*req.data.name",
Values: []string{"suffix"},
},
expected: []string{"JSON_VALUE(data, '$.name') LIKE '%suffix'"},
},
{
name: "Regex NOT REGEXP with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaNotRegex,
Element: "~*req.pattern",
Values: []string{"[a-z]+"},
},
expected: []string{"pattern NOT REGEXP '[a-z]+'"},
},
{
name: "Regex REGEXP with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaRegex,
Element: "~*req.data.pattern",
Values: []string{"[0-9]+"},
},
expected: []string{"JSON_VALUE(data, '$.pattern') REGEXP '[0-9]+'"},
},
{
name: "Not equal with empty beforeSep",
fltrRule: FilterRule{
Type: utils.MetaNotString,
Element: "~*req.status",
Values: []string{"inactive"},
},
expected: []string{"status != 'inactive'"},
},
{
name: "Equal condition with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaString,
Element: "~*req.data.status",
Values: []string{"active"},
},
expected: []string{"JSON_VALUE(data, '$.status') = 'active'"},
},
{
name: "Greater than condition with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaGreaterThan,
Element: "~*req.data.score",
Values: []string{"50"},
},
expected: []string{"JSON_VALUE(data, '$.score') > '50'"},
},
{
name: "Less than or equal condition with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaLessOrEqual,
Element: "~*req.data.score",
Values: []string{"30"},
},
expected: []string{"JSON_VALUE(data, '$.score') <= '30'"},
},
{
name: "Less than condition with JSON_VALUE",
fltrRule: FilterRule{
Type: utils.MetaLessThan,
Element: "~*req.data.score",
Values: []string{"20"},
},
expected: []string{"JSON_VALUE(data, '$.score') < '20'"},
},
{
name: "MetaExists with no values",
fltrRule: FilterRule{
Type: utils.MetaExists,
Element: "~*req.column1",
Values: nil,
},
expected: []string{"column1 IS NULL"},
},
{
name: "MetaNotExists with no values",
fltrRule: FilterRule{
Type: utils.MetaNotExists,
Element: "~*req.json_field.key",
Values: nil,
},
expected: []string{"JSON_VALUE(json_field, '$.key') IS NOT NULL"},
},
{
name: "MetaString with values",
fltrRule: FilterRule{
Type: utils.MetaString,
Element: "~*req.column2",
Values: []string{"value1", "value2"},
},
expected: []string{"column2 = 'value1'", "column2 = 'value2'"},
},
{
name: "MetaPrefix with NOT condition",
fltrRule: FilterRule{
Type: utils.MetaNotPrefix,
Element: "~*req.json_field.key",
Values: []string{"prefix1"},
},
expected: []string{"JSON_VALUE(json_field, '$.key') NOT LIKE 'prefix1%'"},
},
{
name: "MetaRegex with multiple values",
fltrRule: FilterRule{
Type: utils.MetaRegex,
Element: "~*req.column3",
Values: []string{"pattern1", "pattern2"},
},
expected: []string{"column3 REGEXP 'pattern1'", "column3 REGEXP 'pattern2'"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.fltrRule.FilterToSQLQuery()
if len(got) != len(tt.expected) {
t.Errorf("expected %v, got %v", tt.expected, got)
return
}
for i, cond := range got {
if cond != tt.expected[i] {
t.Errorf("expected %v, got %v", tt.expected[i], cond)
}
}
})
}
}

35
engine/libtest.go
View File

@@ -31,6 +31,7 @@ import (
"path/filepath"
"slices"
"strings"
"syscall"
"testing"
"time"
@@ -332,14 +333,15 @@ func NewRPCClient(t testing.TB, cfg *config.ListenCfg) *birpc.Client {
// TestEngine holds the setup parameters and configurations
// required for running integration tests.
type TestEngine struct {
ConfigPath string // path to the main configuration file
ConfigJSON string // JSON cfg content (standalone/overwrites static configs)
DBCfg DBCfg // custom db settings for dynamic setup (overrides static config)
LogBuffer io.Writer // captures log output of the test environment
PreserveDataDB bool // prevents automatic data_db flush when set
PreserveStorDB bool // prevents automatic stor_db flush when set
TpPath string // path to the tariff plans
TpFiles map[string]string // CSV data for tariff plans: filename -> content
ConfigPath string // path to the main configuration file
ConfigJSON string // JSON cfg content (standalone/overwrites static configs)
DBCfg DBCfg // custom db settings for dynamic setup (overrides static config)
LogBuffer io.Writer // captures log output of the test environment
PreserveDataDB bool // prevents automatic data_db flush when set
PreserveStorDB bool // prevents automatic stor_db flush when set
TpPath string // path to the tariff plans
TpFiles map[string]string // CSV data for tariff plans: filename -> content
GracefulShutdown bool // shutdown the engine gracefuly, otherwise use process.Kill
// PreStartHook executes custom logic relying on CGRConfig
// before starting cgr-engine.
@@ -356,7 +358,7 @@ func (ng TestEngine) Run(t testing.TB, extraFlags ...string) (*birpc.Client, *co
if ng.PreStartHook != nil {
ng.PreStartHook(t, cfg)
}
startEngine(t, cfg, ng.LogBuffer)
startEngine(t, cfg, ng.LogBuffer, ng.GracefulShutdown)
client := NewRPCClient(t, cfg.ListenCfg())
LoadCSVs(t, client, ng.TpPath, ng.TpFiles)
return client, cfg
@@ -542,7 +544,7 @@ func FlushDBs(t testing.TB, cfg *config.CGRConfig, flushDataDB, flushStorDB bool
// startEngine starts the CGR engine process with the provided configuration. It writes engine logs to the
// provided logBuffer (if any).
func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, extraFlags ...string) {
func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, gracefulShutdown bool, extraFlags ...string) {
t.Helper()
binPath, err := exec.LookPath("cgr-engine")
if err != nil {
@@ -564,8 +566,17 @@ func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, extra
t.Fatalf("cgr-engine command failed: %v", err)
}
t.Cleanup(func() {
if err := engine.Process.Kill(); err != nil {
t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
if gracefulShutdown {
if err := engine.Process.Signal(syscall.SIGTERM); err != nil {
t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
}
if err := engine.Wait(); err != nil {
t.Errorf("cgr-engine process failed to exit cleanly: %v", err)
}
} else {
if err := engine.Process.Kill(); err != nil {
t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
}
}
})
backoff := utils.FibDuration(time.Millisecond, 0)

2
engine/storage_mysql.go
View File

@@ -45,7 +45,7 @@ func NewMySQLStorage(host, port, name, user, password string,
if mySQLStorage.Db, err = db.DB(); err != nil {
return nil, err
}
if mySQLStorage.Db.Ping(); err != nil {
if err := mySQLStorage.Db.Ping(); err != nil {
return nil, err
}
mySQLStorage.Db.SetMaxIdleConns(maxIdleConn)