uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-21 23:28:44 +05:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to ERS to update or move ERS SQL events to a new table & add *export flag for ERS readers

Browse Source
This commit is contained in:
arberkatellari
2024-12-10 20:05:23 +02:00
committed by Dan Christian Bogos
parent a2b86e1a02
commit d35b14b6b9
30 changed files with 1898 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
utils/consts.go
View File

@@ -2809,6 +2809,7 @@ const (
SQLDBNameOpt = "sqlDBName"
SQLTableNameOpt = "sqlTableName"
SQLDeleteIndexedFieldsOpt = "sqlDeleteIndexedFields"
SQLUpdateIndexedFieldsOpt = "sqlUpdateIndexedFields"
SQLMaxOpenConns = "sqlMaxOpenConns"
SQLConnMaxLifetime = "sqlConnMaxLifetime"

7
utils/coreutils.go
View File

@@ -294,6 +294,13 @@ func ParseTimeDetectLayout(tmStr string, timezone string) (time.Time, error) {
} else {
return time.Now().Add(tmStrTmp), nil
}
case strings.HasPrefix(tmStr, "-"):
tmStr = strings.TrimPrefix(tmStr, "-")
if tmStrTmp, err := time.ParseDuration(tmStr); err != nil {
return nilTime, err
} else {
return time.Now().Add(-tmStrTmp), nil
}
case utcFormat.MatchString(tmStr):
return time.ParseInLocation("2006-01-02T15:04:05", tmStr, loc)

7
utils/coreutils_test.go
View File

@@ -451,6 +451,13 @@ func TestParseTimeDetectLayout(t *testing.T) {
if err != nil || date.Sub(expected).Seconds() > 20 || date.Sub(expected).Seconds() < 19 {
t.Error("error parsing date: ", date.Sub(expected).Seconds())
}
date, err = ParseTimeDetectLayout("-20s", "")
expected = time.Now()
if err != nil || expected.Sub(date).Seconds() > 21 || expected.Sub(date).Seconds() < 20 {
t.Error("error parsing date: ", expected.Sub(date).Seconds())
}
expected = time.Now().AddDate(0, 0, 1)
if date, err := ParseTimeDetectLayout("*daily", ""); err != nil {
t.Error(err)

156
utils/dbutils.go
View File

@@ -1,156 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package utils
import "fmt"
// Creates mysql conditions used in WHERE statement out of filters
func FilterToSQLQuery(ruleType, beforeSep, afterSep string, values []string, not bool) (conditions []string) {
// here are for the filters that their values are empty: *exists, *notexists, *empty, *notempty..
if len(values) == 0 {
switch ruleType {
case MetaExists, MetaNotExists:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", afterSep))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", beforeSep, afterSep))
return
}
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s IS NULL", afterSep))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", beforeSep, afterSep))
case MetaEmpty, MetaNotEmpty:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s != ''", afterSep))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != ''", beforeSep, afterSep))
return
}
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s == ''", afterSep))
return
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') == ''", beforeSep, afterSep))
}
return
}
// here are for the filters that can have more than one value: *string, *prefix, *suffix ..
for _, value := range values {
switch value { // in case we have boolean values, it should be queried over 1 or 0
case "true":
value = "1"
case "false":
value = "0"
}
var singleCond string
switch ruleType {
case MetaString, MetaNotString, MetaEqual, MetaNotEqual:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s != '%s'", afterSep, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != '%s'",
beforeSep, afterSep, value))
continue
}
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s = '%s'", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') = '%s'", beforeSep, afterSep, value)
}
case MetaLessThan, MetaLessOrEqual, MetaGreaterThan, MetaGreaterOrEqual:
if ruleType == MetaGreaterOrEqual {
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s >= %s", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') >= %s", beforeSep, afterSep, value)
}
} else if ruleType == MetaGreaterThan {
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s > %s", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') > %s", beforeSep, afterSep, value)
}
} else if ruleType == MetaLessOrEqual {
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s <= %s", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') <= %s", beforeSep, afterSep, value)
}
} else if ruleType == MetaLessThan {
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s < %s", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') < %s", beforeSep, afterSep, value)
}
}
case MetaPrefix, MetaNotPrefix:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%s%%'", afterSep, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%s%%'", beforeSep, afterSep, value))
continue
}
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s LIKE '%s%%'", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%s%%'", beforeSep, afterSep, value)
}
case MetaSuffix, MetaNotSuffix:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%%%s'", afterSep, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%%%s'", beforeSep, afterSep, value))
continue
}
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s LIKE '%%%s'", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%%%s'", beforeSep, afterSep, value)
}
case MetaRegex, MetaNotRegex:
if not {
if beforeSep == EmptyString {
conditions = append(conditions, fmt.Sprintf("%s NOT REGEXP '%s'", afterSep, value))
continue
}
conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT REGEXP '%s'", beforeSep, afterSep, value))
continue
}
if beforeSep == EmptyString {
singleCond = fmt.Sprintf("%s REGEXP '%s'", afterSep, value)
} else {
singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') REGEXP '%s'", beforeSep, afterSep, value)
}
}
conditions = append(conditions, singleCond)
}
return
}

309
utils/dbutils_test.go
View File

@@ -1,309 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package utils
import "testing"
func TestFilterToSQLQuery(t *testing.T) {
tests := []struct {
name string
ruleType string
beforeSep string
afterSep string
values []string
not bool
expected []string
}{
{"MetaGreaterThan with values", MetaGreaterThan, "", "answer_time", []string{"NOW() - INTERVAL 7 DAY"}, false, []string{"answer_time > NOW() - INTERVAL 7 DAY"}},
{"MetaEqual with values", MetaEqual, "cost_details", "Charges[0].RatingID", []string{"RatingID2"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') = 'RatingID2'"}},
{"MetaExists with no values", MetaExists, "", "answer_time", nil, false, []string{"answer_time IS NULL"}},
{"MetaExists with JSON field", MetaExists, "cost_details", "Charges[0].RatingID", nil, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NULL"}},
{"MetaNotExists with no values", MetaNotExists, "", "answer_time", nil, true, []string{"answer_time IS NOT NULL"}},
{"MetaNotExists with JSON field", MetaNotExists, "cost_details", "Charges[0].RatingID", nil, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NOT NULL"}},
{"MetaString with values", MetaString, "", "answer_time", []string{"value1", "value2"}, false, []string{"answer_time = 'value1'", "answer_time = 'value2'"}},
{"MetaNotString with values", MetaNotString, "cost_details", "Charges[0].RatingID", []string{"value1"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != 'value1'"}},
{"MetaEmpty with no values", MetaEmpty, "", "answer_time", nil, false, []string{"answer_time == ''"}},
{"MetaEmpty with JSON field", MetaEmpty, "cost_details", "Charges[0].RatingID", nil, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') == ''"}},
{"MetaNotEmpty with no values", MetaNotEmpty, "", "answer_time", nil, true, []string{"answer_time != ''"}},
{"MetaNotEmpty with JSON field", MetaNotEmpty, "cost_details", "Charges[0].RatingID", nil, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != ''"}},
{"MetaGreaterOrEqual with values", MetaGreaterOrEqual, "", "answer_time", []string{"10"}, false, []string{"answer_time >= 10"}},
{"MetaGreaterThan with values", MetaGreaterThan, "cost_details", "Charges[0].RatingID", []string{"20"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') > 20"}},
{"MetaLessThan with values", MetaLessThan, "", "answer_time", []string{"5"}, false, []string{"answer_time < 5"}},
{"MetaLessOrEqual with values", MetaLessOrEqual, "cost_details", "Charges[0].RatingID", []string{"15"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') <= 15"}},
{"MetaPrefix with values", MetaPrefix, "", "answer_time", []string{"pre"}, false, []string{"answer_time LIKE 'pre%'"}},
{"MetaNotPrefix with values", MetaNotPrefix, "cost_details", "Charges[0].RatingID", []string{"pre"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE 'pre%'"}},
{"MetaSuffix with values", MetaSuffix, "", "answer_time", []string{"suf"}, false, []string{"answer_time LIKE '%suf'"}},
{"MetaNotSuffix with values", MetaNotSuffix, "cost_details", "Charges[0].RatingID", []string{"suf"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE '%suf'"}},
{"MetaGreaterOrEqual with JSON field", MetaGreaterOrEqual, "cost_details", "Charges[0].RatingID", []string{"100"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') >= 100"}},
{"MetaRegex with values", MetaRegex, "", "answer_time", []string{"pattern1", "pattern2"}, false, []string{"answer_time REGEXP 'pattern1'", "answer_time REGEXP 'pattern2'"}},
{"MetaNotRegex with values", MetaNotRegex, "cost_details", "Charges[0].RatingID", []string{"pattern"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT REGEXP 'pattern'"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := FilterToSQLQuery(tt.ruleType, tt.beforeSep, tt.afterSep, tt.values, tt.not)
if len(got) != len(tt.expected) {
t.Errorf("expected %v, got %v", tt.expected, got)
return
}
for i, cond := range got {
if cond != tt.expected[i] {
t.Errorf("expected %v, got %v", tt.expected[i], cond)
}
}
})
}
}
func TestFilterToSQLQueryValidations(t *testing.T) {
tests := []struct {
name string
ruleType string
beforeSep string
afterSep string
values []string
not bool
expected []string
}{
{
name: "Boolean true value",
ruleType: MetaString,
beforeSep: EmptyString,
afterSep: "active",
values: []string{"true"},
not: false,
expected: []string{"active = '1'"},
},
{
name: "Boolean false value",
ruleType: MetaString,
beforeSep: EmptyString,
afterSep: "active",
values: []string{"false"},
not: false,
expected: []string{"active = '0'"},
},
{
name: "Greater than or equal with empty beforeSep",
ruleType: MetaGreaterOrEqual,
beforeSep: EmptyString,
afterSep: "score",
values: []string{"10"},
not: false,
expected: []string{"score >= 10"},
},
{
name: "Greater than with empty beforeSep",
ruleType: MetaGreaterThan,
beforeSep: EmptyString,
afterSep: "score",
values: []string{"20"},
not: false,
expected: []string{"score > 20"},
},
{
name: "Less than or equal with empty beforeSep",
ruleType: MetaLessOrEqual,
beforeSep: EmptyString,
afterSep: "score",
values: []string{"30"},
not: false,
expected: []string{"score <= 30"},
},
{
name: "Less than with empty beforeSep",
ruleType: MetaLessThan,
beforeSep: EmptyString,
afterSep: "score",
values: []string{"40"},
not: false,
expected: []string{"score < 40"},
},
{
name: "Prefix NOT LIKE with empty beforeSep",
ruleType: MetaNotPrefix,
beforeSep: EmptyString,
afterSep: "name",
values: []string{"prefix"},
not: true,
expected: []string{"name NOT LIKE 'prefix%'"},
},
{
name: "Prefix LIKE with JSON_VALUE",
ruleType: MetaPrefix,
beforeSep: "data",
afterSep: "name",
values: []string{"prefix"},
not: false,
expected: []string{"JSON_VALUE(data, '$.name') LIKE 'prefix%'"},
},
{
name: "Suffix NOT LIKE with empty beforeSep",
ruleType: MetaNotSuffix,
beforeSep: EmptyString,
afterSep: "name",
values: []string{"suffix"},
not: true,
expected: []string{"name NOT LIKE '%suffix'"},
},
{
name: "Suffix LIKE with JSON_VALUE",
ruleType: MetaSuffix,
beforeSep: "data",
afterSep: "name",
values: []string{"suffix"},
not: false,
expected: []string{"JSON_VALUE(data, '$.name') LIKE '%suffix'"},
},
{
name: "Regex NOT REGEXP with empty beforeSep",
ruleType: MetaNotRegex,
beforeSep: EmptyString,
afterSep: "pattern",
values: []string{"[a-z]+"},
not: true,
expected: []string{"pattern NOT REGEXP '[a-z]+'"},
},
{
name: "Regex REGEXP with JSON_VALUE",
ruleType: MetaRegex,
beforeSep: "data",
afterSep: "pattern",
values: []string{"[0-9]+"},
not: false,
expected: []string{"JSON_VALUE(data, '$.pattern') REGEXP '[0-9]+'"},
},
{
name: "Not equal with empty beforeSep",
ruleType: MetaString,
beforeSep: EmptyString,
afterSep: "status",
values: []string{"inactive"},
not: true,
expected: []string{"status != 'inactive'"},
},
{
name: "Equal condition with JSON_VALUE",
ruleType: MetaString,
beforeSep: "data",
afterSep: "status",
values: []string{"active"},
not: false,
expected: []string{"JSON_VALUE(data, '$.status') = 'active'"},
},
{
name: "Greater than condition with JSON_VALUE",
ruleType: MetaGreaterThan,
beforeSep: "data",
afterSep: "score",
values: []string{"50"},
not: false,
expected: []string{"JSON_VALUE(data, '$.score') > 50"},
},
{
name: "Less than or equal condition with JSON_VALUE",
ruleType: MetaLessOrEqual,
beforeSep: "data",
afterSep: "score",
values: []string{"30"},
not: false,
expected: []string{"JSON_VALUE(data, '$.score') <= 30"},
},
{
name: "Less than condition with JSON_VALUE",
ruleType: MetaLessThan,
beforeSep: "data",
afterSep: "score",
values: []string{"20"},
not: false,
expected: []string{"JSON_VALUE(data, '$.score') < 20"},
},
{
name: "MetaExists with no values",
ruleType: MetaExists,
beforeSep: "",
afterSep: "column1",
values: nil,
not: false,
expected: []string{"column1 IS NULL"},
},
{
name: "MetaNotExists with no values",
ruleType: MetaNotExists,
beforeSep: "json_field",
afterSep: "key",
values: nil,
not: true,
expected: []string{"JSON_VALUE(json_field, '$.key') IS NOT NULL"},
},
{
name: "MetaString with values",
ruleType: MetaString,
beforeSep: "",
afterSep: "column2",
values: []string{"value1", "value2"},
not: false,
expected: []string{"column2 = 'value1'", "column2 = 'value2'"},
},
{
name: "MetaPrefix with NOT condition",
ruleType: MetaNotPrefix,
beforeSep: "json_field",
afterSep: "key",
values: []string{"prefix1"},
not: true,
expected: []string{"JSON_VALUE(json_field, '$.key') NOT LIKE 'prefix1%'"},
},
{
name: "MetaRegex with multiple values",
ruleType: MetaRegex,
beforeSep: "",
afterSep: "column3",
values: []string{"pattern1", "pattern2"},
not: false,
expected: []string{"column3 REGEXP 'pattern1'", "column3 REGEXP 'pattern2'"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := FilterToSQLQuery(tt.ruleType, tt.beforeSep, tt.afterSep, tt.values, tt.not)
if len(got) != len(tt.expected) {
t.Errorf("expected %v, got %v", tt.expected, got)
return
}
for i, cond := range got {
if cond != tt.expected[i] {
t.Errorf("expected %v, got %v", tt.expected[i], cond)
}
}
})
}
}

9
utils/rsrfilters_test.go
View File

@@ -247,9 +247,6 @@ func TestRSRFilterPass(t *testing.T) {
if err != nil {
t.Error(err)
}
if !fltr.Pass("-1s") {
t.Error("not passing!")
}
if fltr.Pass("0s") {
t.Error("passing!")
}
@@ -274,9 +271,6 @@ func TestRSRFilterPass(t *testing.T) {
if fltr.Pass("12s") {
t.Error("passing!")
}
if !fltr.Pass("-12s") {
t.Error("not passing!")
}
// compare not lessThan
fltr, err = NewRSRFilter("!<0s")
@@ -301,9 +295,6 @@ func TestRSRFilterPass(t *testing.T) {
if err != nil {
t.Error(err)
}
if !fltr.Pass("-1s") {
t.Error("not passing!")
}
if !fltr.Pass("0s") {
t.Error("not passing!")
}