mirror of
https://github.com/cgrates/cgrates.git
synced 2026-02-11 18:16:24 +05:00
Improvements to ERS SQL filters
This commit is contained in:
arberkatellari
committed by
Dan Christian Bogos
Dan Christian Bogos
parent
321910d181
commit
041b14fa03
@@ -38,6 +38,7 @@ import (
|
||||
var (
|
||||
db *gorm.DB
|
||||
dbConnString = "cgrates:CGRateS.org@tcp(127.0.0.1:3306)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'"
|
||||
timeStart = time.Now()
|
||||
cdr1 = &engine.CDR{ // sample with values not realisticy calculated
|
||||
CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()),
|
||||
OrderID: 123,
|
||||
@@ -131,13 +132,12 @@ var (
|
||||
},
|
||||
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
|
||||
}
|
||||
timeStart = time.Now()
|
||||
cgrID = utils.Sha1("dsafdsaf", timeStart.String())
|
||||
cdr2 = &engine.CDR{ // sample with values not realisticy calculated
|
||||
cgrID = utils.Sha1("oid2", timeStart.String())
|
||||
cdr2 = &engine.CDR{ // sample with values not realisticy calculated
|
||||
CGRID: cgrID,
|
||||
OrderID: 123,
|
||||
ToR: utils.MetaVoice,
|
||||
OriginID: "dsafdsaf",
|
||||
OriginID: "oid2",
|
||||
OriginHost: "192.168.1.1",
|
||||
Source: "test",
|
||||
RequestType: utils.MetaRated,
|
||||
@@ -224,7 +224,32 @@ var (
|
||||
},
|
||||
},
|
||||
},
|
||||
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
|
||||
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
|
||||
Cost: 1.01,
|
||||
}
|
||||
cdr3 = &engine.CDR{ // sample with values not realisticy calculated
|
||||
CGRID: utils.Sha1("oid3", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()),
|
||||
OrderID: 123,
|
||||
ToR: utils.MetaVoice,
|
||||
OriginID: "oid3",
|
||||
OriginHost: "192.168.1.1",
|
||||
Source: "test",
|
||||
RequestType: utils.MetaRated,
|
||||
Tenant: "cgrates.org",
|
||||
Category: "call",
|
||||
Account: "1001",
|
||||
Subject: "1001",
|
||||
Destination: "1002",
|
||||
SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
|
||||
AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
|
||||
RunID: utils.MetaDefault,
|
||||
Usage: 10 * time.Second,
|
||||
ExtraInfo: "extraInfo",
|
||||
Partial: false,
|
||||
PreRated: true,
|
||||
CostSource: "cost source",
|
||||
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
|
||||
Cost: 1.01,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -232,7 +257,7 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
var dbcfg engine.DBCfg
|
||||
switch *utils.DBType {
|
||||
case utils.MetaInternal:
|
||||
dbcfg = engine.InternalDBCfg
|
||||
t.SkipNow()
|
||||
case utils.MetaMySQL:
|
||||
case utils.MetaMongo:
|
||||
dbcfg = engine.MongoDBCfg
|
||||
@@ -305,8 +330,10 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
tx = tx.Table(utils.CDRsTBL)
|
||||
cdrSql := cdr1.AsCDRsql()
|
||||
cdrSql2 := cdr2.AsCDRsql()
|
||||
cdrsql3 := cdr3.AsCDRsql()
|
||||
cdrSql.CreatedAt = time.Now()
|
||||
cdrSql2.CreatedAt = time.Now()
|
||||
cdrsql3.CreatedAt = time.Now()
|
||||
saved := tx.Save(cdrSql)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
@@ -317,12 +344,17 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved = tx.Save(cdrsql3)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx.Commit()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 2 {
|
||||
t.Error("Expected table to have only one result ", result)
|
||||
if result != 3 {
|
||||
t.Error("Expected table to have 3 results but got ", result)
|
||||
}
|
||||
})
|
||||
defer t.Run("StopSQL", func(t *testing.T) {
|
||||
@@ -348,11 +380,12 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
"apiers": {
|
||||
"enabled": true
|
||||
},
|
||||
|
||||
"filters": {
|
||||
"apiers_conns": ["*localhost"]
|
||||
},
|
||||
"ers": {
|
||||
"enabled": true,
|
||||
"sessions_conns":["*localhost"],
|
||||
"apiers_conns": ["*localhost"],
|
||||
"readers": [
|
||||
{
|
||||
"id": "mysql",
|
||||
@@ -361,12 +394,16 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
"source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306",
|
||||
"opts": {
|
||||
"sqlDBName":"cgrates2",
|
||||
"sqlDeleteIndexedFields": ["id"],
|
||||
},
|
||||
"start_delay": "500ms", // wait for db to be populated before starting reader
|
||||
"processed_path": "",
|
||||
"tenant": "cgrates.org",
|
||||
"filters": [
|
||||
"*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days)
|
||||
"*eq:~*req.cost_details.Charges[0].RatingID:RatingID2",
|
||||
"FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2",
|
||||
"*string:~*vars.*readerID:mysql",
|
||||
"FLTR_VARS", // "*string:~*vars.*readerID:mysql",
|
||||
],
|
||||
"flags": ["*dryrun"],
|
||||
"fields":[
|
||||
@@ -390,20 +427,29 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
|
||||
}`
|
||||
|
||||
tpFiles := map[string]string{
|
||||
utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5]
|
||||
cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2,
|
||||
cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`,
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
ng := engine.TestEngine{
|
||||
ConfigJSON: content,
|
||||
DBCfg: dbcfg,
|
||||
TpFiles: tpFiles,
|
||||
LogBuffer: buf,
|
||||
}
|
||||
ng.Run(t)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) {
|
||||
time.Sleep(100 * time.Millisecond) // give enough time to process from sql table
|
||||
records := 0
|
||||
scanner := bufio.NewScanner(strings.NewReader(buf.String()))
|
||||
timeStartFormated := timeStart.Format("2006-01-02T15:04:05-07:00")
|
||||
expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"dsafdsaf\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated)
|
||||
timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00")
|
||||
expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated)
|
||||
var ersLogsCount int
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.Contains(line, "<ERs> DRYRUN, reader: <mysql>") {
|
||||
@@ -413,6 +459,9 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
if !strings.Contains(line, expectedLog) {
|
||||
t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line)
|
||||
}
|
||||
if strings.Contains(line, "[INFO] <ERs> DRYRUN") {
|
||||
ersLogsCount++
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Errorf("error reading input: %v", err)
|
||||
@@ -420,5 +469,504 @@ func TestERSSQLFilters(t *testing.T) {
|
||||
if records != 1 {
|
||||
t.Errorf("expected ERs to process 1 records, but it processed %d records", records)
|
||||
}
|
||||
if ersLogsCount != 1 {
|
||||
t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("VerifyRowsNotDeleted", func(t *testing.T) {
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 2 {
|
||||
t.Fatal("Expected 2 rows in table ", result)
|
||||
}
|
||||
var rslt []map[string]interface{}
|
||||
if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil {
|
||||
t.Fatalf("failed to query table: %v", err)
|
||||
}
|
||||
|
||||
// Print the entire table as a string
|
||||
for _, row := range rslt {
|
||||
for col, value := range row {
|
||||
if strings.Contains(fmt.Sprintln(value), "RatingID2") {
|
||||
t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%s>, value <%s>", col, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestERSSQLFiltersWithoutDelete(t *testing.T) {
|
||||
var dbcfg engine.DBCfg
|
||||
switch *utils.DBType {
|
||||
case utils.MetaInternal:
|
||||
t.SkipNow()
|
||||
case utils.MetaMySQL:
|
||||
case utils.MetaMongo:
|
||||
dbcfg = engine.MongoDBCfg
|
||||
case utils.MetaPostgres:
|
||||
dbcfg = engine.PostgresDBCfg
|
||||
default:
|
||||
t.Fatal("unsupported dbtype value")
|
||||
}
|
||||
|
||||
t.Run("InitSQLDB", func(t *testing.T) {
|
||||
var err error
|
||||
var db2 *gorm.DB
|
||||
if db2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")),
|
||||
&gorm.Config{
|
||||
AllowGlobalUpdate: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = db2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
type testModelSql struct {
|
||||
ID int64
|
||||
Cgrid string
|
||||
RunID string
|
||||
OriginHost string
|
||||
Source string
|
||||
OriginID string
|
||||
ToR string
|
||||
RequestType string
|
||||
Tenant string
|
||||
Category string
|
||||
Account string
|
||||
Subject string
|
||||
Destination string
|
||||
SetupTime time.Time
|
||||
AnswerTime time.Time
|
||||
Usage int64
|
||||
ExtraFields string
|
||||
CostSource string
|
||||
Cost float64
|
||||
CostDetails string
|
||||
ExtraInfo string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
DeletedAt *time.Time
|
||||
}
|
||||
t.Run("PutCDRsInDataBase", func(t *testing.T) {
|
||||
var err error
|
||||
if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")),
|
||||
&gorm.Config{
|
||||
AllowGlobalUpdate: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx := db.Begin()
|
||||
if !tx.Migrator().HasTable("cdrs") {
|
||||
if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tx.Commit()
|
||||
tx = db.Begin()
|
||||
tx = tx.Table(utils.CDRsTBL)
|
||||
cdrSql := cdr1.AsCDRsql()
|
||||
cdrSql2 := cdr2.AsCDRsql()
|
||||
cdrsql3 := cdr3.AsCDRsql()
|
||||
cdrSql.CreatedAt = time.Now()
|
||||
cdrSql2.CreatedAt = time.Now()
|
||||
cdrsql3.CreatedAt = time.Now()
|
||||
saved := tx.Save(cdrSql)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved = tx.Save(cdrSql2)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved = tx.Save(cdrsql3)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx.Commit()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 3 {
|
||||
t.Error("Expected table to have 3 results but got ", result)
|
||||
}
|
||||
})
|
||||
defer t.Run("StopSQL", func(t *testing.T) {
|
||||
if err := db.Migrator().DropTable("cdrs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if db2, err := db.DB(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err = db2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
content := `{
|
||||
|
||||
"general": {
|
||||
"log_level": 7
|
||||
},
|
||||
|
||||
"apiers": {
|
||||
"enabled": true
|
||||
},
|
||||
"filters": {
|
||||
"apiers_conns": ["*localhost"]
|
||||
},
|
||||
"ers": {
|
||||
"enabled": true,
|
||||
"sessions_conns":["*localhost"],
|
||||
"readers": [
|
||||
{
|
||||
"id": "mysql",
|
||||
"type": "*sql",
|
||||
"run_delay": "1m",
|
||||
"source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306",
|
||||
"opts": {
|
||||
"sqlDBName":"cgrates2",
|
||||
},
|
||||
"start_delay": "500ms", // wait for db to be populated before starting reader
|
||||
"processed_path": "",
|
||||
"tenant": "cgrates.org",
|
||||
"filters": [
|
||||
"*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days)
|
||||
"FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2",
|
||||
"*string:~*vars.*readerID:mysql",
|
||||
"FLTR_VARS", // "*string:~*vars.*readerID:mysql",
|
||||
],
|
||||
"flags": ["*dryrun"],
|
||||
"fields":[
|
||||
{"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true},
|
||||
{"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true},
|
||||
{"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true},
|
||||
{"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true},
|
||||
{"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true},
|
||||
{"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true},
|
||||
{"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true},
|
||||
{"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true},
|
||||
{"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true},
|
||||
{"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true},
|
||||
{"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true},
|
||||
{"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true},
|
||||
{"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
}`
|
||||
|
||||
tpFiles := map[string]string{
|
||||
utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5]
|
||||
cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2,
|
||||
cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`,
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
ng := engine.TestEngine{
|
||||
ConfigJSON: content,
|
||||
DBCfg: dbcfg,
|
||||
TpFiles: tpFiles,
|
||||
LogBuffer: buf,
|
||||
}
|
||||
ng.Run(t)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) {
|
||||
time.Sleep(100 * time.Millisecond) // give enough time to process from sql table
|
||||
records := 0
|
||||
scanner := bufio.NewScanner(strings.NewReader(buf.String()))
|
||||
timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00")
|
||||
expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated)
|
||||
var ersLogsCount int
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.Contains(line, "<ERs> DRYRUN, reader: <mysql>") {
|
||||
continue
|
||||
}
|
||||
records++
|
||||
if !strings.Contains(line, expectedLog) {
|
||||
t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line)
|
||||
}
|
||||
if strings.Contains(line, "[INFO] <ERs> DRYRUN") {
|
||||
ersLogsCount++
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Errorf("error reading input: %v", err)
|
||||
}
|
||||
if records != 1 {
|
||||
t.Errorf("expected ERs to process 1 records, but it processed %d records", records)
|
||||
}
|
||||
if ersLogsCount != 1 {
|
||||
t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("VerifyRowsNotDeleted", func(t *testing.T) {
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 3 {
|
||||
t.Error("Expected 3 rows in table, got: ", result)
|
||||
}
|
||||
var rslt []map[string]interface{}
|
||||
if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil {
|
||||
t.Fatalf("failed to query table: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestERSSQLFiltersWithMetaDelete(t *testing.T) {
|
||||
var dbcfg engine.DBCfg
|
||||
switch *utils.DBType {
|
||||
case utils.MetaInternal:
|
||||
t.SkipNow()
|
||||
case utils.MetaMySQL:
|
||||
case utils.MetaMongo:
|
||||
dbcfg = engine.MongoDBCfg
|
||||
case utils.MetaPostgres:
|
||||
dbcfg = engine.PostgresDBCfg
|
||||
default:
|
||||
t.Fatal("unsupported dbtype value")
|
||||
}
|
||||
|
||||
t.Run("InitSQLDB", func(t *testing.T) {
|
||||
var err error
|
||||
var db2 *gorm.DB
|
||||
if db2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")),
|
||||
&gorm.Config{
|
||||
AllowGlobalUpdate: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = db2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
type testModelSql struct {
|
||||
ID int64
|
||||
Cgrid string
|
||||
RunID string
|
||||
OriginHost string
|
||||
Source string
|
||||
OriginID string
|
||||
ToR string
|
||||
RequestType string
|
||||
Tenant string
|
||||
Category string
|
||||
Account string
|
||||
Subject string
|
||||
Destination string
|
||||
SetupTime time.Time
|
||||
AnswerTime time.Time
|
||||
Usage int64
|
||||
ExtraFields string
|
||||
CostSource string
|
||||
Cost float64
|
||||
CostDetails string
|
||||
ExtraInfo string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
DeletedAt *time.Time
|
||||
}
|
||||
t.Run("PutCDRsInDataBase", func(t *testing.T) {
|
||||
var err error
|
||||
if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")),
|
||||
&gorm.Config{
|
||||
AllowGlobalUpdate: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx := db.Begin()
|
||||
if !tx.Migrator().HasTable("cdrs") {
|
||||
if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tx.Commit()
|
||||
tx = db.Begin()
|
||||
tx = tx.Table(utils.CDRsTBL)
|
||||
cdrSql := cdr1.AsCDRsql()
|
||||
cdrSql2 := cdr2.AsCDRsql()
|
||||
cdrsql3 := cdr3.AsCDRsql()
|
||||
cdrSql.CreatedAt = time.Now()
|
||||
cdrSql2.CreatedAt = time.Now()
|
||||
cdrsql3.CreatedAt = time.Now()
|
||||
saved := tx.Save(cdrSql)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved = tx.Save(cdrSql2)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved = tx.Save(cdrsql3)
|
||||
if saved.Error != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx.Commit()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 3 {
|
||||
t.Error("Expected table to have 3 results but got ", result)
|
||||
}
|
||||
})
|
||||
defer t.Run("StopSQL", func(t *testing.T) {
|
||||
if err := db.Migrator().DropTable("cdrs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if db2, err := db.DB(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err = db2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
content := `{
|
||||
|
||||
"general": {
|
||||
"log_level": 7
|
||||
},
|
||||
|
||||
"apiers": {
|
||||
"enabled": true
|
||||
},
|
||||
"filters": {
|
||||
"apiers_conns": ["*localhost"]
|
||||
},
|
||||
"ers": {
|
||||
"enabled": true,
|
||||
"sessions_conns":["*localhost"],
|
||||
"readers": [
|
||||
{
|
||||
"id": "mysql",
|
||||
"type": "*sql",
|
||||
"run_delay": "1m",
|
||||
"source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306",
|
||||
"opts": {
|
||||
"sqlDBName":"cgrates2",
|
||||
},
|
||||
"start_delay": "500ms", // wait for db to be populated before starting reader
|
||||
"processed_path": "*delete",
|
||||
"tenant": "cgrates.org",
|
||||
"filters": [
|
||||
"*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days)
|
||||
"FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2",
|
||||
"*string:~*vars.*readerID:mysql",
|
||||
"FLTR_VARS", // "*string:~*vars.*readerID:mysql",
|
||||
],
|
||||
"flags": ["*dryrun"],
|
||||
"fields":[
|
||||
{"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true},
|
||||
{"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true},
|
||||
{"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true},
|
||||
{"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true},
|
||||
{"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true},
|
||||
{"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true},
|
||||
{"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true},
|
||||
{"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true},
|
||||
{"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true},
|
||||
{"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true},
|
||||
{"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true},
|
||||
{"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true},
|
||||
{"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
}`
|
||||
|
||||
tpFiles := map[string]string{
|
||||
utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5]
|
||||
cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2,
|
||||
cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`,
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
ng := engine.TestEngine{
|
||||
ConfigJSON: content,
|
||||
DBCfg: dbcfg,
|
||||
TpFiles: tpFiles,
|
||||
LogBuffer: buf,
|
||||
}
|
||||
ng.Run(t)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) {
|
||||
time.Sleep(100 * time.Millisecond) // give enough time to process from sql table
|
||||
records := 0
|
||||
scanner := bufio.NewScanner(strings.NewReader(buf.String()))
|
||||
timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00")
|
||||
expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated)
|
||||
var ersLogsCount int
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.Contains(line, "<ERs> DRYRUN, reader: <mysql>") {
|
||||
continue
|
||||
}
|
||||
records++
|
||||
if !strings.Contains(line, expectedLog) {
|
||||
t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line)
|
||||
}
|
||||
if strings.Contains(line, "[INFO] <ERs> DRYRUN") {
|
||||
ersLogsCount++
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Errorf("error reading input: %v", err)
|
||||
}
|
||||
if records != 1 {
|
||||
t.Errorf("expected ERs to process 1 records, but it processed %d records", records)
|
||||
}
|
||||
if ersLogsCount != 1 {
|
||||
t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("VerifyRowsNotDeleted", func(t *testing.T) {
|
||||
var result int64
|
||||
db.Table(utils.CDRsTBL).Count(&result)
|
||||
if result != 2 {
|
||||
t.Error("Expected 2 rows in table, got: ", result)
|
||||
}
|
||||
var rslt []map[string]interface{}
|
||||
if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil {
|
||||
t.Fatalf("failed to query table: %v", err)
|
||||
}
|
||||
|
||||
// Print the entire table as a string
|
||||
for _, row := range rslt {
|
||||
for col, value := range row {
|
||||
if strings.Contains(fmt.Sprintln(value), "RatingID2") {
|
||||
t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%s>, value <%s>", col, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user