diff --git a/config/config_defaults.go b/config/config_defaults.go index e9cb55ec6..6fefcac67 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -554,6 +554,7 @@ const CGRATES_CFG_JSON = ` // "sqlMaxIdleConns": 0, // SQLMaxIdleConns // "sqlMaxOpenConns": 0, // SQLMaxOpenConns // "sqlConnMaxLifetime": "0", // SQLConnMaxLifetime + // "sqlUpdateIndexedFields": [], // list of field names used for indexing UPDATE queries from the table // "mysqlDSNParams": {}, // DSN params diff --git a/config/configsanity.go b/config/configsanity.go index fbdc66e08..cefdb3acc 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -923,10 +923,6 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> nonexistent folder: %s for exporter with ID: %s", utils.EEs, dir, exp.ID) } } - case utils.MetaSQL: - if len(exp.ContentFields()) == 0 { - return fmt.Errorf("<%s> empty content fields for exporter with ID: %s", utils.EEs, exp.ID) - } case utils.MetaElastic: elsOpts := exp.Opts.Els if elsOpts.Logger != nil { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 4799e103d..87bbc5036 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1487,12 +1487,6 @@ func TestConfigSanityEventExporter(t *testing.T) { t.Errorf("Expecting: %+q received: %+q", expected, err) } - cfg.eesCfg.Exporters[0].Type = utils.MetaSQL - expected = " empty content fields for exporter with ID: " - if err := cfg.CheckConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.eesCfg.Exporters[0].Type = utils.MetaHTTPPost cfg.eesCfg.Exporters[0].Fields[0].Path = "~Field1..Field2[0]" expected = " Empty field path for ~Field1..Field2[0] at Path" diff --git a/config/eescfg.go b/config/eescfg.go index cb378777e..a45474536 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -192,13 +192,14 @@ type ElsOpts struct { } type SQLOpts struct { - MaxIdleConns *int - MaxOpenConns *int - ConnMaxLifetime *time.Duration - MYSQLDSNParams map[string]string - TableName *string - DBName *string - PgSSLMode *string + MaxIdleConns *int + MaxOpenConns *int + ConnMaxLifetime *time.Duration + MYSQLDSNParams map[string]string + TableName *string + DBName *string + UpdateIndexedFields *[]string + PgSSLMode *string } type AMQPOpts struct { @@ -423,6 +424,11 @@ func (sqlOpts *SQLOpts) loadFromJSONCfg(jsnCfg *EventExporterOptsJson) (err erro if jsnCfg.SQLDBName != nil { sqlOpts.DBName = jsnCfg.SQLDBName } + if jsnCfg.SQLUpdateIndexedFields != nil { + uif := make([]string, len(*jsnCfg.SQLUpdateIndexedFields)) + copy(uif, *jsnCfg.SQLUpdateIndexedFields) + sqlOpts.UpdateIndexedFields = &uif + } if jsnCfg.PgSSLMode != nil { sqlOpts.PgSSLMode = jsnCfg.PgSSLMode } @@ -825,6 +831,11 @@ func (sqlOpts *SQLOpts) Clone() *SQLOpts { cln.DBName = new(string) *cln.DBName = *sqlOpts.DBName } + if sqlOpts.UpdateIndexedFields != nil { + idx := make([]string, len(*sqlOpts.UpdateIndexedFields)) + copy(idx, *sqlOpts.UpdateIndexedFields) + cln.UpdateIndexedFields = &idx + } if sqlOpts.PgSSLMode != nil { cln.PgSSLMode = new(string) *cln.PgSSLMode = *sqlOpts.PgSSLMode @@ -1146,6 +1157,11 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str if sqlOpts.DBName != nil { opts[utils.SQLDBNameOpt] = *sqlOpts.DBName } + if sqlOpts.UpdateIndexedFields != nil { + updateIndexedFields := make([]string, len(*sqlOpts.UpdateIndexedFields)) + copy(updateIndexedFields, *sqlOpts.UpdateIndexedFields) + opts[utils.SQLUpdateIndexedFieldsOpt] = updateIndexedFields + } if sqlOpts.PgSSLMode != nil { opts[utils.PgSSLModeCfg] = *sqlOpts.PgSSLMode } diff --git a/config/eescfg_test.go b/config/eescfg_test.go index 480f5be1b..aee74b139 100644 --- a/config/eescfg_test.go +++ b/config/eescfg_test.go @@ -57,6 +57,7 @@ func TestEESClone(t *testing.T) { "sqlConnMaxLifetime":"1m", "sqlTableName":"table", "sqlDBName":"db", + "sqlUpdateIndexedFields": ["id"], "pgSSLMode":"pg", "awsToken":"token", "s3FolderPath":"s3", @@ -232,12 +233,13 @@ func TestEESClone(t *testing.T) { "allowOldPasswords": "true", "allowNativePasswords": "true", }, - MaxIdleConns: utils.IntPointer(4), - ConnMaxLifetime: utils.DurationPointer(1 * time.Minute), - TableName: utils.StringPointer("table"), - DBName: utils.StringPointer("db"), - PgSSLMode: utils.StringPointer("pg"), - MaxOpenConns: utils.IntPointer(6), + MaxIdleConns: utils.IntPointer(4), + ConnMaxLifetime: utils.DurationPointer(1 * time.Minute), + TableName: utils.StringPointer("table"), + DBName: utils.StringPointer("db"), + UpdateIndexedFields: utils.SliceStringPointer([]string{"id"}), + PgSSLMode: utils.StringPointer("pg"), + MaxOpenConns: utils.IntPointer(6), }, Els: &ElsOpts{ Index: utils.StringPointer("test"), @@ -898,6 +900,7 @@ func TestEEsCfgAsMapInterface(t *testing.T) { "sqlConnMaxLifetime":"1m", "sqlTableName":"table", "sqlDBName":"db", + "sqlUpdateIndexedFields": ["id"], "pgSSLMode":"pg", "awsToken":"token", "s3FolderPath":"s3", @@ -968,49 +971,50 @@ func TestEEsCfgAsMapInterface(t *testing.T) { utils.TypeCfg: "*file_csv", utils.ExportPathCfg: "/tmp/testCSV", utils.OptsCfg: map[string]any{ - utils.KafkaTopic: "test", - utils.ElsIndex: "test", - utils.ElsRefresh: "true", - utils.ElsOpType: "test2", - utils.ElsPipeline: "test3", - utils.ElsRouting: "test4", - utils.ElsTimeout: "1m0s", - utils.ElsWaitForActiveShards: "test6", - utils.SQLMaxIdleConnsCfg: 4, - utils.SQLMaxOpenConns: 6, - utils.SQLConnMaxLifetime: "1m0s", - utils.SQLTableNameOpt: "table", - utils.SQLDBNameOpt: "db", - utils.PgSSLModeCfg: "pg", - utils.AWSToken: "token", - utils.S3FolderPath: "s3", - utils.NatsJetStream: true, - utils.NatsSubject: "nat", - utils.NatsJWTFile: "jwt", - utils.NatsSeedFile: "seed", - utils.NatsCertificateAuthority: "NATS", - utils.NatsClientCertificate: "NATSClient", - utils.NatsClientKey: "key", - utils.NatsJetStreamMaxWait: "1m0s", - utils.AMQPQueueID: "id", - utils.AMQPRoutingKey: "key", - utils.AMQPExchangeType: "type", - utils.AMQPExchange: "exchange", - utils.AWSRegion: "eu", - utils.AWSKey: "key", - utils.AWSSecret: "secretkey", - utils.SQSQueueID: "sqsid", - utils.S3Bucket: "s3", - utils.RpcCodec: "rpc", - utils.ServiceMethod: "service", - utils.KeyPath: "path", - utils.CertPath: "certpath", - utils.CaPath: "capath", - utils.Tls: true, - utils.ConnIDs: []string{"id1", "id2"}, - utils.RpcConnTimeout: "1m0s", - utils.RpcReplyTimeout: "1m0s", - utils.CSVFieldSepOpt: ",", + utils.KafkaTopic: "test", + utils.ElsIndex: "test", + utils.ElsRefresh: "true", + utils.ElsOpType: "test2", + utils.ElsPipeline: "test3", + utils.ElsRouting: "test4", + utils.ElsTimeout: "1m0s", + utils.ElsWaitForActiveShards: "test6", + utils.SQLMaxIdleConnsCfg: 4, + utils.SQLMaxOpenConns: 6, + utils.SQLConnMaxLifetime: "1m0s", + utils.SQLTableNameOpt: "table", + utils.SQLDBNameOpt: "db", + utils.SQLUpdateIndexedFieldsOpt: []string{"id"}, + utils.PgSSLModeCfg: "pg", + utils.AWSToken: "token", + utils.S3FolderPath: "s3", + utils.NatsJetStream: true, + utils.NatsSubject: "nat", + utils.NatsJWTFile: "jwt", + utils.NatsSeedFile: "seed", + utils.NatsCertificateAuthority: "NATS", + utils.NatsClientCertificate: "NATSClient", + utils.NatsClientKey: "key", + utils.NatsJetStreamMaxWait: "1m0s", + utils.AMQPQueueID: "id", + utils.AMQPRoutingKey: "key", + utils.AMQPExchangeType: "type", + utils.AMQPExchange: "exchange", + utils.AWSRegion: "eu", + utils.AWSKey: "key", + utils.AWSSecret: "secretkey", + utils.SQSQueueID: "sqsid", + utils.S3Bucket: "s3", + utils.RpcCodec: "rpc", + utils.ServiceMethod: "service", + utils.KeyPath: "path", + utils.CertPath: "certpath", + utils.CaPath: "capath", + utils.Tls: true, + utils.ConnIDs: []string{"id1", "id2"}, + utils.RpcConnTimeout: "1m0s", + utils.RpcReplyTimeout: "1m0s", + utils.CSVFieldSepOpt: ",", utils.MYSQLDSNParams: map[string]string{ "key": "param", }, @@ -1138,6 +1142,7 @@ func TestEEsCfgloadFromJSONCfg(t *testing.T) { SQLConnMaxLifetime: &str, MYSQLDSNParams: map[string]string{str: str}, SQLTableName: &str, + SQLUpdateIndexedFields: &[]string{"id"}, SQLDBName: &str, PgSSLMode: &str, KafkaTopic: &str, diff --git a/config/erscfg.go b/config/erscfg.go index b02fa3bea..9c1b907a0 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -811,7 +811,7 @@ func (er EventReaderCfg) Clone() (cln *EventReaderCfg) { Flags: er.Flags.Clone(), Reconnects: er.Reconnects, MaxReconnectInterval: er.MaxReconnectInterval, - EEsSuccessIDs: slices.Clone(er.EEsFailedIDs), + EEsSuccessIDs: slices.Clone(er.EEsSuccessIDs), EEsFailedIDs: slices.Clone(er.EEsFailedIDs), Opts: er.Opts.Clone(), } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 8cfa554cf..f497e927e 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -331,6 +331,7 @@ type EventExporterOptsJson struct { MYSQLDSNParams map[string]string `json:"mysqlDSNParams"` SQLTableName *string `json:"sqlTableName"` SQLDBName *string `json:"sqlDBName"` + SQLUpdateIndexedFields *[]string `json:"sqlUpdateIndexedFields"` PgSSLMode *string `json:"pgSSLMode"` KafkaTopic *string `json:"kafkaTopic"` KafkaBatchSize *int `json:"kafkaBatchSize"` diff --git a/data/conf/samples/ers_mysql_delete_indexed_fields/cgrates.json b/data/conf/samples/ers_mysql_delete_indexed_fields/cgrates.json new file mode 100644 index 000000000..cd7f90923 --- /dev/null +++ b/data/conf/samples/ers_mysql_delete_indexed_fields/cgrates.json @@ -0,0 +1,61 @@ +{ + + "general": { + "log_level": 7 + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "readers": [ + { + "id": "mysql", + "type": "*sql", + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + "sqlDeleteIndexedFields": ["id"], + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "processed_path": "*delete", + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysql", + "FLTR_VARS", // "*string:~*vars.*readerID:mysql", + ], + "flags": ["*dryrun"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/data/conf/samples/ers_mysql_filters/cgrates.json b/data/conf/samples/ers_mysql_filters/cgrates.json new file mode 100644 index 000000000..0233f4641 --- /dev/null +++ b/data/conf/samples/ers_mysql_filters/cgrates.json @@ -0,0 +1,59 @@ +{ + + "general": { + "log_level": 7 + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "readers": [ + { + "id": "mysql", + "type": "*sql", + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysql", + "FLTR_VARS", // "*string:~*vars.*readerID:mysql", + ], + "flags": ["*dryrun"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/data/conf/samples/ers_mysql_meta_delete/cgrates.json b/data/conf/samples/ers_mysql_meta_delete/cgrates.json new file mode 100644 index 000000000..3f769da49 --- /dev/null +++ b/data/conf/samples/ers_mysql_meta_delete/cgrates.json @@ -0,0 +1,60 @@ +{ + + "general": { + "log_level": 7 + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "readers": [ + { + "id": "mysql", + "type": "*sql", + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "processed_path": "*delete", + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysql", + "FLTR_VARS", // "*string:~*vars.*readerID:mysql", + ], + "flags": ["*dryrun"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/data/conf/samples/ers_mysql_move/cgrates.json b/data/conf/samples/ers_mysql_move/cgrates.json new file mode 100644 index 000000000..8288fdc1d --- /dev/null +++ b/data/conf/samples/ers_mysql_move/cgrates.json @@ -0,0 +1,77 @@ +{ + + "general": { + "log_level": 7 + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ees": { + "enabled": true, + "exporters": [{ + "id": "SQLExporter", + "type": "*sql", + "export_path": "mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "attempts": 1, + "opts": { + "sqlDBName": "cgrates2", + "sqlTableName":"cdrsProcessed", + }, + "flags": ["*log"], + }] + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "ees_conns": ["*internal"], + "readers": [ + { + "id": "mysql", + "type": "*sql", + "ees_success_ids": ["SQLExporter"], + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + "sqlDeleteIndexedFields": ["id"], + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "processed_path": "*delete", + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysql", + "FLTR_VARS", // "*string:~*vars.*readerID:mysql", + ], + "flags": ["*dryrun"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/data/conf/samples/ers_mysql_raw_update/cgrates.json b/data/conf/samples/ers_mysql_raw_update/cgrates.json new file mode 100644 index 000000000..c61b63238 --- /dev/null +++ b/data/conf/samples/ers_mysql_raw_update/cgrates.json @@ -0,0 +1,80 @@ +{ + "general": { + "log_level": 7, + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ees": { + "enabled": true, + "exporters": [{ + "id": "SQLExporter", + "type": "*sql", + "export_path": "mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "attempts": 1, + "opts": { + "sqlDBName": "cgrates2", + "sqlTableName":"cdrs", + "sqlUpdateIndexedFields": ["id", "cgrid"], + }, + "flags": ["*log"], + "fields": [ + {"tag": "SetupTime", "path": "*exp.setup_time", "type": "*constant", "value": "2018-11-27 14:21:26"}, + {"tag": "Account", "path": "*exp.account", "type": "*variable", "value": "~*req.extra_info"}, + {"tag": "ID", "path": "*exp.id", "type": "*variable", "value": "~*req.id"}, + {"tag": "CGRID", "path": "*exp.cgrid", "type": "*variable", "value": "~*req.cgrid"}, + ] + }] + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "ees_conns": ["*localhost"], + "readers": [{ + "id": "mysql", + "type": "*sql", + "ees_success_ids": ["SQLExporter"], + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysql", + "FLTR_VARS", // "*string:~*vars.*readerID:mysql", + ], + "flags": ["*dryrun"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/data/conf/samples/ers_mysql_update/cgrates.json b/data/conf/samples/ers_mysql_update/cgrates.json new file mode 100644 index 000000000..9f43fd065 --- /dev/null +++ b/data/conf/samples/ers_mysql_update/cgrates.json @@ -0,0 +1,84 @@ +{ + + "general": { + "log_level": 7, + }, + + "apiers": { + "enabled": true + }, + "filters": { + "apiers_conns": ["*localhost"] + }, + "stor_db": { + "opts": { + "sqlConnMaxLifetime": "5s", // needed while running all integration tests + }, + }, + "ees": { + "enabled": true, + "exporters": [{ + "id": "SQLExporter", + "type": "*sql", + "export_path": "mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "attempts": 1, + "opts": { + "sqlDBName": "cgrates2", + "sqlTableName":"cdrs", + "sqlUpdateIndexedFields": ["id", "cgrid"], + }, + "filters":["*string:~*req.ReaderID:mysqlReaderID"], + "flags": ["*log"], + "fields": [ + {"tag": "SetupTime", "path": "*exp.setup_time", "type": "*constant", "value": "2018-11-27 14:21:26"}, + {"tag": "Account", "path": "*exp.account", "type": "*variable", "value": "~*req.ExtraInfo"}, + {"tag": "ID", "path": "*exp.id", "type": "*variable", "value": "~*req.Id"}, + {"tag": "CGRID", "path": "*exp.cgrid", "type": "*variable", "value": "~*req.CGRID"}, + ] + }] + }, + "ers": { + "enabled": true, + "sessions_conns":["*localhost"], + "ees_conns": ["*localhost"], + "readers": [{ + "id": "mysqlReaderID", + "type": "*sql", + "run_delay": "1m", + "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", + "opts": { + "sqlDBName":"cgrates2", + "sqlTableName":"cdrs", + }, + "start_delay": "500ms", // wait for db to be populated before starting reader + "tenant": "cgrates.org", + "filters": [ + "*gt:~*req.answer_time:-168h", // dont process cdrs with answer_time older than 7 days ago + "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", + "*string:~*vars.*readerID:mysqlReaderID", + "FLTR_VARS", // "*string:~*vars.*readerID:mysqlReaderID", + ], + "flags": ["*dryrun", "*export"], + "fields":[ + {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, + {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, + {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, + {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, + {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, + {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, + {"tag": "ExtraInfo", "path": "*cgreq.ExtraInfo", "type": "*variable", "value": "~*req.extra_info", "mandatory": true}, + {"tag": "ID", "path": "*cgreq.Id", "type": "*variable", "value": "~*req.id", "mandatory": true}, + {"tag": "ReaderID", "path": "*cgreq.ReaderID", "type": "*variable", "value": "~*vars.*readerID", "mandatory": true}, + ], + }, + ], + }, + +} \ No newline at end of file diff --git a/ees/ees.go b/ees/ees.go index e4b093bb4..cfbd8419d 100644 --- a/ees/ees.go +++ b/ees/ees.go @@ -250,6 +250,9 @@ func (eeS *EventExporterS) V1ProcessEvent(ctx *context.Context, cgrEv *engine.CG } go func(evict, sync bool, ee EventExporter) { if err := exportEventWithExporter(ee, exportEvent, evict, eeS.cfg, eeS.filterS); err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> Exporter <%s> error : <%s>", + utils.EEs, ee.Cfg().ID, err.Error())) withErr = true } if sync { diff --git a/ees/sql.go b/ees/sql.go index 8514e78f8..4957927c3 100644 --- a/ees/sql.go +++ b/ees/sql.go @@ -158,11 +158,34 @@ func (sqlEe *SQLEe) Close() (err error) { func (sqlEe *SQLEe) GetMetrics() *utils.SafeMapStorage { return sqlEe.dc } -func (sqlEe *SQLEe) PrepareMap(*utils.CGREvent) (any, error) { return nil, nil } +// Create the sqlPosterRequest used to instert the map into the table +func (sqlEe *SQLEe) PrepareMap(cgrEv *utils.CGREvent) (any, error) { + colNames := make([]string, 0, len(cgrEv.Event)) // slice with all column names to be insterted + vals := make([]any, 0, len(cgrEv.Event)) // slice with all values to be insterted + for colName, value := range cgrEv.Event { + colNames = append(colNames, colName) + vals = append(vals, value) + } + sqlValues := make([]string, len(vals)) // values to be inserted as "?" for the query + for i := range vals { + sqlValues[i] = "?" + } + sqlQuery := fmt.Sprintf("INSERT INTO %s (`%s`) VALUES (%s);", + sqlEe.tableName, + strings.Join(colNames, "`, `"), // back ticks added to include special characters + strings.Join(sqlValues, ","), + ) + return &sqlPosterRequest{ + Querry: sqlQuery, + Values: vals, + }, nil +} func (sqlEe *SQLEe) PrepareOrderMap(mp *utils.OrderedNavigableMap) (any, error) { var vals []any var colNames []string + var whereVars []string // key-value parts of WHERE clause used on UPDATE + var whereVals []any // will hold the values replacing "?" used on WHERE part of UPDATE query for el := mp.GetFirstElement(); el != nil; el = el.Next() { nmIt, _ := mp.Field(el.Value) pathWithoutIndex := strings.Join(el.Value[:len(el.Value)-1], utils.NestingSep) // remove the index path.index @@ -170,21 +193,46 @@ func (sqlEe *SQLEe) PrepareOrderMap(mp *utils.OrderedNavigableMap) (any, error) colNames = append(colNames, pathWithoutIndex) } vals = append(vals, nmIt.Data) + if sqlEe.cfg.Opts.SQL.UpdateIndexedFields != nil { + for _, updateFields := range *sqlEe.cfg.Opts.SQL.UpdateIndexedFields { + if pathWithoutIndex == updateFields { + whereVars = append(whereVars, fmt.Sprintf("%s = ?", updateFields)) + whereVals = append(whereVals, nmIt.Data) + } + } + } } - sqlValues := make([]string, len(vals)) + sqlValues := make([]string, len(vals)+len(whereVals)) for i := range vals { sqlValues[i] = "?" } var sqlQuery string - if len(colNames) != len(vals) { - sqlQuery = fmt.Sprintf("INSERT INTO %s VALUES (%s); ", + if sqlEe.cfg.Opts.SQL.UpdateIndexedFields != nil { + if len(whereVars) == 0 { + return nil, fmt.Errorf("%w: no usable sqlUpdateIndexedFields found <%v>", utils.ErrNotFound, *sqlEe.cfg.Opts.SQL.UpdateIndexedFields) + } + setClauses := []string{} // used in SET part of UPDATE query + for _, col := range colNames { + setClauses = append(setClauses, fmt.Sprintf("%s = ?", col)) + } + sqlQuery = fmt.Sprintf("UPDATE %s SET %s WHERE %s;", sqlEe.tableName, - strings.Join(sqlValues, ",")) + strings.Join(setClauses, ", "), + strings.Join(whereVars, " AND ")) + for _, val := range whereVals { + vals = append(vals, val) + } } else { - sqlQuery = fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s); ", - sqlEe.tableName, - strings.Join(colNames, ", "), - strings.Join(sqlValues, ",")) + if len(colNames) != len(vals) { + sqlQuery = fmt.Sprintf("INSERT INTO %s VALUES (%s); ", + sqlEe.tableName, + strings.Join(sqlValues, ",")) + } else { + sqlQuery = fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s); ", + sqlEe.tableName, + strings.Join(colNames, ", "), + strings.Join(sqlValues, ",")) + } } return &sqlPosterRequest{ Querry: sqlQuery, diff --git a/ees/sql_test.go b/ees/sql_test.go index e87e305a1..9cf378d42 100644 --- a/ees/sql_test.go +++ b/ees/sql_test.go @@ -155,8 +155,12 @@ func TestPrepareMap(t *testing.T) { if err != nil { t.Errorf("PrepareMap() returned an error: %v", err) } - if result != nil { - t.Errorf("PrepareMap() returned a non-nil result: %v", result) + exp := &sqlPosterRequest{ + Querry: "INSERT INTO (``) VALUES ();", + Values: make([]any, 0), + } + if !reflect.DeepEqual(exp, result) { + t.Errorf("Expected <%+v>, Received <%+v>", utils.ToJSON(exp), utils.ToJSON(result)) } } diff --git a/engine/filters.go b/engine/filters.go index 2c17d1386..da0b7193a 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -795,3 +795,150 @@ func (fltr *FilterRule) passHttp(dDP utils.DataProvider) (bool, error) { func (fltr *FilterRule) ElementItems() []string { return strings.Split(fltr.Element, utils.NestingSep) } + +// Creates mysql conditions used in WHERE statement out of filters +func (fltr *FilterRule) FilterToSQLQuery() (conditions []string) { + var firstItem string // Excluding ~*req, hold the first item of an element, left empty if no more than 1 item in element. e.g. "cost_details" out of ~*req.cost_details.Charges[0].RatingID or "" out of ~*req.answer_time + var restOfItems string // Excluding ~*req, hold the rest of the items past the first one. If only 1 item in all element, holds that item. e.g. "Charges[0].RatingID" out of ~*req.cost_details.Charges[0].RatingID or "answer_time" out of ~*req.answer_time + not := strings.HasPrefix(fltr.Type, utils.MetaNot) + elementItems := fltr.ElementItems()[1:] // exclude first item: ~*req + if len(elementItems) > 1 { + firstItem = elementItems[0] + restOfItems = strings.Join(elementItems[1:], utils.NestingSep) + } else { + restOfItems = elementItems[0] + } + + // here are for the filters that their values are empty: *exists, *notexists, *empty, *notempty.. + if len(fltr.Values) == 0 { + switch fltr.Type { + case utils.MetaExists, utils.MetaNotExists: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", restOfItems)) + return + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", firstItem, restOfItems)) + return + } + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s IS NULL", restOfItems)) + return + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", firstItem, restOfItems)) + case utils.MetaEmpty, utils.MetaNotEmpty: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s != ''", restOfItems)) + return + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != ''", firstItem, restOfItems)) + return + } + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s == ''", restOfItems)) + return + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') == ''", firstItem, restOfItems)) + } + return + } + // here are for the filters that can have more than one value: *string, *prefix, *suffix .. + for _, value := range fltr.Values { + switch value { // in case we have boolean values, it should be queried over 1 or 0 + case "true": + value = "1" + case "false": + value = "0" + } + var singleCond string + switch fltr.Type { + case utils.MetaString, utils.MetaNotString, utils.MetaEqual, utils.MetaNotEqual: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s != '%s'", restOfItems, value)) + continue + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != '%s'", + firstItem, restOfItems, value)) + continue + } + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s = '%s'", restOfItems, value) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') = '%s'", firstItem, restOfItems, value) + } + case utils.MetaLessThan, utils.MetaLessOrEqual, utils.MetaGreaterThan, utils.MetaGreaterOrEqual: + parsedValAny := utils.StringToInterface(value) + if fltr.Type == utils.MetaGreaterOrEqual { + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s >= '%v'", restOfItems, parsedValAny) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') >= '%v'", firstItem, restOfItems, parsedValAny) + } + } else if fltr.Type == utils.MetaGreaterThan { + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s > '%v'", restOfItems, parsedValAny) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') > '%v'", firstItem, restOfItems, parsedValAny) + } + } else if fltr.Type == utils.MetaLessOrEqual { + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s <= '%v'", restOfItems, parsedValAny) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') <= '%v'", firstItem, restOfItems, parsedValAny) + } + } else if fltr.Type == utils.MetaLessThan { + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s < '%v'", restOfItems, parsedValAny) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') < '%v'", firstItem, restOfItems, parsedValAny) + } + } + case utils.MetaPrefix, utils.MetaNotPrefix: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%s%%'", restOfItems, value)) + continue + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%s%%'", firstItem, restOfItems, value)) + continue + } + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s LIKE '%s%%'", restOfItems, value) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%s%%'", firstItem, restOfItems, value) + } + case utils.MetaSuffix, utils.MetaNotSuffix: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%%%s'", restOfItems, value)) + continue + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%%%s'", firstItem, restOfItems, value)) + continue + } + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s LIKE '%%%s'", restOfItems, value) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%%%s'", firstItem, restOfItems, value) + } + case utils.MetaRegex, utils.MetaNotRegex: + if not { + if firstItem == utils.EmptyString { + conditions = append(conditions, fmt.Sprintf("%s NOT REGEXP '%s'", restOfItems, value)) + continue + } + conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT REGEXP '%s'", firstItem, restOfItems, value)) + continue + } + if firstItem == utils.EmptyString { + singleCond = fmt.Sprintf("%s REGEXP '%s'", restOfItems, value) + } else { + singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') REGEXP '%s'", firstItem, restOfItems, value) + } + } + conditions = append(conditions, singleCond) + } + return +} diff --git a/engine/filters_test.go b/engine/filters_test.go index 7f63b2ee1..9b369aabf 100644 --- a/engine/filters_test.go +++ b/engine/filters_test.go @@ -3115,3 +3115,294 @@ func TestNewFilterRule(t *testing.T) { } }) } + +func TestFilterToSQLQuery(t *testing.T) { + tests := []struct { + name string + fltrRule FilterRule + expected []string + }{ + {"MetaEqual with values", FilterRule{Type: utils.MetaEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"RatingID2"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') = 'RatingID2'"}}, + + {"MetaExists with no values", FilterRule{Type: utils.MetaExists, Element: "~*req.answer_time", Values: nil}, []string{"answer_time IS NULL"}}, + + {"MetaExists with JSON field", FilterRule{Type: utils.MetaExists, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NULL"}}, + + {"MetaNotExists with no values", FilterRule{Type: utils.MetaNotExists, Element: "~*req.answer_time", Values: nil}, []string{"answer_time IS NOT NULL"}}, + + {"MetaNotExists with JSON field", FilterRule{Type: utils.MetaNotExists, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NOT NULL"}}, + + {"MetaString with values", FilterRule{Type: utils.MetaString, Element: "~*req.answer_time", Values: []string{"value1", "value2"}}, []string{"answer_time = 'value1'", "answer_time = 'value2'"}}, + + {"MetaNotString with values", FilterRule{Type: utils.MetaNotString, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"value1"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != 'value1'"}}, + + {"MetaEmpty with no values", FilterRule{Type: utils.MetaEmpty, Element: "~*req.answer_time", Values: nil}, []string{"answer_time == ''"}}, + + {"MetaEmpty with JSON field", FilterRule{Type: utils.MetaEmpty, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') == ''"}}, + + {"MetaNotEmpty with no values", FilterRule{Type: utils.MetaNotEmpty, Element: "~*req.answer_time", Values: nil}, []string{"answer_time != ''"}}, + + {"MetaNotEmpty with JSON field", FilterRule{Type: utils.MetaNotEmpty, Element: "~*req.cost_details.Charges[0].RatingID", Values: nil}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != ''"}}, + + {"MetaGreaterOrEqual with values", FilterRule{Type: utils.MetaGreaterOrEqual, Element: "~*req.answer_time", Values: []string{"10"}}, []string{"answer_time >= '10'"}}, + + {"MetaGreaterThan with values", FilterRule{Type: utils.MetaGreaterThan, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"20"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') > '20'"}}, + + {"MetaLessThan with values", FilterRule{Type: utils.MetaLessThan, Element: "~*req.answer_time", Values: []string{"5"}}, []string{"answer_time < '5'"}}, + + {"MetaLessOrEqual with values", FilterRule{Type: utils.MetaLessOrEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"15"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') <= '15'"}}, + + {"MetaPrefix with values", FilterRule{Type: utils.MetaPrefix, Element: "~*req.answer_time", Values: []string{"pre"}}, []string{"answer_time LIKE 'pre%'"}}, + + {"MetaNotPrefix with values", FilterRule{Type: utils.MetaNotPrefix, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"pre"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE 'pre%'"}}, + + {"MetaSuffix with values", FilterRule{Type: utils.MetaSuffix, Element: "~*req.answer_time", Values: []string{"suf"}}, []string{"answer_time LIKE '%suf'"}}, + + {"MetaNotSuffix with values", FilterRule{Type: utils.MetaNotSuffix, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"suf"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE '%suf'"}}, + + {"MetaGreaterOrEqual with JSON field", FilterRule{Type: utils.MetaGreaterOrEqual, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"100"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') >= '100'"}}, + + {"MetaRegex with values", FilterRule{Type: utils.MetaRegex, Element: "~*req.answer_time", Values: []string{"pattern1", "pattern2"}}, []string{"answer_time REGEXP 'pattern1'", "answer_time REGEXP 'pattern2'"}}, + + {"MetaNotRegex with values", FilterRule{Type: utils.MetaNotRegex, Element: "~*req.cost_details.Charges[0].RatingID", Values: []string{"pattern"}}, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT REGEXP 'pattern'"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.fltrRule.FilterToSQLQuery() + if len(got) != len(tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, got) + return + } + for i, cond := range got { + if cond != tt.expected[i] { + t.Errorf("expected %v, got %v", tt.expected[i], cond) + } + } + }) + } +} + +func TestFilterToSQLQueryValidations(t *testing.T) { + tests := []struct { + name string + fltrRule FilterRule + expected []string + }{ + { + name: "Boolean true value", + fltrRule: FilterRule{ + Type: utils.MetaString, + Element: "~*req.active", + Values: []string{"true"}, + }, + expected: []string{"active = '1'"}, + }, + { + name: "Boolean false value", + fltrRule: FilterRule{ + Type: utils.MetaString, + Element: "~*req.active", + Values: []string{"false"}, + }, + expected: []string{"active = '0'"}, + }, + { + name: "Greater than or equal with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaGreaterOrEqual, + Element: "~*req.score", + Values: []string{"10"}, + }, + expected: []string{"score >= '10'"}, + }, + { + name: "Greater than with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaGreaterThan, + Element: "~*req.score", + Values: []string{"20"}, + }, + expected: []string{"score > '20'"}, + }, + { + name: "Less than or equal with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaLessOrEqual, + Element: "~*req.score", + Values: []string{"30"}, + }, + expected: []string{"score <= '30'"}, + }, + { + name: "Less than with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaLessThan, + Element: "~*req.score", + Values: []string{"40"}, + }, + expected: []string{"score < '40'"}, + }, + { + name: "Prefix NOT LIKE with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaNotPrefix, + Element: "~*req.name", + Values: []string{"prefix"}, + }, + expected: []string{"name NOT LIKE 'prefix%'"}, + }, + { + name: "Prefix LIKE with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaPrefix, + Element: "~*req.data.name", + Values: []string{"prefix"}, + }, + expected: []string{"JSON_VALUE(data, '$.name') LIKE 'prefix%'"}, + }, + { + name: "Suffix NOT LIKE with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaNotSuffix, + Element: "~*req.name", + Values: []string{"suffix"}, + }, + expected: []string{"name NOT LIKE '%suffix'"}, + }, + { + name: "Suffix LIKE with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaSuffix, + Element: "~*req.data.name", + Values: []string{"suffix"}, + }, + expected: []string{"JSON_VALUE(data, '$.name') LIKE '%suffix'"}, + }, + { + name: "Regex NOT REGEXP with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaNotRegex, + Element: "~*req.pattern", + Values: []string{"[a-z]+"}, + }, + expected: []string{"pattern NOT REGEXP '[a-z]+'"}, + }, + { + name: "Regex REGEXP with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaRegex, + Element: "~*req.data.pattern", + Values: []string{"[0-9]+"}, + }, + expected: []string{"JSON_VALUE(data, '$.pattern') REGEXP '[0-9]+'"}, + }, + + { + name: "Not equal with empty beforeSep", + fltrRule: FilterRule{ + Type: utils.MetaNotString, + Element: "~*req.status", + Values: []string{"inactive"}, + }, + expected: []string{"status != 'inactive'"}, + }, + { + name: "Equal condition with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaString, + Element: "~*req.data.status", + Values: []string{"active"}, + }, + expected: []string{"JSON_VALUE(data, '$.status') = 'active'"}, + }, + { + name: "Greater than condition with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaGreaterThan, + Element: "~*req.data.score", + Values: []string{"50"}, + }, + expected: []string{"JSON_VALUE(data, '$.score') > '50'"}, + }, + { + name: "Less than or equal condition with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaLessOrEqual, + Element: "~*req.data.score", + Values: []string{"30"}, + }, + expected: []string{"JSON_VALUE(data, '$.score') <= '30'"}, + }, + { + name: "Less than condition with JSON_VALUE", + fltrRule: FilterRule{ + Type: utils.MetaLessThan, + Element: "~*req.data.score", + Values: []string{"20"}, + }, + expected: []string{"JSON_VALUE(data, '$.score') < '20'"}, + }, + + { + name: "MetaExists with no values", + fltrRule: FilterRule{ + Type: utils.MetaExists, + Element: "~*req.column1", + Values: nil, + }, + expected: []string{"column1 IS NULL"}, + }, + { + name: "MetaNotExists with no values", + fltrRule: FilterRule{ + Type: utils.MetaNotExists, + Element: "~*req.json_field.key", + Values: nil, + }, + expected: []string{"JSON_VALUE(json_field, '$.key') IS NOT NULL"}, + }, + { + name: "MetaString with values", + fltrRule: FilterRule{ + Type: utils.MetaString, + Element: "~*req.column2", + Values: []string{"value1", "value2"}, + }, + expected: []string{"column2 = 'value1'", "column2 = 'value2'"}, + }, + { + name: "MetaPrefix with NOT condition", + fltrRule: FilterRule{ + Type: utils.MetaNotPrefix, + Element: "~*req.json_field.key", + Values: []string{"prefix1"}, + }, + expected: []string{"JSON_VALUE(json_field, '$.key') NOT LIKE 'prefix1%'"}, + }, + { + name: "MetaRegex with multiple values", + fltrRule: FilterRule{ + Type: utils.MetaRegex, + Element: "~*req.column3", + Values: []string{"pattern1", "pattern2"}, + }, + expected: []string{"column3 REGEXP 'pattern1'", "column3 REGEXP 'pattern2'"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.fltrRule.FilterToSQLQuery() + if len(got) != len(tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, got) + return + } + for i, cond := range got { + if cond != tt.expected[i] { + t.Errorf("expected %v, got %v", tt.expected[i], cond) + } + } + }) + } +} diff --git a/engine/libtest.go b/engine/libtest.go index 31d07842a..485b4f79b 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -31,6 +31,7 @@ import ( "path/filepath" "slices" "strings" + "syscall" "testing" "time" @@ -332,14 +333,15 @@ func NewRPCClient(t testing.TB, cfg *config.ListenCfg) *birpc.Client { // TestEngine holds the setup parameters and configurations // required for running integration tests. type TestEngine struct { - ConfigPath string // path to the main configuration file - ConfigJSON string // JSON cfg content (standalone/overwrites static configs) - DBCfg DBCfg // custom db settings for dynamic setup (overrides static config) - LogBuffer io.Writer // captures log output of the test environment - PreserveDataDB bool // prevents automatic data_db flush when set - PreserveStorDB bool // prevents automatic stor_db flush when set - TpPath string // path to the tariff plans - TpFiles map[string]string // CSV data for tariff plans: filename -> content + ConfigPath string // path to the main configuration file + ConfigJSON string // JSON cfg content (standalone/overwrites static configs) + DBCfg DBCfg // custom db settings for dynamic setup (overrides static config) + LogBuffer io.Writer // captures log output of the test environment + PreserveDataDB bool // prevents automatic data_db flush when set + PreserveStorDB bool // prevents automatic stor_db flush when set + TpPath string // path to the tariff plans + TpFiles map[string]string // CSV data for tariff plans: filename -> content + GracefulShutdown bool // shutdown the engine gracefuly, otherwise use process.Kill // PreStartHook executes custom logic relying on CGRConfig // before starting cgr-engine. @@ -356,7 +358,7 @@ func (ng TestEngine) Run(t testing.TB, extraFlags ...string) (*birpc.Client, *co if ng.PreStartHook != nil { ng.PreStartHook(t, cfg) } - startEngine(t, cfg, ng.LogBuffer) + startEngine(t, cfg, ng.LogBuffer, ng.GracefulShutdown) client := NewRPCClient(t, cfg.ListenCfg()) LoadCSVs(t, client, ng.TpPath, ng.TpFiles) return client, cfg @@ -542,7 +544,7 @@ func FlushDBs(t testing.TB, cfg *config.CGRConfig, flushDataDB, flushStorDB bool // startEngine starts the CGR engine process with the provided configuration. It writes engine logs to the // provided logBuffer (if any). -func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, extraFlags ...string) { +func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, gracefulShutdown bool, extraFlags ...string) { t.Helper() binPath, err := exec.LookPath("cgr-engine") if err != nil { @@ -564,8 +566,17 @@ func startEngine(t testing.TB, cfg *config.CGRConfig, logBuffer io.Writer, extra t.Fatalf("cgr-engine command failed: %v", err) } t.Cleanup(func() { - if err := engine.Process.Kill(); err != nil { - t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err) + if gracefulShutdown { + if err := engine.Process.Signal(syscall.SIGTERM); err != nil { + t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err) + } + if err := engine.Wait(); err != nil { + t.Errorf("cgr-engine process failed to exit cleanly: %v", err) + } + } else { + if err := engine.Process.Kill(); err != nil { + t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err) + } } }) backoff := utils.FibDuration(time.Millisecond, 0) diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 6dcf8465f..d3d08f3bf 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -45,7 +45,7 @@ func NewMySQLStorage(host, port, name, user, password string, if mySQLStorage.Db, err = db.DB(); err != nil { return nil, err } - if mySQLStorage.Db.Ping(); err != nil { + if err := mySQLStorage.Db.Ping(); err != nil { return nil, err } mySQLStorage.Db.SetMaxIdleConns(maxIdleConn) diff --git a/ers/ers.go b/ers/ers.go index 1ab27897e..d4d218136 100644 --- a/ers/ers.go +++ b/ers/ers.go @@ -41,6 +41,7 @@ import ( // erEvent is passed from reader to ERs type erEvent struct { + rawEvent map[string]any cgrEvent *utils.CGREvent rdrCfg *config.EventReaderCfg } @@ -152,7 +153,7 @@ func (erS *ERService) ListenAndServe(stopChan, cfgRldChan chan struct{}) error { fmt.Sprintf("<%s> reading event: <%s> from reader: <%s> got error: <%v>", utils.ERs, utils.ToJSON(erEv.cgrEvent), erEv.rdrCfg.ID, err)) } - if err = erS.exportEvent(erEv, err != nil); err != nil { + if err = erS.exportRawEvent(erEv, err != nil); err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> exporting event: <%s> from reader: <%s> got error: <%v>", utils.ERs, utils.ToJSON(erEv.cgrEvent), erEv.rdrCfg.ID, err)) @@ -166,7 +167,7 @@ func (erS *ERService) ListenAndServe(stopChan, cfgRldChan chan struct{}) error { fmt.Sprintf("<%s> reading partial event: <%s> from reader: <%s> got error: <%v>", utils.ERs, utils.ToJSON(pEv.cgrEvent), pEv.rdrCfg.ID, err)) } - if err = erS.exportEvent(pEv, err != nil); err != nil { + if err = erS.exportRawEvent(pEv, err != nil); err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> exporting partial event: <%s> from reader: <%s> got error: <%v>", utils.ERs, utils.ToJSON(pEv.cgrEvent), pEv.rdrCfg.ID, err)) @@ -244,7 +245,7 @@ func (erS *ERService) processEvent(cgrEv *utils.CGREvent, utils.MetaDryRun, utils.MetaAuthorize, utils.MetaInitiate, utils.MetaUpdate, utils.MetaTerminate, utils.MetaMessage, - utils.MetaCDRs, utils.MetaEvent, utils.MetaNone} { + utils.MetaCDRs, utils.MetaEvent, utils.MetaNone, utils.MetaExport} { if rdrCfg.Flags.Has(typ) { // request type is identified through flags reqType = typ break @@ -359,6 +360,7 @@ func (erS *ERService) processEvent(cgrEv *utils.CGREvent, err = erS.connMgr.Call(context.TODO(), erS.cfg.ERsCfg().SessionSConns, utils.SessionSv1ProcessEvent, evArgs, rply) case utils.MetaCDRs: // allow CDR processing + case utils.MetaExport: // allow event exporting } if err != nil { return @@ -370,6 +372,13 @@ func (erS *ERService) processEvent(cgrEv *utils.CGREvent, err = erS.connMgr.Call(context.TODO(), erS.cfg.ERsCfg().SessionSConns, utils.SessionSv1ProcessCDR, cgrEv, rplyCDRs) } + if rdrCfg.Flags.Has(utils.MetaExport) { + var reply map[string]map[string]any + return erS.connMgr.Call(context.TODO(), erS.cfg.ERsCfg().EEsConns, utils.EeSv1ProcessEvent, + &engine.CGREventWithEeIDs{ + CGREvent: cgrEv, + }, &reply) + } return } @@ -593,9 +602,9 @@ func (erS *ERService) onEvicted(id string, value any) { } -// exportEvent exports the given event. If the processing of the event failed, +// exportRawEvent exports the given event. If the processing of the event failed, // it uses ees_failed_ids; otherwise, it uses ees_success_ids. -func (erS *ERService) exportEvent(event *erEvent, processingFailed bool) error { +func (erS *ERService) exportRawEvent(event *erEvent, processingFailed bool) error { var exporterIDs []string if processingFailed { if len(event.rdrCfg.EEsFailedIDs) == 0 { @@ -608,11 +617,13 @@ func (erS *ERService) exportEvent(event *erEvent, processingFailed bool) error { } exporterIDs = event.rdrCfg.EEsSuccessIDs } - var reply map[string]map[string]any return erS.connMgr.Call(context.TODO(), erS.cfg.ERsCfg().EEsConns, utils.EeSv1ProcessEvent, &engine.CGREventWithEeIDs{ - EeIDs: exporterIDs, - CGREvent: event.cgrEvent, + EeIDs: exporterIDs, + CGREvent: &utils.CGREvent{ + Tenant: erS.cfg.GeneralCfg().DefaultTenant, + Event: event.rawEvent, + }, }, &reply) } diff --git a/ers/nats_it_test.go b/ers/nats_it_test.go index d0b302525..cf6213deb 100644 --- a/ers/nats_it_test.go +++ b/ers/nats_it_test.go @@ -150,8 +150,7 @@ var natsCfg string = `{ "id": "nats_reader", "type": "*nats_json_map", "source_path": "%s", - "ees_success_ids": ["nats_processed"], - "flags": ["*dryrun"], + "flags": ["*dryrun", "*export"], "opts": { %s }, diff --git a/ers/sql.go b/ers/sql.go index 215632cca..0a26bd0c6 100644 --- a/ers/sql.go +++ b/ers/sql.go @@ -148,33 +148,24 @@ func (rdr *SQLEventReader) readLoop(db *gorm.DB, sqlDB io.Closer) { for _, filterObj := range filtersObjList { // seperate filters used for WHERE clause from other filters, and build query conditions out of them var lazyFltrPopulated bool // Track if a lazyFilter is already populated by the previous filterObj.Rules, so we dont store the same lazy filter more than once for _, rule := range filterObj.Rules { - var firstItem string // Excluding ~*req, hold the first item of an element, left empty if no more than 1 item in element. e.g. "cost_details" out of ~*req.cost_details.Charges[0].RatingID or "" out of ~*req.answer_time - var restOfItems string // Excluding ~*req, hold the rest of the items past the first one. If only 1 item in all element, holds that item. e.g. "Charges[0].RatingID" out of ~*req.cost_details.Charges[0].RatingID or "answer_time" out of ~*req.answer_time - switch { - case strings.HasPrefix(rule.Element, utils.MetaDynReq+utils.NestingSep): // convert filter to WHERE condition only on filters with ~*req. - elementItems := rule.ElementItems()[1:] // exclude first item: ~*req - if len(elementItems) > 1 { - firstItem = elementItems[0] - restOfItems = strings.Join(elementItems[1:], utils.NestingSep) - } else { - restOfItems = elementItems[0] - } - default: // If not used in the WHERE condition, put the filter in rdr.lazyFilters - if !lazyFltrPopulated { - rdr.lazyFilters = append(rdr.lazyFilters, filterObj.ID) - lazyFltrPopulated = true - } + if strings.HasPrefix(rule.Element, utils.MetaDynReq+utils.NestingSep) { // convert filter to WHERE condition only on filters with ~*req. + rdr.dbFilters = append(rdr.dbFilters, strings.Join(rule.FilterToSQLQuery(), " OR ")) continue } - conditions := utils.FilterToSQLQuery(rule.Type, firstItem, restOfItems, rule.Values, strings.HasPrefix(rule.Type, utils.MetaNot)) - rdr.dbFilters = append(rdr.dbFilters, strings.Join(conditions, " OR ")) + // If not used in the WHERE condition, put the filter in rdr.lazyFilters + if !lazyFltrPopulated { + rdr.lazyFilters = append(rdr.lazyFilters, filterObj.ID) + lazyFltrPopulated = true + } } } - tm := time.NewTimer(0) // Timer matching rdr.Config().RunDelay, will delay the for loop until timer expires. It doesnt wait for the loop to finish an iteration to start. + selectWhereQuery := strings.Join(rdr.dbFilters, " AND ") // the whole WHERE query gotten from filters + tm := time.NewTimer(0) // Timer matching rdr.Config().RunDelay, will delay the for loop until timer expires. It doesnt wait for the loop to finish an iteration to start. for { - tx := db.Table(rdr.tableName).Select(utils.Meta) // Select everything from the table - for _, whereQ := range rdr.dbFilters { - tx = tx.Where(whereQ) // apply WHERE conditions to the select if any + tx := db.Table(rdr.tableName).Select(utils.Meta) // Select everything from the table + if err := tx.Where(selectWhereQuery).Error; err != nil { // apply WHERE conditions to the select if any + rdr.rdrErr <- err + return } rows, err := tx.Rows() // get all rows selected if err != nil { @@ -220,23 +211,24 @@ func (rdr *SQLEventReader) readLoop(db *gorm.DB, sqlDB io.Closer) { for i, colName := range colNames { // populate ev from columns ev[colName] = columns[i] } + sqlClauseVars := make(map[string]any) // map used for conditioning queries used for marking processed events if rdr.Config().ProcessedPath == utils.MetaDelete { - sqlWhereVars := make(map[string]any) // map used for conditioning the DELETE query if rdr.Config().Opts.SQL.DeleteIndexedFields != nil { for _, fieldName := range *rdr.Config().Opts.SQL.DeleteIndexedFields { - if _, has := ev[fieldName]; has && fieldName != createdAt && fieldName != updatedAt && fieldName != deletedAt { // ignore the sql colums for filter only - addValidFieldToSQLWHEREVars(sqlWhereVars, fieldName, ev[fieldName]) + if _, has := ev[fieldName]; has && fieldName != createdAt && + fieldName != updatedAt && fieldName != deletedAt { // ignore the sql colums for sqlWhereVars only + addValidFieldToSQLWhereVars(sqlClauseVars, fieldName, ev[fieldName]) } } } - if len(sqlWhereVars) == 0 { + if len(sqlClauseVars) == 0 { for i, colName := range colNames { - if colName != createdAt && colName != updatedAt && colName != deletedAt { // ignore the sql colums for filter only - addValidFieldToSQLWHEREVars(sqlWhereVars, colName, columns[i]) + if colName != createdAt && colName != updatedAt && colName != deletedAt { // ignore the sql colums for sqlWhereVars only + addValidFieldToSQLWhereVars(sqlClauseVars, colName, columns[i]) } } } - if err = tx.Delete(nil, sqlWhereVars).Error; err != nil { // to ensure we don't read it again + if err = tx.Delete(nil, sqlClauseVars).Error; err != nil { // to ensure we don't read it again utils.Logger.Warning( fmt.Sprintf("<%s> deleting message %s error: %s", utils.ERs, utils.ToJSON(ev), err.Error())) @@ -245,7 +237,6 @@ func (rdr *SQLEventReader) readLoop(db *gorm.DB, sqlDB io.Closer) { return } } - go func(ev map[string]any) { if err := rdr.processMessage(ev); err != nil { utils.Logger.Warning( @@ -272,7 +263,7 @@ func (rdr *SQLEventReader) readLoop(db *gorm.DB, sqlDB io.Closer) { } // Helper function to add valid time and non-time values to the sqlWhereVars map -func addValidFieldToSQLWHEREVars(sqlWhereVars map[string]any, fieldName string, value any) { +func addValidFieldToSQLWhereVars(sqlWhereVars map[string]any, fieldName string, value any) { switch dateTimeCol := value.(type) { case time.Time: if dateTimeCol.IsZero() { @@ -291,10 +282,10 @@ func addValidFieldToSQLWHEREVars(sqlWhereVars map[string]any, fieldName string, } } -func (rdr *SQLEventReader) processMessage(msg map[string]any) (err error) { +func (rdr *SQLEventReader) processMessage(ev map[string]any) (err error) { reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.MetaReaderID: utils.NewLeafNode(rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].ID)}} agReq := agents.NewAgentRequest( - utils.MapStorage(msg), reqVars, + utils.MapStorage(ev), reqVars, nil, nil, nil, rdr.Config().Tenant, rdr.cgrCfg.GeneralCfg().DefaultTenant, utils.FirstNonEmpty(rdr.Config().Timezone, @@ -313,8 +304,19 @@ func (rdr *SQLEventReader) processMessage(msg map[string]any) (err error) { if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } + rawEvent := make(map[string]any, len(ev)) + if len(rdr.Config().EEsSuccessIDs) != 0 { + for key, value := range ev { + if val, ok := value.([]uint8); ok { // convert byte values to string to comply with the INSERT INTO query on SQL exporter. Converted before sent to rpc call to avoid unnecesary decoding and converting on SQL exporter side + rawEvent[key] = string(val) + continue + } + rawEvent[key] = value + } + } rdrEv <- &erEvent{ cgrEvent: cgrEv, + rawEvent: rawEvent, rdrCfg: rdr.Config(), } return diff --git a/general_tests/ers_sql_filters_it_test.go b/general_tests/ers_sql_filters_it_test.go index 53217da86..24fb47d9c 100644 --- a/general_tests/ers_sql_filters_it_test.go +++ b/general_tests/ers_sql_filters_it_test.go @@ -24,6 +24,7 @@ import ( "bufio" "bytes" "fmt" + "path" "strings" "testing" "time" @@ -36,7 +37,6 @@ import ( ) var ( - db *gorm.DB dbConnString = "cgrates:CGRateS.org@tcp(127.0.0.1:3306)/%s?charset=utf8&loc=Local&parseTime=true&sql_mode='ALLOW_INVALID_DATES'" timeStart = time.Now() cdr1 = &engine.CDR{ // sample with values not realisticy calculated @@ -88,7 +88,7 @@ var ( }, AccountSummary: &engine.AccountSummary{ Tenant: "cgrates.org", - ID: "testV1CDRsRefundOutOfSessionCost", + ID: "1001", BalanceSummaries: []*engine.BalanceSummary{ { UUID: "uuid1", @@ -110,12 +110,12 @@ var ( }, Accounting: engine.Accounting{ "a012888": &engine.BalanceCharge{ - AccountID: "cgrates.org:testV1CDRsRefundOutOfSessionCost", + AccountID: "cgrates.org:1001", BalanceUUID: "uuid1", Units: 120.7, }, "44d6c02": &engine.BalanceCharge{ - AccountID: "cgrates.org:testV1CDRsRefundOutOfSessionCost", + AccountID: "cgrates.org:1001", BalanceUUID: "uuid1", Units: 120.7, }, @@ -182,7 +182,7 @@ var ( }, AccountSummary: &engine.AccountSummary{ Tenant: "cgrates.org", - ID: "testV1CDRsRefundOutOfSessionCost", + ID: "1001", BalanceSummaries: []*engine.BalanceSummary{ { UUID: "uuid1", @@ -204,12 +204,12 @@ var ( }, Accounting: engine.Accounting{ "a012888": &engine.BalanceCharge{ - AccountID: "cgrates.org:testV1CDRsRefundOutOfSessionCost", + AccountID: "cgrates.org:1001", BalanceUUID: "uuid1", Units: 120.7, }, "44d6c02": &engine.BalanceCharge{ - AccountID: "cgrates.org:testV1CDRsRefundOutOfSessionCost", + AccountID: "cgrates.org:1001", BalanceUUID: "uuid1", Units: 120.7, }, @@ -267,54 +267,31 @@ func TestERSSQLFilters(t *testing.T) { t.Fatal("unsupported dbtype value") } + var db, cdb2 *gorm.DB t.Run("InitSQLDB", func(t *testing.T) { var err error - var db2 *gorm.DB - if db2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), &gorm.Config{ AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), }); err != nil { t.Fatal(err) } - if err = db2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { t.Fatal(err) } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather }) - type testModelSql struct { - ID int64 - Cgrid string - RunID string - OriginHost string - Source string - OriginID string - ToR string - RequestType string - Tenant string - Category string - Account string - Subject string - Destination string - SetupTime time.Time - AnswerTime time.Time - Usage int64 - ExtraFields string - CostSource string - Cost float64 - CostDetails string - ExtraInfo string - CreatedAt time.Time - UpdatedAt time.Time - DeletedAt *time.Time - } t.Run("PutCDRsInDataBase", func(t *testing.T) { var err error if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), &gorm.Config{ AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), }); err != nil { t.Fatal(err) } @@ -369,64 +346,13 @@ func TestERSSQLFilters(t *testing.T) { } else if err = db2.Close(); err != nil { t.Fatal(err) } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } }) - content := `{ - - "general": { - "log_level": 7 - }, - - "apiers": { - "enabled": true - }, - "filters": { - "apiers_conns": ["*localhost"] - }, - "ers": { - "enabled": true, - "sessions_conns":["*localhost"], - "readers": [ - { - "id": "mysql", - "type": "*sql", - "run_delay": "1m", - "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", - "opts": { - "sqlDBName":"cgrates2", - "sqlDeleteIndexedFields": ["id"], - }, - "start_delay": "500ms", // wait for db to be populated before starting reader - "processed_path": "*delete", - "tenant": "cgrates.org", - "filters": [ - "*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days) - "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", - "*string:~*vars.*readerID:mysql", - "FLTR_VARS", // "*string:~*vars.*readerID:mysql", - ], - "flags": ["*dryrun"], - "fields":[ - {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, - {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, - {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, - {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, - ], - }, - ], - }, - - }` - tpFiles := map[string]string{ utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, @@ -435,10 +361,11 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, buf := &bytes.Buffer{} ng := engine.TestEngine{ - ConfigJSON: content, - DBCfg: dbcfg, - TpFiles: tpFiles, - LogBuffer: buf, + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_filters"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, } ng.Run(t) time.Sleep(1 * time.Second) @@ -448,7 +375,7 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, records := 0 scanner := bufio.NewScanner(strings.NewReader(buf.String())) timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") - expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) var ersLogsCount int for scanner.Scan() { line := scanner.Text() @@ -457,7 +384,170 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, } records++ if !strings.Contains(line, expectedLog) { - t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line) + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) + } + if strings.Contains(line, "[INFO] DRYRUN") { + ersLogsCount++ + } + } + if err := scanner.Err(); err != nil { + t.Errorf("error reading input: %v", err) + } + if records != 1 { + t.Errorf("expected ERs to process 1 records, but it processed %d records", records) + } + if ersLogsCount != 1 { + t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount) + } + }) + + t.Run("VerifyRowsNotDeleted", func(t *testing.T) { + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected 3 rows in table, got: ", result) + } + var rslt []map[string]interface{} + if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil { + t.Fatalf("failed to query table: %v", err) + } + }) +} + +func TestERSSQLFiltersDeleteIndexedFields(t *testing.T) { + var dbcfg engine.DBCfg + switch *utils.DBType { + case utils.MetaInternal: + t.SkipNow() + case utils.MetaMySQL: + case utils.MetaMongo: + dbcfg = engine.MongoDBCfg + case utils.MetaPostgres: + dbcfg = engine.PostgresDBCfg + default: + t.Fatal("unsupported dbtype value") + } + + var db, cdb2 *gorm.DB + t.Run("InitSQLDB", func(t *testing.T) { + var err error + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + t.Fatal(err) + } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather + + }) + + t.Run("PutCDRsInDataBase", func(t *testing.T) { + var err error + if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + tx := db.Begin() + if !tx.Migrator().HasTable("cdrs") { + if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + tx.Commit() + tx = db.Begin() + tx = tx.Table(utils.CDRsTBL) + cdrSql := cdr1.AsCDRsql() + cdrSql2 := cdr2.AsCDRsql() + cdrsql3 := cdr3.AsCDRsql() + cdrSql.CreatedAt = time.Now() + cdrSql2.CreatedAt = time.Now() + cdrsql3.CreatedAt = time.Now() + saved := tx.Save(cdrSql) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrSql2) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrsql3) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + tx.Commit() + time.Sleep(10 * time.Millisecond) + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected table to have 3 results but got ", result) + } + }) + defer t.Run("StopSQL", func(t *testing.T) { + if err := db.Migrator().DropTable("cdrs"); err != nil { + t.Fatal(err) + } + if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil { + t.Fatal(err) + } + if db2, err := db.DB(); err != nil { + t.Fatal(err) + } else if err = db2.Close(); err != nil { + t.Fatal(err) + } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } + + }) + + tpFiles := map[string]string{ + utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] +cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, +cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, + } + + buf := &bytes.Buffer{} + ng := engine.TestEngine{ + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_delete_indexed_fields"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, + } + ng.Run(t) + time.Sleep(1 * time.Second) + + t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) { + time.Sleep(100 * time.Millisecond) // give enough time to process from sql table + records := 0 + scanner := bufio.NewScanner(strings.NewReader(buf.String())) + timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + var ersLogsCount int + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, " DRYRUN, reader: ") { + continue + } + records++ + if !strings.Contains(line, expectedLog) { + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) } if strings.Contains(line, "[INFO] DRYRUN") { ersLogsCount++ @@ -489,246 +579,12 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, for _, row := range rslt { for col, value := range row { if strings.Contains(fmt.Sprintln(value), "RatingID2") { - t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%s>, value <%s>", col, value) + t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%q>, value <%q>", col, value) } } } }) } - -func TestERSSQLFiltersWithoutDelete(t *testing.T) { - var dbcfg engine.DBCfg - switch *utils.DBType { - case utils.MetaInternal: - t.SkipNow() - case utils.MetaMySQL: - case utils.MetaMongo: - dbcfg = engine.MongoDBCfg - case utils.MetaPostgres: - dbcfg = engine.PostgresDBCfg - default: - t.Fatal("unsupported dbtype value") - } - - t.Run("InitSQLDB", func(t *testing.T) { - var err error - var db2 *gorm.DB - if db2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), - &gorm.Config{ - AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), - }); err != nil { - t.Fatal(err) - } - - if err = db2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { - t.Fatal(err) - } - }) - - type testModelSql struct { - ID int64 - Cgrid string - RunID string - OriginHost string - Source string - OriginID string - ToR string - RequestType string - Tenant string - Category string - Account string - Subject string - Destination string - SetupTime time.Time - AnswerTime time.Time - Usage int64 - ExtraFields string - CostSource string - Cost float64 - CostDetails string - ExtraInfo string - CreatedAt time.Time - UpdatedAt time.Time - DeletedAt *time.Time - } - t.Run("PutCDRsInDataBase", func(t *testing.T) { - var err error - if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), - &gorm.Config{ - AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), - }); err != nil { - t.Fatal(err) - } - tx := db.Begin() - if !tx.Migrator().HasTable("cdrs") { - if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil { - tx.Rollback() - t.Fatal(err) - } - } - tx.Commit() - tx = db.Begin() - tx = tx.Table(utils.CDRsTBL) - cdrSql := cdr1.AsCDRsql() - cdrSql2 := cdr2.AsCDRsql() - cdrsql3 := cdr3.AsCDRsql() - cdrSql.CreatedAt = time.Now() - cdrSql2.CreatedAt = time.Now() - cdrsql3.CreatedAt = time.Now() - saved := tx.Save(cdrSql) - if saved.Error != nil { - tx.Rollback() - t.Fatal(err) - } - saved = tx.Save(cdrSql2) - if saved.Error != nil { - tx.Rollback() - t.Fatal(err) - } - saved = tx.Save(cdrsql3) - if saved.Error != nil { - tx.Rollback() - t.Fatal(err) - } - tx.Commit() - time.Sleep(10 * time.Millisecond) - var result int64 - db.Table(utils.CDRsTBL).Count(&result) - if result != 3 { - t.Error("Expected table to have 3 results but got ", result) - } - }) - defer t.Run("StopSQL", func(t *testing.T) { - if err := db.Migrator().DropTable("cdrs"); err != nil { - t.Fatal(err) - } - if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil { - t.Fatal(err) - } - if db2, err := db.DB(); err != nil { - t.Fatal(err) - } else if err = db2.Close(); err != nil { - t.Fatal(err) - } - }) - - content := `{ - - "general": { - "log_level": 7 - }, - - "apiers": { - "enabled": true - }, - "filters": { - "apiers_conns": ["*localhost"] - }, - "ers": { - "enabled": true, - "sessions_conns":["*localhost"], - "readers": [ - { - "id": "mysql", - "type": "*sql", - "run_delay": "1m", - "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", - "opts": { - "sqlDBName":"cgrates2", - }, - "start_delay": "500ms", // wait for db to be populated before starting reader - "processed_path": "", - "tenant": "cgrates.org", - "filters": [ - "*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days) - "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", - "*string:~*vars.*readerID:mysql", - "FLTR_VARS", // "*string:~*vars.*readerID:mysql", - ], - "flags": ["*dryrun"], - "fields":[ - {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, - {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, - {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, - {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, - ], - }, - ], - }, - - }` - - tpFiles := map[string]string{ - utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] -cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, -cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, - } - - buf := &bytes.Buffer{} - ng := engine.TestEngine{ - ConfigJSON: content, - DBCfg: dbcfg, - TpFiles: tpFiles, - LogBuffer: buf, - } - ng.Run(t) - time.Sleep(1 * time.Second) - - t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) { - time.Sleep(100 * time.Millisecond) // give enough time to process from sql table - records := 0 - scanner := bufio.NewScanner(strings.NewReader(buf.String())) - timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") - expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) - var ersLogsCount int - for scanner.Scan() { - line := scanner.Text() - if !strings.Contains(line, " DRYRUN, reader: ") { - continue - } - records++ - if !strings.Contains(line, expectedLog) { - t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line) - } - if strings.Contains(line, "[INFO] DRYRUN") { - ersLogsCount++ - } - } - if err := scanner.Err(); err != nil { - t.Errorf("error reading input: %v", err) - } - if records != 1 { - t.Errorf("expected ERs to process 1 records, but it processed %d records", records) - } - if ersLogsCount != 1 { - t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount) - } - }) - - t.Run("VerifyRowsNotDeleted", func(t *testing.T) { - var result int64 - db.Table(utils.CDRsTBL).Count(&result) - if result != 3 { - t.Error("Expected 3 rows in table, got: ", result) - } - var rslt []map[string]interface{} - if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil { - t.Fatalf("failed to query table: %v", err) - } - }) -} - func TestERSSQLFiltersWithMetaDelete(t *testing.T) { var dbcfg engine.DBCfg switch *utils.DBType { @@ -743,54 +599,31 @@ func TestERSSQLFiltersWithMetaDelete(t *testing.T) { t.Fatal("unsupported dbtype value") } + var db, cdb2 *gorm.DB t.Run("InitSQLDB", func(t *testing.T) { var err error - var db2 *gorm.DB - if db2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), &gorm.Config{ AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), }); err != nil { t.Fatal(err) } - if err = db2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { t.Fatal(err) } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather }) - type testModelSql struct { - ID int64 - Cgrid string - RunID string - OriginHost string - Source string - OriginID string - ToR string - RequestType string - Tenant string - Category string - Account string - Subject string - Destination string - SetupTime time.Time - AnswerTime time.Time - Usage int64 - ExtraFields string - CostSource string - Cost float64 - CostDetails string - ExtraInfo string - CreatedAt time.Time - UpdatedAt time.Time - DeletedAt *time.Time - } t.Run("PutCDRsInDataBase", func(t *testing.T) { var err error if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), &gorm.Config{ AllowGlobalUpdate: true, - Logger: logger.Default.LogMode(logger.Silent), }); err != nil { t.Fatal(err) } @@ -845,63 +678,13 @@ func TestERSSQLFiltersWithMetaDelete(t *testing.T) { } else if err = db2.Close(); err != nil { t.Fatal(err) } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } }) - content := `{ - - "general": { - "log_level": 7 - }, - - "apiers": { - "enabled": true - }, - "filters": { - "apiers_conns": ["*localhost"] - }, - "ers": { - "enabled": true, - "sessions_conns":["*localhost"], - "readers": [ - { - "id": "mysql", - "type": "*sql", - "run_delay": "1m", - "source_path": "*mysql://cgrates:CGRateS.org@127.0.0.1:3306", - "opts": { - "sqlDBName":"cgrates2", - }, - "start_delay": "500ms", // wait for db to be populated before starting reader - "processed_path": "*delete", - "tenant": "cgrates.org", - "filters": [ - "*gt:~*req.answer_time:NOW() - INTERVAL 7 DAY", // dont process cdrs with answer_time older than 7 days ago (continue if answer_time > now-7days) - "FLTR_SQL_RatingID", // "*eq:~*req.cost_details.Charges[0].RatingID:RatingID2", - "*string:~*vars.*readerID:mysql", - "FLTR_VARS", // "*string:~*vars.*readerID:mysql", - ], - "flags": ["*dryrun"], - "fields":[ - {"tag": "CGRID", "path": "*cgreq.CGRID", "type": "*variable", "value": "~*req.cgrid", "mandatory": true}, - {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.tor", "mandatory": true}, - {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.origin_id", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.request_type", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*variable", "value": "~*req.tenant", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*variable", "value": "~*req.category", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.account", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.subject", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.destination", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.setup_time", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.answer_time", "mandatory": true}, - {"tag": "CostDetails", "path": "*cgreq.CostDetails", "type": "*variable", "value": "~*req.cost_details", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.usage", "mandatory": true}, - ], - }, - ], - }, - - }` - tpFiles := map[string]string{ utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, @@ -910,10 +693,11 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, buf := &bytes.Buffer{} ng := engine.TestEngine{ - ConfigJSON: content, - DBCfg: dbcfg, - TpFiles: tpFiles, - LogBuffer: buf, + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_meta_delete"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, } ng.Run(t) time.Sleep(1 * time.Second) @@ -923,7 +707,7 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, records := 0 scanner := bufio.NewScanner(strings.NewReader(buf.String())) timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") - expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:testV1CDRsRefundOutOfSessionCost\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) var ersLogsCount int for scanner.Scan() { line := scanner.Text() @@ -932,7 +716,7 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, } records++ if !strings.Contains(line, expectedLog) { - t.Errorf("expected \n<%s>, \nreceived\n<%s>", expectedLog, line) + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) } if strings.Contains(line, "[INFO] DRYRUN") { ersLogsCount++ @@ -964,9 +748,596 @@ cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, for _, row := range rslt { for col, value := range row { if strings.Contains(fmt.Sprintln(value), "RatingID2") { - t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%s>, value <%s>", col, value) + t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%q>, value <%q>", col, value) } } } }) } + +type mockTableName struct { + ID int64 + Cgrid string + RunID string + OriginHost string + Source string + OriginID string + TOR string + RequestType string + Tenant string + Category string + Account string + Subject string + Destination string + SetupTime time.Time + AnswerTime *time.Time + Usage int64 + ExtraFields string + CostSource string + Cost float64 + CostDetails string + ExtraInfo string + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time +} + +func (mtn mockTableName) TableName() string { + return "cdrsProcessed" +} + +func TestERSSQLFiltersMove(t *testing.T) { + var dbcfg engine.DBCfg + switch *utils.DBType { + case utils.MetaInternal: + t.SkipNow() + case utils.MetaMySQL: + case utils.MetaMongo: + dbcfg = engine.MongoDBCfg + case utils.MetaPostgres: + dbcfg = engine.PostgresDBCfg + default: + t.Fatal("unsupported dbtype value") + } + + var db, cdb2 *gorm.DB + t.Run("InitSQLDB", func(t *testing.T) { + var err error + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + t.Fatal(err) + } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather + }) + t.Run("PutCDRsInDataBase", func(t *testing.T) { + var err error + if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + tx := db.Begin() + if !tx.Migrator().HasTable("cdrs") { + if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + if !tx.Migrator().HasTable("cdrsProcessed") { + if err = tx.Migrator().CreateTable(new(mockTableName)); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + tx.Commit() + tx = db.Begin() + tx = tx.Table(utils.CDRsTBL) + cdrSql := cdr1.AsCDRsql() + cdrSql2 := cdr2.AsCDRsql() + cdrsql3 := cdr3.AsCDRsql() + cdrSql.CreatedAt = time.Now() + cdrSql2.CreatedAt = time.Now() + cdrsql3.CreatedAt = time.Now() + saved := tx.Save(cdrSql) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrSql2) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrsql3) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + tx.Commit() + time.Sleep(10 * time.Millisecond) + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected table to have 3 results but got ", result) + } + }) + defer t.Run("StopSQL", func(t *testing.T) { + if err := db.Migrator().DropTable("cdrs"); err != nil { + t.Fatal(err) + } + if err := db.Migrator().DropTable("cdrsProcessed"); err != nil { + t.Fatal(err) + } + if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil { + t.Fatal(err) + } + if db2, err := db.DB(); err != nil { + t.Fatal(err) + } else if err = db2.Close(); err != nil { + t.Fatal(err) + } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } + }) + + tpFiles := map[string]string{ + utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] +cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, +cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, + } + + buf := &bytes.Buffer{} + ng := engine.TestEngine{ + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_move"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, + } + ng.Run(t) + time.Sleep(1 * time.Second) + + t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) { + time.Sleep(100 * time.Millisecond) // give enough time to process from sql table + records := 0 + scanner := bufio.NewScanner(strings.NewReader(buf.String())) + timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + var ersLogsCount int + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, " DRYRUN, reader: ") { + continue + } + records++ + if !strings.Contains(line, expectedLog) { + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) + } + if strings.Contains(line, "[INFO] DRYRUN") { + ersLogsCount++ + } + } + if err := scanner.Err(); err != nil { + t.Errorf("error reading input: %v", err) + } + if records != 1 { + t.Errorf("expected ERs to process 1 records, but it processed %d records", records) + } + if ersLogsCount != 1 { + t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount) + } + }) + + t.Run("VerifyRowsCount", func(t *testing.T) { + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 2 { + t.Fatal("Expected 2 rows in table, got: ", result) + } + var rslt []map[string]interface{} + if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil { + t.Errorf("failed to query table: %v", err) + } + // Print the entire table as a string + for _, row := range rslt { + for col, value := range row { + if strings.Contains(fmt.Sprintln(value), "RatingID2") { + t.Fatalf("Expected CDR with RatingID: \"RatingID2\" to be deleted. Received column <%q>, value <%q>", col, value) + } + } + } + + var result2 int64 + db.Table("cdrsProcessed").Count(&result2) + if result2 != 1 { + t.Fatal("Expected 1 rows in table, got: ", result2) + } + var rslt2 []map[string]interface{} + if err := db.Raw("SELECT * FROM " + "cdrsProcessed").Scan(&rslt2).Error; err != nil { + t.Errorf("failed to query table: %v", err) + } + timeStartFormated := rslt2[0]["answer_time"] + createdAt := rslt2[0]["created_at"] + updatedAt := rslt2[0]["updated_at"] + exp := fmt.Sprintf("map[account:1001 answer_time:%s category:call cgrid:%s cost:1.01 cost_details:{\"CGRID\":\"test1\",\"RunID\":\"*default\",\"StartTime\":\"2017-01-09T16:18:21Z\",\"Usage\":180000000000,\"Cost\":2.3,\"Charges\":[{\"RatingID\":\"RatingID2\",\"Increments\":[{\"Usage\":120000000000,\"Cost\":2,\"AccountingID\":\"a012888\",\"CompressFactor\":1},{\"Usage\":1000000000,\"Cost\":0.005,\"AccountingID\":\"44d6c02\",\"CompressFactor\":60}],\"CompressFactor\":1}],\"AccountSummary\":{\"Tenant\":\"cgrates.org\",\"ID\":\"1001\",\"BalanceSummaries\":[{\"UUID\":\"uuid1\",\"ID\":\"\",\"Type\":\"*monetary\",\"Initial\":0,\"Value\":50,\"Disabled\":false}],\"AllowNegative\":false,\"Disabled\":false},\"Rating\":{\"c1a5ab9\":{\"ConnectFee\":0.1,\"RoundingMethod\":\"*up\",\"RoundingDecimals\":5,\"MaxCost\":0,\"MaxCostStrategy\":\"\",\"TimingID\":\"\",\"RatesID\":\"ec1a177\",\"RatingFiltersID\":\"43e77dc\"}},\"Accounting\":{\"44d6c02\":{\"AccountID\":\"cgrates.org:1001\",\"BalanceUUID\":\"uuid1\",\"RatingID\":\"\",\"Units\":120.7,\"ExtraChargeID\":\"\"},\"a012888\":{\"AccountID\":\"cgrates.org:1001\",\"BalanceUUID\":\"uuid1\",\"RatingID\":\"\",\"Units\":120.7,\"ExtraChargeID\":\"\"}},\"RatingFilters\":null,\"Rates\":{\"ec1a177\":[{\"GroupIntervalStart\":0,\"Value\":0.01,\"RateIncrement\":60000000000,\"RateUnit\":1000000000}]},\"Timings\":null} cost_source:cost source created_at:%s deleted_at: destination:1002 extra_fields:{\"field_extr1\":\"val_extr1\",\"fieldextr2\":\"valextr2\"} extra_info:extraInfo id:2 origin_host:192.168.1.1 origin_id:oid2 request_type:*rated run_id:*default setup_time:%s source:test subject:1001 tenant:cgrates.org tor:*voice updated_at:%s usage:10000000000]", timeStartFormated, cgrID, createdAt, timeStartFormated, updatedAt) + // Print the entire table as a string + for _, row := range rslt2 { + if !strings.Contains(fmt.Sprintf("%+v", row), exp) { + t.Errorf("Expected <%v>, \nReceived <%v>", exp, fmt.Sprintf("%+v", row)) + } + } + }) +} + +func TestERSSQLFiltersUpdate(t *testing.T) { + var dbcfg engine.DBCfg + switch *utils.DBType { + case utils.MetaInternal: + t.SkipNow() + case utils.MetaMySQL: + case utils.MetaMongo: + dbcfg = engine.MongoDBCfg + case utils.MetaPostgres: + dbcfg = engine.PostgresDBCfg + default: + t.Fatal("unsupported dbtype value") + } + + var db, cdb2 *gorm.DB + t.Run("InitSQLDB", func(t *testing.T) { + var err error + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + t.Fatal(err) + } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather + }) + + t.Run("PutCDRsInDataBase", func(t *testing.T) { + var err error + if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), + &gorm.Config{ + AllowGlobalUpdate: true, + }); err != nil { + t.Fatal(err) + } + tx := db.Begin() + if !tx.Migrator().HasTable("cdrs") { + if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + tx.Commit() + tx = db.Begin() + tx = tx.Table(utils.CDRsTBL) + cdrSql := cdr1.AsCDRsql() + cdrSql2 := cdr2.AsCDRsql() + cdrsql3 := cdr3.AsCDRsql() + cdrSql.CreatedAt = time.Now() + cdrSql2.CreatedAt = time.Now() + cdrsql3.CreatedAt = time.Now() + saved := tx.Save(cdrSql) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrSql2) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrsql3) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + tx.Commit() + time.Sleep(10 * time.Millisecond) + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected table to have 3 results but got ", result) + } + }) + defer t.Run("StopSQL", func(t *testing.T) { + if err := db.Migrator().DropTable("cdrs"); err != nil { + t.Fatal(err) + } + if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil { + t.Fatal(err) + } + if db2, err := db.DB(); err != nil { + t.Fatal(err) + } else if err = db2.Close(); err != nil { + t.Fatal(err) + } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } + }) + + tpFiles := map[string]string{ + utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] +cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, +cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysqlReaderID,`, + } + + buf := &bytes.Buffer{} + ng := engine.TestEngine{ + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_update"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, + } + ng.Run(t) + time.Sleep(1 * time.Second) + + t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) { + time.Sleep(100 * time.Millisecond) // give enough time to process from sql table + records := 0 + scanner := bufio.NewScanner(strings.NewReader(buf.String())) + timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"ExtraInfo\":\"extraInfo\",\"Id\":\"2\",\"OriginID\":\"oid2\",\"ReaderID\":\"mysqlReaderID\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + var ersLogsCount int + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, " DRYRUN, reader: ") { + continue + } + records++ + if !strings.Contains(line, expectedLog) { + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) + } + if strings.Contains(line, "[INFO] DRYRUN") { + ersLogsCount++ + } + } + if err := scanner.Err(); err != nil { + t.Errorf("error reading input: %v", err) + } + if records != 1 { + t.Errorf("expected ERs to process 1 records, but it processed %d records", records) + } + if ersLogsCount != 1 { + t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount) + } + }) + + t.Run("VerifyRowsCount", func(t *testing.T) { + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected 3 rows in table, got: ", result) + } + var rslt []map[string]interface{} + if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil { + t.Errorf("failed to query table: %v", err) + } + var countST int + // Print the entire table as a string + for _, row := range rslt { + for col, value := range row { + if col == "setup_time" { + if value.(time.Time).UTC().Equal(time.Date(2018, 11, 27, 14, 21, 26, 0, time.Local).UTC()) { + countST++ + if utils.IfaceAsString(row["account"]) != "extraInfo" { + t.Errorf("Expected CDR to be updated with empty cost_details. Received value <%q>", utils.IfaceAsString(row["cost_details"])) + } + } + } + } + } + if countST != 1 { + t.Errorf("Expected CDR with origin_id:oid2 to have updated setup_time: <%+v>. Received <%v> \nCounted <%v> with expected setup_time", time.Date(2018, 11, 27, 14, 21, 26, 0, time.Local).UTC(), utils.ToJSON(rslt), countST) + } + }) +} + +// Using the raw event to update the rows means you dont need to define in the reader's template fields, the fields which you want to send to EES since all of the fields read in that row will be sent to EES. It also means that the request field names used for exporter template fields will be as read on the row; meaning instead of ~*req.SetupTime, it will be written as ~*req.setup_time +func TestERSSQLFiltersRawUpdate(t *testing.T) { + var dbcfg engine.DBCfg + switch *utils.DBType { + case utils.MetaInternal: + t.SkipNow() + case utils.MetaMySQL: + case utils.MetaMongo: + dbcfg = engine.MongoDBCfg + case utils.MetaPostgres: + dbcfg = engine.PostgresDBCfg + default: + t.Fatal("unsupported dbtype value") + } + + var db, cdb2 *gorm.DB + t.Run("InitSQLDB", func(t *testing.T) { + var err error + if cdb2, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates")), + &gorm.Config{ + AllowGlobalUpdate: true, + Logger: logger.Default.LogMode(logger.Silent), + }); err != nil { + t.Fatal(err) + } + + if err = cdb2.Exec(`CREATE DATABASE IF NOT EXISTS cgrates2;`).Error; err != nil { + t.Fatal(err) + } + sqlDB, err := cdb2.DB() + if err != nil { + t.Fatal(err) + } + sqlDB.SetConnMaxLifetime(5 * time.Second) // connections will stay idle even if you close the database. Set MaxLifetime to 5 seconds so that we dont get too many connection attempts error when ran with other tests togather + }) + + t.Run("PutCDRsInDataBase", func(t *testing.T) { + var err error + if db, err = gorm.Open(mysql.Open(fmt.Sprintf(dbConnString, "cgrates2")), + &gorm.Config{ + AllowGlobalUpdate: true, + Logger: logger.Default.LogMode(logger.Silent), + }); err != nil { + t.Fatal(err) + } + tx := db.Begin() + if !tx.Migrator().HasTable("cdrs") { + if err = tx.Migrator().CreateTable(new(engine.CDRsql)); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + tx.Commit() + tx = db.Begin() + tx = tx.Table(utils.CDRsTBL) + cdrSql := cdr1.AsCDRsql() + cdrSql2 := cdr2.AsCDRsql() + cdrsql3 := cdr3.AsCDRsql() + cdrSql.CreatedAt = time.Now() + cdrSql2.CreatedAt = time.Now() + cdrsql3.CreatedAt = time.Now() + saved := tx.Save(cdrSql) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrSql2) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + saved = tx.Save(cdrsql3) + if saved.Error != nil { + tx.Rollback() + t.Fatal(err) + } + tx.Commit() + time.Sleep(10 * time.Millisecond) + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected table to have 3 results but got ", result) + } + }) + defer t.Run("StopSQL", func(t *testing.T) { + if err := db.Migrator().DropTable("cdrs"); err != nil { + t.Fatal(err) + } + if err := db.Exec(`DROP DATABASE cgrates2;`).Error; err != nil { + t.Fatal(err) + } + if db2, err := db.DB(); err != nil { + t.Fatal(err) + } else if err = db2.Close(); err != nil { + t.Fatal(err) + } + if cdb2, err := cdb2.DB(); err != nil { + t.Fatal(err) + } else if err = cdb2.Close(); err != nil { + t.Fatal(err) + } + }) + + tpFiles := map[string]string{ + utils.FiltersCsv: `#Tenant[0],ID[1],Type[2],Path[3],Values[4],ActivationInterval[5] +cgrates.org,FLTR_SQL_RatingID,*eq,~*req.cost_details.Charges[0].RatingID,RatingID2, +cgrates.org,FLTR_VARS,*string,~*vars.*readerID,mysql,`, + } + + buf := &bytes.Buffer{} + ng := engine.TestEngine{ + ConfigPath: path.Join(*utils.DataDir, "conf", "samples", "ers_mysql_raw_update"), + DBCfg: dbcfg, + TpFiles: tpFiles, + LogBuffer: buf, + GracefulShutdown: true, + } + ng.Run(t) + time.Sleep(1 * time.Second) + + t.Run("VerifyProcessedFieldsFromLogs", func(t *testing.T) { + time.Sleep(100 * time.Millisecond) // give enough time to process from sql table + records := 0 + scanner := bufio.NewScanner(strings.NewReader(buf.String())) + timeStartFormated := timeStart.Format("2006-01-02T15:04:05Z07:00") + expectedLog := fmt.Sprintf("\"Event\":{\"Account\":\"1001\",\"AnswerTime\":\"%s\",\"CGRID\":\"%s\",\"Category\":\"call\",\"CostDetails\":\"{\\\"CGRID\\\":\\\"test1\\\",\\\"RunID\\\":\\\"*default\\\",\\\"StartTime\\\":\\\"2017-01-09T16:18:21Z\\\",\\\"Usage\\\":180000000000,\\\"Cost\\\":2.3,\\\"Charges\\\":[{\\\"RatingID\\\":\\\"RatingID2\\\",\\\"Increments\\\":[{\\\"Usage\\\":120000000000,\\\"Cost\\\":2,\\\"AccountingID\\\":\\\"a012888\\\",\\\"CompressFactor\\\":1},{\\\"Usage\\\":1000000000,\\\"Cost\\\":0.005,\\\"AccountingID\\\":\\\"44d6c02\\\",\\\"CompressFactor\\\":60}],\\\"CompressFactor\\\":1}],\\\"AccountSummary\\\":{\\\"Tenant\\\":\\\"cgrates.org\\\",\\\"ID\\\":\\\"1001\\\",\\\"BalanceSummaries\\\":[{\\\"UUID\\\":\\\"uuid1\\\",\\\"ID\\\":\\\"\\\",\\\"Type\\\":\\\"*monetary\\\",\\\"Initial\\\":0,\\\"Value\\\":50,\\\"Disabled\\\":false}],\\\"AllowNegative\\\":false,\\\"Disabled\\\":false},\\\"Rating\\\":{\\\"c1a5ab9\\\":{\\\"ConnectFee\\\":0.1,\\\"RoundingMethod\\\":\\\"*up\\\",\\\"RoundingDecimals\\\":5,\\\"MaxCost\\\":0,\\\"MaxCostStrategy\\\":\\\"\\\",\\\"TimingID\\\":\\\"\\\",\\\"RatesID\\\":\\\"ec1a177\\\",\\\"RatingFiltersID\\\":\\\"43e77dc\\\"}},\\\"Accounting\\\":{\\\"44d6c02\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"},\\\"a012888\\\":{\\\"AccountID\\\":\\\"cgrates.org:1001\\\",\\\"BalanceUUID\\\":\\\"uuid1\\\",\\\"RatingID\\\":\\\"\\\",\\\"Units\\\":120.7,\\\"ExtraChargeID\\\":\\\"\\\"}},\\\"RatingFilters\\\":null,\\\"Rates\\\":{\\\"ec1a177\\\":[{\\\"GroupIntervalStart\\\":0,\\\"Value\\\":0.01,\\\"RateIncrement\\\":60000000000,\\\"RateUnit\\\":1000000000}]},\\\"Timings\\\":null}\",\"Destination\":\"1002\",\"OriginID\":\"oid2\",\"RequestType\":\"*rated\",\"SetupTime\":\"%s\",\"Subject\":\"1001\",\"Tenant\":\"cgrates.org\",\"ToR\":\"*voice\",\"Usage\":\"10000000000\"},\"APIOpts\":{}}>", timeStartFormated, cgrID, timeStartFormated) + var ersLogsCount int + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, " DRYRUN, reader: ") { + continue + } + records++ + if !strings.Contains(line, expectedLog) { + t.Errorf("expected \n<%q>, \nreceived\n<%q>", expectedLog, line) + } + if strings.Contains(line, "[INFO] DRYRUN") { + ersLogsCount++ + } + } + if err := scanner.Err(); err != nil { + t.Errorf("error reading input: %v", err) + } + if records != 1 { + t.Errorf("expected ERs to process 1 records, but it processed %d records", records) + } + if ersLogsCount != 1 { + t.Error("Expected only 1 ERS Dryrun log, received: ", ersLogsCount) + } + }) + + t.Run("VerifyRowsCount", func(t *testing.T) { + var result int64 + db.Table(utils.CDRsTBL).Count(&result) + if result != 3 { + t.Error("Expected 3 rows in table, got: ", result) + } + var rslt []map[string]interface{} + if err := db.Raw("SELECT * FROM " + utils.CDRsTBL).Scan(&rslt).Error; err != nil { + t.Errorf("failed to query table: %v", err) + } + var countST int + // Print the entire table as a string + for _, row := range rslt { + for col, value := range row { + if col == "setup_time" { + if value.(time.Time).UTC().Equal((time.Date(2018, 11, 27, 14, 21, 26, 0, time.Local).UTC())) { + countST++ + if utils.IfaceAsString(row["account"]) != "extraInfo" { + t.Errorf("Expected CDR to be updated with empty cost_details. Received value <%q>", utils.IfaceAsString(row["cost_details"])) + } + } + } + } + } + if countST != 1 { + t.Errorf("Expected CDR with origin_id:oid2 to have updated setup_time: <%v>. Received <%v> \nCounted <%v> with expected setup_time", (time.Date(2018, 11, 27, 14, 21, 26, 0, time.Local).UTC()), utils.ToJSON(rslt), countST) + } + }) +} diff --git a/utils/consts.go b/utils/consts.go index e56cf4b2c..ad7cbd3d0 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -2809,6 +2809,7 @@ const ( SQLDBNameOpt = "sqlDBName" SQLTableNameOpt = "sqlTableName" SQLDeleteIndexedFieldsOpt = "sqlDeleteIndexedFields" + SQLUpdateIndexedFieldsOpt = "sqlUpdateIndexedFields" SQLMaxOpenConns = "sqlMaxOpenConns" SQLConnMaxLifetime = "sqlConnMaxLifetime" diff --git a/utils/coreutils.go b/utils/coreutils.go index ec4de61c2..744a0c685 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -294,6 +294,13 @@ func ParseTimeDetectLayout(tmStr string, timezone string) (time.Time, error) { } else { return time.Now().Add(tmStrTmp), nil } + case strings.HasPrefix(tmStr, "-"): + tmStr = strings.TrimPrefix(tmStr, "-") + if tmStrTmp, err := time.ParseDuration(tmStr); err != nil { + return nilTime, err + } else { + return time.Now().Add(-tmStrTmp), nil + } case utcFormat.MatchString(tmStr): return time.ParseInLocation("2006-01-02T15:04:05", tmStr, loc) diff --git a/utils/coreutils_test.go b/utils/coreutils_test.go index 5422b53b8..682529f48 100644 --- a/utils/coreutils_test.go +++ b/utils/coreutils_test.go @@ -451,6 +451,13 @@ func TestParseTimeDetectLayout(t *testing.T) { if err != nil || date.Sub(expected).Seconds() > 20 || date.Sub(expected).Seconds() < 19 { t.Error("error parsing date: ", date.Sub(expected).Seconds()) } + + date, err = ParseTimeDetectLayout("-20s", "") + expected = time.Now() + if err != nil || expected.Sub(date).Seconds() > 21 || expected.Sub(date).Seconds() < 20 { + t.Error("error parsing date: ", expected.Sub(date).Seconds()) + } + expected = time.Now().AddDate(0, 0, 1) if date, err := ParseTimeDetectLayout("*daily", ""); err != nil { t.Error(err) diff --git a/utils/dbutils.go b/utils/dbutils.go deleted file mode 100644 index 3ae2cdd6e..000000000 --- a/utils/dbutils.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package utils - -import "fmt" - -// Creates mysql conditions used in WHERE statement out of filters -func FilterToSQLQuery(ruleType, beforeSep, afterSep string, values []string, not bool) (conditions []string) { - // here are for the filters that their values are empty: *exists, *notexists, *empty, *notempty.. - if len(values) == 0 { - switch ruleType { - case MetaExists, MetaNotExists: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s IS NOT NULL", afterSep)) - return - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NOT NULL", beforeSep, afterSep)) - return - } - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s IS NULL", afterSep)) - return - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') IS NULL", beforeSep, afterSep)) - case MetaEmpty, MetaNotEmpty: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s != ''", afterSep)) - return - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != ''", beforeSep, afterSep)) - return - } - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s == ''", afterSep)) - return - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') == ''", beforeSep, afterSep)) - } - return - } - // here are for the filters that can have more than one value: *string, *prefix, *suffix .. - for _, value := range values { - switch value { // in case we have boolean values, it should be queried over 1 or 0 - case "true": - value = "1" - case "false": - value = "0" - } - var singleCond string - switch ruleType { - case MetaString, MetaNotString, MetaEqual, MetaNotEqual: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s != '%s'", afterSep, value)) - continue - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') != '%s'", - beforeSep, afterSep, value)) - continue - } - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s = '%s'", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') = '%s'", beforeSep, afterSep, value) - } - case MetaLessThan, MetaLessOrEqual, MetaGreaterThan, MetaGreaterOrEqual: - if ruleType == MetaGreaterOrEqual { - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s >= %s", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') >= %s", beforeSep, afterSep, value) - } - } else if ruleType == MetaGreaterThan { - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s > %s", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') > %s", beforeSep, afterSep, value) - } - } else if ruleType == MetaLessOrEqual { - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s <= %s", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') <= %s", beforeSep, afterSep, value) - } - } else if ruleType == MetaLessThan { - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s < %s", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') < %s", beforeSep, afterSep, value) - } - } - case MetaPrefix, MetaNotPrefix: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%s%%'", afterSep, value)) - continue - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%s%%'", beforeSep, afterSep, value)) - continue - } - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s LIKE '%s%%'", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%s%%'", beforeSep, afterSep, value) - } - case MetaSuffix, MetaNotSuffix: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s NOT LIKE '%%%s'", afterSep, value)) - continue - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT LIKE '%%%s'", beforeSep, afterSep, value)) - continue - } - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s LIKE '%%%s'", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') LIKE '%%%s'", beforeSep, afterSep, value) - } - case MetaRegex, MetaNotRegex: - if not { - if beforeSep == EmptyString { - conditions = append(conditions, fmt.Sprintf("%s NOT REGEXP '%s'", afterSep, value)) - continue - } - conditions = append(conditions, fmt.Sprintf("JSON_VALUE(%s, '$.%s') NOT REGEXP '%s'", beforeSep, afterSep, value)) - continue - } - if beforeSep == EmptyString { - singleCond = fmt.Sprintf("%s REGEXP '%s'", afterSep, value) - } else { - singleCond = fmt.Sprintf("JSON_VALUE(%s, '$.%s') REGEXP '%s'", beforeSep, afterSep, value) - } - } - conditions = append(conditions, singleCond) - } - return -} diff --git a/utils/dbutils_test.go b/utils/dbutils_test.go deleted file mode 100644 index 28e070379..000000000 --- a/utils/dbutils_test.go +++ /dev/null @@ -1,309 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package utils - -import "testing" - -func TestFilterToSQLQuery(t *testing.T) { - tests := []struct { - name string - ruleType string - beforeSep string - afterSep string - values []string - not bool - expected []string - }{ - {"MetaGreaterThan with values", MetaGreaterThan, "", "answer_time", []string{"NOW() - INTERVAL 7 DAY"}, false, []string{"answer_time > NOW() - INTERVAL 7 DAY"}}, - {"MetaEqual with values", MetaEqual, "cost_details", "Charges[0].RatingID", []string{"RatingID2"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') = 'RatingID2'"}}, - - {"MetaExists with no values", MetaExists, "", "answer_time", nil, false, []string{"answer_time IS NULL"}}, - {"MetaExists with JSON field", MetaExists, "cost_details", "Charges[0].RatingID", nil, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NULL"}}, - {"MetaNotExists with no values", MetaNotExists, "", "answer_time", nil, true, []string{"answer_time IS NOT NULL"}}, - {"MetaNotExists with JSON field", MetaNotExists, "cost_details", "Charges[0].RatingID", nil, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') IS NOT NULL"}}, - - {"MetaString with values", MetaString, "", "answer_time", []string{"value1", "value2"}, false, []string{"answer_time = 'value1'", "answer_time = 'value2'"}}, - {"MetaNotString with values", MetaNotString, "cost_details", "Charges[0].RatingID", []string{"value1"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != 'value1'"}}, - - {"MetaEmpty with no values", MetaEmpty, "", "answer_time", nil, false, []string{"answer_time == ''"}}, - {"MetaEmpty with JSON field", MetaEmpty, "cost_details", "Charges[0].RatingID", nil, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') == ''"}}, - {"MetaNotEmpty with no values", MetaNotEmpty, "", "answer_time", nil, true, []string{"answer_time != ''"}}, - {"MetaNotEmpty with JSON field", MetaNotEmpty, "cost_details", "Charges[0].RatingID", nil, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') != ''"}}, - - {"MetaGreaterOrEqual with values", MetaGreaterOrEqual, "", "answer_time", []string{"10"}, false, []string{"answer_time >= 10"}}, - {"MetaGreaterThan with values", MetaGreaterThan, "cost_details", "Charges[0].RatingID", []string{"20"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') > 20"}}, - - {"MetaLessThan with values", MetaLessThan, "", "answer_time", []string{"5"}, false, []string{"answer_time < 5"}}, - {"MetaLessOrEqual with values", MetaLessOrEqual, "cost_details", "Charges[0].RatingID", []string{"15"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') <= 15"}}, - - {"MetaPrefix with values", MetaPrefix, "", "answer_time", []string{"pre"}, false, []string{"answer_time LIKE 'pre%'"}}, - {"MetaNotPrefix with values", MetaNotPrefix, "cost_details", "Charges[0].RatingID", []string{"pre"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE 'pre%'"}}, - - {"MetaSuffix with values", MetaSuffix, "", "answer_time", []string{"suf"}, false, []string{"answer_time LIKE '%suf'"}}, - {"MetaNotSuffix with values", MetaNotSuffix, "cost_details", "Charges[0].RatingID", []string{"suf"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT LIKE '%suf'"}}, - - {"MetaGreaterOrEqual with JSON field", MetaGreaterOrEqual, "cost_details", "Charges[0].RatingID", []string{"100"}, false, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') >= 100"}}, - - {"MetaRegex with values", MetaRegex, "", "answer_time", []string{"pattern1", "pattern2"}, false, []string{"answer_time REGEXP 'pattern1'", "answer_time REGEXP 'pattern2'"}}, - {"MetaNotRegex with values", MetaNotRegex, "cost_details", "Charges[0].RatingID", []string{"pattern"}, true, []string{"JSON_VALUE(cost_details, '$.Charges[0].RatingID') NOT REGEXP 'pattern'"}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := FilterToSQLQuery(tt.ruleType, tt.beforeSep, tt.afterSep, tt.values, tt.not) - if len(got) != len(tt.expected) { - t.Errorf("expected %v, got %v", tt.expected, got) - return - } - for i, cond := range got { - if cond != tt.expected[i] { - t.Errorf("expected %v, got %v", tt.expected[i], cond) - } - } - }) - } -} - -func TestFilterToSQLQueryValidations(t *testing.T) { - tests := []struct { - name string - ruleType string - beforeSep string - afterSep string - values []string - not bool - expected []string - }{ - { - name: "Boolean true value", - ruleType: MetaString, - beforeSep: EmptyString, - afterSep: "active", - values: []string{"true"}, - not: false, - expected: []string{"active = '1'"}, - }, - { - name: "Boolean false value", - ruleType: MetaString, - beforeSep: EmptyString, - afterSep: "active", - values: []string{"false"}, - not: false, - expected: []string{"active = '0'"}, - }, - { - name: "Greater than or equal with empty beforeSep", - ruleType: MetaGreaterOrEqual, - beforeSep: EmptyString, - afterSep: "score", - values: []string{"10"}, - not: false, - expected: []string{"score >= 10"}, - }, - { - name: "Greater than with empty beforeSep", - ruleType: MetaGreaterThan, - beforeSep: EmptyString, - afterSep: "score", - values: []string{"20"}, - not: false, - expected: []string{"score > 20"}, - }, - { - name: "Less than or equal with empty beforeSep", - ruleType: MetaLessOrEqual, - beforeSep: EmptyString, - afterSep: "score", - values: []string{"30"}, - not: false, - expected: []string{"score <= 30"}, - }, - { - name: "Less than with empty beforeSep", - ruleType: MetaLessThan, - beforeSep: EmptyString, - afterSep: "score", - values: []string{"40"}, - not: false, - expected: []string{"score < 40"}, - }, - { - name: "Prefix NOT LIKE with empty beforeSep", - ruleType: MetaNotPrefix, - beforeSep: EmptyString, - afterSep: "name", - values: []string{"prefix"}, - not: true, - expected: []string{"name NOT LIKE 'prefix%'"}, - }, - { - name: "Prefix LIKE with JSON_VALUE", - ruleType: MetaPrefix, - beforeSep: "data", - afterSep: "name", - values: []string{"prefix"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.name') LIKE 'prefix%'"}, - }, - { - name: "Suffix NOT LIKE with empty beforeSep", - ruleType: MetaNotSuffix, - beforeSep: EmptyString, - afterSep: "name", - values: []string{"suffix"}, - not: true, - expected: []string{"name NOT LIKE '%suffix'"}, - }, - { - name: "Suffix LIKE with JSON_VALUE", - ruleType: MetaSuffix, - beforeSep: "data", - afterSep: "name", - values: []string{"suffix"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.name') LIKE '%suffix'"}, - }, - { - name: "Regex NOT REGEXP with empty beforeSep", - ruleType: MetaNotRegex, - beforeSep: EmptyString, - afterSep: "pattern", - values: []string{"[a-z]+"}, - not: true, - expected: []string{"pattern NOT REGEXP '[a-z]+'"}, - }, - { - name: "Regex REGEXP with JSON_VALUE", - ruleType: MetaRegex, - beforeSep: "data", - afterSep: "pattern", - values: []string{"[0-9]+"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.pattern') REGEXP '[0-9]+'"}, - }, - - { - name: "Not equal with empty beforeSep", - ruleType: MetaString, - beforeSep: EmptyString, - afterSep: "status", - values: []string{"inactive"}, - not: true, - expected: []string{"status != 'inactive'"}, - }, - { - name: "Equal condition with JSON_VALUE", - ruleType: MetaString, - beforeSep: "data", - afterSep: "status", - values: []string{"active"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.status') = 'active'"}, - }, - { - name: "Greater than condition with JSON_VALUE", - ruleType: MetaGreaterThan, - beforeSep: "data", - afterSep: "score", - values: []string{"50"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.score') > 50"}, - }, - { - name: "Less than or equal condition with JSON_VALUE", - ruleType: MetaLessOrEqual, - beforeSep: "data", - afterSep: "score", - values: []string{"30"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.score') <= 30"}, - }, - { - name: "Less than condition with JSON_VALUE", - ruleType: MetaLessThan, - beforeSep: "data", - afterSep: "score", - values: []string{"20"}, - not: false, - expected: []string{"JSON_VALUE(data, '$.score') < 20"}, - }, - - { - name: "MetaExists with no values", - ruleType: MetaExists, - beforeSep: "", - afterSep: "column1", - values: nil, - not: false, - expected: []string{"column1 IS NULL"}, - }, - { - name: "MetaNotExists with no values", - ruleType: MetaNotExists, - beforeSep: "json_field", - afterSep: "key", - values: nil, - not: true, - expected: []string{"JSON_VALUE(json_field, '$.key') IS NOT NULL"}, - }, - { - name: "MetaString with values", - ruleType: MetaString, - beforeSep: "", - afterSep: "column2", - values: []string{"value1", "value2"}, - not: false, - expected: []string{"column2 = 'value1'", "column2 = 'value2'"}, - }, - { - name: "MetaPrefix with NOT condition", - ruleType: MetaNotPrefix, - beforeSep: "json_field", - afterSep: "key", - values: []string{"prefix1"}, - not: true, - expected: []string{"JSON_VALUE(json_field, '$.key') NOT LIKE 'prefix1%'"}, - }, - { - name: "MetaRegex with multiple values", - ruleType: MetaRegex, - beforeSep: "", - afterSep: "column3", - values: []string{"pattern1", "pattern2"}, - not: false, - expected: []string{"column3 REGEXP 'pattern1'", "column3 REGEXP 'pattern2'"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := FilterToSQLQuery(tt.ruleType, tt.beforeSep, tt.afterSep, tt.values, tt.not) - if len(got) != len(tt.expected) { - t.Errorf("expected %v, got %v", tt.expected, got) - return - } - for i, cond := range got { - if cond != tt.expected[i] { - t.Errorf("expected %v, got %v", tt.expected[i], cond) - } - } - }) - } -} diff --git a/utils/rsrfilters_test.go b/utils/rsrfilters_test.go index 09d9278d0..7db50f924 100644 --- a/utils/rsrfilters_test.go +++ b/utils/rsrfilters_test.go @@ -247,9 +247,6 @@ func TestRSRFilterPass(t *testing.T) { if err != nil { t.Error(err) } - if !fltr.Pass("-1s") { - t.Error("not passing!") - } if fltr.Pass("0s") { t.Error("passing!") } @@ -274,9 +271,6 @@ func TestRSRFilterPass(t *testing.T) { if fltr.Pass("12s") { t.Error("passing!") } - if !fltr.Pass("-12s") { - t.Error("not passing!") - } // compare not lessThan fltr, err = NewRSRFilter("!<0s") @@ -301,9 +295,6 @@ func TestRSRFilterPass(t *testing.T) { if err != nil { t.Error(err) } - if !fltr.Pass("-1s") { - t.Error("not passing!") - } if !fltr.Pass("0s") { t.Error("not passing!") }