From 01d3c5d0fa7fa194cfa9773a5c8b65af2f7eb973 Mon Sep 17 00:00:00 2001 From: gezimbll Date: Fri, 23 Jun 2023 09:21:55 -0400 Subject: [PATCH] Separating opts in ERS && fixing tests --- config/config.go | 9 +- config/config_it_test.go | 44 +- config/config_test.go | 133 ++-- config/configsanity.go | 8 +- config/configsanity_test.go | 14 +- config/eescfg.go | 8 +- config/eescfg_test.go | 145 ++-- config/erscfg.go | 1411 +++++++++++++++++++---------------- config/erscfg_test.go | 390 ++++++---- ees/elastic_test.go | 18 +- ees/libcdre_test.go | 11 + ers/amqp.go | 34 +- ers/amqp_test.go | 12 +- ers/amqpv1.go | 15 +- ers/ers.go | 4 +- ers/ers_it_test.go | 8 +- ers/filecsv.go | 16 +- ers/kafka.go | 18 +- ers/kafka_test.go | 16 +- ers/libers.go | 255 ++++--- ers/libers_test.go | 4 +- ers/nats.go | 46 +- ers/readers_test.go | 17 +- ers/s3.go | 30 +- ers/s3_test.go | 22 +- ers/sql.go | 21 +- ers/sql_it_test.go | 4 +- ers/sql_test.go | 42 +- ers/sqs.go | 31 +- ers/sqs_test.go | 16 +- 30 files changed, 1623 insertions(+), 1179 deletions(-) diff --git a/config/config.go b/config/config.go index 651fdda8d..0c2bd05f7 100644 --- a/config/config.go +++ b/config/config.go @@ -185,7 +185,14 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { RPC: new(RPCOpts), Kafka: new(KafkaOpts), }} - cfg.dfltEvRdr = &EventReaderCfg{Opts: &EventReaderOpts{}} + cfg.dfltEvRdr = &EventReaderCfg{Opts: &EventReaderOpts{ + SQLOpts: new(SQLROpts), + CSVOpts: new(CSVROpts), + AWSOpts: new(AWSROpts), + AMQPOpts: new(AMQPROpts), + KafkaOpts: new(KafkaROpts), + NATSOpts: new(NATSROpts), + }} cfg.cacheDP = make(map[string]utils.MapStorage) diff --git a/config/config_it_test.go b/config/config_it_test.go index 201fbfb9c..ae17beb60 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -592,13 +592,21 @@ func testCGRConfigReloadERs(t *testing.T) { CacheDumpFields: []*FCTemplate{}, PartialCommitFields: []*FCTemplate{}, Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - XMLRootPath: utils.StringPointer(utils.EmptyString), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + XMLRootPath: utils.StringPointer(utils.EmptyString), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -614,13 +622,21 @@ func testCGRConfigReloadERs(t *testing.T) { CacheDumpFields: []*FCTemplate{}, PartialCommitFields: []*FCTemplate{}, Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - XMLRootPath: utils.StringPointer(utils.EmptyString), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + KafkaOpts: &KafkaROpts{}, + SQLOpts: &SQLROpts{}, + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + XMLRootPath: utils.StringPointer(utils.EmptyString), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, diff --git a/config/config_test.go b/config/config_test.go index e6bec7d6e..592113d19 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2352,13 +2352,21 @@ func TestERSConfig(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - XMLRootPath: utils.StringPointer(utils.EmptyString), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + XMLRootPath: utils.StringPointer(utils.EmptyString), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -2388,19 +2396,27 @@ func TestEEsNoLksConfig(t *testing.T) { }, Exporters: []*EventExporterCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - ExportPath: "/var/spool/cgrates/ees", - Attempts: 1, - Timezone: utils.EmptyString, - Filters: []string{}, - AttributeSIDs: []string{}, - Flags: utils.FlagsWithParams{}, - contentFields: []*FCTemplate{}, - Fields: []*FCTemplate{}, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + ExportPath: "/var/spool/cgrates/ees", + Attempts: 1, + Timezone: utils.EmptyString, + Filters: []string{}, + AttributeSIDs: []string{}, + Flags: utils.FlagsWithParams{}, + contentFields: []*FCTemplate{}, + Fields: []*FCTemplate{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + SQL: &SQLOpts{}, + Kafka: &KafkaOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + NATS: &NATSOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, }, @@ -5371,13 +5387,21 @@ func TestCgrCdfEventReader(t *testing.T) { CacheDumpFields: []*FCTemplate{}, PartialCommitFields: []*FCTemplate{}, Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - XMLRootPath: utils.StringPointer(utils.EmptyString), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, + PartialCacheAction: utils.StringPointer(utils.MetaNone), + XMLRootPath: utils.StringPointer(utils.EmptyString), }, }, }, @@ -5406,19 +5430,27 @@ func TestCgrCdfEventExporter(t *testing.T) { }, Exporters: []*EventExporterCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - ExportPath: "/var/spool/cgrates/ees", - Attempts: 1, - Timezone: utils.EmptyString, - Filters: []string{}, - AttributeSIDs: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*FCTemplate{}, - contentFields: []*FCTemplate{}, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + ExportPath: "/var/spool/cgrates/ees", + Attempts: 1, + Timezone: utils.EmptyString, + Filters: []string{}, + AttributeSIDs: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{}, + contentFields: []*FCTemplate{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + Els: &ElsOpts{}, + NATS: &NATSOpts{}, + SQL: &SQLOpts{}, + AMQP: &AMQPOpts{}, + RPC: &RPCOpts{}, + Kafka: &KafkaOpts{}, + AWS: &AWSOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, }, @@ -5467,13 +5499,20 @@ func TestCgrCfgEventReaderDefault(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - XMLRootPath: utils.StringPointer(utils.EmptyString), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0)}, + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + XMLRootPath: utils.StringPointer(utils.EmptyString), + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, } for _, v := range eCfg.Fields { diff --git a/config/configsanity.go b/config/configsanity.go index c36a72423..786b67c9b 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -725,8 +725,8 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> nonexistent partial folder: %s for reader with ID: %s", utils.ERs, path, rdr.ID) } if pAct == utils.MetaDumpToFile { - if rdr.Opts.PartialCSVFieldSeparator != nil && // the separtor must not be empty - *rdr.Opts.PartialCSVFieldSeparator == utils.EmptyString { + if rdr.Opts.CSVOpts.PartialCSVFieldSeparator != nil && // the separtor must not be empty + *rdr.Opts.CSVOpts.PartialCSVFieldSeparator == utils.EmptyString { return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.PartialCSVFieldSepartorOpt, rdr.ID) } } @@ -742,8 +742,8 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) } } - if rdr.Opts.CSVFieldSeparator != nil && - *rdr.Opts.CSVFieldSeparator == utils.EmptyString { + if rdr.Opts.CSVOpts.CSVFieldSeparator != nil && + *rdr.Opts.CSVOpts.CSVFieldSeparator == utils.EmptyString { return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.CSVFieldSepOpt, rdr.ID) } case utils.MetaKafkajsonMap: diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 191df1630..7c3ff5490 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1249,7 +1249,9 @@ func TestConfigSanityEventReader(t *testing.T) { ProcessedPath: "/", SourcePath: "/", Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.EmptyString), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.EmptyString), + }, PartialCacheAction: utils.StringPointer(utils.MetaNone), }, }} @@ -1262,7 +1264,9 @@ func TestConfigSanityEventReader(t *testing.T) { Type: utils.MetaKafkajsonMap, RunDelay: 1, Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + }, PartialCacheAction: utils.StringPointer(utils.MetaNone), }, } @@ -2013,8 +2017,10 @@ func TestConfigSanityErs(t *testing.T) { } cfg.ersCfg.Readers[0].Opts = &EventReaderOpts{ - PartialCacheAction: utils.StringPointer(utils.MetaDumpToFile), - PartialCSVFieldSeparator: utils.StringPointer(utils.EmptyString), + PartialCacheAction: utils.StringPointer(utils.MetaDumpToFile), + CSVOpts: &CSVROpts{ + PartialCSVFieldSeparator: utils.StringPointer(utils.EmptyString), + }, } cfg.ersCfg.Readers[0].ProcessedPath = "/tmp" expected = " empty partialcsvFieldSeparator for reader with ID: rdrID" diff --git a/config/eescfg.go b/config/eescfg.go index 96d29b2c5..5c30ec0e3 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -986,16 +986,16 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str opts[utils.ElsOpType] = *elsOpts.OpType } if elsOpts.Pipeline != nil { - opts[utils.ElsPipeline] = *elsOpts.OpType + opts[utils.ElsPipeline] = *elsOpts.Pipeline } if elsOpts.Routing != nil { opts[utils.ElsRouting] = *elsOpts.Routing } if elsOpts.Timeout != nil { - opts[utils.ElsTimeout] = *elsOpts.Timeout + opts[utils.ElsTimeout] = elsOpts.Timeout.String() } if elsOpts.Version != nil { - opts[utils.ElsVersionLow] = *elsOpts.VersionType + opts[utils.ElsVersionLow] = *elsOpts.Version } if elsOpts.VersionType != nil { opts[utils.ElsVersionType] = *elsOpts.VersionType @@ -1012,7 +1012,7 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str opts[utils.SQLMaxOpenConns] = *sqlOpts.MaxOpenConns } if sqlOpts.ConnMaxLifetime != nil { - opts[utils.SQLConnMaxLifetime] = *sqlOpts.ConnMaxLifetime + opts[utils.SQLConnMaxLifetime] = sqlOpts.ConnMaxLifetime.String() } if sqlOpts.MYSQLDSNParams != nil { opts[utils.MYSQLDSNParams] = sqlOpts.MYSQLDSNParams diff --git a/config/eescfg_test.go b/config/eescfg_test.go index 8133ee729..ced96a419 100644 --- a/config/eescfg_test.go +++ b/config/eescfg_test.go @@ -127,21 +127,29 @@ func TestEESClone(t *testing.T) { }, Exporters: []*EventExporterCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - Synchronous: false, - ExportPath: "/var/spool/cgrates/ees", - Attempts: 1, - Timezone: utils.EmptyString, - AttributeSCtx: utils.EmptyString, - Filters: []string{}, - AttributeSIDs: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*FCTemplate{}, - contentFields: []*FCTemplate{}, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + Synchronous: false, + ExportPath: "/var/spool/cgrates/ees", + Attempts: 1, + Timezone: utils.EmptyString, + AttributeSCtx: utils.EmptyString, + Filters: []string{}, + AttributeSIDs: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{}, + contentFields: []*FCTemplate{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + Kafka: &KafkaOpts{}, + NATS: &NATSOpts{}, + SQL: &SQLOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, { @@ -420,6 +428,9 @@ func TestEventExporterOptsloadFromJsonCfg(t *testing.T) { Token: utils.StringPointer("token"), S3FolderPath: utils.StringPointer("s3"), }, + AMQP: &AMQPOpts{}, + Kafka: &KafkaOpts{}, + RPC: &RPCOpts{}, NATS: &NATSOpts{ JetStream: utils.BoolPointer(true), Subject: utils.StringPointer("nat"), @@ -445,7 +456,7 @@ func TestEventExporterOptsloadFromJsonCfg(t *testing.T) { if err := eventExporter.Opts.loadFromJSONCfg(eventExporterOptsJSON); err != nil { t.Error(expected) } else if !reflect.DeepEqual(expected, eventExporter.Opts) { - t.Errorf("expected %v received %v", expected, eventExporter.Opts) + t.Errorf("expected %v received %v", utils.ToJSON(expected), utils.ToJSON(eventExporter.Opts)) } if err := eventExporter.Opts.loadFromJSONCfg(nil); err != nil { t.Error(err) @@ -502,19 +513,27 @@ func TestEventExporterSameID(t *testing.T) { }, Exporters: []*EventExporterCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - ExportPath: "/var/spool/cgrates/ees", - Attempts: 1, - Timezone: utils.EmptyString, - Filters: []string{}, - AttributeSIDs: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*FCTemplate{}, - contentFields: []*FCTemplate{}, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + ExportPath: "/var/spool/cgrates/ees", + Attempts: 1, + Timezone: utils.EmptyString, + Filters: []string{}, + AttributeSIDs: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{}, + contentFields: []*FCTemplate{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + Kafka: &KafkaOpts{}, + NATS: &NATSOpts{}, + SQL: &SQLOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, { @@ -534,9 +553,17 @@ func TestEventExporterSameID(t *testing.T) { {Tag: "CustomTag2", Path: "*exp.CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + Kafka: &KafkaOpts{}, + NATS: &NATSOpts{}, + SQL: &SQLOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, }, @@ -624,19 +651,27 @@ func TestEEsCfgloadFromJsonCfgCase1(t *testing.T) { }, Exporters: []*EventExporterCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - ExportPath: "/var/spool/cgrates/ees", - Attempts: 1, - Timezone: utils.EmptyString, - Filters: []string{}, - AttributeSIDs: []string{}, - Flags: utils.FlagsWithParams{}, - contentFields: []*FCTemplate{}, - Fields: []*FCTemplate{}, - headerFields: []*FCTemplate{}, - trailerFields: []*FCTemplate{}, - Opts: &EventExporterOpts{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + ExportPath: "/var/spool/cgrates/ees", + Attempts: 1, + Timezone: utils.EmptyString, + Filters: []string{}, + AttributeSIDs: []string{}, + Flags: utils.FlagsWithParams{}, + contentFields: []*FCTemplate{}, + Fields: []*FCTemplate{}, + headerFields: []*FCTemplate{}, + trailerFields: []*FCTemplate{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + SQL: &SQLOpts{}, + Kafka: &KafkaOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + NATS: &NATSOpts{}, + }, FailedPostsDir: "/var/spool/cgrates/failed_posts", }, { @@ -660,7 +695,15 @@ func TestEEsCfgloadFromJsonCfgCase1(t *testing.T) { Layout: time.RFC3339, }, }, - Opts: &EventExporterOpts{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + SQL: &SQLOpts{}, + Kafka: &KafkaOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + NATS: &NATSOpts{}, + }, Fields: []*FCTemplate{ {Tag: utils.CGRID, Path: "*exp.CGRID", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.CGRID", utils.InfieldSep), Layout: time.RFC3339}, }, @@ -782,7 +825,15 @@ func TestEEsCfgloadFromJsonCfgCase2(t *testing.T) { }, }, FailedPostsDir: "/var/spool/cgrates/failed_posts", - Opts: &EventExporterOpts{}, + Opts: &EventExporterOpts{ + AMQP: &AMQPOpts{}, + AWS: &AWSOpts{}, + SQL: &SQLOpts{}, + Kafka: &KafkaOpts{}, + RPC: &RPCOpts{}, + Els: &ElsOpts{}, + NATS: &NATSOpts{}, + }, Fields: []*FCTemplate{ { Tag: utils.CGRID, diff --git a/config/erscfg.go b/config/erscfg.go index 583ae5f1c..6aa7ac2fa 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -135,52 +135,184 @@ func (erS *ERsCfg) AsMapInterface(separator string) (initialMP map[string]any) { return } -type EventReaderOpts struct { - PartialPath *string - PartialCacheAction *string - PartialOrderField *string - PartialCSVFieldSeparator *string - CSVRowLength *int - CSVFieldSeparator *string - CSVHeaderDefineChar *string - CSVLazyQuotes *bool - XMLRootPath *string - AMQPQueueID *string - AMQPQueueIDProcessed *string - AMQPUsername *string - AMQPPassword *string - AMQPUsernameProcessed *string - AMQPPasswordProcessed *string - AMQPConsumerTag *string - AMQPExchange *string - AMQPExchangeType *string - AMQPRoutingKey *string - AMQPExchangeProcessed *string - AMQPExchangeTypeProcessed *string - AMQPRoutingKeyProcessed *string - KafkaTopic *string - KafkaGroupID *string - KafkaMaxWait *time.Duration - KafkaTopicProcessed *string - SQLDBName *string - SQLTableName *string - PgSSLMode *string - SQLDBNameProcessed *string - SQLTableNameProcessed *string - PgSSLModeProcessed *string - AWSRegion *string - AWSKey *string - AWSSecret *string - AWSToken *string - AWSRegionProcessed *string - AWSKeyProcessed *string - AWSSecretProcessed *string - AWSTokenProcessed *string - SQSQueueID *string - SQSQueueIDProcessed *string - S3BucketID *string - S3FolderPathProcessed *string - S3BucketIDProcessed *string +type AMQPROpts struct { + AMQPQueueID *string + AMQPQueueIDProcessed *string + AMQPUsername *string + AMQPPassword *string + AMQPUsernameProcessed *string + AMQPPasswordProcessed *string + AMQPConsumerTag *string + AMQPExchange *string + AMQPExchangeType *string + AMQPRoutingKey *string + AMQPExchangeProcessed *string + AMQPExchangeTypeProcessed *string + AMQPRoutingKeyProcessed *string +} + +func (amqpr *AMQPROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + if jsnCfg.AMQPQueueID != nil { + amqpr.AMQPQueueID = jsnCfg.AMQPQueueID + } + if jsnCfg.AMQPQueueIDProcessed != nil { + amqpr.AMQPQueueIDProcessed = jsnCfg.AMQPQueueIDProcessed + } + if jsnCfg.AMQPUsername != nil { + amqpr.AMQPUsername = jsnCfg.AMQPUsername + } + if jsnCfg.AMQPPassword != nil { + amqpr.AMQPPassword = jsnCfg.AMQPPassword + } + if jsnCfg.AMQPUsernameProcessed != nil { + amqpr.AMQPUsernameProcessed = jsnCfg.AMQPUsernameProcessed + } + if jsnCfg.AMQPPasswordProcessed != nil { + amqpr.AMQPPasswordProcessed = jsnCfg.AMQPPasswordProcessed + } + if jsnCfg.AMQPConsumerTag != nil { + amqpr.AMQPConsumerTag = jsnCfg.AMQPConsumerTag + } + if jsnCfg.AMQPExchange != nil { + amqpr.AMQPExchange = jsnCfg.AMQPExchange + } + if jsnCfg.AMQPExchangeType != nil { + amqpr.AMQPExchangeType = jsnCfg.AMQPExchangeType + } + if jsnCfg.AMQPRoutingKey != nil { + amqpr.AMQPRoutingKey = jsnCfg.AMQPRoutingKey + } + if jsnCfg.AMQPExchangeProcessed != nil { + amqpr.AMQPExchangeProcessed = jsnCfg.AMQPExchangeProcessed + } + if jsnCfg.AMQPExchangeTypeProcessed != nil { + amqpr.AMQPExchangeTypeProcessed = jsnCfg.AMQPExchangeTypeProcessed + } + if jsnCfg.AMQPRoutingKeyProcessed != nil { + amqpr.AMQPRoutingKeyProcessed = jsnCfg.AMQPRoutingKeyProcessed + } + return +} + +type KafkaROpts struct { + KafkaTopic *string + KafkaGroupID *string + KafkaMaxWait *time.Duration + KafkaTopicProcessed *string +} + +func (kafkaROpts *KafkaROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + if jsnCfg.KafkaTopic != nil { + kafkaROpts.KafkaTopic = jsnCfg.KafkaTopic + } + if jsnCfg.KafkaGroupID != nil { + kafkaROpts.KafkaGroupID = jsnCfg.KafkaGroupID + } + if jsnCfg.KafkaMaxWait != nil { + var kafkaMaxWait time.Duration + if kafkaMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.KafkaMaxWait); err != nil { + return + } + kafkaROpts.KafkaMaxWait = utils.DurationPointer(kafkaMaxWait) + } + if jsnCfg.KafkaTopicProcessed != nil { + kafkaROpts.KafkaTopicProcessed = jsnCfg.KafkaTopicProcessed + } + return +} + +type SQLROpts struct { + SQLDBName *string + SQLTableName *string + PgSSLMode *string + SQLDBNameProcessed *string + SQLTableNameProcessed *string + PgSSLModeProcessed *string +} + +func (sqlOpts *SQLROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + if jsnCfg.SQLDBName != nil { + sqlOpts.SQLDBName = jsnCfg.SQLDBName + } + if jsnCfg.SQLTableName != nil { + sqlOpts.SQLTableName = jsnCfg.SQLTableName + } + if jsnCfg.PgSSLMode != nil { + sqlOpts.PgSSLMode = jsnCfg.PgSSLMode + } + if jsnCfg.SQLDBNameProcessed != nil { + sqlOpts.SQLDBNameProcessed = jsnCfg.SQLDBNameProcessed + } + if jsnCfg.SQLTableNameProcessed != nil { + sqlOpts.SQLTableNameProcessed = jsnCfg.SQLTableNameProcessed + } + if jsnCfg.PgSSLModeProcessed != nil { + sqlOpts.PgSSLModeProcessed = jsnCfg.PgSSLModeProcessed + } + return +} + +type AWSROpts struct { + AWSRegion *string + AWSKey *string + AWSSecret *string + AWSToken *string + AWSRegionProcessed *string + AWSKeyProcessed *string + AWSSecretProcessed *string + AWSTokenProcessed *string + SQSQueueID *string + SQSQueueIDProcessed *string + S3BucketID *string + S3FolderPathProcessed *string + S3BucketIDProcessed *string +} + +func (awsROpts *AWSROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + + if jsnCfg.AWSRegion != nil { + awsROpts.AWSRegion = jsnCfg.AWSRegion + } + if jsnCfg.AWSKey != nil { + awsROpts.AWSKey = jsnCfg.AWSKey + } + if jsnCfg.AWSSecret != nil { + awsROpts.AWSSecret = jsnCfg.AWSSecret + } + if jsnCfg.AWSToken != nil { + awsROpts.AWSToken = jsnCfg.AWSToken + } + if jsnCfg.AWSRegionProcessed != nil { + awsROpts.AWSRegionProcessed = jsnCfg.AWSRegionProcessed + } + if jsnCfg.AWSKeyProcessed != nil { + awsROpts.AWSKeyProcessed = jsnCfg.AWSKeyProcessed + } + if jsnCfg.AWSSecretProcessed != nil { + awsROpts.AWSSecretProcessed = jsnCfg.AWSSecretProcessed + } + if jsnCfg.AWSTokenProcessed != nil { + awsROpts.AWSTokenProcessed = jsnCfg.AWSTokenProcessed + } + if jsnCfg.SQSQueueID != nil { + awsROpts.SQSQueueID = jsnCfg.SQSQueueID + } + if jsnCfg.SQSQueueIDProcessed != nil { + awsROpts.SQSQueueIDProcessed = jsnCfg.SQSQueueIDProcessed + } + if jsnCfg.S3BucketID != nil { + awsROpts.S3BucketID = jsnCfg.S3BucketID + } + if jsnCfg.S3FolderPathProcessed != nil { + awsROpts.S3FolderPathProcessed = jsnCfg.S3FolderPathProcessed + } + if jsnCfg.S3BucketIDProcessed != nil { + awsROpts.S3BucketIDProcessed = jsnCfg.S3BucketIDProcessed + } + return +} + +type NATSROpts struct { NATSJetStream *bool NATSConsumerName *string NATSSubject *string @@ -201,6 +333,112 @@ type EventReaderOpts struct { NATSJetStreamMaxWaitProcessed *time.Duration } +func (natsOpts *NATSROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + if jsnCfg.NATSJetStream != nil { + natsOpts.NATSJetStream = jsnCfg.NATSJetStream + } + if jsnCfg.NATSConsumerName != nil { + natsOpts.NATSConsumerName = jsnCfg.NATSConsumerName + } + if jsnCfg.NATSSubject != nil { + natsOpts.NATSSubject = jsnCfg.NATSSubject + } + if jsnCfg.NATSQueueID != nil { + natsOpts.NATSQueueID = jsnCfg.NATSQueueID + } + if jsnCfg.NATSJWTFile != nil { + natsOpts.NATSJWTFile = jsnCfg.NATSJWTFile + } + if jsnCfg.NATSSeedFile != nil { + natsOpts.NATSSeedFile = jsnCfg.NATSSeedFile + } + if jsnCfg.NATSCertificateAuthority != nil { + natsOpts.NATSCertificateAuthority = jsnCfg.NATSCertificateAuthority + } + if jsnCfg.NATSClientCertificate != nil { + natsOpts.NATSClientCertificate = jsnCfg.NATSClientCertificate + } + if jsnCfg.NATSClientKey != nil { + natsOpts.NATSClientKey = jsnCfg.NATSClientKey + } + if jsnCfg.NATSJetStreamMaxWait != nil { + var jetStreamMaxWait time.Duration + if jetStreamMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.NATSJetStreamMaxWait); err != nil { + return + } + natsOpts.NATSJetStreamMaxWait = utils.DurationPointer(jetStreamMaxWait) + } + if jsnCfg.NATSJetStreamProcessed != nil { + natsOpts.NATSJetStreamProcessed = jsnCfg.NATSJetStreamProcessed + } + if jsnCfg.NATSSubjectProcessed != nil { + natsOpts.NATSSubjectProcessed = jsnCfg.NATSSubjectProcessed + } + if jsnCfg.NATSJWTFileProcessed != nil { + natsOpts.NATSJWTFileProcessed = jsnCfg.NATSJWTFileProcessed + } + if jsnCfg.NATSSeedFileProcessed != nil { + natsOpts.NATSSeedFileProcessed = jsnCfg.NATSSeedFileProcessed + } + if jsnCfg.NATSCertificateAuthorityProcessed != nil { + natsOpts.NATSCertificateAuthorityProcessed = jsnCfg.NATSCertificateAuthorityProcessed + } + if jsnCfg.NATSClientCertificateProcessed != nil { + natsOpts.NATSClientCertificateProcessed = jsnCfg.NATSClientCertificateProcessed + } + if jsnCfg.NATSClientKeyProcessed != nil { + natsOpts.NATSClientKeyProcessed = jsnCfg.NATSClientKeyProcessed + } + if jsnCfg.NATSJetStreamMaxWaitProcessed != nil { + var jetStreamMaxWait time.Duration + if jetStreamMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.NATSJetStreamMaxWaitProcessed); err != nil { + return + } + natsOpts.NATSJetStreamMaxWaitProcessed = utils.DurationPointer(jetStreamMaxWait) + } + return +} + +type CSVROpts struct { + PartialCSVFieldSeparator *string + CSVRowLength *int + CSVFieldSeparator *string + CSVHeaderDefineChar *string + CSVLazyQuotes *bool +} + +func (csvROpts *CSVROpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err error) { + if jsnCfg.PartialCSVFieldSeparator != nil { + csvROpts.PartialCSVFieldSeparator = jsnCfg.PartialCSVFieldSeparator + } + if jsnCfg.CSVRowLength != nil { + csvROpts.CSVRowLength = jsnCfg.CSVRowLength + } + if jsnCfg.CSVFieldSeparator != nil { + csvROpts.CSVFieldSeparator = jsnCfg.CSVFieldSeparator + } + if jsnCfg.CSVHeaderDefineChar != nil { + csvROpts.CSVHeaderDefineChar = jsnCfg.CSVHeaderDefineChar + } + if jsnCfg.CSVLazyQuotes != nil { + csvROpts.CSVLazyQuotes = jsnCfg.CSVLazyQuotes + } + return +} + +type EventReaderOpts struct { + PartialPath *string + PartialCacheAction *string + PartialOrderField *string + XMLRootPath *string + CSVOpts *CSVROpts + AMQPOpts *AMQPROpts + AWSOpts *AWSROpts + NATSOpts *NATSROpts + KafkaOpts *KafkaROpts + SQLOpts *SQLROpts +} + // EventReaderCfg the event for the Event Reader type EventReaderCfg struct { ID string @@ -223,6 +461,24 @@ func (erOpts *EventReaderOpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err if jsnCfg == nil { return } + if err = erOpts.AMQPOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } + if err = erOpts.AWSOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } + if err = erOpts.KafkaOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } + if err = erOpts.NATSOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } + if err = erOpts.SQLOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } + if err = erOpts.CSVOpts.loadFromJSONCfg(jsnCfg); err != nil { + return + } if jsnCfg.PartialPath != nil { erOpts.PartialPath = jsnCfg.PartialPath } @@ -232,198 +488,11 @@ func (erOpts *EventReaderOpts) loadFromJSONCfg(jsnCfg *EventReaderOptsJson) (err if jsnCfg.PartialOrderField != nil { erOpts.PartialOrderField = jsnCfg.PartialOrderField } - if jsnCfg.PartialCSVFieldSeparator != nil { - erOpts.PartialCSVFieldSeparator = jsnCfg.PartialCSVFieldSeparator - } - if jsnCfg.CSVRowLength != nil { - erOpts.CSVRowLength = jsnCfg.CSVRowLength - } - if jsnCfg.CSVFieldSeparator != nil { - erOpts.CSVFieldSeparator = jsnCfg.CSVFieldSeparator - } - if jsnCfg.CSVHeaderDefineChar != nil { - erOpts.CSVHeaderDefineChar = jsnCfg.CSVHeaderDefineChar - } - if jsnCfg.CSVLazyQuotes != nil { - erOpts.CSVLazyQuotes = jsnCfg.CSVLazyQuotes - } + if jsnCfg.XMLRootPath != nil { erOpts.XMLRootPath = jsnCfg.XMLRootPath } - if jsnCfg.AMQPQueueID != nil { - erOpts.AMQPQueueID = jsnCfg.AMQPQueueID - } - if jsnCfg.AMQPQueueIDProcessed != nil { - erOpts.AMQPQueueIDProcessed = jsnCfg.AMQPQueueIDProcessed - } - if jsnCfg.AMQPUsername != nil { - erOpts.AMQPUsername = jsnCfg.AMQPUsername - } - if jsnCfg.AMQPPassword != nil { - erOpts.AMQPPassword = jsnCfg.AMQPPassword - } - if jsnCfg.AMQPUsernameProcessed != nil { - erOpts.AMQPUsernameProcessed = jsnCfg.AMQPUsernameProcessed - } - if jsnCfg.AMQPPasswordProcessed != nil { - erOpts.AMQPPasswordProcessed = jsnCfg.AMQPPasswordProcessed - } - if jsnCfg.AMQPConsumerTag != nil { - erOpts.AMQPConsumerTag = jsnCfg.AMQPConsumerTag - } - if jsnCfg.AMQPExchange != nil { - erOpts.AMQPExchange = jsnCfg.AMQPExchange - } - if jsnCfg.AMQPExchangeType != nil { - erOpts.AMQPExchangeType = jsnCfg.AMQPExchangeType - } - if jsnCfg.AMQPRoutingKey != nil { - erOpts.AMQPRoutingKey = jsnCfg.AMQPRoutingKey - } - if jsnCfg.AMQPExchangeProcessed != nil { - erOpts.AMQPExchangeProcessed = jsnCfg.AMQPExchangeProcessed - } - if jsnCfg.AMQPExchangeTypeProcessed != nil { - erOpts.AMQPExchangeTypeProcessed = jsnCfg.AMQPExchangeTypeProcessed - } - if jsnCfg.AMQPRoutingKeyProcessed != nil { - erOpts.AMQPRoutingKeyProcessed = jsnCfg.AMQPRoutingKeyProcessed - } - if jsnCfg.KafkaTopic != nil { - erOpts.KafkaTopic = jsnCfg.KafkaTopic - } - if jsnCfg.KafkaGroupID != nil { - erOpts.KafkaGroupID = jsnCfg.KafkaGroupID - } - if jsnCfg.KafkaMaxWait != nil { - var kafkaMaxWait time.Duration - if kafkaMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.KafkaMaxWait); err != nil { - return - } - erOpts.KafkaMaxWait = utils.DurationPointer(kafkaMaxWait) - } - if jsnCfg.KafkaTopicProcessed != nil { - erOpts.KafkaTopicProcessed = jsnCfg.KafkaTopicProcessed - } - if jsnCfg.SQLDBName != nil { - erOpts.SQLDBName = jsnCfg.SQLDBName - } - if jsnCfg.SQLTableName != nil { - erOpts.SQLTableName = jsnCfg.SQLTableName - } - if jsnCfg.PgSSLMode != nil { - erOpts.PgSSLMode = jsnCfg.PgSSLMode - } - if jsnCfg.SQLDBNameProcessed != nil { - erOpts.SQLDBNameProcessed = jsnCfg.SQLDBNameProcessed - } - if jsnCfg.SQLTableNameProcessed != nil { - erOpts.SQLTableNameProcessed = jsnCfg.SQLTableNameProcessed - } - if jsnCfg.PgSSLModeProcessed != nil { - erOpts.PgSSLModeProcessed = jsnCfg.PgSSLModeProcessed - } - if jsnCfg.AWSRegion != nil { - erOpts.AWSRegion = jsnCfg.AWSRegion - } - if jsnCfg.AWSKey != nil { - erOpts.AWSKey = jsnCfg.AWSKey - } - if jsnCfg.AWSSecret != nil { - erOpts.AWSSecret = jsnCfg.AWSSecret - } - if jsnCfg.AWSToken != nil { - erOpts.AWSToken = jsnCfg.AWSToken - } - if jsnCfg.AWSRegionProcessed != nil { - erOpts.AWSRegionProcessed = jsnCfg.AWSRegionProcessed - } - if jsnCfg.AWSKeyProcessed != nil { - erOpts.AWSKeyProcessed = jsnCfg.AWSKeyProcessed - } - if jsnCfg.AWSSecretProcessed != nil { - erOpts.AWSSecretProcessed = jsnCfg.AWSSecretProcessed - } - if jsnCfg.AWSTokenProcessed != nil { - erOpts.AWSTokenProcessed = jsnCfg.AWSTokenProcessed - } - if jsnCfg.SQSQueueID != nil { - erOpts.SQSQueueID = jsnCfg.SQSQueueID - } - if jsnCfg.SQSQueueIDProcessed != nil { - erOpts.SQSQueueIDProcessed = jsnCfg.SQSQueueIDProcessed - } - if jsnCfg.S3BucketID != nil { - erOpts.S3BucketID = jsnCfg.S3BucketID - } - if jsnCfg.S3FolderPathProcessed != nil { - erOpts.S3FolderPathProcessed = jsnCfg.S3FolderPathProcessed - } - if jsnCfg.S3BucketIDProcessed != nil { - erOpts.S3BucketIDProcessed = jsnCfg.S3BucketIDProcessed - } - if jsnCfg.NATSJetStream != nil { - erOpts.NATSJetStream = jsnCfg.NATSJetStream - } - if jsnCfg.NATSConsumerName != nil { - erOpts.NATSConsumerName = jsnCfg.NATSConsumerName - } - if jsnCfg.NATSSubject != nil { - erOpts.NATSSubject = jsnCfg.NATSSubject - } - if jsnCfg.NATSQueueID != nil { - erOpts.NATSQueueID = jsnCfg.NATSQueueID - } - if jsnCfg.NATSJWTFile != nil { - erOpts.NATSJWTFile = jsnCfg.NATSJWTFile - } - if jsnCfg.NATSSeedFile != nil { - erOpts.NATSSeedFile = jsnCfg.NATSSeedFile - } - if jsnCfg.NATSCertificateAuthority != nil { - erOpts.NATSCertificateAuthority = jsnCfg.NATSCertificateAuthority - } - if jsnCfg.NATSClientCertificate != nil { - erOpts.NATSClientCertificate = jsnCfg.NATSClientCertificate - } - if jsnCfg.NATSClientKey != nil { - erOpts.NATSClientKey = jsnCfg.NATSClientKey - } - if jsnCfg.NATSJetStreamMaxWait != nil { - var jetStreamMaxWait time.Duration - if jetStreamMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.NATSJetStreamMaxWait); err != nil { - return - } - erOpts.NATSJetStreamMaxWait = utils.DurationPointer(jetStreamMaxWait) - } - if jsnCfg.NATSJetStreamProcessed != nil { - erOpts.NATSJetStreamProcessed = jsnCfg.NATSJetStreamProcessed - } - if jsnCfg.NATSSubjectProcessed != nil { - erOpts.NATSSubjectProcessed = jsnCfg.NATSSubjectProcessed - } - if jsnCfg.NATSJWTFileProcessed != nil { - erOpts.NATSJWTFileProcessed = jsnCfg.NATSJWTFileProcessed - } - if jsnCfg.NATSSeedFileProcessed != nil { - erOpts.NATSSeedFileProcessed = jsnCfg.NATSSeedFileProcessed - } - if jsnCfg.NATSCertificateAuthorityProcessed != nil { - erOpts.NATSCertificateAuthorityProcessed = jsnCfg.NATSCertificateAuthorityProcessed - } - if jsnCfg.NATSClientCertificateProcessed != nil { - erOpts.NATSClientCertificateProcessed = jsnCfg.NATSClientCertificateProcessed - } - if jsnCfg.NATSClientKeyProcessed != nil { - erOpts.NATSClientKeyProcessed = jsnCfg.NATSClientKeyProcessed - } - if jsnCfg.NATSJetStreamMaxWaitProcessed != nil { - var jetStreamMaxWait time.Duration - if jetStreamMaxWait, err = utils.ParseDurationWithNanosecs(*jsnCfg.NATSJetStreamMaxWaitProcessed); err != nil { - return - } - erOpts.NATSJetStreamMaxWaitProcessed = utils.DurationPointer(jetStreamMaxWait) - } + return } @@ -503,6 +572,269 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat } return } +func (amqpOpts *AMQPROpts) Clone() *AMQPROpts { + cln := &AMQPROpts{} + if amqpOpts.AMQPQueueID != nil { + cln.AMQPQueueID = new(string) + *cln.AMQPQueueID = *amqpOpts.AMQPQueueID + } + if amqpOpts.AMQPQueueIDProcessed != nil { + cln.AMQPQueueIDProcessed = new(string) + *cln.AMQPQueueIDProcessed = *amqpOpts.AMQPQueueIDProcessed + } + if amqpOpts.AMQPUsername != nil { + cln.AMQPUsername = new(string) + *cln.AMQPUsername = *amqpOpts.AMQPUsername + } + if amqpOpts.AMQPPassword != nil { + cln.AMQPPassword = new(string) + *cln.AMQPPassword = *amqpOpts.AMQPPassword + } + if amqpOpts.AMQPUsernameProcessed != nil { + cln.AMQPUsernameProcessed = new(string) + *cln.AMQPUsernameProcessed = *amqpOpts.AMQPUsernameProcessed + } + if amqpOpts.AMQPPasswordProcessed != nil { + cln.AMQPPasswordProcessed = new(string) + *cln.AMQPPasswordProcessed = *amqpOpts.AMQPPasswordProcessed + } + if amqpOpts.AMQPConsumerTag != nil { + cln.AMQPConsumerTag = new(string) + *cln.AMQPConsumerTag = *amqpOpts.AMQPConsumerTag + } + if amqpOpts.AMQPExchange != nil { + cln.AMQPExchange = new(string) + *cln.AMQPExchange = *amqpOpts.AMQPExchange + } + if amqpOpts.AMQPExchangeType != nil { + cln.AMQPExchangeType = new(string) + *cln.AMQPExchangeType = *amqpOpts.AMQPExchangeType + } + if amqpOpts.AMQPRoutingKey != nil { + cln.AMQPRoutingKey = new(string) + *cln.AMQPRoutingKey = *amqpOpts.AMQPRoutingKey + } + if amqpOpts.AMQPExchangeProcessed != nil { + cln.AMQPExchangeProcessed = new(string) + *cln.AMQPExchangeProcessed = *amqpOpts.AMQPExchangeProcessed + } + if amqpOpts.AMQPExchangeTypeProcessed != nil { + cln.AMQPExchangeTypeProcessed = new(string) + *cln.AMQPExchangeTypeProcessed = *amqpOpts.AMQPExchangeTypeProcessed + } + if amqpOpts.AMQPRoutingKeyProcessed != nil { + cln.AMQPRoutingKeyProcessed = new(string) + *cln.AMQPRoutingKeyProcessed = *amqpOpts.AMQPRoutingKeyProcessed + } + return cln +} + +func (csvOpts *CSVROpts) Clone() *CSVROpts { + cln := &CSVROpts{} + if csvOpts.PartialCSVFieldSeparator != nil { + cln.PartialCSVFieldSeparator = new(string) + *cln.PartialCSVFieldSeparator = *csvOpts.PartialCSVFieldSeparator + } + if csvOpts.CSVRowLength != nil { + cln.CSVRowLength = new(int) + *cln.CSVRowLength = *csvOpts.CSVRowLength + } + if csvOpts.CSVFieldSeparator != nil { + cln.CSVFieldSeparator = new(string) + *cln.CSVFieldSeparator = *csvOpts.CSVFieldSeparator + } + if csvOpts.CSVHeaderDefineChar != nil { + cln.CSVHeaderDefineChar = new(string) + *cln.CSVHeaderDefineChar = *csvOpts.CSVHeaderDefineChar + } + if csvOpts.CSVLazyQuotes != nil { + cln.CSVLazyQuotes = new(bool) + *cln.CSVLazyQuotes = *csvOpts.CSVLazyQuotes + } + return cln +} +func (kafkaOpts *KafkaROpts) Clone() *KafkaROpts { + cln := &KafkaROpts{} + if kafkaOpts.KafkaTopic != nil { + cln.KafkaTopic = new(string) + *cln.KafkaTopic = *kafkaOpts.KafkaTopic + } + if kafkaOpts.KafkaGroupID != nil { + cln.KafkaGroupID = new(string) + *cln.KafkaGroupID = *kafkaOpts.KafkaGroupID + } + if kafkaOpts.KafkaMaxWait != nil { + cln.KafkaMaxWait = new(time.Duration) + *cln.KafkaMaxWait = *kafkaOpts.KafkaMaxWait + } + if kafkaOpts.KafkaTopicProcessed != nil { + cln.KafkaTopicProcessed = new(string) + *cln.KafkaTopicProcessed = *kafkaOpts.KafkaTopicProcessed + } + return cln +} + +func (sqlOpts *SQLROpts) Clone() *SQLROpts { + cln := &SQLROpts{} + if sqlOpts.SQLDBName != nil { + cln.SQLDBName = new(string) + *cln.SQLDBName = *sqlOpts.SQLDBName + } + if sqlOpts.SQLTableName != nil { + cln.SQLTableName = new(string) + *cln.SQLTableName = *sqlOpts.SQLTableName + } + if sqlOpts.PgSSLMode != nil { + cln.PgSSLMode = new(string) + *cln.PgSSLMode = *sqlOpts.PgSSLMode + } + if sqlOpts.SQLDBNameProcessed != nil { + cln.SQLDBNameProcessed = new(string) + *cln.SQLDBNameProcessed = *sqlOpts.SQLDBNameProcessed + } + if sqlOpts.SQLTableNameProcessed != nil { + cln.SQLTableNameProcessed = new(string) + *cln.SQLTableNameProcessed = *sqlOpts.SQLTableNameProcessed + } + if sqlOpts.PgSSLModeProcessed != nil { + cln.PgSSLModeProcessed = new(string) + *cln.PgSSLModeProcessed = *sqlOpts.PgSSLModeProcessed + } + return cln +} + +func (awsOpt *AWSROpts) Clone() *AWSROpts { + cln := &AWSROpts{} + if awsOpt.AWSRegion != nil { + cln.AWSRegion = new(string) + *cln.AWSRegion = *awsOpt.AWSRegion + } + if awsOpt.AWSKey != nil { + cln.AWSKey = new(string) + *cln.AWSKey = *awsOpt.AWSKey + } + if awsOpt.AWSSecret != nil { + cln.AWSSecret = new(string) + *cln.AWSSecret = *awsOpt.AWSSecret + } + if awsOpt.AWSToken != nil { + cln.AWSToken = new(string) + *cln.AWSToken = *awsOpt.AWSToken + } + if awsOpt.AWSRegionProcessed != nil { + cln.AWSRegionProcessed = new(string) + *cln.AWSRegionProcessed = *awsOpt.AWSRegionProcessed + } + if awsOpt.AWSKeyProcessed != nil { + cln.AWSKeyProcessed = new(string) + *cln.AWSKeyProcessed = *awsOpt.AWSKeyProcessed + } + if awsOpt.AWSSecretProcessed != nil { + cln.AWSSecretProcessed = new(string) + *cln.AWSSecretProcessed = *awsOpt.AWSSecretProcessed + } + if awsOpt.AWSTokenProcessed != nil { + cln.AWSTokenProcessed = new(string) + *cln.AWSTokenProcessed = *awsOpt.AWSTokenProcessed + } + if awsOpt.SQSQueueID != nil { + cln.SQSQueueID = new(string) + *cln.SQSQueueID = *awsOpt.SQSQueueID + } + if awsOpt.SQSQueueIDProcessed != nil { + cln.SQSQueueIDProcessed = new(string) + *cln.SQSQueueIDProcessed = *awsOpt.SQSQueueIDProcessed + } + if awsOpt.S3BucketID != nil { + cln.S3BucketID = new(string) + *cln.S3BucketID = *awsOpt.S3BucketID + } + if awsOpt.S3FolderPathProcessed != nil { + cln.S3FolderPathProcessed = new(string) + *cln.S3FolderPathProcessed = *awsOpt.S3FolderPathProcessed + } + if awsOpt.S3BucketIDProcessed != nil { + cln.S3BucketIDProcessed = new(string) + *cln.S3BucketIDProcessed = *awsOpt.S3BucketIDProcessed + } + return cln +} +func (natOpts *NATSROpts) Clone() *NATSROpts { + cln := &NATSROpts{} + if natOpts.NATSJetStream != nil { + cln.NATSJetStream = new(bool) + *cln.NATSJetStream = *natOpts.NATSJetStream + } + if natOpts.NATSConsumerName != nil { + cln.NATSConsumerName = new(string) + *cln.NATSConsumerName = *natOpts.NATSConsumerName + } + if natOpts.NATSSubject != nil { + cln.NATSSubject = new(string) + *cln.NATSSubject = *natOpts.NATSSubject + } + if natOpts.NATSQueueID != nil { + cln.NATSQueueID = new(string) + *cln.NATSQueueID = *natOpts.NATSQueueID + } + if natOpts.NATSJWTFile != nil { + cln.NATSJWTFile = new(string) + *cln.NATSJWTFile = *natOpts.NATSJWTFile + } + if natOpts.NATSSeedFile != nil { + cln.NATSSeedFile = new(string) + *cln.NATSSeedFile = *natOpts.NATSSeedFile + } + if natOpts.NATSCertificateAuthority != nil { + cln.NATSCertificateAuthority = new(string) + *cln.NATSCertificateAuthority = *natOpts.NATSCertificateAuthority + } + if natOpts.NATSClientCertificate != nil { + cln.NATSClientCertificate = new(string) + *cln.NATSClientCertificate = *natOpts.NATSClientCertificate + } + if natOpts.NATSClientKey != nil { + cln.NATSClientKey = new(string) + *cln.NATSClientKey = *natOpts.NATSClientKey + } + if natOpts.NATSJetStreamMaxWait != nil { + cln.NATSJetStreamMaxWait = new(time.Duration) + *cln.NATSJetStreamMaxWait = *natOpts.NATSJetStreamMaxWait + } + if natOpts.NATSJetStreamProcessed != nil { + cln.NATSJetStreamProcessed = new(bool) + *cln.NATSJetStreamProcessed = *natOpts.NATSJetStreamProcessed + } + if natOpts.NATSSubjectProcessed != nil { + cln.NATSSubjectProcessed = new(string) + *cln.NATSSubjectProcessed = *natOpts.NATSSubjectProcessed + } + if natOpts.NATSJWTFileProcessed != nil { + cln.NATSJWTFileProcessed = new(string) + *cln.NATSJWTFileProcessed = *natOpts.NATSJWTFileProcessed + } + if natOpts.NATSSeedFileProcessed != nil { + cln.NATSSeedFileProcessed = new(string) + *cln.NATSSeedFileProcessed = *natOpts.NATSSeedFileProcessed + } + if natOpts.NATSCertificateAuthorityProcessed != nil { + cln.NATSCertificateAuthorityProcessed = new(string) + *cln.NATSCertificateAuthorityProcessed = *natOpts.NATSCertificateAuthorityProcessed + } + if natOpts.NATSClientCertificateProcessed != nil { + cln.NATSClientCertificateProcessed = new(string) + *cln.NATSClientCertificateProcessed = *natOpts.NATSClientCertificateProcessed + } + if natOpts.NATSClientKeyProcessed != nil { + cln.NATSClientKeyProcessed = new(string) + *cln.NATSClientKeyProcessed = *natOpts.NATSClientKeyProcessed + } + if natOpts.NATSJetStreamMaxWaitProcessed != nil { + cln.NATSJetStreamMaxWaitProcessed = new(time.Duration) + *cln.NATSJetStreamMaxWaitProcessed = *natOpts.NATSJetStreamMaxWaitProcessed + } + return cln +} func (erOpts *EventReaderOpts) Clone() *EventReaderOpts { cln := &EventReaderOpts{} @@ -518,246 +850,29 @@ func (erOpts *EventReaderOpts) Clone() *EventReaderOpts { cln.PartialOrderField = new(string) *cln.PartialOrderField = *erOpts.PartialOrderField } - if erOpts.PartialCSVFieldSeparator != nil { - cln.PartialCSVFieldSeparator = new(string) - *cln.PartialCSVFieldSeparator = *erOpts.PartialCSVFieldSeparator - } - if erOpts.CSVRowLength != nil { - cln.CSVRowLength = new(int) - *cln.CSVRowLength = *erOpts.CSVRowLength - } - if erOpts.CSVFieldSeparator != nil { - cln.CSVFieldSeparator = new(string) - *cln.CSVFieldSeparator = *erOpts.CSVFieldSeparator - } - if erOpts.CSVHeaderDefineChar != nil { - cln.CSVHeaderDefineChar = new(string) - *cln.CSVHeaderDefineChar = *erOpts.CSVHeaderDefineChar - } - if erOpts.CSVLazyQuotes != nil { - cln.CSVLazyQuotes = new(bool) - *cln.CSVLazyQuotes = *erOpts.CSVLazyQuotes + if erOpts.CSVOpts != nil { + cln.CSVOpts = erOpts.CSVOpts.Clone() } if erOpts.XMLRootPath != nil { cln.XMLRootPath = new(string) *cln.XMLRootPath = *erOpts.XMLRootPath } - if erOpts.AMQPQueueID != nil { - cln.AMQPQueueID = new(string) - *cln.AMQPQueueID = *erOpts.AMQPQueueID + if erOpts.AMQPOpts != nil { + cln.AMQPOpts = erOpts.AMQPOpts.Clone() } - if erOpts.AMQPQueueIDProcessed != nil { - cln.AMQPQueueIDProcessed = new(string) - *cln.AMQPQueueIDProcessed = *erOpts.AMQPQueueIDProcessed + if erOpts.NATSOpts != nil { + cln.NATSOpts = erOpts.NATSOpts.Clone() } - if erOpts.AMQPUsername != nil { - cln.AMQPUsername = new(string) - *cln.AMQPUsername = *erOpts.AMQPUsername + if erOpts.KafkaOpts != nil { + cln.KafkaOpts = erOpts.KafkaOpts.Clone() } - if erOpts.AMQPPassword != nil { - cln.AMQPPassword = new(string) - *cln.AMQPPassword = *erOpts.AMQPPassword + if erOpts.SQLOpts != nil { + cln.SQLOpts = erOpts.SQLOpts.Clone() } - if erOpts.AMQPUsernameProcessed != nil { - cln.AMQPUsernameProcessed = new(string) - *cln.AMQPUsernameProcessed = *erOpts.AMQPUsernameProcessed - } - if erOpts.AMQPPasswordProcessed != nil { - cln.AMQPPasswordProcessed = new(string) - *cln.AMQPPasswordProcessed = *erOpts.AMQPPasswordProcessed - } - if erOpts.AMQPConsumerTag != nil { - cln.AMQPConsumerTag = new(string) - *cln.AMQPConsumerTag = *erOpts.AMQPConsumerTag - } - if erOpts.AMQPExchange != nil { - cln.AMQPExchange = new(string) - *cln.AMQPExchange = *erOpts.AMQPExchange - } - if erOpts.AMQPExchangeType != nil { - cln.AMQPExchangeType = new(string) - *cln.AMQPExchangeType = *erOpts.AMQPExchangeType - } - if erOpts.AMQPRoutingKey != nil { - cln.AMQPRoutingKey = new(string) - *cln.AMQPRoutingKey = *erOpts.AMQPRoutingKey - } - if erOpts.AMQPExchangeProcessed != nil { - cln.AMQPExchangeProcessed = new(string) - *cln.AMQPExchangeProcessed = *erOpts.AMQPExchangeProcessed - } - if erOpts.AMQPExchangeTypeProcessed != nil { - cln.AMQPExchangeTypeProcessed = new(string) - *cln.AMQPExchangeTypeProcessed = *erOpts.AMQPExchangeTypeProcessed - } - if erOpts.AMQPRoutingKeyProcessed != nil { - cln.AMQPRoutingKeyProcessed = new(string) - *cln.AMQPRoutingKeyProcessed = *erOpts.AMQPRoutingKeyProcessed - } - if erOpts.KafkaTopic != nil { - cln.KafkaTopic = new(string) - *cln.KafkaTopic = *erOpts.KafkaTopic - } - if erOpts.KafkaGroupID != nil { - cln.KafkaGroupID = new(string) - *cln.KafkaGroupID = *erOpts.KafkaGroupID - } - if erOpts.KafkaMaxWait != nil { - cln.KafkaMaxWait = new(time.Duration) - *cln.KafkaMaxWait = *erOpts.KafkaMaxWait - } - if erOpts.KafkaTopicProcessed != nil { - cln.KafkaTopicProcessed = new(string) - *cln.KafkaTopicProcessed = *erOpts.KafkaTopicProcessed - } - if erOpts.SQLDBName != nil { - cln.SQLDBName = new(string) - *cln.SQLDBName = *erOpts.SQLDBName - } - if erOpts.SQLTableName != nil { - cln.SQLTableName = new(string) - *cln.SQLTableName = *erOpts.SQLTableName - } - if erOpts.PgSSLMode != nil { - cln.PgSSLMode = new(string) - *cln.PgSSLMode = *erOpts.PgSSLMode - } - if erOpts.SQLDBNameProcessed != nil { - cln.SQLDBNameProcessed = new(string) - *cln.SQLDBNameProcessed = *erOpts.SQLDBNameProcessed - } - if erOpts.SQLTableNameProcessed != nil { - cln.SQLTableNameProcessed = new(string) - *cln.SQLTableNameProcessed = *erOpts.SQLTableNameProcessed - } - if erOpts.PgSSLModeProcessed != nil { - cln.PgSSLModeProcessed = new(string) - *cln.PgSSLModeProcessed = *erOpts.PgSSLModeProcessed - } - if erOpts.AWSRegion != nil { - cln.AWSRegion = new(string) - *cln.AWSRegion = *erOpts.AWSRegion - } - if erOpts.AWSKey != nil { - cln.AWSKey = new(string) - *cln.AWSKey = *erOpts.AWSKey - } - if erOpts.AWSSecret != nil { - cln.AWSSecret = new(string) - *cln.AWSSecret = *erOpts.AWSSecret - } - if erOpts.AWSToken != nil { - cln.AWSToken = new(string) - *cln.AWSToken = *erOpts.AWSToken - } - if erOpts.AWSRegionProcessed != nil { - cln.AWSRegionProcessed = new(string) - *cln.AWSRegionProcessed = *erOpts.AWSRegionProcessed - } - if erOpts.AWSKeyProcessed != nil { - cln.AWSKeyProcessed = new(string) - *cln.AWSKeyProcessed = *erOpts.AWSKeyProcessed - } - if erOpts.AWSSecretProcessed != nil { - cln.AWSSecretProcessed = new(string) - *cln.AWSSecretProcessed = *erOpts.AWSSecretProcessed - } - if erOpts.AWSTokenProcessed != nil { - cln.AWSTokenProcessed = new(string) - *cln.AWSTokenProcessed = *erOpts.AWSTokenProcessed - } - if erOpts.SQSQueueID != nil { - cln.SQSQueueID = new(string) - *cln.SQSQueueID = *erOpts.SQSQueueID - } - if erOpts.SQSQueueIDProcessed != nil { - cln.SQSQueueIDProcessed = new(string) - *cln.SQSQueueIDProcessed = *erOpts.SQSQueueIDProcessed - } - if erOpts.S3BucketID != nil { - cln.S3BucketID = new(string) - *cln.S3BucketID = *erOpts.S3BucketID - } - if erOpts.S3FolderPathProcessed != nil { - cln.S3FolderPathProcessed = new(string) - *cln.S3FolderPathProcessed = *erOpts.S3FolderPathProcessed - } - if erOpts.S3BucketIDProcessed != nil { - cln.S3BucketIDProcessed = new(string) - *cln.S3BucketIDProcessed = *erOpts.S3BucketIDProcessed - } - if erOpts.NATSJetStream != nil { - cln.NATSJetStream = new(bool) - *cln.NATSJetStream = *erOpts.NATSJetStream - } - if erOpts.NATSConsumerName != nil { - cln.NATSConsumerName = new(string) - *cln.NATSConsumerName = *erOpts.NATSConsumerName - } - if erOpts.NATSSubject != nil { - cln.NATSSubject = new(string) - *cln.NATSSubject = *erOpts.NATSSubject - } - if erOpts.NATSQueueID != nil { - cln.NATSQueueID = new(string) - *cln.NATSQueueID = *erOpts.NATSQueueID - } - if erOpts.NATSJWTFile != nil { - cln.NATSJWTFile = new(string) - *cln.NATSJWTFile = *erOpts.NATSJWTFile - } - if erOpts.NATSSeedFile != nil { - cln.NATSSeedFile = new(string) - *cln.NATSSeedFile = *erOpts.NATSSeedFile - } - if erOpts.NATSCertificateAuthority != nil { - cln.NATSCertificateAuthority = new(string) - *cln.NATSCertificateAuthority = *erOpts.NATSCertificateAuthority - } - if erOpts.NATSClientCertificate != nil { - cln.NATSClientCertificate = new(string) - *cln.NATSClientCertificate = *erOpts.NATSClientCertificate - } - if erOpts.NATSClientKey != nil { - cln.NATSClientKey = new(string) - *cln.NATSClientKey = *erOpts.NATSClientKey - } - if erOpts.NATSJetStreamMaxWait != nil { - cln.NATSJetStreamMaxWait = new(time.Duration) - *cln.NATSJetStreamMaxWait = *erOpts.NATSJetStreamMaxWait - } - if erOpts.NATSJetStreamProcessed != nil { - cln.NATSJetStreamProcessed = new(bool) - *cln.NATSJetStreamProcessed = *erOpts.NATSJetStreamProcessed - } - if erOpts.NATSSubjectProcessed != nil { - cln.NATSSubjectProcessed = new(string) - *cln.NATSSubjectProcessed = *erOpts.NATSSubjectProcessed - } - if erOpts.NATSJWTFileProcessed != nil { - cln.NATSJWTFileProcessed = new(string) - *cln.NATSJWTFileProcessed = *erOpts.NATSJWTFileProcessed - } - if erOpts.NATSSeedFileProcessed != nil { - cln.NATSSeedFileProcessed = new(string) - *cln.NATSSeedFileProcessed = *erOpts.NATSSeedFileProcessed - } - if erOpts.NATSCertificateAuthorityProcessed != nil { - cln.NATSCertificateAuthorityProcessed = new(string) - *cln.NATSCertificateAuthorityProcessed = *erOpts.NATSCertificateAuthorityProcessed - } - if erOpts.NATSClientCertificateProcessed != nil { - cln.NATSClientCertificateProcessed = new(string) - *cln.NATSClientCertificateProcessed = *erOpts.NATSClientCertificateProcessed - } - if erOpts.NATSClientKeyProcessed != nil { - cln.NATSClientKeyProcessed = new(string) - *cln.NATSClientKeyProcessed = *erOpts.NATSClientKeyProcessed - } - if erOpts.NATSJetStreamMaxWaitProcessed != nil { - cln.NATSJetStreamMaxWaitProcessed = new(time.Duration) - *cln.NATSJetStreamMaxWaitProcessed = *erOpts.NATSJetStreamMaxWaitProcessed + if erOpts.AWSOpts != nil { + cln.AWSOpts = erOpts.AWSOpts.Clone() } + return cln } @@ -815,187 +930,203 @@ func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string if er.Opts.PartialOrderField != nil { opts[utils.PartialOrderFieldOpt] = *er.Opts.PartialOrderField } - if er.Opts.PartialCSVFieldSeparator != nil { - opts[utils.PartialCSVFieldSepartorOpt] = *er.Opts.PartialCSVFieldSeparator - } - if er.Opts.CSVRowLength != nil { - opts[utils.CSVRowLengthOpt] = *er.Opts.CSVRowLength - } - if er.Opts.CSVFieldSeparator != nil { - opts[utils.CSVFieldSepOpt] = *er.Opts.CSVFieldSeparator - } - if er.Opts.CSVHeaderDefineChar != nil { - opts[utils.HeaderDefineCharOpt] = *er.Opts.CSVHeaderDefineChar - } - if er.Opts.CSVLazyQuotes != nil { - opts[utils.CSVLazyQuotes] = *er.Opts.CSVLazyQuotes + + if csvOpts := er.Opts.CSVOpts; csvOpts != nil { + if csvOpts.PartialCSVFieldSeparator != nil { + opts[utils.PartialCSVFieldSepartorOpt] = *csvOpts.PartialCSVFieldSeparator + } + if csvOpts.CSVRowLength != nil { + opts[utils.CSVRowLengthOpt] = *csvOpts.CSVRowLength + } + if csvOpts.CSVFieldSeparator != nil { + opts[utils.CSVFieldSepOpt] = *csvOpts.CSVFieldSeparator + } + if csvOpts.CSVHeaderDefineChar != nil { + opts[utils.HeaderDefineCharOpt] = *csvOpts.CSVHeaderDefineChar + } + if csvOpts.CSVLazyQuotes != nil { + opts[utils.CSVLazyQuotes] = *csvOpts.CSVLazyQuotes + } } if er.Opts.XMLRootPath != nil { opts[utils.XMLRootPathOpt] = *er.Opts.XMLRootPath } - if er.Opts.AMQPQueueID != nil { - opts[utils.AMQPQueueID] = *er.Opts.AMQPQueueID - } - if er.Opts.AMQPQueueIDProcessed != nil { - opts[utils.AMQPQueueIDProcessedCfg] = *er.Opts.AMQPQueueIDProcessed - } - if er.Opts.AMQPUsername != nil { - opts[utils.AMQPUsername] = *er.Opts.AMQPUsername - } - if er.Opts.AMQPPassword != nil { - opts[utils.AMQPPassword] = *er.Opts.AMQPPassword - } - if er.Opts.AMQPUsernameProcessed != nil { - opts[utils.AMQPUsernameProcessedCfg] = *er.Opts.AMQPUsernameProcessed - } - if er.Opts.AMQPPasswordProcessed != nil { - opts[utils.AMQPPasswordProcessedCfg] = *er.Opts.AMQPPasswordProcessed - } - if er.Opts.AMQPConsumerTag != nil { - opts[utils.AMQPConsumerTag] = *er.Opts.AMQPConsumerTag - } - if er.Opts.AMQPExchange != nil { - opts[utils.AMQPExchange] = *er.Opts.AMQPExchange - } - if er.Opts.AMQPExchangeType != nil { - opts[utils.AMQPExchangeType] = *er.Opts.AMQPExchangeType - } - if er.Opts.AMQPRoutingKey != nil { - opts[utils.AMQPRoutingKey] = *er.Opts.AMQPRoutingKey - } - if er.Opts.AMQPExchangeProcessed != nil { - opts[utils.AMQPExchangeProcessedCfg] = *er.Opts.AMQPExchangeProcessed - } - if er.Opts.AMQPExchangeTypeProcessed != nil { - opts[utils.AMQPExchangeTypeProcessedCfg] = *er.Opts.AMQPExchangeTypeProcessed - } - if er.Opts.AMQPRoutingKeyProcessed != nil { - opts[utils.AMQPRoutingKeyProcessedCfg] = *er.Opts.AMQPRoutingKeyProcessed - } - if er.Opts.KafkaTopic != nil { - opts[utils.KafkaTopic] = *er.Opts.KafkaTopic - } - if er.Opts.KafkaGroupID != nil { - opts[utils.KafkaGroupID] = *er.Opts.KafkaGroupID - } - if er.Opts.KafkaMaxWait != nil { - opts[utils.KafkaMaxWait] = er.Opts.KafkaMaxWait.String() - } - if er.Opts.KafkaTopicProcessed != nil { - opts[utils.KafkaTopicProcessedCfg] = *er.Opts.KafkaTopicProcessed - } - if er.Opts.SQLDBName != nil { - opts[utils.SQLDBNameOpt] = *er.Opts.SQLDBName - } - if er.Opts.SQLTableName != nil { - opts[utils.SQLTableNameOpt] = *er.Opts.SQLTableName - } - if er.Opts.PgSSLMode != nil { - opts[utils.PgSSLModeCfg] = *er.Opts.PgSSLMode - } - if er.Opts.SQLDBNameProcessed != nil { - opts[utils.SQLDBNameProcessedCfg] = *er.Opts.SQLDBNameProcessed - } - if er.Opts.SQLTableNameProcessed != nil { - opts[utils.SQLTableNameProcessedCfg] = *er.Opts.SQLTableNameProcessed - } - if er.Opts.PgSSLModeProcessed != nil { - opts[utils.PgSSLModeProcessedCfg] = *er.Opts.PgSSLModeProcessed - } - if er.Opts.AWSRegion != nil { - opts[utils.AWSRegion] = *er.Opts.AWSRegion - } - if er.Opts.AWSKey != nil { - opts[utils.AWSKey] = *er.Opts.AWSKey - } - if er.Opts.AWSSecret != nil { - opts[utils.AWSSecret] = *er.Opts.AWSSecret - } - if er.Opts.AWSToken != nil { - opts[utils.AWSToken] = *er.Opts.AWSToken - } - if er.Opts.AWSRegionProcessed != nil { - opts[utils.AWSRegionProcessedCfg] = *er.Opts.AWSRegionProcessed - } - if er.Opts.AWSKeyProcessed != nil { - opts[utils.AWSKeyProcessedCfg] = *er.Opts.AWSKeyProcessed - } - if er.Opts.AWSSecretProcessed != nil { - opts[utils.AWSSecretProcessedCfg] = *er.Opts.AWSSecretProcessed - } - if er.Opts.AWSTokenProcessed != nil { - opts[utils.AWSTokenProcessedCfg] = *er.Opts.AWSTokenProcessed - } - if er.Opts.SQSQueueID != nil { - opts[utils.SQSQueueID] = *er.Opts.SQSQueueID - } - if er.Opts.SQSQueueIDProcessed != nil { - opts[utils.SQSQueueIDProcessedCfg] = *er.Opts.SQSQueueIDProcessed - } - if er.Opts.S3BucketID != nil { - opts[utils.S3Bucket] = *er.Opts.S3BucketID - } - if er.Opts.S3FolderPathProcessed != nil { - opts[utils.S3FolderPathProcessedCfg] = *er.Opts.S3FolderPathProcessed - } - if er.Opts.S3BucketIDProcessed != nil { - opts[utils.S3BucketIDProcessedCfg] = *er.Opts.S3BucketIDProcessed - } - if er.Opts.NATSJetStream != nil { - opts[utils.NatsJetStream] = *er.Opts.NATSJetStream - } - if er.Opts.NATSConsumerName != nil { - opts[utils.NatsConsumerName] = *er.Opts.NATSConsumerName - } - if er.Opts.NATSSubject != nil { - opts[utils.NatsSubject] = *er.Opts.NATSSubject - } - if er.Opts.NATSQueueID != nil { - opts[utils.NatsQueueID] = *er.Opts.NATSQueueID - } - if er.Opts.NATSJWTFile != nil { - opts[utils.NatsJWTFile] = *er.Opts.NATSJWTFile - } - if er.Opts.NATSSeedFile != nil { - opts[utils.NatsSeedFile] = *er.Opts.NATSSeedFile - } - if er.Opts.NATSCertificateAuthority != nil { - opts[utils.NatsCertificateAuthority] = *er.Opts.NATSCertificateAuthority - } - if er.Opts.NATSClientCertificate != nil { - opts[utils.NatsClientCertificate] = *er.Opts.NATSClientCertificate - } - if er.Opts.NATSClientKey != nil { - opts[utils.NatsClientKey] = *er.Opts.NATSClientKey - } - if er.Opts.NATSJetStreamMaxWait != nil { - opts[utils.NatsJetStreamMaxWait] = er.Opts.NATSJetStreamMaxWait.String() - } - if er.Opts.NATSJetStreamProcessed != nil { - opts[utils.NATSJetStreamProcessedCfg] = *er.Opts.NATSJetStreamProcessed - } - if er.Opts.NATSSubjectProcessed != nil { - opts[utils.NATSSubjectProcessedCfg] = *er.Opts.NATSSubjectProcessed - } - if er.Opts.NATSJWTFileProcessed != nil { - opts[utils.NATSJWTFileProcessedCfg] = *er.Opts.NATSJWTFileProcessed - } - if er.Opts.NATSSeedFileProcessed != nil { - opts[utils.NATSSeedFileProcessedCfg] = *er.Opts.NATSSeedFileProcessed - } - if er.Opts.NATSCertificateAuthorityProcessed != nil { - opts[utils.NATSCertificateAuthorityProcessedCfg] = *er.Opts.NATSCertificateAuthorityProcessed - } - if er.Opts.NATSClientCertificateProcessed != nil { - opts[utils.NATSClientCertificateProcessed] = *er.Opts.NATSClientCertificateProcessed - } - if er.Opts.NATSClientKeyProcessed != nil { - opts[utils.NATSClientKeyProcessedCfg] = *er.Opts.NATSClientKeyProcessed - } - if er.Opts.NATSJetStreamMaxWaitProcessed != nil { - opts[utils.NATSJetStreamMaxWaitProcessedCfg] = er.Opts.NATSJetStreamMaxWaitProcessed.String() + if amqpOpts := er.Opts.AMQPOpts; amqpOpts != nil { + if amqpOpts.AMQPQueueID != nil { + opts[utils.AMQPQueueID] = *amqpOpts.AMQPQueueID + } + if amqpOpts.AMQPQueueIDProcessed != nil { + opts[utils.AMQPQueueIDProcessedCfg] = *amqpOpts.AMQPQueueIDProcessed + } + if amqpOpts.AMQPUsername != nil { + opts[utils.AMQPUsername] = *amqpOpts.AMQPUsername + } + if amqpOpts.AMQPPassword != nil { + opts[utils.AMQPPassword] = *amqpOpts.AMQPPassword + } + if amqpOpts.AMQPUsernameProcessed != nil { + opts[utils.AMQPUsernameProcessedCfg] = *amqpOpts.AMQPUsernameProcessed + } + if amqpOpts.AMQPPasswordProcessed != nil { + opts[utils.AMQPPasswordProcessedCfg] = *amqpOpts.AMQPPasswordProcessed + } + if amqpOpts.AMQPConsumerTag != nil { + opts[utils.AMQPConsumerTag] = *amqpOpts.AMQPConsumerTag + } + if amqpOpts.AMQPExchange != nil { + opts[utils.AMQPExchange] = *amqpOpts.AMQPExchange + } + if amqpOpts.AMQPExchangeType != nil { + opts[utils.AMQPExchangeType] = *amqpOpts.AMQPExchangeType + } + if amqpOpts.AMQPRoutingKey != nil { + opts[utils.AMQPRoutingKey] = *amqpOpts.AMQPRoutingKey + } + if amqpOpts.AMQPExchangeProcessed != nil { + opts[utils.AMQPExchangeProcessedCfg] = *amqpOpts.AMQPExchangeProcessed + } + if amqpOpts.AMQPExchangeTypeProcessed != nil { + opts[utils.AMQPExchangeTypeProcessedCfg] = *amqpOpts.AMQPExchangeTypeProcessed + } + if amqpOpts.AMQPRoutingKeyProcessed != nil { + opts[utils.AMQPRoutingKeyProcessedCfg] = *amqpOpts.AMQPRoutingKeyProcessed + } } + if kafkaOpts := er.Opts.KafkaOpts; kafkaOpts != nil { + if kafkaOpts.KafkaTopic != nil { + opts[utils.KafkaTopic] = *kafkaOpts.KafkaTopic + } + if kafkaOpts.KafkaGroupID != nil { + opts[utils.KafkaGroupID] = *kafkaOpts.KafkaGroupID + } + if kafkaOpts.KafkaMaxWait != nil { + opts[utils.KafkaMaxWait] = kafkaOpts.KafkaMaxWait.String() + } + if kafkaOpts.KafkaTopicProcessed != nil { + opts[utils.KafkaTopicProcessedCfg] = *kafkaOpts.KafkaTopicProcessed + } + } + + if sqlOpts := er.Opts.SQLOpts; sqlOpts != nil { + if sqlOpts.SQLDBName != nil { + opts[utils.SQLDBNameOpt] = *sqlOpts.SQLDBName + } + if sqlOpts.SQLTableName != nil { + opts[utils.SQLTableNameOpt] = *sqlOpts.SQLTableName + } + if sqlOpts.PgSSLMode != nil { + opts[utils.PgSSLModeCfg] = *sqlOpts.PgSSLMode + } + if sqlOpts.SQLDBNameProcessed != nil { + opts[utils.SQLDBNameProcessedCfg] = *sqlOpts.SQLDBNameProcessed + } + if sqlOpts.SQLTableNameProcessed != nil { + opts[utils.SQLTableNameProcessedCfg] = *sqlOpts.SQLTableNameProcessed + } + if sqlOpts.PgSSLModeProcessed != nil { + opts[utils.PgSSLModeProcessedCfg] = *sqlOpts.PgSSLModeProcessed + } + } + + if awsOpts := er.Opts.AWSOpts; awsOpts != nil { + if awsOpts.AWSRegion != nil { + opts[utils.AWSRegion] = *awsOpts.AWSRegion + } + if awsOpts.AWSKey != nil { + opts[utils.AWSKey] = *awsOpts.AWSKey + } + if awsOpts.AWSSecret != nil { + opts[utils.AWSSecret] = *awsOpts.AWSSecret + } + if awsOpts.AWSToken != nil { + opts[utils.AWSToken] = *awsOpts.AWSToken + } + if awsOpts.AWSRegionProcessed != nil { + opts[utils.AWSRegionProcessedCfg] = *awsOpts.AWSRegionProcessed + } + if awsOpts.AWSKeyProcessed != nil { + opts[utils.AWSKeyProcessedCfg] = *awsOpts.AWSKeyProcessed + } + if awsOpts.AWSSecretProcessed != nil { + opts[utils.AWSSecretProcessedCfg] = *awsOpts.AWSSecretProcessed + } + if awsOpts.AWSTokenProcessed != nil { + opts[utils.AWSTokenProcessedCfg] = *awsOpts.AWSTokenProcessed + } + if awsOpts.SQSQueueID != nil { + opts[utils.SQSQueueID] = *awsOpts.SQSQueueID + } + if awsOpts.SQSQueueIDProcessed != nil { + opts[utils.SQSQueueIDProcessedCfg] = *awsOpts.SQSQueueIDProcessed + } + if awsOpts.S3BucketID != nil { + opts[utils.S3Bucket] = *awsOpts.S3BucketID + } + if awsOpts.S3FolderPathProcessed != nil { + opts[utils.S3FolderPathProcessedCfg] = *awsOpts.S3FolderPathProcessed + } + if awsOpts.S3BucketIDProcessed != nil { + opts[utils.S3BucketIDProcessedCfg] = *awsOpts.S3BucketIDProcessed + } + } + + if natsOpts := er.Opts.NATSOpts; natsOpts != nil { + if natsOpts.NATSJetStream != nil { + opts[utils.NatsJetStream] = *natsOpts.NATSJetStream + } + if natsOpts.NATSConsumerName != nil { + opts[utils.NatsConsumerName] = *natsOpts.NATSConsumerName + } + if natsOpts.NATSSubject != nil { + opts[utils.NatsSubject] = *natsOpts.NATSSubject + } + if natsOpts.NATSQueueID != nil { + opts[utils.NatsQueueID] = *natsOpts.NATSQueueID + } + if natsOpts.NATSJWTFile != nil { + opts[utils.NatsJWTFile] = *natsOpts.NATSJWTFile + } + if natsOpts.NATSSeedFile != nil { + opts[utils.NatsSeedFile] = *natsOpts.NATSSeedFile + } + if natsOpts.NATSCertificateAuthority != nil { + opts[utils.NatsCertificateAuthority] = *natsOpts.NATSCertificateAuthority + } + if natsOpts.NATSClientCertificate != nil { + opts[utils.NatsClientCertificate] = *natsOpts.NATSClientCertificate + } + if natsOpts.NATSClientKey != nil { + opts[utils.NatsClientKey] = *natsOpts.NATSClientKey + } + if natsOpts.NATSJetStreamMaxWait != nil { + opts[utils.NatsJetStreamMaxWait] = natsOpts.NATSJetStreamMaxWait.String() + } + if natsOpts.NATSJetStreamProcessed != nil { + opts[utils.NATSJetStreamProcessedCfg] = *natsOpts.NATSJetStreamProcessed + } + if natsOpts.NATSSubjectProcessed != nil { + opts[utils.NATSSubjectProcessedCfg] = *natsOpts.NATSSubjectProcessed + } + if natsOpts.NATSJWTFileProcessed != nil { + opts[utils.NATSJWTFileProcessedCfg] = *natsOpts.NATSJWTFileProcessed + } + if natsOpts.NATSSeedFileProcessed != nil { + opts[utils.NATSSeedFileProcessedCfg] = *natsOpts.NATSSeedFileProcessed + } + if natsOpts.NATSCertificateAuthorityProcessed != nil { + opts[utils.NATSCertificateAuthorityProcessedCfg] = *natsOpts.NATSCertificateAuthorityProcessed + } + if natsOpts.NATSClientCertificateProcessed != nil { + opts[utils.NATSClientCertificateProcessed] = *natsOpts.NATSClientCertificateProcessed + } + if natsOpts.NATSClientKeyProcessed != nil { + opts[utils.NATSClientKeyProcessedCfg] = *natsOpts.NATSClientKeyProcessed + } + if natsOpts.NATSJetStreamMaxWaitProcessed != nil { + opts[utils.NATSJetStreamMaxWaitProcessedCfg] = natsOpts.NATSJetStreamMaxWaitProcessed.String() + } + } initialMP = map[string]any{ utils.IDCfg: er.ID, utils.TypeCfg: er.Type, diff --git a/config/erscfg_test.go b/config/erscfg_test.go index b588de95b..268edd4dc 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -100,13 +100,21 @@ func TestERSClone(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -132,13 +140,21 @@ func TestERSClone(t *testing.T) { }, PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -252,13 +268,21 @@ func TestERSLoadFromjsonCfg(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + KafkaOpts: &KafkaROpts{}, + SQLOpts: &SQLROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -299,13 +323,21 @@ func TestERSLoadFromjsonCfg(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -479,13 +511,21 @@ func TestERSloadFromJsonCase3(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + KafkaOpts: &KafkaROpts{}, + SQLOpts: &SQLROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -510,13 +550,21 @@ func TestERSloadFromJsonCase3(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -614,13 +662,21 @@ func TestERSloadFromJsonCase4(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + KafkaOpts: &KafkaROpts{}, + SQLOpts: &SQLROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -645,13 +701,21 @@ func TestERSloadFromJsonCase4(t *testing.T) { }, PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AWSOpts: &AWSROpts{}, + AMQPOpts: &AMQPROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -744,13 +808,21 @@ func TestEventReaderSameID(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + KafkaOpts: &KafkaROpts{}, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -771,13 +843,21 @@ func TestEventReaderSameID(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -1222,13 +1302,21 @@ func TestERsloadFromJsonCfg(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + KafkaOpts: &KafkaROpts{}, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, { @@ -1255,13 +1343,21 @@ func TestERsloadFromJsonCfg(t *testing.T) { CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ - CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), - CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), - CSVRowLength: utils.IntPointer(0), - XMLRootPath: utils.StringPointer(utils.EmptyString), - PartialOrderField: utils.StringPointer("~*req.AnswerTime"), - PartialCacheAction: utils.StringPointer(utils.MetaNone), - NATSSubject: utils.StringPointer("cgrates_cdrs"), + CSVOpts: &CSVROpts{ + CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), + CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), + CSVRowLength: utils.IntPointer(0), + }, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + SQLOpts: &SQLROpts{}, + KafkaOpts: &KafkaROpts{}, + XMLRootPath: utils.StringPointer(utils.EmptyString), + PartialOrderField: utils.StringPointer("~*req.AnswerTime"), + PartialCacheAction: utils.StringPointer(utils.MetaNone), + NATSOpts: &NATSROpts{ + NATSSubject: utils.StringPointer("cgrates_cdrs"), + }, }, }, }, @@ -1352,7 +1448,14 @@ func TestEventReaderOptsCfg(t *testing.T) { KafkaGroupID: utils.StringPointer("groupId"), } eventReader := &EventReaderCfg{ - Opts: &EventReaderOpts{}, + Opts: &EventReaderOpts{ + CSVOpts: &CSVROpts{}, + AMQPOpts: &AMQPROpts{}, + AWSOpts: &AWSROpts{}, + NATSOpts: &NATSROpts{}, + KafkaOpts: &KafkaROpts{}, + SQLOpts: &SQLROpts{}, + }, } if err := eventReader.Opts.loadFromJSONCfg(eventReaderOptsJson); err != nil { t.Error(err) @@ -1395,58 +1498,71 @@ func TestEventReaderCfgClone(t *testing.T) { Timezone: "time.Utc", Flags: utils.FlagsWithParams{}, Opts: &EventReaderOpts{ - PartialPath: utils.StringPointer("path"), - PartialCSVFieldSeparator: utils.StringPointer("/"), - CSVLazyQuotes: utils.BoolPointer(false), - AMQPQueueID: utils.StringPointer("id"), - AMQPQueueIDProcessed: utils.StringPointer("id"), - AMQPConsumerTag: utils.StringPointer("tag"), - AMQPExchange: utils.StringPointer("exchange"), - AMQPExchangeType: utils.StringPointer("type"), - AMQPRoutingKey: utils.StringPointer("key1"), - AMQPExchangeProcessed: utils.StringPointer("amq"), - AMQPExchangeTypeProcessed: utils.StringPointer("amqtype"), - AMQPRoutingKeyProcessed: utils.StringPointer("key"), - KafkaTopic: utils.StringPointer("kafka"), - KafkaMaxWait: utils.DurationPointer(1 * time.Minute), - KafkaTopicProcessed: utils.StringPointer("kafkaproc"), - SQLDBName: utils.StringPointer("dbname"), - SQLTableName: utils.StringPointer("tablename"), - PgSSLMode: utils.StringPointer("sslmode"), - SQLDBNameProcessed: utils.StringPointer("dbnameproc"), - SQLTableNameProcessed: utils.StringPointer("tablenameproc"), - PgSSLModeProcessed: utils.StringPointer("sslproc"), - AWSRegion: utils.StringPointer("eu"), - AWSKey: utils.StringPointer("key"), - AWSSecret: utils.StringPointer("secret"), - AWSToken: utils.StringPointer("token"), - AWSKeyProcessed: utils.StringPointer("secret"), - AWSSecretProcessed: utils.StringPointer("secret"), - AWSTokenProcessed: utils.StringPointer("token"), - SQSQueueID: utils.StringPointer("SQSQueue"), - SQSQueueIDProcessed: utils.StringPointer("SQSQueueId"), - S3BucketID: utils.StringPointer("S3BucketID"), - S3FolderPathProcessed: utils.StringPointer("S3Path"), - S3BucketIDProcessed: utils.StringPointer("S3BucketProc"), - NATSJetStream: utils.BoolPointer(false), - NATSConsumerName: utils.StringPointer("user"), - NATSQueueID: utils.StringPointer("id"), - NATSJWTFile: utils.StringPointer("jwt"), - NATSSeedFile: utils.StringPointer("seed"), - NATSCertificateAuthority: utils.StringPointer("authority"), - NATSClientCertificate: utils.StringPointer("certificate"), - NATSClientKey: utils.StringPointer("key5"), - NATSJetStreamMaxWait: utils.DurationPointer(1 * time.Minute), - NATSJetStreamProcessed: utils.BoolPointer(true), - NATSJWTFileProcessed: utils.StringPointer("file"), - NATSSeedFileProcessed: utils.StringPointer("natseed"), - NATSCertificateAuthorityProcessed: utils.StringPointer("natsauth"), - NATSClientCertificateProcessed: utils.StringPointer("natcertificate"), - NATSClientKeyProcessed: utils.StringPointer("natsprocess"), - NATSJetStreamMaxWaitProcessed: utils.DurationPointer(1 * time.Minute), - AWSRegionProcessed: utils.StringPointer("eu"), - NATSSubjectProcessed: utils.StringPointer("process"), - KafkaGroupID: utils.StringPointer("groupId"), + PartialPath: utils.StringPointer("path"), + CSVOpts: &CSVROpts{ + PartialCSVFieldSeparator: utils.StringPointer("/"), + CSVLazyQuotes: utils.BoolPointer(false), + }, + AMQPOpts: &AMQPROpts{ + AMQPQueueID: utils.StringPointer("id"), + AMQPQueueIDProcessed: utils.StringPointer("id"), + AMQPConsumerTag: utils.StringPointer("tag"), + AMQPExchange: utils.StringPointer("exchange"), + AMQPExchangeType: utils.StringPointer("type"), + AMQPRoutingKey: utils.StringPointer("key1"), + AMQPExchangeProcessed: utils.StringPointer("amq"), + AMQPExchangeTypeProcessed: utils.StringPointer("amqtype"), + AMQPRoutingKeyProcessed: utils.StringPointer("key"), + }, + SQLOpts: &SQLROpts{ + SQLDBName: utils.StringPointer("dbname"), + SQLTableName: utils.StringPointer("tablename"), + PgSSLMode: utils.StringPointer("sslmode"), + SQLDBNameProcessed: utils.StringPointer("dbnameproc"), + SQLTableNameProcessed: utils.StringPointer("tablenameproc"), + PgSSLModeProcessed: utils.StringPointer("sslproc"), + }, + AWSOpts: &AWSROpts{ + + AWSRegion: utils.StringPointer("eu"), + AWSKey: utils.StringPointer("key"), + AWSSecret: utils.StringPointer("secret"), + AWSToken: utils.StringPointer("token"), + AWSKeyProcessed: utils.StringPointer("secret"), + AWSRegionProcessed: utils.StringPointer("eu"), + AWSSecretProcessed: utils.StringPointer("secret"), + AWSTokenProcessed: utils.StringPointer("token"), + SQSQueueID: utils.StringPointer("SQSQueue"), + SQSQueueIDProcessed: utils.StringPointer("SQSQueueId"), + S3BucketID: utils.StringPointer("S3BucketID"), + S3FolderPathProcessed: utils.StringPointer("S3Path"), + S3BucketIDProcessed: utils.StringPointer("S3BucketProc"), + }, + NATSOpts: &NATSROpts{ + NATSJetStream: utils.BoolPointer(false), + NATSConsumerName: utils.StringPointer("user"), + NATSQueueID: utils.StringPointer("id"), + NATSJWTFile: utils.StringPointer("jwt"), + NATSSeedFile: utils.StringPointer("seed"), + NATSCertificateAuthority: utils.StringPointer("authority"), + NATSClientCertificate: utils.StringPointer("certificate"), + NATSClientKey: utils.StringPointer("key5"), + NATSJetStreamMaxWait: utils.DurationPointer(1 * time.Minute), + NATSJetStreamProcessed: utils.BoolPointer(true), + NATSJWTFileProcessed: utils.StringPointer("file"), + NATSSeedFileProcessed: utils.StringPointer("natseed"), + NATSCertificateAuthorityProcessed: utils.StringPointer("natsauth"), + NATSClientCertificateProcessed: utils.StringPointer("natcertificate"), + NATSClientKeyProcessed: utils.StringPointer("natsprocess"), + NATSJetStreamMaxWaitProcessed: utils.DurationPointer(1 * time.Minute), + NATSSubjectProcessed: utils.StringPointer("process"), + }, + KafkaOpts: &KafkaROpts{ + KafkaTopic: utils.StringPointer("kafka"), + KafkaMaxWait: utils.DurationPointer(1 * time.Minute), + KafkaTopicProcessed: utils.StringPointer("kafkaproc"), + KafkaGroupID: utils.StringPointer("groupId"), + }, }, } rcv := ban.Clone() diff --git a/ees/elastic_test.go b/ees/elastic_test.go index 16b9c30d3..54976dfab 100644 --- a/ees/elastic_test.go +++ b/ees/elastic_test.go @@ -46,9 +46,16 @@ func TestInitClient(t *testing.T) { ee := &ElasticEE{ cfg: &config.EventExporterCfg{ ExportPath: "/\x00", + Opts: &config.EventExporterOpts{ + Els: &config.ElsOpts{}, + RPC: &config.RPCOpts{}, + }, }, } - errExpect := `cannot create client: parse "/\x00": net/url: invalid control character in URL` + if err := ee.prepareOpts(); err != nil { + t.Error(err) + } + errExpect := `cannot create client: cannot parse url: parse "/\x00": net/url: invalid control character in URL` if err := ee.Connect(); err == nil || err.Error() != errExpect { t.Errorf("Expected %+v \n but got %+v", errExpect, err) } @@ -308,14 +315,17 @@ func TestElasticExportEvent3(t *testing.T) { if err != nil { t.Error(err) } + if err := eEe.prepareOpts(); err != nil { + t.Error(err) + } if err = eEe.Connect(); err != nil { t.Error(err) } eEe.eClnt.Transport = new(mockClient) - // errExpect := `unsupported protocol scheme ""` + errExpect := `the client noticed that the server is not Elasticsearch and we do not support this unknown product` cgrCfg.EEsCfg().Exporters[0].ComputeFields() - if err := eEe.ExportEvent([]byte{}, ""); err == nil { - t.Error(err) + if err := eEe.ExportEvent([]byte{}, ""); err == nil || err.Error() != errExpect { + t.Errorf("Expected %q but got %q", errExpect, err) } } diff --git a/ees/libcdre_test.go b/ees/libcdre_test.go index dd7b148df..93f7dd944 100644 --- a/ees/libcdre_test.go +++ b/ees/libcdre_test.go @@ -46,6 +46,7 @@ func TestAddFldPost(t *testing.T) { NATS: &config.NATSOpts{}, Kafka: &config.KafkaOpts{}, RPC: &config.RPCOpts{}, + SQL: &config.SQLOpts{}, }) x, ok := failedPostCache.Get(utils.ConcatenatedKey("", "path1", "format1")) if !ok { @@ -70,6 +71,7 @@ func TestAddFldPost(t *testing.T) { NATS: &config.NATSOpts{}, Kafka: &config.KafkaOpts{}, RPC: &config.RPCOpts{}, + SQL: &config.SQLOpts{}, }, } if !reflect.DeepEqual(eOut, failedPost) { @@ -82,6 +84,7 @@ func TestAddFldPost(t *testing.T) { NATS: &config.NATSOpts{}, Kafka: &config.KafkaOpts{}, RPC: &config.RPCOpts{}, + SQL: &config.SQLOpts{}, }) AddFailedPost("", "path2", "format2", "3", &config.EventExporterOpts{ AWS: &config.AWSOpts{ @@ -92,6 +95,7 @@ func TestAddFldPost(t *testing.T) { RPC: &config.RPCOpts{}, AMQP: &config.AMQPOpts{}, Els: &config.ElsOpts{}, + SQL: &config.SQLOpts{}, }) x, ok = failedPostCache.Get(utils.ConcatenatedKey("", "path1", "format1")) if !ok { @@ -115,6 +119,7 @@ func TestAddFldPost(t *testing.T) { NATS: &config.NATSOpts{}, Kafka: &config.KafkaOpts{}, RPC: &config.RPCOpts{}, + SQL: &config.SQLOpts{}, }, } if !reflect.DeepEqual(eOut, failedPost) { @@ -136,6 +141,12 @@ func TestAddFldPost(t *testing.T) { Format: "format2", Events: []any{"3"}, Opts: &config.EventExporterOpts{ + Els: &config.ElsOpts{}, + NATS: &config.NATSOpts{}, + SQL: &config.SQLOpts{}, + AMQP: &config.AMQPOpts{}, + RPC: &config.RPCOpts{}, + Kafka: &config.KafkaOpts{}, AWS: &config.AWSOpts{ SQSQueueID: utils.StringPointer("qID")}, }, diff --git a/ers/amqp.go b/ers/amqp.go index 7816be9cf..7ad7f41ec 100644 --- a/ers/amqp.go +++ b/ers/amqp.go @@ -215,22 +215,24 @@ func (rdr *AMQPER) processMessage(msg []byte) (err error) { func (rdr *AMQPER) setOpts(opts *config.EventReaderOpts) { rdr.queueID = utils.DefaultQueueID - if opts.AMQPQueueID != nil { - rdr.queueID = *opts.AMQPQueueID - } - rdr.tag = utils.AMQPDefaultConsumerTag - if opts.AMQPConsumerTag != nil { - rdr.tag = *opts.AMQPConsumerTag - } - if opts.AMQPRoutingKey != nil { - rdr.routingKey = *opts.AMQPRoutingKey - } - if opts.AMQPExchange != nil { - rdr.exchange = *opts.AMQPExchange - rdr.exchangeType = utils.DefaultExchangeType - } - if opts.AMQPExchangeType != nil { - rdr.exchangeType = *opts.AMQPExchangeType + if amqpOpts := opts.AMQPOpts; amqpOpts != nil { + if opts.AMQPOpts.AMQPQueueID != nil { + rdr.queueID = *opts.AMQPOpts.AMQPQueueID + } + rdr.tag = utils.AMQPDefaultConsumerTag + if opts.AMQPOpts.AMQPConsumerTag != nil { + rdr.tag = *opts.AMQPOpts.AMQPConsumerTag + } + if opts.AMQPOpts.AMQPRoutingKey != nil { + rdr.routingKey = *opts.AMQPOpts.AMQPRoutingKey + } + if opts.AMQPOpts.AMQPExchange != nil { + rdr.exchange = *opts.AMQPOpts.AMQPExchange + rdr.exchangeType = utils.DefaultExchangeType + } + if opts.AMQPOpts.AMQPExchangeType != nil { + rdr.exchangeType = *opts.AMQPOpts.AMQPExchangeType + } } } diff --git a/ers/amqp_test.go b/ers/amqp_test.go index 39bef3b06..34d948a0f 100644 --- a/ers/amqp_test.go +++ b/ers/amqp_test.go @@ -34,8 +34,9 @@ func TestAMQPSetOpts(t *testing.T) { tag: "new", } if k.setOpts(&config.EventReaderOpts{ - AMQPQueueID: utils.StringPointer("cdrs"), - AMQPConsumerTag: utils.StringPointer("new"), + AMQPOpts: &config.AMQPROpts{ + AMQPQueueID: utils.StringPointer("cdrs"), + AMQPConsumerTag: utils.StringPointer("new")}, }); expKafka.dialURL != k.dialURL { t.Errorf("Expected: %s ,received: %s", expKafka.dialURL, k.dialURL) } else if expKafka.queueID != k.queueID { @@ -50,7 +51,12 @@ func TestAMQPSetOpts(t *testing.T) { queueID: "cgrates_cdrs", tag: "cgrates", } - if k.setOpts(&config.EventReaderOpts{}); expKafka.dialURL != k.dialURL { + if k.setOpts(&config.EventReaderOpts{ + AMQPOpts: &config.AMQPROpts{ + AMQPQueueID: utils.StringPointer("cgrates_cdrs"), + AMQPConsumerTag: utils.StringPointer("cgrates"), + }, + }); expKafka.dialURL != k.dialURL { t.Errorf("Expected: %s ,received: %s", expKafka.dialURL, k.dialURL) } else if expKafka.queueID != k.queueID { t.Errorf("Expected: %s ,received: %s", expKafka.queueID, k.queueID) diff --git a/ers/amqpv1.go b/ers/amqpv1.go index 577736c03..5db2530fe 100644 --- a/ers/amqpv1.go +++ b/ers/amqpv1.go @@ -51,12 +51,15 @@ func NewAMQPv1ER(cfg *config.CGRConfig, cfgIdx int, rdr.cap <- struct{}{} } } - if rdr.Config().Opts.AMQPQueueID != nil { - rdr.queueID = "/" + *rdr.Config().Opts.AMQPQueueID - } - if rdr.Config().Opts.AMQPUsername != nil && rdr.Config().Opts.AMQPPassword != nil { - rdr.connOpts = &amqpv1.ConnOptions{ - SASLType: amqpv1.SASLTypePlain(*rdr.Config().Opts.AMQPUsername, *rdr.Config().Opts.AMQPPassword), + + if amqOpts := rdr.Config().Opts.AMQPOpts; amqOpts != nil { + if amqOpts.AMQPQueueID != nil { + rdr.queueID = "/" + *amqOpts.AMQPQueueID + } + if amqOpts.AMQPUsername != nil && amqOpts.AMQPPassword != nil { + rdr.connOpts = &amqpv1.ConnOptions{ + SASLType: amqpv1.SASLTypePlain(*amqOpts.AMQPUsername, *amqOpts.AMQPPassword), + } } } rdr.createPoster() diff --git a/ers/ers.go b/ers/ers.go index 90fe3473a..4a03809bc 100644 --- a/ers/ers.go +++ b/ers/ers.go @@ -457,8 +457,8 @@ func (erS *ERService) onEvicted(id string, value any) { return } csvWriter := csv.NewWriter(fileOut) - if eEvs.rdrCfg.Opts.PartialCSVFieldSeparator != nil { - csvWriter.Comma = rune((*eEvs.rdrCfg.Opts.PartialCSVFieldSeparator)[0]) + if eEvs.rdrCfg.Opts.CSVOpts.PartialCSVFieldSeparator != nil { + csvWriter.Comma = rune((*eEvs.rdrCfg.Opts.CSVOpts.PartialCSVFieldSeparator)[0]) } if err = csvWriter.Write(record); err != nil { diff --git a/ers/ers_it_test.go b/ers/ers_it_test.go index afadcbadd..e78158430 100644 --- a/ers/ers_it_test.go +++ b/ers/ers_it_test.go @@ -794,6 +794,7 @@ func TestErsOnEvictedMetaDumpToFileOK(t *testing.T) { Opts: &config.EventReaderOpts{ PartialCacheAction: utils.StringPointer(utils.MetaDumpToFile), PartialPath: utils.StringPointer(dirPath), + CSVOpts: &config.CSVROpts{}, }, CacheDumpFields: []*config.FCTemplate{ { @@ -875,9 +876,10 @@ func TestErsOnEvictedMetaDumpToFileCSVWriteErr(t *testing.T) { ID: "ER1", Type: utils.MetaNone, Opts: &config.EventReaderOpts{ - PartialCacheAction: utils.StringPointer(utils.MetaDumpToFile), - PartialPath: utils.StringPointer(dirPath), - PartialCSVFieldSeparator: utils.StringPointer("\""), + PartialCacheAction: utils.StringPointer(utils.MetaDumpToFile), + PartialPath: utils.StringPointer(dirPath), + CSVOpts: &config.CSVROpts{ + PartialCSVFieldSeparator: utils.StringPointer("\"")}, }, }, } diff --git a/ers/filecsv.go b/ers/filecsv.go index 043a2320e..03c782267 100644 --- a/ers/filecsv.go +++ b/ers/filecsv.go @@ -134,17 +134,17 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { defer file.Close() csvReader := csv.NewReader(file) var rowLength int - if rdr.Config().Opts.CSVRowLength != nil { - rowLength = *rdr.Config().Opts.CSVRowLength + if rdr.Config().Opts.CSVOpts.CSVRowLength != nil { + rowLength = *rdr.Config().Opts.CSVOpts.CSVRowLength } csvReader.FieldsPerRecord = rowLength csvReader.Comment = utils.CommentChar csvReader.Comma = utils.CSVSep - if rdr.Config().Opts.CSVFieldSeparator != nil { - csvReader.Comma = rune((*rdr.Config().Opts.CSVFieldSeparator)[0]) + if rdr.Config().Opts.CSVOpts.CSVFieldSeparator != nil { + csvReader.Comma = rune((*rdr.Config().Opts.CSVOpts.CSVFieldSeparator)[0]) } - if rdr.Config().Opts.CSVLazyQuotes != nil { - csvReader.LazyQuotes = *rdr.Config().Opts.CSVLazyQuotes + if rdr.Config().Opts.CSVOpts.CSVLazyQuotes != nil { + csvReader.LazyQuotes = *rdr.Config().Opts.CSVOpts.CSVLazyQuotes } var indxAls map[string]int rowNr := 0 // This counts the rows in the file, not really number of CDRs @@ -152,8 +152,8 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.MetaFileName: utils.NewLeafNode(fName)}} var hdrDefChar string - if rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts.CSVHeaderDefineChar != nil { - hdrDefChar = *rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts.CSVHeaderDefineChar + if rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts.CSVOpts.CSVHeaderDefineChar != nil { + hdrDefChar = *rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts.CSVOpts.CSVHeaderDefineChar } for { var record []string diff --git a/ers/kafka.go b/ers/kafka.go index 1da474fa1..ea118d333 100644 --- a/ers/kafka.go +++ b/ers/kafka.go @@ -188,14 +188,16 @@ func (rdr *KafkaER) setOpts(opts *config.EventReaderOpts) (err error) { rdr.topic = utils.KafkaDefaultTopic rdr.groupID = utils.KafkaDefaultGroupID rdr.maxWait = utils.KafkaDefaultMaxWait - if opts.KafkaTopic != nil { - rdr.topic = *opts.KafkaTopic - } - if opts.KafkaGroupID != nil { - rdr.groupID = *opts.KafkaGroupID - } - if opts.KafkaMaxWait != nil { - rdr.maxWait = *opts.KafkaMaxWait + if kfkOpts := opts.KafkaOpts; kfkOpts != nil { + if kfkOpts.KafkaTopic != nil { + rdr.topic = *kfkOpts.KafkaTopic + } + if kfkOpts.KafkaGroupID != nil { + rdr.groupID = *kfkOpts.KafkaGroupID + } + if kfkOpts.KafkaMaxWait != nil { + rdr.maxWait = *kfkOpts.KafkaMaxWait + } } return } diff --git a/ers/kafka_test.go b/ers/kafka_test.go index 546c17352..faaea6c2e 100644 --- a/ers/kafka_test.go +++ b/ers/kafka_test.go @@ -40,9 +40,11 @@ func TestKafkasetOpts(t *testing.T) { } if err := k.setOpts(&config.EventReaderOpts{ - KafkaTopic: utils.StringPointer("cdrs"), - KafkaGroupID: utils.StringPointer("new"), - KafkaMaxWait: utils.DurationPointer(time.Second), + KafkaOpts: &config.KafkaROpts{ + KafkaTopic: utils.StringPointer("cdrs"), + KafkaGroupID: utils.StringPointer("new"), + KafkaMaxWait: utils.DurationPointer(time.Second), + }, }); err != nil { t.Fatal(err) } else if expKafka.dialURL != k.dialURL { @@ -83,9 +85,11 @@ func TestKafkasetOpts(t *testing.T) { maxWait: time.Second, } if err := k.setOpts(&config.EventReaderOpts{ - KafkaTopic: utils.StringPointer("cdrs"), - KafkaGroupID: utils.StringPointer("new"), - KafkaMaxWait: utils.DurationPointer(time.Second), + KafkaOpts: &config.KafkaROpts{ + KafkaTopic: utils.StringPointer("cdrs"), + KafkaGroupID: utils.StringPointer("new"), + KafkaMaxWait: utils.DurationPointer(time.Second), + }, }); err != nil { t.Fatal(err) } else if expKafka.dialURL != k.dialURL { diff --git a/ers/libers.go b/ers/libers.go index 54c873b25..07cd8803d 100644 --- a/ers/libers.go +++ b/ers/libers.go @@ -32,157 +32,170 @@ import ( // getProcessOptions assigns all non-nil fields ending in "Processed" from EventReaderOpts to their counterparts in EventExporterOpts func getProcessOptions(erOpts *config.EventReaderOpts) (eeOpts *config.EventExporterOpts) { eeOpts = new(config.EventExporterOpts) - if erOpts.AMQPExchangeProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts := erOpts.AMQPOpts; amqOpts != nil { + if amqOpts.AMQPExchangeProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.Exchange = amqOpts.AMQPExchangeProcessed } - eeOpts.AMQP.Exchange = erOpts.AMQPExchangeProcessed - } - if erOpts.AMQPExchangeTypeProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts.AMQPExchangeTypeProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.ExchangeType = amqOpts.AMQPExchangeTypeProcessed } - eeOpts.AMQP.ExchangeType = erOpts.AMQPExchangeTypeProcessed - } - if erOpts.AMQPQueueIDProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts.AMQPQueueIDProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.QueueID = amqOpts.AMQPQueueIDProcessed } - eeOpts.AMQP.QueueID = erOpts.AMQPQueueIDProcessed - } - if erOpts.AMQPRoutingKeyProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts.AMQPRoutingKeyProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.RoutingKey = amqOpts.AMQPRoutingKeyProcessed } - eeOpts.AMQP.RoutingKey = erOpts.AMQPRoutingKeyProcessed - } - if erOpts.AMQPUsernameProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts.AMQPUsernameProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.Username = amqOpts.AMQPUsernameProcessed } - eeOpts.AMQP.Username = erOpts.AMQPUsernameProcessed - } - if erOpts.AMQPPasswordProcessed != nil { - if eeOpts.AMQP == nil { - eeOpts.AMQP = new(config.AMQPOpts) + if amqOpts.AMQPPasswordProcessed != nil { + if eeOpts.AMQP == nil { + eeOpts.AMQP = new(config.AMQPOpts) + } + eeOpts.AMQP.Password = amqOpts.AMQPPasswordProcessed } - eeOpts.AMQP.Password = erOpts.AMQPPasswordProcessed } - if erOpts.AWSKeyProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if awsOpts := erOpts.AWSOpts; awsOpts != nil { + if awsOpts.AWSKeyProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.Key = awsOpts.AWSKeyProcessed } - eeOpts.AWS.Key = erOpts.AWSKeyProcessed - } - if erOpts.AWSRegionProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if awsOpts.AWSRegionProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.Region = awsOpts.AWSRegionProcessed } - eeOpts.AWS.Region = erOpts.AWSRegionProcessed - } - if erOpts.AWSSecretProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if awsOpts.AWSSecretProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.Secret = awsOpts.AWSSecretProcessed } - eeOpts.AWS.Secret = erOpts.AWSSecretProcessed - } - if erOpts.AWSTokenProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if awsOpts.AWSTokenProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.Token = awsOpts.AWSTokenProcessed } - eeOpts.AWS.Token = erOpts.AWSTokenProcessed - } - if erOpts.KafkaTopicProcessed != nil { - if eeOpts.Kafka == nil { - eeOpts.Kafka = new(config.KafkaOpts) + if awsOpts.S3BucketIDProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.S3BucketID = awsOpts.S3BucketIDProcessed } - eeOpts.Kafka.KafkaTopic = erOpts.KafkaTopicProcessed - } - if erOpts.NATSCertificateAuthorityProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if awsOpts.S3FolderPathProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.S3FolderPath = awsOpts.S3FolderPathProcessed } - eeOpts.NATS.CertificateAuthority = erOpts.NATSCertificateAuthorityProcessed - } - if erOpts.NATSClientCertificateProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if awsOpts.SQSQueueIDProcessed != nil { + if eeOpts.AWS == nil { + eeOpts.AWS = new(config.AWSOpts) + } + eeOpts.AWS.SQSQueueID = awsOpts.SQSQueueIDProcessed } - eeOpts.NATS.ClientCertificate = erOpts.NATSClientCertificateProcessed } - if erOpts.NATSClientKeyProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + + if kfkOpts := erOpts.KafkaOpts; kfkOpts != nil { + if kfkOpts.KafkaTopicProcessed != nil { + if eeOpts.Kafka == nil { + eeOpts.Kafka = new(config.KafkaOpts) + } + eeOpts.Kafka.KafkaTopic = kfkOpts.KafkaTopicProcessed } - eeOpts.NATS.ClientKey = erOpts.NATSClientKeyProcessed } - if erOpts.NATSJWTFileProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if natsOpts := erOpts.NATSOpts; natsOpts != nil { + if natsOpts.NATSCertificateAuthorityProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.CertificateAuthority = natsOpts.NATSCertificateAuthorityProcessed } - eeOpts.NATS.JWTFile = erOpts.NATSJWTFileProcessed - } - if erOpts.NATSJetStreamMaxWaitProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if natsOpts.NATSClientCertificateProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.ClientCertificate = natsOpts.NATSClientCertificateProcessed } - eeOpts.NATS.JetStreamMaxWait = erOpts.NATSJetStreamMaxWaitProcessed - } - if erOpts.NATSJetStreamProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if natsOpts.NATSClientKeyProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.ClientKey = natsOpts.NATSClientKeyProcessed } - eeOpts.NATS.JetStream = erOpts.NATSJetStreamProcessed - } - if erOpts.NATSSeedFileProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if natsOpts.NATSJWTFileProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.JWTFile = natsOpts.NATSJWTFileProcessed } - eeOpts.NATS.SeedFile = erOpts.NATSSeedFileProcessed - } - if erOpts.NATSSubjectProcessed != nil { - if eeOpts.NATS == nil { - eeOpts.NATS = new(config.NATSOpts) + if natsOpts.NATSJetStreamMaxWaitProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.JetStreamMaxWait = natsOpts.NATSJetStreamMaxWaitProcessed } - eeOpts.NATS.Subject = erOpts.NATSSubjectProcessed - } - if erOpts.S3BucketIDProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if natsOpts.NATSJetStreamProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.JetStream = natsOpts.NATSJetStreamProcessed } - eeOpts.AWS.S3BucketID = erOpts.S3BucketIDProcessed - } - if erOpts.S3FolderPathProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if natsOpts.NATSSeedFileProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.SeedFile = natsOpts.NATSSeedFileProcessed } - eeOpts.AWS.S3FolderPath = erOpts.S3FolderPathProcessed - } - if erOpts.SQLDBNameProcessed != nil { - if eeOpts.SQL == nil { - eeOpts.SQL = new(config.SQLOpts) + if natsOpts.NATSSubjectProcessed != nil { + if eeOpts.NATS == nil { + eeOpts.NATS = new(config.NATSOpts) + } + eeOpts.NATS.Subject = natsOpts.NATSSubjectProcessed } - eeOpts.SQL.DBName = erOpts.SQLDBNameProcessed } - if erOpts.SQLTableNameProcessed != nil { - if eeOpts.SQL == nil { - eeOpts.SQL = new(config.SQLOpts) + + if sqlOpts := erOpts.SQLOpts; sqlOpts != nil { + if sqlOpts.SQLDBNameProcessed != nil { + if eeOpts.SQL == nil { + eeOpts.SQL = new(config.SQLOpts) + } + eeOpts.SQL.DBName = sqlOpts.SQLDBNameProcessed } - eeOpts.SQL.TableName = erOpts.SQLTableNameProcessed - } - if erOpts.SQSQueueIDProcessed != nil { - if eeOpts.AWS == nil { - eeOpts.AWS = new(config.AWSOpts) + if sqlOpts.SQLTableNameProcessed != nil { + if eeOpts.SQL == nil { + eeOpts.SQL = new(config.SQLOpts) + } + eeOpts.SQL.TableName = sqlOpts.SQLTableNameProcessed } - eeOpts.AWS.SQSQueueID = erOpts.SQSQueueIDProcessed - } - if erOpts.PgSSLModeProcessed != nil { - if eeOpts.SQL == nil { - eeOpts.SQL = new(config.SQLOpts) + if sqlOpts.PgSSLModeProcessed != nil { + if eeOpts.SQL == nil { + eeOpts.SQL = new(config.SQLOpts) + } + eeOpts.SQL.PgSSLMode = sqlOpts.PgSSLModeProcessed } - eeOpts.SQL.PgSSLMode = erOpts.PgSSLModeProcessed + } - fmt.Println(utils.ToJSON(eeOpts)) + return } diff --git a/ers/libers_test.go b/ers/libers_test.go index b8d625e0a..ad6d2e69c 100644 --- a/ers/libers_test.go +++ b/ers/libers_test.go @@ -28,7 +28,9 @@ import ( func TestGetProcessOptions(t *testing.T) { opts := &config.EventReaderOpts{ - AMQPQueueIDProcessed: utils.StringPointer("processed"), + AMQPOpts: &config.AMQPROpts{ + AMQPQueueIDProcessed: utils.StringPointer("processed"), + }, } result := getProcessOptions(opts) expected := &config.EventExporterOpts{ diff --git a/ers/nats.go b/ers/nats.go index 57d334447..af1cac7a8 100644 --- a/ers/nats.go +++ b/ers/nats.go @@ -199,25 +199,25 @@ func (rdr *NatsER) createPoster() (err error) { } func (rdr *NatsER) processOpts() (err error) { - if rdr.Config().Opts.NATSSubject != nil { - rdr.subject = *rdr.Config().Opts.NATSSubject + if rdr.Config().Opts.NATSOpts.NATSSubject != nil { + rdr.subject = *rdr.Config().Opts.NATSOpts.NATSSubject } var queueID string - if rdr.Config().Opts.NATSQueueID != nil { - queueID = *rdr.Config().Opts.NATSQueueID + if rdr.Config().Opts.NATSOpts.NATSQueueID != nil { + queueID = *rdr.Config().Opts.NATSOpts.NATSQueueID } rdr.queueID = utils.FirstNonEmpty(queueID, rdr.cgrCfg.GeneralCfg().NodeID) var consumerName string - if rdr.Config().Opts.NATSConsumerName != nil { - consumerName = *rdr.Config().Opts.NATSConsumerName + if rdr.Config().Opts.NATSOpts.NATSConsumerName != nil { + consumerName = *rdr.Config().Opts.NATSOpts.NATSConsumerName } rdr.consumerName = utils.FirstNonEmpty(consumerName, utils.CGRateSLwr) - if rdr.Config().Opts.NATSJetStream != nil { - rdr.jetStream = *rdr.Config().Opts.NATSJetStream + if rdr.Config().Opts.NATSOpts.NATSJetStream != nil { + rdr.jetStream = *rdr.Config().Opts.NATSOpts.NATSJetStream } if rdr.jetStream { - if rdr.Config().Opts.NATSJetStreamMaxWait != nil { - rdr.jsOpts = []nats.JSOpt{nats.MaxWait(*rdr.Config().Opts.NATSJetStreamMaxWait)} + if rdr.Config().Opts.NATSOpts.NATSJetStreamMaxWait != nil { + rdr.jsOpts = []nats.JSOpt{nats.MaxWait(*rdr.Config().Opts.NATSOpts.NATSJetStreamMaxWait)} } } rdr.opts, err = GetNatsOpts(rdr.Config().Opts, @@ -231,46 +231,46 @@ func GetNatsOpts(opts *config.EventReaderOpts, nodeID string, connTimeout time.D nop = append(nop, nats.Name(utils.CGRateSLwr+nodeID), nats.Timeout(connTimeout), nats.DrainTimeout(time.Second)) - if opts.NATSJWTFile != nil { + if opts.NATSOpts.NATSJWTFile != nil { keys := make([]string, 0, 1) - if opts.NATSSeedFile != nil { - keys = append(keys, *opts.NATSSeedFile) + if opts.NATSOpts.NATSSeedFile != nil { + keys = append(keys, *opts.NATSOpts.NATSSeedFile) } - nop = append(nop, nats.UserCredentials(*opts.NATSJWTFile, keys...)) + nop = append(nop, nats.UserCredentials(*opts.NATSOpts.NATSJWTFile, keys...)) } - if opts.NATSSeedFile != nil { - opt, err := nats.NkeyOptionFromSeed(*opts.NATSSeedFile) + if opts.NATSOpts.NATSSeedFile != nil { + opt, err := nats.NkeyOptionFromSeed(*opts.NATSOpts.NATSSeedFile) if err != nil { return nil, err } nop = append(nop, opt) } - if opts.NATSClientCertificate != nil { - if opts.NATSClientKey == nil { + if opts.NATSOpts.NATSClientCertificate != nil { + if opts.NATSOpts.NATSClientKey == nil { err = fmt.Errorf("has certificate but no key") return } - nop = append(nop, nats.ClientCert(*opts.NATSClientCertificate, *opts.NATSClientKey)) - } else if opts.NATSClientKey != nil { + nop = append(nop, nats.ClientCert(*opts.NATSOpts.NATSClientCertificate, *opts.NATSOpts.NATSClientKey)) + } else if opts.NATSOpts.NATSClientKey != nil { err = fmt.Errorf("has key but no certificate") return } - if opts.NATSCertificateAuthority != nil { + if opts.NATSOpts.NATSCertificateAuthority != nil { nop = append(nop, func(o *nats.Options) error { pool, err := x509.SystemCertPool() if err != nil { return err } - rootPEM, err := ioutil.ReadFile(*opts.NATSCertificateAuthority) + rootPEM, err := ioutil.ReadFile(*opts.NATSOpts.NATSCertificateAuthority) if err != nil || rootPEM == nil { return fmt.Errorf("nats: error loading or parsing rootCA file: %v", err) } ok := pool.AppendCertsFromPEM(rootPEM) if !ok { return fmt.Errorf("nats: failed to parse root certificate from %q", - *opts.NATSCertificateAuthority) + *opts.NATSOpts.NATSCertificateAuthority) } if o.TLSConfig == nil { o.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} diff --git a/ers/readers_test.go b/ers/readers_test.go index f1a7bab9e..9bd802fb7 100644 --- a/ers/readers_test.go +++ b/ers/readers_test.go @@ -103,7 +103,9 @@ func TestNewSQLReader(t *testing.T) { reader.ID = "file_reader" reader.ConcurrentReqs = -1 reader.Opts = &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), + SQLOpts: &config.SQLROpts{ + SQLDBName: utils.StringPointer("cgrates2"), + }, } reader.SourcePath = "*mysql://cgrates:CGRateS.org@127.0.0.1:3306" reader.ProcessedPath = "" @@ -130,7 +132,7 @@ func TestNewSQLReaderError(t *testing.T) { reader.ID = "file_reader" reader.ConcurrentReqs = -1 reader.Opts = &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), + SQLOpts: &config.SQLROpts{SQLDBName: utils.StringPointer("cgrates2")}, } reader.SourcePath = "#" reader.ProcessedPath = "" @@ -216,14 +218,21 @@ func TestNewAMQPReader(t *testing.T) { } exp.dialURL = exp.Config().SourcePath exp.Config().ProcessedPath = "" - exp.setOpts(&config.EventReaderOpts{}) + exp.setOpts(&config.EventReaderOpts{ + CSVOpts: &config.CSVROpts{}, + AMQPOpts: &config.AMQPROpts{}, + SQLOpts: &config.SQLROpts{}, + AWSOpts: &config.AWSROpts{}, + NATSOpts: &config.NATSROpts{}, + KafkaOpts: &config.KafkaROpts{}, + }) exp.createPoster() var expected EventReader = exp rcv, err := NewEventReader(cfg, 0, nil, nil, nil, fltr, nil) if err != nil { t.Error(err) } else if !reflect.DeepEqual(expected, rcv) { - t.Errorf("Expected %v but received %v", expected, rcv) + t.Errorf("Expected %v but received %v", utils.ToJSON(expected), utils.ToJSON(rcv)) } } diff --git a/ers/s3.go b/ers/s3.go index 28f037dd2..e43a83975 100644 --- a/ers/s3.go +++ b/ers/s3.go @@ -152,20 +152,22 @@ func (rdr *S3ER) processMessage(body []byte) (err error) { func (rdr *S3ER) parseOpts(opts *config.EventReaderOpts) { rdr.bucket = utils.DefaultQueueID - if opts.S3BucketID != nil { - rdr.bucket = *opts.S3BucketID - } - if opts.AWSRegion != nil { - rdr.awsRegion = *opts.AWSRegion - } - if opts.AWSKey != nil { - rdr.awsID = *opts.AWSKey - } - if opts.AWSSecret != nil { - rdr.awsKey = *opts.AWSSecret - } - if opts.AWSToken != nil { - rdr.awsToken = *opts.AWSToken + if awsOpts := opts.AWSOpts; awsOpts != nil { + if awsOpts.S3BucketID != nil { + rdr.bucket = *awsOpts.S3BucketID + } + if awsOpts.AWSRegion != nil { + rdr.awsRegion = *awsOpts.AWSRegion + } + if awsOpts.AWSKey != nil { + rdr.awsID = *awsOpts.AWSKey + } + if awsOpts.AWSSecret != nil { + rdr.awsKey = *awsOpts.AWSSecret + } + if awsOpts.AWSToken != nil { + rdr.awsToken = *awsOpts.AWSToken + } } } diff --git a/ers/s3_test.go b/ers/s3_test.go index 1ce545e78..cf52ed9dd 100644 --- a/ers/s3_test.go +++ b/ers/s3_test.go @@ -227,18 +227,20 @@ func TestS3ERParseOpts(t *testing.T) { } opts := &config.EventReaderOpts{ - S3BucketID: utils.StringPointer("QueueID"), - AWSRegion: utils.StringPointer("AWSRegion"), - AWSKey: utils.StringPointer("AWSKey"), - AWSSecret: utils.StringPointer("AWSSecret"), - AWSToken: utils.StringPointer("AWSToken"), + AWSOpts: &config.AWSROpts{ + S3BucketID: utils.StringPointer("QueueID"), + AWSRegion: utils.StringPointer("AWSRegion"), + AWSKey: utils.StringPointer("AWSKey"), + AWSSecret: utils.StringPointer("AWSSecret"), + AWSToken: utils.StringPointer("AWSToken"), + }, } rdr.parseOpts(opts) - if rdr.bucket != *opts.S3BucketID || - rdr.awsRegion != *opts.AWSRegion || - rdr.awsID != *opts.AWSKey || - rdr.awsKey != *opts.AWSSecret || - rdr.awsToken != *opts.AWSToken { + if rdr.bucket != *opts.AWSOpts.S3BucketID || + rdr.awsRegion != *opts.AWSOpts.AWSRegion || + rdr.awsID != *opts.AWSOpts.AWSKey || + rdr.awsKey != *opts.AWSOpts.AWSSecret || + rdr.awsToken != *opts.AWSOpts.AWSToken { t.Error("Fields do not corespond") } rdr.Config().Opts = &config.EventReaderOpts{} diff --git a/ers/sql.go b/ers/sql.go index ec3fcaee6..97c112cb7 100644 --- a/ers/sql.go +++ b/ers/sql.go @@ -275,17 +275,20 @@ func (rdr *SQLEventReader) setURL(inURL, outURL string, opts *config.EventReader rdr.connType = u.Scheme dbname := utils.SQLDefaultDBName - if opts.SQLDBName != nil { - dbname = *opts.SQLDBName - } ssl := utils.SQLDefaultSSLMode - if opts.PgSSLMode != nil { - ssl = *opts.PgSSLMode - } + if sqlOpts := opts.SQLOpts; sqlOpts != nil { + if sqlOpts.SQLDBName != nil { + dbname = *sqlOpts.SQLDBName + } - rdr.tableName = utils.CDRsTBL - if opts.SQLTableName != nil { - rdr.tableName = *opts.SQLTableName + if sqlOpts.PgSSLMode != nil { + ssl = *sqlOpts.PgSSLMode + } + + rdr.tableName = utils.CDRsTBL + if sqlOpts.SQLTableName != nil { + rdr.tableName = *sqlOpts.SQLTableName + } } switch rdr.connType { case utils.MySQL: diff --git a/ers/sql_it_test.go b/ers/sql_it_test.go index 5ce92b38e..81256102c 100644 --- a/ers/sql_it_test.go +++ b/ers/sql_it_test.go @@ -677,7 +677,9 @@ func TestErsSqlPostCDRS(t *testing.T) { reader.ID = "file_reader" reader.ConcurrentReqs = -1 reader.Opts = &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), + SQLOpts: &config.SQLROpts{ + SQLDBName: utils.StringPointer("cgrates2"), + }, } reader.SourcePath = "*mysql://cgrates:CGRateS.org@127.0.0.1:3306" reader.ProcessedPath = "" diff --git a/ers/sql_test.go b/ers/sql_test.go index 124fb26ee..e20fd52e7 100644 --- a/ers/sql_test.go +++ b/ers/sql_test.go @@ -41,13 +41,13 @@ func TestSQLSetURL(t *testing.T) { inURL := "*mysql://cgrates:CGRateS.org@127.0.0.1:3306" outURL := "*mysql://cgrates:CGRateS.org@127.0.0.1:3306" if err := sql.setURL(inURL, outURL, &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), - SQLTableName: utils.StringPointer("cdrs2"), - PgSSLMode: utils.StringPointer("enabled"), - - SQLDBNameProcessed: utils.StringPointer("cgrates3"), - SQLTableNameProcessed: utils.StringPointer("cdrs3"), - PgSSLModeProcessed: utils.StringPointer("enabled"), + SQLOpts: &config.SQLROpts{ + SQLDBName: utils.StringPointer("cgrates2"), + SQLTableName: utils.StringPointer("cdrs2"), + PgSSLMode: utils.StringPointer("enabled"), + SQLDBNameProcessed: utils.StringPointer("cgrates3"), + SQLTableNameProcessed: utils.StringPointer("cdrs3"), + PgSSLModeProcessed: utils.StringPointer("enabled")}, }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -76,13 +76,13 @@ func TestSQLSetURL(t *testing.T) { inURL = "*postgres://cgrates:CGRateS.org@127.0.0.1:3306" outURL = "*postgres://cgrates:CGRateS.org@127.0.0.1:3306" if err := sql.setURL(inURL, outURL, &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), - SQLTableName: utils.StringPointer("cdrs2"), - PgSSLMode: utils.StringPointer("enabled"), - - SQLDBNameProcessed: utils.StringPointer("cgrates3"), - SQLTableNameProcessed: utils.StringPointer("cdrs3"), - PgSSLModeProcessed: utils.StringPointer("enabled"), + SQLOpts: &config.SQLROpts{ + SQLDBName: utils.StringPointer("cgrates2"), + SQLTableName: utils.StringPointer("cdrs2"), + PgSSLMode: utils.StringPointer("enabled"), + SQLDBNameProcessed: utils.StringPointer("cgrates3"), + SQLTableNameProcessed: utils.StringPointer("cdrs3"), + PgSSLModeProcessed: utils.StringPointer("enabled")}, }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -111,13 +111,13 @@ func TestSQLSetURL(t *testing.T) { inURL = "*postgres://cgrates:CGRateS.org@127.0.0.1:3306" outURL = "" if err := sql.setURL(inURL, outURL, &config.EventReaderOpts{ - SQLDBName: utils.StringPointer("cgrates2"), - SQLTableName: utils.StringPointer("cdrs2"), - PgSSLMode: utils.StringPointer("enabled"), - - SQLDBNameProcessed: utils.StringPointer("cgrates2"), - SQLTableNameProcessed: utils.StringPointer("cdrs2"), - PgSSLModeProcessed: utils.StringPointer("enabled"), + SQLOpts: &config.SQLROpts{ + SQLDBName: utils.StringPointer("cgrates2"), + SQLTableName: utils.StringPointer("cdrs2"), + PgSSLMode: utils.StringPointer("enabled"), + SQLDBNameProcessed: utils.StringPointer("cgrates2"), + SQLTableNameProcessed: utils.StringPointer("cdrs2"), + PgSSLModeProcessed: utils.StringPointer("enabled")}, }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { diff --git a/ers/sqs.go b/ers/sqs.go index fae5169f5..c1c84ce1a 100644 --- a/ers/sqs.go +++ b/ers/sqs.go @@ -140,20 +140,23 @@ func (rdr *SQSER) processMessage(body []byte) (err error) { func (rdr *SQSER) parseOpts(opts *config.EventReaderOpts) { rdr.queueID = utils.DefaultQueueID - if opts.SQSQueueID != nil { - rdr.queueID = *opts.SQSQueueID - } - if opts.AWSRegion != nil { - rdr.awsRegion = *opts.AWSRegion - } - if opts.AWSKey != nil { - rdr.awsID = *opts.AWSKey - } - if opts.AWSSecret != nil { - rdr.awsKey = *opts.AWSSecret - } - if opts.AWSToken != nil { - rdr.awsToken = *opts.AWSToken + + if sqOpts := opts.AWSOpts; sqOpts != nil { + if opts.AWSOpts.SQSQueueID != nil { + rdr.queueID = *opts.AWSOpts.SQSQueueID + } + if opts.AWSOpts.AWSRegion != nil { + rdr.awsRegion = *opts.AWSOpts.AWSRegion + } + if opts.AWSOpts.AWSKey != nil { + rdr.awsID = *opts.AWSOpts.AWSKey + } + if opts.AWSOpts.AWSSecret != nil { + rdr.awsKey = *opts.AWSOpts.AWSSecret + } + if opts.AWSOpts.AWSToken != nil { + rdr.awsToken = *opts.AWSOpts.AWSToken + } } rdr.getQueueURL() } diff --git a/ers/sqs_test.go b/ers/sqs_test.go index 603c7cb68..695049f1b 100644 --- a/ers/sqs_test.go +++ b/ers/sqs_test.go @@ -277,15 +277,17 @@ func TestSQSERParseOpts(t *testing.T) { } opts := &config.EventReaderOpts{ - SQSQueueID: utils.StringPointer("QueueID"), - AWSRegion: utils.StringPointer("AWSRegion"), - AWSKey: utils.StringPointer("AWSKey"), - AWSSecret: utils.StringPointer("AWSSecret"), - AWSToken: utils.StringPointer("AWSToken"), + AWSOpts: &config.AWSROpts{ + SQSQueueID: utils.StringPointer("QueueID"), + AWSRegion: utils.StringPointer("AWSRegion"), + AWSKey: utils.StringPointer("AWSKey"), + AWSSecret: utils.StringPointer("AWSSecret"), + AWSToken: utils.StringPointer("AWSToken"), + }, } rdr.parseOpts(opts) - if rdr.queueID != *opts.SQSQueueID || rdr.awsRegion != *opts.AWSRegion || rdr.awsID != *opts.AWSKey || rdr.awsKey != *opts.AWSSecret || - rdr.awsToken != *opts.AWSToken { + if rdr.queueID != *opts.AWSOpts.SQSQueueID || rdr.awsRegion != *opts.AWSOpts.AWSRegion || rdr.awsID != *opts.AWSOpts.AWSKey || rdr.awsKey != *opts.AWSOpts.AWSSecret || + rdr.awsToken != *opts.AWSOpts.AWSToken { t.Error("Fields do not corespond") } rdr.Config().Opts = &config.EventReaderOpts{}