/* Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments Copyright (C) ITsysCOM GmbH This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see */ package config import ( "reflect" "testing" "time" "github.com/cgrates/cgrates/utils" ) func TestERSClone(t *testing.T) { cfgJSONStr := `{ "ers": { "enabled": true, "sessions_conns":["*internal"], "readers": [ { "id": "file_reader1", "run_delay": "-1", "type": "*file_csv", "flags": ["*dryrun"], "source_path": "/tmp/ers/in", "processed_path": "/tmp/ers/out", "opts": {}, "xml_root_path": "", "tenant": "~*req.Destination1", "timezone": "", "filters": ["randomFiletrs"], "flags": [], "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.2", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.3", "mandatory": true}, ], "cache_dump_fields": [ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.2", "mandatory": true}, ], "failed_calls_prefix": "randomPrefix", "partial_record_cache": "1s", "partial_cache_expiry_action": "randomAction" }, ], }, }` expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: "*file_csv", RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), Timezone: utils.EmptyString, Filters: []string{"randomFiletrs"}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { v.ComputePath() } for _, v := range profile.CacheDumpFields { v.ComputePath() } } if jsonCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else { clonedErs := jsonCfg.ersCfg.Clone() if !reflect.DeepEqual(clonedErs, expectedERsCfg) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(clonedErs)) } } } func TestEventReaderloadFromJsonCfg(t *testing.T) { jsonCfg := NewDefaultCGRConfig() eventReader := new(EventReaderCfg) if err = eventReader.loadFromJSONCfg(nil, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err != nil { t.Error(err) } } func TestEventReaderloadFromJsonCase1(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Run_delay: utils.StringPointer("1ss"), }, }, } expected := "time: unknown unit \"ss\" in duration \"1ss\"" jsoncfg := NewDefaultCGRConfig() if err = jsoncfg.ersCfg.loadFromJSONCfg(cfgJSON, jsoncfg.templates, jsoncfg.generalCfg.RSRSep, jsoncfg.dfltEvRdr, jsoncfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } cfgJson := &ERsJsonCfg{ Partial_cache_ttl: utils.StringPointer("test"), } if err := jsoncfg.ersCfg.loadFromJSONCfg(cfgJson, jsoncfg.templates, jsoncfg.generalCfg.RSRSep, jsoncfg.dfltEvRdr, jsoncfg.generalCfg.RSRSep); err == nil { t.Error(err) } } func TestEventReaderloadFromJsonCase3(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Tenant: utils.StringPointer("a{*"), }, }, } expected := "invalid converter terminator in rule: " jsoncfg := NewDefaultCGRConfig() if err = jsoncfg.ersCfg.loadFromJSONCfg(cfgJSON, jsoncfg.templates, jsoncfg.generalCfg.RSRSep, jsoncfg.dfltEvRdr, jsoncfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } } func TestERSLoadFromjsonCfg(t *testing.T) { expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"conn1", "conn3"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, KafkaOpts: &KafkaROpts{}, SQLOpts: &SQLROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: utils.MetaFileCSV, RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { v.ComputePath() } } cfgJSONStr := `{ "ers": { "enabled": true, "sessions_conns":["conn1","conn3"], "readers": [ { "id": "file_reader1", "run_delay": "-1", "type": "*file_csv", "source_path": "/tmp/ers/in", "processed_path": "/tmp/ers/out", "cache_dump_fields": [], }, ], } }` if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedERsCfg, cfg.ersCfg) { t.Errorf("Expected: %+v ,\n received: %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cfg.ersCfg)) } } func TestERSloadFromJsonCfg(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: nil, } jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err != nil { t.Error(err) } } func TestEventReaderFieldsloadFromJsonCfg(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Fields: &[]*FcTemplateJsonCfg{ { Value: utils.StringPointer("a{*"), }, }, }, }, } expected := "invalid converter terminator in rule: " jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } } func TestERSloadFromJsonCase1(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Fields: &[]*FcTemplateJsonCfg{ { Type: utils.StringPointer(utils.MetaTemplate), }, }, }, }, } expected := "no template with id: <>" jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } } func TestERSloadFromJsonCase2(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Cache_dump_fields: &[]*FcTemplateJsonCfg{ { Type: utils.StringPointer(utils.MetaTemplate), }, }, }, }, } expected := "no template with id: <>" jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } } func TestERSloadFromJsonCase3(t *testing.T) { cfgJSON := &ERsJsonCfg{ Enabled: utils.BoolPointer(true), Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { Id: utils.StringPointer("file_reader1"), Type: utils.StringPointer(utils.MetaFileCSV), Run_delay: utils.StringPointer("-1"), Concurrent_requests: utils.IntPointer(1024), Source_path: utils.StringPointer("/tmp/ers/in"), Processed_path: utils.StringPointer("/tmp/ers/out"), Tenant: nil, Timezone: utils.StringPointer(""), Filters: nil, Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.AnswerTime), Path: utils.StringPointer("*cgreq.AnswerTime"), Type: utils.StringPointer(utils.MetaTemplate), Value: utils.StringPointer("randomTemplate"), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00"), }, }, }, }, } expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, KafkaOpts: &KafkaROpts{}, SQLOpts: &SQLROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: utils.MetaFileCSV, RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, Path: "*exp.CGRID", Type: utils.MetaVariable, Layout: time.RFC3339, }, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } msgTemplates := map[string][]*FCTemplate{ "randomTemplate": { { Tag: utils.CGRID, Path: "*exp.CGRID", Type: utils.MetaVariable, Layout: time.RFC3339, }, }, } for _, v := range expectedERsCfg.Readers[0].Fields { v.ComputePath() } jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, msgTemplates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedERsCfg, jsonCfg.ersCfg) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(jsonCfg.ersCfg)) } } func TestERSloadFromJsonCase4(t *testing.T) { cfgJSON := &ERsJsonCfg{ Enabled: utils.BoolPointer(true), Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { Id: utils.StringPointer("file_reader1"), Type: utils.StringPointer(utils.MetaFileCSV), Run_delay: utils.StringPointer("-1"), Concurrent_requests: utils.IntPointer(1024), Source_path: utils.StringPointer("/tmp/ers/in"), Processed_path: utils.StringPointer("/tmp/ers/out"), Tenant: nil, Timezone: utils.StringPointer(""), Filters: nil, Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{}, Cache_dump_fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer("OriginID"), Path: utils.StringPointer("*exp.OriginID"), Type: utils.StringPointer(utils.MetaTemplate), Value: utils.StringPointer("randomTemplate"), }, }, }, }, } expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, KafkaOpts: &KafkaROpts{}, SQLOpts: &SQLROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: utils.MetaFileCSV, RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{}, CacheDumpFields: []*FCTemplate{ { Tag: "OrderID", Path: "*exp.OrderID", Type: "*variable", Value: NewRSRParsersMustCompile("~*req.OrderID", utils.InfieldSep), }, }, PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AWSOpts: &AWSROpts{}, AMQPOpts: &AMQPROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } msgTemplates := map[string][]*FCTemplate{ "randomTemplate": { { Tag: "OrderID", Path: "*exp.OrderID", Type: "*variable", Value: NewRSRParsersMustCompile("~*req.OrderID", utils.InfieldSep), }, }, } for _, v := range expectedERsCfg.Readers[0].CacheDumpFields { v.ComputePath() } for _, v := range expectedERsCfg.Readers[0].Fields { v.ComputePath() } jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, msgTemplates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedERsCfg, jsonCfg.ersCfg) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(jsonCfg.ersCfg)) } } func TestEventReaderCacheDumpFieldsloadFromJsonCfg(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ { Cache_dump_fields: &[]*FcTemplateJsonCfg{ { Value: utils.StringPointer("a{*"), }, }, }, }, } expected := "invalid converter terminator in rule: " jsonCfg := NewDefaultCGRConfig() if err = jsonCfg.ersCfg.loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep, jsonCfg.dfltEvRdr, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { t.Errorf("Expected %+v, received %+v", expected, err) } } func TestEventReaderSameID(t *testing.T) { expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"conn1"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, KafkaOpts: &KafkaROpts{}, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: utils.MetaFileCSV, RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { v.ComputePath() } } cfgJSONStr := `{ "ers": { "enabled": true, "sessions_conns":["conn1"], "readers": [ { "id": "file_reader1", "run_delay": "-1", "type": "*file_csv", "row_length" : 5, "source_path": "/tmp/ers/in", "processed_path": "/tmp/ers/out", "fields":[ {"tag": "CustomTag1", "path": "CustomPath1", "type": "*variable", "value": "CustomValue1", "mandatory": true}, ], "cache_dump_fields": [], }, { "id": "file_reader1", "run_delay": "-1", "type": "*file_csv", "source_path": "/tmp/ers/in", "processed_path": "/tmp/ers/out", "fields":[ {"tag": "CustomTag2", "path": "CustomPath2", "type": "*variable", "value": "CustomValue2", "mandatory": true}, ], "cache_dump_fields": [], }, ], } }` if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedERsCfg, cfg.ersCfg) { t.Errorf("Expected: %+v ,\n received: %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cfg.ersCfg)) } } func TestERsCfgAsMapInterfaceCase1(t *testing.T) { cfgJSONStr := `{ "ers": { "enabled": true, "sessions_conns":["conn1","conn3"], "readers": [ { "id": "file_reader1", "run_delay": "-1", "tenant": "~*req.Destination1", "type": "*file_csv", "source_path": "/tmp/ers/in", "processed_path": "/tmp/ers/out", "cache_dump_fields": [], }, ], } }` eMap := map[string]any{ utils.EnabledCfg: true, utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]any{ { utils.FiltersCfg: []string{}, utils.FlagsCfg: []string{}, utils.IDCfg: "*default", utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", utils.RunDelayCfg: "0", utils.SourcePathCfg: "/var/spool/cgrates/ers/in", utils.TenantCfg: "", utils.TimezoneCfg: "", utils.CacheDumpFieldsCfg: []map[string]any{}, utils.PartialCommitFieldsCfg: []map[string]any{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]any{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.OptsCfg: map[string]any{ "csvFieldSeparator": ",", "csvHeaderDefineChar": ":", "csvRowLength": 0, "xmlRootPath": "", "partialOrderField": "~*req.AnswerTime", "partialCacheAction": utils.MetaNone, "natsSubject": "cgrates_cdrs", }, }, { utils.CacheDumpFieldsCfg: []map[string]any{}, utils.PartialCommitFieldsCfg: []map[string]any{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", utils.FieldsCfg: []map[string]any{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.FiltersCfg: []string{}, utils.FlagsCfg: []string{}, utils.IDCfg: "file_reader1", utils.ProcessedPathCfg: "/tmp/ers/out", utils.RunDelayCfg: "-1", utils.SourcePathCfg: "/tmp/ers/in", utils.TenantCfg: "~*req.Destination1", utils.TimezoneCfg: "", utils.OptsCfg: map[string]any{ "csvFieldSeparator": ",", "csvHeaderDefineChar": ":", "csvRowLength": 0, "xmlRootPath": "", "partialOrderField": "~*req.AnswerTime", "partialCacheAction": utils.MetaNone, "natsSubject": "cgrates_cdrs", }, }, }, utils.PartialCacheTTLCfg: "1s", } if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if rcv := cfg.ersCfg.AsMapInterface(utils.InfieldSep); !reflect.DeepEqual(eMap, rcv) { t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(eMap), utils.ToJSON(rcv)) } } func TestERSCfgAsMapInterfaceCase2(t *testing.T) { cfgJSONStr := `{ "ers": { "enabled": true, "sessions_conns":["conn1","conn3"], "readers": [ { "id": "file_reader1", "run_delay": "10s", "tenant": "~*req.Destination1", "type": "*file_csv", "flags": ["randomFlag"], "filters": ["randomFilter"], "source_path": "/tmp/ers/in", "partial_record_cache": "1s", "processed_path": "/tmp/ers/out", "cache_dump_fields": [ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*variable", "value": "~*req.2", "mandatory": true} ], "opts":{ "kafkaGroupID": "test", "csvLazyQuotes": false, "amqpQueueID": "id", "amqpQueueIDProcessed":"queue", "amqpConsumerTag": "tag", "amqpExchange":"exchange", "amqpExchangeType":"type", "amqpRoutingKey":"key", "amqpExchangeProcessed":"amqprocess", "amqpExchangeTypeProcessed":"amqtype", "amqpRoutingKeyProcessed":"routekey", "partialPath":"optpath", "partialcsvFieldSeparator":",", "kafkaTopic":"topic", "kafkaMaxWait":"1m", "kafkaTopicProcessed":"process", "sqlDBName":"db", "sqlTableName":"table", "pgSSLMode":"pg", "sqlDBNameProcessed":"dbname", "sqlTableNameProcessed":"tablename", "pgSSLModeProcessed":"sslprocess", "awsRegion":"eu", "awsKey":"key", "awsSecret":"secret", "awsToken":"token", "awsRegionProcessed":"awsRegion", "awsKeyProcessed":"keyprocess", "awsSecretProcessed":"secretkey", "awsTokenProcessed": "tokenprocess", "sqsQueueID":"sqs", "sqsQueueIDProcessed":"sqsid", "s3BucketID":"s3bucket", "s3FolderPathProcessed": "s3folder", "s3BucketIDProcessed":"s3bucketid", "natsJetStream":true, "natsConsumerName": "NATConsumer", "natsQueueID":"NATid", "natsJWTFile":"jwt", "natsCertificateAuthority":"auth", "natsClientCertificate":"certificate", "natsSeedFile":"seed", "natsClientKey":"clientkey", "natsJetStreamMaxWait":"1m", "natsJetStreamProcessed":false, "natsSubjectProcessed":"NATProcess", "natsJWTFileProcessed":"jwtfile", "natsSeedFileProcessed":"NATSeed", "natsCertificateAuthorityProcessed":"CertificateAuth", "natsClientCertificateProcessed":"ClientCertificate", "natsClientKeyProcessed":"ClientKey", "natsJetStreamMaxWaitProcessed":"1m", }, }, ], } }` eMap := map[string]any{ utils.EnabledCfg: true, utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]any{ { utils.FiltersCfg: []string{}, utils.FlagsCfg: []string{}, utils.IDCfg: "*default", utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", utils.RunDelayCfg: "0", utils.SourcePathCfg: "/var/spool/cgrates/ers/in", utils.TenantCfg: "", utils.TimezoneCfg: "", utils.CacheDumpFieldsCfg: []map[string]any{}, utils.PartialCommitFieldsCfg: []map[string]any{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]any{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.OptsCfg: map[string]any{ "csvFieldSeparator": ",", "csvHeaderDefineChar": ":", "csvRowLength": 0, "xmlRootPath": "", "partialOrderField": "~*req.AnswerTime", "partialCacheAction": utils.MetaNone, "natsSubject": "cgrates_cdrs", }, }, { utils.CacheDumpFieldsCfg: []map[string]any{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, }, utils.PartialCommitFieldsCfg: []map[string]any{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", utils.FieldsCfg: []map[string]any{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.FiltersCfg: []string{"randomFilter"}, utils.FlagsCfg: []string{"randomFlag"}, utils.IDCfg: "file_reader1", utils.ProcessedPathCfg: "/tmp/ers/out", utils.RunDelayCfg: "10s", utils.SourcePathCfg: "/tmp/ers/in", utils.TenantCfg: "~*req.Destination1", utils.TimezoneCfg: "", utils.OptsCfg: map[string]any{ utils.CSVLazyQuotes: false, utils.KafkaGroupID: "test", utils.CSVFieldSepOpt: ",", utils.XMLRootPathOpt: "", "csvHeaderDefineChar": ":", "csvRowLength": 0, "partialOrderField": "~*req.AnswerTime", "partialCacheAction": utils.MetaNone, "natsSubject": "cgrates_cdrs", utils.AMQPQueueID: "id", utils.AMQPQueueIDProcessedCfg: "queue", utils.AMQPConsumerTag: "tag", utils.AMQPExchange: "exchange", utils.AMQPExchangeType: "type", utils.AMQPRoutingKey: "key", utils.AMQPExchangeProcessedCfg: "amqprocess", utils.AMQPExchangeTypeProcessedCfg: "amqtype", utils.AMQPRoutingKeyProcessedCfg: "routekey", utils.PartialPathOpt: "optpath", utils.PartialCSVFieldSepartorOpt: ",", utils.KafkaTopic: "topic", utils.KafkaMaxWait: "1m0s", utils.KafkaTopicProcessedCfg: "process", utils.SQLDBNameOpt: "db", utils.SQLTableNameOpt: "table", utils.PgSSLModeCfg: "pg", utils.SQLDBNameProcessedCfg: "dbname", utils.SQLTableNameProcessedCfg: "tablename", utils.PgSSLModeProcessedCfg: "sslprocess", utils.AWSRegion: "eu", utils.AWSKey: "key", utils.AWSSecret: "secret", utils.AWSToken: "token", utils.AWSRegionProcessedCfg: "awsRegion", utils.AWSKeyProcessedCfg: "keyprocess", utils.AWSSecretProcessedCfg: "secretkey", utils.AWSTokenProcessedCfg: "tokenprocess", utils.SQSQueueID: "sqs", utils.SQSQueueIDProcessedCfg: "sqsid", utils.S3Bucket: "s3bucket", utils.S3FolderPathProcessedCfg: "s3folder", utils.S3BucketIDProcessedCfg: "s3bucketid", utils.NatsJetStream: true, utils.NatsConsumerName: "NATConsumer", utils.NatsQueueID: "NATid", utils.NatsJWTFile: "jwt", utils.NatsSeedFile: "seed", utils.NatsCertificateAuthority: "auth", utils.NatsClientCertificate: "certificate", utils.NatsClientKey: "clientkey", utils.NatsJetStreamMaxWait: "1m0s", utils.NATSJetStreamProcessedCfg: false, utils.NATSSubjectProcessedCfg: "NATProcess", utils.NATSJWTFileProcessedCfg: "jwtfile", utils.NATSSeedFileProcessedCfg: "NATSeed", utils.NATSCertificateAuthorityProcessedCfg: "CertificateAuth", utils.NATSClientCertificateProcessed: "ClientCertificate", utils.NATSClientKeyProcessedCfg: "ClientKey", utils.NATSJetStreamMaxWaitProcessedCfg: "1m0s", }, }, }, utils.PartialCacheTTLCfg: "1s", } if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if rcv := cfg.ersCfg.AsMapInterface(utils.InfieldSep); !reflect.DeepEqual(eMap, rcv) { t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(eMap), utils.ToJSON(rcv)) } } func TestERsloadFromJsonCfg(t *testing.T) { cfgJSON := &ERsJsonCfg{ Enabled: utils.BoolPointer(true), Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { Id: utils.StringPointer("file_reader1"), Type: utils.StringPointer(utils.MetaFileCSV), Run_delay: utils.StringPointer("-1"), Concurrent_requests: utils.IntPointer(1024), Source_path: utils.StringPointer("/tmp/ers/in"), Processed_path: utils.StringPointer("/tmp/ers/out"), Tenant: nil, Timezone: utils.StringPointer(""), Filters: nil, Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.CGRID), Path: utils.StringPointer("*exp.CGRID"), Type: utils.StringPointer(utils.MetaVariable), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00"), }, {Tag: utils.StringPointer("CustomTag2"), Path: utils.StringPointer("CustomPath2"), Type: utils.StringPointer(utils.MetaVariable), Value: utils.StringPointer("CustomValue2"), Mandatory: utils.BoolPointer(true), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, }, }, }, } expectedERsCfg := &ERsCfg{ Enabled: true, SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { ID: utils.MetaDefault, Type: utils.MetaNone, RunDelay: 0, ConcurrentReqs: 1024, SourcePath: "/var/spool/cgrates/ers/in", ProcessedPath: "/var/spool/cgrates/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AccountField, Path: utils.MetaCgreq + utils.NestingSep + utils.AccountField, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, KafkaOpts: &KafkaROpts{}, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, { ID: "file_reader1", Type: utils.MetaFileCSV, RunDelay: -1, ConcurrentReqs: 1024, SourcePath: "/tmp/ers/in", ProcessedPath: "/tmp/ers/out", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, Path: "*exp.CGRID", Type: utils.MetaVariable, Layout: time.RFC3339, }, {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), PartialCommitFields: make([]*FCTemplate, 0), Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{ CSVFieldSeparator: utils.StringPointer(utils.FieldsSep), CSVHeaderDefineChar: utils.StringPointer(utils.InInFieldSep), CSVRowLength: utils.IntPointer(0), }, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, SQLOpts: &SQLROpts{}, KafkaOpts: &KafkaROpts{}, XMLRootPath: utils.StringPointer(utils.EmptyString), PartialOrderField: utils.StringPointer("~*req.AnswerTime"), PartialCacheAction: utils.StringPointer(utils.MetaNone), NATSOpts: &NATSROpts{ Subject: utils.StringPointer("cgrates_cdrs"), }, }, }, }, PartialCacheTTL: time.Second, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { v.ComputePath() } for _, v := range profile.CacheDumpFields { v.ComputePath() } } cfgCgr := NewDefaultCGRConfig() if err := cfgCgr.ersCfg.loadFromJSONCfg(cfgJSON, cfgCgr.templates, cfgCgr.generalCfg.RSRSep, cfgCgr.dfltEvRdr, cfgCgr.generalCfg.RSRSep); err != nil { t.Error(err) } else if !reflect.DeepEqual(cfgCgr.ersCfg, expectedERsCfg) { t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cgrCfg.ersCfg)) } } func TestGetDefaultExporter(t *testing.T) { ees := new(EEsCfg) if dft := ees.GetDefaultExporter(); dft != nil { t.Fatalf("Expected no default cfg, received: %s", utils.ToJSON(dft)) } cfgCgr := NewDefaultCGRConfig() if dft := cfgCgr.EEsCfg().GetDefaultExporter(); dft == nil || dft.ID != utils.MetaDefault { t.Fatalf("Unexpected default cfg returned: %s", utils.ToJSON(dft)) } } func TestEventReaderOptsCfg(t *testing.T) { erCfg := new(EventReaderCfg) if err := erCfg.Opts.loadFromJSONCfg(nil); err != nil { t.Error(err) } eventReaderOptsJson := &EventReaderOptsJson{ PartialPath: utils.StringPointer("path"), PartialCSVFieldSeparator: utils.StringPointer("/"), CSVLazyQuotes: utils.BoolPointer(false), AMQPQueueID: utils.StringPointer("id"), AMQPQueueIDProcessed: utils.StringPointer("id"), AMQPConsumerTag: utils.StringPointer("tag"), AMQPExchange: utils.StringPointer("exchange"), AMQPExchangeType: utils.StringPointer("type"), AMQPRoutingKey: utils.StringPointer("key1"), AMQPExchangeProcessed: utils.StringPointer("amq"), AMQPExchangeTypeProcessed: utils.StringPointer("amqtype"), AMQPRoutingKeyProcessed: utils.StringPointer("key"), KafkaTopic: utils.StringPointer("kafka"), KafkaMaxWait: utils.StringPointer("1m"), KafkaTopicProcessed: utils.StringPointer("kafkaproc"), SQLDBName: utils.StringPointer("dbname"), SQLTableName: utils.StringPointer("tablename"), PgSSLMode: utils.StringPointer("sslmode"), SQLDBNameProcessed: utils.StringPointer("dbnameproc"), SQLTableNameProcessed: utils.StringPointer("tablenameproc"), PgSSLModeProcessed: utils.StringPointer("sslproc"), AWSRegion: utils.StringPointer("eu"), AWSKey: utils.StringPointer("key"), AWSSecret: utils.StringPointer("secret"), AWSToken: utils.StringPointer("token"), AWSKeyProcessed: utils.StringPointer("secret"), AWSSecretProcessed: utils.StringPointer("secret"), AWSTokenProcessed: utils.StringPointer("token"), SQSQueueID: utils.StringPointer("SQSQueue"), SQSQueueIDProcessed: utils.StringPointer("SQSQueueId"), S3BucketID: utils.StringPointer("S3BucketID"), S3FolderPathProcessed: utils.StringPointer("S3Path"), S3BucketIDProcessed: utils.StringPointer("S3BucketProc"), NATSJetStream: utils.BoolPointer(false), NATSConsumerName: utils.StringPointer("user"), NATSQueueID: utils.StringPointer("id"), NATSJWTFile: utils.StringPointer("jwt"), NATSSeedFile: utils.StringPointer("seed"), NATSCertificateAuthority: utils.StringPointer("authority"), NATSClientCertificate: utils.StringPointer("certificate"), NATSClientKey: utils.StringPointer("key5"), NATSJetStreamMaxWait: utils.StringPointer("1m"), NATSJetStreamProcessed: utils.BoolPointer(true), NATSJWTFileProcessed: utils.StringPointer("file"), NATSSeedFileProcessed: utils.StringPointer("natseed"), NATSCertificateAuthorityProcessed: utils.StringPointer("natsauth"), NATSClientCertificateProcessed: utils.StringPointer("natcertificate"), NATSClientKeyProcessed: utils.StringPointer("natsprocess"), NATSJetStreamMaxWaitProcessed: utils.StringPointer("1m"), AWSRegionProcessed: utils.StringPointer("eu"), NATSSubjectProcessed: utils.StringPointer("process"), KafkaGroupID: utils.StringPointer("groupId"), } eventReader := &EventReaderCfg{ Opts: &EventReaderOpts{ CSVOpts: &CSVROpts{}, AMQPOpts: &AMQPROpts{}, AWSOpts: &AWSROpts{}, NATSOpts: &NATSROpts{}, KafkaOpts: &KafkaROpts{}, SQLOpts: &SQLROpts{}, }, } if err := eventReader.Opts.loadFromJSONCfg(eventReaderOptsJson); err != nil { t.Error(err) } if err := eventReader.Opts.loadFromJSONCfg(&EventReaderOptsJson{ KafkaMaxWait: utils.StringPointer("test"), }); err == nil { t.Error(err) } else if err := eventReader.Opts.loadFromJSONCfg(&EventReaderOptsJson{ NATSJetStreamMaxWait: utils.StringPointer("nil"), }); err == nil { t.Error(err) } else if err := eventReader.Opts.loadFromJSONCfg(&EventReaderOptsJson{ NATSJetStreamMaxWaitProcessed: utils.StringPointer("nil"), }); err == nil { t.Error(err) } } func TestEventReaderCfgClone(t *testing.T) { ban := &EventReaderCfg{ ID: "2", Type: "type", RunDelay: 1 * time.Minute, ConcurrentReqs: 5, PartialCommitFields: []*FCTemplate{ { Tag: "tag1", Type: "type1", }, { Tag: "tag2", Type: "type2", }, }, SourcePath: "/", ProcessedPath: "/path", Tenant: RSRParsers{}, Timezone: "time.Utc", Flags: utils.FlagsWithParams{}, Opts: &EventReaderOpts{ PartialPath: utils.StringPointer("path"), CSVOpts: &CSVROpts{ PartialCSVFieldSeparator: utils.StringPointer("/"), CSVLazyQuotes: utils.BoolPointer(false), }, AMQPOpts: &AMQPROpts{ AMQPQueueID: utils.StringPointer("id"), AMQPQueueIDProcessed: utils.StringPointer("id"), AMQPConsumerTag: utils.StringPointer("tag"), AMQPExchange: utils.StringPointer("exchange"), AMQPExchangeType: utils.StringPointer("type"), AMQPRoutingKey: utils.StringPointer("key1"), AMQPExchangeProcessed: utils.StringPointer("amq"), AMQPExchangeTypeProcessed: utils.StringPointer("amqtype"), AMQPRoutingKeyProcessed: utils.StringPointer("key"), }, SQLOpts: &SQLROpts{ SQLDBName: utils.StringPointer("dbname"), SQLTableName: utils.StringPointer("tablename"), PgSSLMode: utils.StringPointer("sslmode"), SQLDBNameProcessed: utils.StringPointer("dbnameproc"), SQLTableNameProcessed: utils.StringPointer("tablenameproc"), PgSSLModeProcessed: utils.StringPointer("sslproc"), }, AWSOpts: &AWSROpts{ AWSRegion: utils.StringPointer("eu"), AWSKey: utils.StringPointer("key"), AWSSecret: utils.StringPointer("secret"), AWSToken: utils.StringPointer("token"), AWSKeyProcessed: utils.StringPointer("secret"), AWSRegionProcessed: utils.StringPointer("eu"), AWSSecretProcessed: utils.StringPointer("secret"), AWSTokenProcessed: utils.StringPointer("token"), SQSQueueID: utils.StringPointer("SQSQueue"), SQSQueueIDProcessed: utils.StringPointer("SQSQueueId"), S3BucketID: utils.StringPointer("S3BucketID"), S3FolderPathProcessed: utils.StringPointer("S3Path"), S3BucketIDProcessed: utils.StringPointer("S3BucketProc"), }, NATSOpts: &NATSROpts{ JetStream: utils.BoolPointer(false), ConsumerName: utils.StringPointer("user"), QueueID: utils.StringPointer("id"), JWTFile: utils.StringPointer("jwt"), SeedFile: utils.StringPointer("seed"), CertificateAuthority: utils.StringPointer("authority"), ClientCertificate: utils.StringPointer("certificate"), ClientKey: utils.StringPointer("key5"), JetStreamMaxWait: utils.DurationPointer(1 * time.Minute), JetStreamProcessed: utils.BoolPointer(true), JWTFileProcessed: utils.StringPointer("file"), SeedFileProcessed: utils.StringPointer("natseed"), CertificateAuthorityProcessed: utils.StringPointer("natsauth"), ClientCertificateProcessed: utils.StringPointer("natcertificate"), ClientKeyProcessed: utils.StringPointer("natsprocess"), JetStreamMaxWaitProcessed: utils.DurationPointer(1 * time.Minute), SubjectProcessed: utils.StringPointer("process"), }, KafkaOpts: &KafkaROpts{ KafkaTopic: utils.StringPointer("kafka"), KafkaMaxWait: utils.DurationPointer(1 * time.Minute), KafkaTopicProcessed: utils.StringPointer("kafkaproc"), KafkaGroupID: utils.StringPointer("groupId"), }, }, } rcv := ban.Clone() if !reflect.DeepEqual(rcv, ban) { t.Errorf("expected %v received %v", utils.ToJSON(ban), utils.ToJSON(rcv)) } } func TestEventReaderCfgloadFromJSONCfg(t *testing.T) { str := "test" amqpr := &AMQPROpts{} exp := &AMQPROpts{ AMQPUsername: &str, AMQPPassword: &str, AMQPUsernameProcessed: &str, AMQPPasswordProcessed: &str, } jsnCfg := &EventReaderOptsJson{ AMQPUsername: &str, AMQPPassword: &str, AMQPUsernameProcessed: &str, AMQPPasswordProcessed: &str, } err := amqpr.loadFromJSONCfg(jsnCfg) if err != nil { t.Error(err) } if !reflect.DeepEqual(amqpr, exp) { t.Errorf("expected %v received %v", utils.ToJSON(exp), utils.ToJSON(amqpr)) } } func TestEventReaderCfgClone2(t *testing.T) { str := "test" amqpr := &AMQPROpts{ AMQPUsername: &str, AMQPPassword: &str, AMQPUsernameProcessed: &str, AMQPPasswordProcessed: &str, } rcv := amqpr.Clone() if !reflect.DeepEqual(amqpr, rcv) { t.Errorf("expected %v received %v", utils.ToJSON(amqpr), utils.ToJSON(rcv)) } } func TestEventReaderCfgAsMapInterface(t *testing.T) { str := "test" amqpr := &AMQPROpts{ AMQPUsername: &str, AMQPPassword: &str, AMQPUsernameProcessed: &str, AMQPPasswordProcessed: &str, } er := &EventReaderCfg{ Opts: &EventReaderOpts{ AMQPOpts: amqpr, }, } opts := map[string]any{ utils.AMQPUsername: str, utils.AMQPPassword: str, utils.AMQPUsernameProcessedCfg: str, utils.AMQPPasswordProcessedCfg: str, } exp := map[string]any{ utils.IDCfg: er.ID, utils.TypeCfg: er.Type, utils.ConcurrentRequestsCfg: er.ConcurrentReqs, utils.SourcePathCfg: er.SourcePath, utils.ProcessedPathCfg: er.ProcessedPath, utils.TenantCfg: er.Tenant.GetRule(""), utils.TimezoneCfg: er.Timezone, utils.FiltersCfg: er.Filters, utils.FlagsCfg: []string{}, utils.RunDelayCfg: "0", utils.OptsCfg: opts, } rcv := er.AsMapInterface("") if !reflect.DeepEqual(exp, rcv) { t.Errorf("expected %v received %v", utils.ToJSON(exp), utils.ToJSON(rcv)) } } func TestERsCfgappendERsReaders(t *testing.T) { tm := 1 * time.Second nm := 1 str := "test" bl := true tms := "1s" fc := []*FCTemplate{{ Tag: str, Type: str, Path: str, Filters: []string{str}, Value: RSRParsers{}, Width: nm, Strip: str, Padding: str, Mandatory: bl, AttributeID: str, NewBranch: bl, Timezone: str, Blocker: bl, Layout: str, CostShiftDigits: nm, RoundingDecimals: &nm, MaskDestID: str, MaskLen: nm, pathSlice: []string{}, }} erS := &ERsCfg{ Enabled: bl, SessionSConns: []string{str}, Readers: []*EventReaderCfg{ { ID: str, Type: str, RunDelay: tm, ConcurrentReqs: nm, SourcePath: str, ProcessedPath: str, Opts: &EventReaderOpts{}, Tenant: RSRParsers{}, Timezone: str, Filters: []string{str}, Flags: utils.FlagsWithParams{}, Fields: fc, PartialCommitFields: fc, CacheDumpFields: fc, }, }, PartialCacheTTL: tm, } fcj := &[]*FcTemplateJsonCfg{} jsnReader := &EventReaderJsonCfg{ Type: &str, Run_delay: &tms, Concurrent_requests: &nm, Source_path: &str, Processed_path: &str, Opts: &EventReaderOptsJson{}, Tenant: &str, Timezone: &str, Filters: &[]string{str}, Flags: &[]string{str}, Fields: fcj, Partial_commit_fields: fcj, Cache_dump_fields: fcj, } jsnReaders := []*EventReaderJsonCfg{jsnReader} exp := &EventReaderCfg{ Type: str, RunDelay: tm, ConcurrentReqs: nm, SourcePath: str, ProcessedPath: str, Opts: &EventReaderOpts{}, Tenant: RSRParsers{}, Timezone: str, Filters: []string{str}, Flags: utils.FlagsWithParams{}, Fields: fc, PartialCommitFields: fc, CacheDumpFields: fc, } err := exp.loadFromJSONCfg(jsnReader, map[string][]*FCTemplate{}, "") if err != nil { t.Error(err) } err = erS.appendERsReaders(&jsnReaders, map[string][]*FCTemplate{}, "", nil) if err != nil { t.Error(err) } if !reflect.DeepEqual(erS.Readers[1], exp) { t.Errorf("\nexpected %s\nreceived %s\n", utils.ToJSON(exp), utils.ToJSON(erS.Readers[1])) } } func TestErsCfgloadFromJSONCfg(t *testing.T) { tm := 1 * time.Second nm := 1 str := "test" str2 := "~test)`" bl := true tms := "1s" fc := []*FCTemplate{{ Tag: str, Type: str, Path: str, Filters: []string{str}, Value: RSRParsers{}, Width: nm, Strip: str, Padding: str, Mandatory: bl, AttributeID: str, NewBranch: bl, Timezone: str, Blocker: bl, Layout: str, CostShiftDigits: nm, RoundingDecimals: &nm, MaskDestID: str, MaskLen: nm, pathSlice: []string{str}, }} fc2 := []*FCTemplate{{ Tag: str2, Type: utils.MetaTemplate, Path: str2, Filters: []string{str2}, Value: RSRParsers{{ Rules: str2, path: str2, }}, Width: nm, Strip: str2, Padding: str2, Mandatory: bl, AttributeID: str2, NewBranch: bl, Timezone: str2, Blocker: bl, Layout: str2, CostShiftDigits: nm, RoundingDecimals: &nm, MaskDestID: str2, MaskLen: nm, pathSlice: []string{str2}, }} er := &EventReaderCfg{ ID: str, Type: str, RunDelay: tm, ConcurrentReqs: nm, SourcePath: str, ProcessedPath: str, Opts: &EventReaderOpts{}, Tenant: RSRParsers{}, Timezone: str, Filters: []string{str}, Flags: utils.FlagsWithParams{}, Fields: fc, PartialCommitFields: fc2, CacheDumpFields: fc, } fcj := &[]*FcTemplateJsonCfg{} fcj2 := &[]*FcTemplateJsonCfg{ { Tag: &str2, Type: &str2, Path: &str2, Attribute_id: &str2, Filters: &[]string{str2}, Value: &str2, Width: &nm, Strip: &str2, Padding: &str2, Mandatory: &bl, New_branch: &bl, Timezone: &str2, Blocker: &bl, Layout: &str2, Cost_shift_digits: &nm, Rounding_decimals: &nm, Mask_destinationd_id: &str2, Mask_length: &nm, }, } jsnCfg := &EventReaderJsonCfg{ Type: &str, Run_delay: &tms, Concurrent_requests: &nm, Source_path: &str, Processed_path: &str, Opts: &EventReaderOptsJson{}, Tenant: &str, Timezone: &str, Filters: &[]string{str}, Flags: &[]string{str}, Fields: fcj, Partial_commit_fields: fcj2, Cache_dump_fields: fcj, } err := er.loadFromJSONCfg(jsnCfg, nil, "") if err != nil { if err.Error() != "Unclosed unspilit syntax" { t.Error(err) } } else { t.Error("was expecting an error") } jsnCfg2 := &EventReaderJsonCfg{ Type: &str, Run_delay: &tms, Concurrent_requests: &nm, Source_path: &str, Processed_path: &str, Opts: &EventReaderOptsJson{}, Tenant: &str, Timezone: &str, Filters: &[]string{str}, Flags: &[]string{str}, Fields: fcj, Partial_commit_fields: fcj, Cache_dump_fields: fcj, } msgTemplates := map[string][]*FCTemplate{ str2: fc2, } err = er.loadFromJSONCfg(jsnCfg2, msgTemplates, "") if err != nil { if err.Error() != "Unclosed unspilit syntax" { t.Error(err) } } }