From 645fba0a1e29e23d57c3f0a4766308e19a2ef9a2 Mon Sep 17 00:00:00 2001 From: porosnicuadrian Date: Mon, 21 Sep 2020 14:45:16 +0300 Subject: [PATCH] Updated AsMapInterface and load method for erscfg --- config/eescfg.go | 1 + config/eescfg_test.go | 1 - config/erscfg.go | 13 +-- config/erscfg_test.go | 260 +++++++++++++++++++++++++++++++----------- 4 files changed, 202 insertions(+), 73 deletions(-) diff --git a/config/eescfg.go b/config/eescfg.go index fffd55ae3..c76c4bb55 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -322,6 +322,7 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str tenant = strings.Join(values, separator) initialMP[utils.TenantCfg] = tenant } + if eeC.Fields != nil { fields := make([]map[string]interface{}, 0, len(eeC.Fields)) for _, fld := range eeC.Fields { diff --git a/config/eescfg_test.go b/config/eescfg_test.go index b7d07a81c..500272ad3 100644 --- a/config/eescfg_test.go +++ b/config/eescfg_test.go @@ -735,7 +735,6 @@ func TestEEsCfgloadFromJsonCfg(t *testing.T) { } else if err := cgrCfg.eesCfg.loadFromJsonCfg(jsonCfg, cgrCfg.templates, cgrCfg.generalCfg.RSRSep, cgrCfg.dfltEvExp); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCfg, cgrCfg.eesCfg) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedCfg), utils.ToJSON(cgrCfg.eesCfg)) } } diff --git a/config/erscfg.go b/config/erscfg.go index 8dbd78553..85df0995b 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -298,14 +298,13 @@ func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string initialMP[utils.XmlRootPathCfg] = xmlRootPath } var tenant string - if er.Tenant != nil { - values := make([]string, len(er.Tenant)) - for i, item := range er.Tenant { - values[i] = item.Rules - } - tenant = strings.Join(values, separator) - initialMP[utils.TenantCfg] = tenant + values := make([]string, len(er.Tenant)) + for i, item := range er.Tenant { + values[i] = item.Rules } + tenant = strings.Join(values, separator) + initialMP[utils.TenantCfg] = tenant + if er.Fields != nil { fields := make([]map[string]interface{}, len(er.Fields)) for i, item := range er.Fields { diff --git a/config/erscfg_test.go b/config/erscfg_test.go index 4257eef52..2d4bcc554 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -375,76 +375,76 @@ func TestERsCfgAsMapInterface(t *testing.T) { }` var filters []string eMap := map[string]interface{}{ - "enabled": true, - "sessions_conns": []string{"conn1", "conn3"}, - "readers": []map[string]interface{}{ + utils.EnabledCfg: true, + utils.SessionSConnsCfg: []string{"conn1", "conn3"}, + utils.ReadersCfg: []map[string]interface{}{ { - "filters": []string{}, - "flags": filters, - "id": "*default", - "partial_record_cache": "0", - "processed_path": "/var/spool/cgrates/ers/out", - "row_length": 0, - "run_delay": "0", - "partial_cache_expiry_action": "", - "source_path": "/var/spool/cgrates/ers/in", - "tenant": "", - "timezone": "", - "xml_root_path": []string{""}, - "cache_dump_fields": []map[string]interface{}{}, - "concurrent_requests": 1024, - "type": "*none", - "failed_calls_prefix": "", - "field_separator": ",", - utils.HeaderDefCharCfg: ":", - "fields": []map[string]interface{}{ - {"mandatory": true, "path": "*cgreq.ToR", "tag": "ToR", "type": "*variable", "value": "~*req.2"}, - {"mandatory": true, "path": "*cgreq.OriginID", "tag": "OriginID", "type": "*variable", "value": "~*req.3"}, - {"mandatory": true, "path": "*cgreq.RequestType", "tag": "RequestType", "type": "*variable", "value": "~*req.4"}, - {"mandatory": true, "path": "*cgreq.Tenant", "tag": "Tenant", "type": "*variable", "value": "~*req.6"}, - {"mandatory": true, "path": "*cgreq.Category", "tag": "Category", "type": "*variable", "value": "~*req.7"}, - {"mandatory": true, "path": "*cgreq.Account", "tag": "Account", "type": "*variable", "value": "~*req.8"}, - {"mandatory": true, "path": "*cgreq.Subject", "tag": "Subject", "type": "*variable", "value": "~*req.9"}, - {"mandatory": true, "path": "*cgreq.Destination", "tag": "Destination", "type": "*variable", "value": "~*req.10"}, - {"mandatory": true, "path": "*cgreq.SetupTime", "tag": "SetupTime", "type": "*variable", "value": "~*req.11"}, - {"mandatory": true, "path": "*cgreq.AnswerTime", "tag": "AnswerTime", "type": "*variable", "value": "~*req.12"}, - {"mandatory": true, "path": "*cgreq.Usage", "tag": "Usage", "type": "*variable", "value": "~*req.13"}, + utils.FiltersCfg: []string{}, + utils.FlagsCfg: filters, + utils.IdCfg: "*default", + utils.PartialRecordCacheCfg: "0", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RowLengthCfg: 0, + utils.RunDelayCfg: "0", + utils.PartialCacheExpiryActionCfg: "", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.XmlRootPathCfg: []string{""}, + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", + utils.FailedCallsPrefixCfg: "", + utils.FieldSeparatorCfg: ",", + utils.HeaderDefCharCfg: ":", + utils.FieldsCfg: []map[string]interface{}{ + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - "opts": make(map[string]interface{}), + utils.OptsCfg: make(map[string]interface{}), }, { - "cache_dump_fields": []map[string]interface{}{}, - "concurrent_requests": 1024, - "type": "*file_csv", - "failed_calls_prefix": "", - "field_separator": ",", - utils.HeaderDefCharCfg: ":", - "fields": []map[string]interface{}{ - {"mandatory": true, "path": "*cgreq.ToR", "tag": "ToR", "type": "*variable", "value": "~*req.2"}, - {"mandatory": true, "path": "*cgreq.OriginID", "tag": "OriginID", "type": "*variable", "value": "~*req.3"}, - {"mandatory": true, "path": "*cgreq.RequestType", "tag": "RequestType", "type": "*variable", "value": "~*req.4"}, - {"mandatory": true, "path": "*cgreq.Tenant", "tag": "Tenant", "type": "*variable", "value": "~*req.6"}, - {"mandatory": true, "path": "*cgreq.Category", "tag": "Category", "type": "*variable", "value": "~*req.7"}, - {"mandatory": true, "path": "*cgreq.Account", "tag": "Account", "type": "*variable", "value": "~*req.8"}, - {"mandatory": true, "path": "*cgreq.Subject", "tag": "Subject", "type": "*variable", "value": "~*req.9"}, - {"mandatory": true, "path": "*cgreq.Destination", "tag": "Destination", "type": "*variable", "value": "~*req.10"}, - {"mandatory": true, "path": "*cgreq.SetupTime", "tag": "SetupTime", "type": "*variable", "value": "~*req.11"}, - {"mandatory": true, "path": "*cgreq.AnswerTime", "tag": "AnswerTime", "type": "*variable", "value": "~*req.12"}, - {"mandatory": true, "path": "*cgreq.Usage", "tag": "Usage", "type": "*variable", "value": "~*req.13"}, + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*file_csv", + utils.FailedCallsPrefixCfg: "", + utils.FieldSeparatorCfg: ",", + utils.HeaderDefCharCfg: ":", + utils.FieldsCfg: []map[string]interface{}{ + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.RequestType", utils.TagCfg: "RequestType", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.4"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Tenant", utils.TagCfg: "Tenant", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.6"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Category", utils.TagCfg: "Category", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.7"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Account", utils.TagCfg: "Account", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.8"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Subject", utils.TagCfg: "Subject", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.9"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Destination", utils.TagCfg: "Destination", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.10"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.SetupTime", utils.TagCfg: "SetupTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.11"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, + {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - "filters": filters, - "flags": filters, - "id": "file_reader1", - "partial_record_cache": "0", - "processed_path": "/tmp/ers/out", - "row_length": 0, - "run_delay": "-1", - "partial_cache_expiry_action": "", - "source_path": "/tmp/ers/in", - "tenant": "", - "timezone": "", - "xml_root_path": []string{""}, - "opts": make(map[string]interface{}), + utils.FiltersCfg: filters, + utils.FlagsCfg: filters, + utils.IDCfg: "file_reader1", + utils.PartialRecordCacheCfg: "0", + utils.ProcessedPathCfg: "/tmp/ers/out", + utils.RowLengthCfg: 0, + utils.RunDelayCfg: "-1", + utils.PartialCacheExpiryActionCfg: "", + utils.SourcePathCfg: "/tmp/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.XmlRootPathCfg: []string{""}, + utils.OptsCfg: make(map[string]interface{}), }, }, } @@ -454,3 +454,133 @@ func TestERsCfgAsMapInterface(t *testing.T) { t.Errorf("\nExpected: %+v\nRecived: %+v", utils.ToIJSON(eMap), utils.ToIJSON(rcv)) } } + +func TestERsloadFromJsonCfg(t *testing.T) { + cfgJSON := &ERsJsonCfg{ + Enabled: utils.BoolPointer(true), + Sessions_conns: &[]string{"*conn1"}, + Readers: &[]*EventReaderJsonCfg{ + { + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Row_length: utils.IntPointer(5), + Field_separator: utils.StringPointer(","), + Header_define_character: utils.StringPointer(":"), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Xml_root_path: utils.StringPointer(""), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, + Failed_calls_prefix: nil, + Partial_cache_expiry_action: utils.StringPointer(""), + Partial_record_cache: utils.StringPointer("0"), + Fields: &[]*FcTemplateJsonCfg{ + { + Tag: utils.StringPointer(utils.CGRID), + Path: utils.StringPointer("*exp.CGRID"), + Type: utils.StringPointer(utils.MetaVariable), + Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00"), + }, + {Tag: utils.StringPointer("CustomTag2"), Path: utils.StringPointer("CustomPath2"), Type: utils.StringPointer(utils.MetaVariable), + Value: utils.StringPointer("CustomValue2"), Mandatory: utils.BoolPointer(true), Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, + }, + }, + }, + } + expectedERsCfg := &ERsCfg{ + Enabled: true, + SessionSConns: []string{"*conn1"}, + Readers: []*EventReaderCfg{ + { + ID: utils.MetaDefault, + Type: utils.META_NONE, + FieldSep: ",", + HeaderDefineChar: ":", + RunDelay: time.Duration(0), + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + XmlRootPath: utils.HierarchyPath{utils.EmptyString}, + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{ + {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.OriginID, Path: utils.MetaCgreq + utils.NestingSep + utils.OriginID, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.RequestType, Path: utils.MetaCgreq + utils.NestingSep + utils.RequestType, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Tenant, Path: utils.MetaCgreq + utils.NestingSep + utils.Tenant, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Category, Path: utils.MetaCgreq + utils.NestingSep + utils.Category, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Account, Path: utils.MetaCgreq + utils.NestingSep + utils.Account, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Subject, Path: utils.MetaCgreq + utils.NestingSep + utils.Subject, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Destination, Path: utils.MetaCgreq + utils.NestingSep + utils.Destination, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.SetupTime, Path: utils.MetaCgreq + utils.NestingSep + utils.SetupTime, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.AnswerTime, Path: utils.MetaCgreq + utils.NestingSep + utils.AnswerTime, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.12", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("~*req.13", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + }, + CacheDumpFields: make([]*FCTemplate, 0), + Opts: make(map[string]interface{}), + }, + { + ID: "file_reader1", + Type: utils.MetaFileCSV, + RowLength: 5, + FieldSep: ",", + HeaderDefineChar: ":", + RunDelay: time.Duration(-1), + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + XmlRootPath: utils.HierarchyPath{utils.EmptyString}, + Tenant: nil, + Timezone: utils.EmptyString, + Filters: nil, + Flags: utils.FlagsWithParams{}, + PartialCacheExpiryAction: "", + PartialRecordCache: time.Duration(0), + Fields: []*FCTemplate{ + { + Tag: utils.CGRID, + Path: "*exp.CGRID", + Type: utils.MetaVariable, + Layout: time.RFC3339, + }, + {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, + Value: NewRSRParsersMustCompile("CustomValue2", utils.INFIELD_SEP), Mandatory: true, Layout: time.RFC3339}, + }, + CacheDumpFields: make([]*FCTemplate, 0), + Opts: make(map[string]interface{}), + }, + }, + } + for _, profile := range expectedERsCfg.Readers { + for _, v := range profile.Fields { + v.ComputePath() + } + for _, v := range profile.CacheDumpFields { + v.ComputePath() + } + } + if cfgCgr, err := NewDefaultCGRConfig(); err != nil { + t.Error(err) + } else if err := cfgCgr.ersCfg.loadFromJsonCfg(cfgJSON, cfgCgr.templates, cfgCgr.generalCfg.RSRSep, cfgCgr.dfltEvRdr, cfgCgr.generalCfg.RSRSep); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(cfgCgr.ersCfg, expectedERsCfg) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cgrCfg.ersCfg)) + } +}