From 32ec2bdc6f57f66beee2dd44c08d7b4524ecb3ab Mon Sep 17 00:00:00 2001 From: TeoV Date: Sun, 1 Sep 2019 15:35:37 +0300 Subject: [PATCH] Update loding config for EventReader --- config/config.go | 16 ++++---- config/erscfg.go | 9 ++++- config/erscfg_test.go | 77 ++++++++++++++++++++++++++++++++++++ config/generalcfg.go | 4 +- config/multifiles_it_test.go | 2 +- engine/action.go | 4 +- engine/model_helpers.go | 2 +- migrator/attributes.go | 2 +- 8 files changed, 99 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 3d052aa84..c21863d24 100755 --- a/config/config.go +++ b/config/config.go @@ -881,7 +881,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if err != nil { return err } - if err := self.diameterAgentCfg.loadFromJsonCfg(jsnDACfg, self.generalCfg.RsrSepatarot); err != nil { + if err := self.diameterAgentCfg.loadFromJsonCfg(jsnDACfg, self.generalCfg.RSRSep); err != nil { return err } @@ -889,7 +889,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if err != nil { return err } - if err := self.radiusAgentCfg.loadFromJsonCfg(jsnRACfg, self.generalCfg.RsrSepatarot); err != nil { + if err := self.radiusAgentCfg.loadFromJsonCfg(jsnRACfg, self.generalCfg.RSRSep); err != nil { return err } @@ -897,7 +897,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if err != nil { return err } - if err := self.dnsAgentCfg.loadFromJsonCfg(jsnDNSCfg, self.generalCfg.RsrSepatarot); err != nil { + if err := self.dnsAgentCfg.loadFromJsonCfg(jsnDNSCfg, self.generalCfg.RSRSep); err != nil { return err } @@ -905,7 +905,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if err != nil { return err } - if err := self.httpAgentCfg.loadFromJsonCfg(jsnHttpAgntCfg, self.generalCfg.RsrSepatarot); err != nil { + if err := self.httpAgentCfg.loadFromJsonCfg(jsnHttpAgntCfg, self.generalCfg.RSRSep); err != nil { return err } @@ -1029,7 +1029,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if err != nil { return nil } - if err := self.ersCfg.loadFromJsonCfg(jsnERsCfg, self.generalCfg.RsrSepatarot); err != nil { + if err := self.ersCfg.loadFromJsonCfg(jsnERsCfg, self.generalCfg.RSRSep, self.dfltEvRdr); err != nil { return err } @@ -1041,7 +1041,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { self.CdreProfiles[profileName] = self.dfltCdreProfile.Clone() // Clone default so we do not inherit pointers } } - if err = self.CdreProfiles[profileName].loadFromJsonCfg(jsnCdre1Cfg, self.generalCfg.RsrSepatarot); err != nil { // Update the existing profile with content from json config + if err = self.CdreProfiles[profileName].loadFromJsonCfg(jsnCdre1Cfg, self.generalCfg.RSRSep); err != nil { // Update the existing profile with content from json config return err } } @@ -1051,7 +1051,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { // self.loaderCfg = make([]*LoaderSCfg, len(jsnLoaderCfg)) for _, profile := range jsnLoaderCfg { loadSCfgp := NewDfltLoaderSCfg() - loadSCfgp.loadFromJsonCfg(profile, self.generalCfg.RsrSepatarot) + loadSCfgp.loadFromJsonCfg(profile, self.generalCfg.RSRSep) self.loaderCfg = append(self.loaderCfg, loadSCfgp) // use apend so the loaderS profile to be loaded from multiple files } } @@ -1082,7 +1082,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { if cdrcInstCfg == nil { cdrcInstCfg = self.dfltCdrcProfile.Clone() } - if err := cdrcInstCfg.loadFromJsonCfg(jsnCrc1Cfg, self.generalCfg.RsrSepatarot); err != nil { + if err := cdrcInstCfg.loadFromJsonCfg(jsnCrc1Cfg, self.generalCfg.RSRSep); err != nil { return err } if cdrcInstCfg.CDRInPath == "" { diff --git a/config/erscfg.go b/config/erscfg.go index 065926e90..aeef9196a 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -30,7 +30,7 @@ type ERsCfg struct { Readers []*EventReaderCfg } -func (erS *ERsCfg) loadFromJsonCfg(jsnCfg *ERsJsonCfg, sep string) (err error) { +func (erS *ERsCfg) loadFromJsonCfg(jsnCfg *ERsJsonCfg, sep string, dfltRdrCfg *EventReaderCfg) (err error) { if jsnCfg == nil { return } @@ -47,7 +47,12 @@ func (erS *ERsCfg) loadFromJsonCfg(jsnCfg *ERsJsonCfg, sep string) (err error) { if jsnCfg.Readers != nil { erS.Readers = make([]*EventReaderCfg, len(*jsnCfg.Readers)) for idx, rdrs := range *jsnCfg.Readers { - erS.Readers[idx] = new(EventReaderCfg) + if dfltRdrCfg == nil { // when loading defaults dfltRdrCfg is nil ( e.g line 195) + erS.Readers[idx] = new(EventReaderCfg) + } else { + erS.Readers[idx] = dfltRdrCfg.Clone() + } + if err = erS.Readers[idx].loadFromJsonCfg(rdrs, sep); err != nil { return err } diff --git a/config/erscfg_test.go b/config/erscfg_test.go index 4eab5da4b..24774c442 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -20,6 +20,7 @@ package config import ( "reflect" "testing" + "time" "github.com/cgrates/cgrates/utils" ) @@ -97,3 +98,79 @@ func TestEventRedearClone(t *testing.T) { t.Errorf("expected: %s \n,received: %s", utils.ToJSON(initialOrig), utils.ToJSON(cloned)) } } + +func TestEventReaderLoadFromJSON(t *testing.T) { + expectedERsCfg := &ERsCfg{ + Enabled: true, + SessionSConns: []*RemoteHost{ + { + Address: utils.MetaInternal, + }, + }, + Readers: []*EventReaderCfg{ + &EventReaderCfg{ + ID: "file_reader1", + Type: utils.MetaFileCSV, + FieldSep: ",", + RunDelay: time.Duration(-1), + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + XmlRootPath: utils.EmptyString, + SourceID: "ers_csv", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: nil, + Flags: utils.FlagsWithParams{}, + Header_fields: make([]*FCTemplate, 0), + Content_fields: []*FCTemplate{ + {Tag: "TOR", FieldId: "ToR", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~2", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "OriginID", FieldId: "OriginID", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~3", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "RequestType", FieldId: "RequestType", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~4", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Tenant", FieldId: "Tenant", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Category", FieldId: "Category", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Account", FieldId: "Account", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Subject", FieldId: "Subject", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Destination", FieldId: "Destination", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~10", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "SetupTime", FieldId: "SetupTime", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~11", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "AnswerTime", FieldId: "AnswerTime", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~12", true, utils.INFIELD_SEP), Mandatory: true}, + {Tag: "Usage", FieldId: "Usage", Type: utils.META_COMPOSED, + Value: NewRSRParsersMustCompile("~13", true, utils.INFIELD_SEP), Mandatory: true}, + }, + Trailer_fields: make([]*FCTemplate, 0), + }, + }, + } + + cfgJSONStr := `{ +"ers": { + "enabled": true, + "readers": [ + { + "id": "file_reader1", + "run_delay": -1, + "type": "*file_csv", + "source_path": "/tmp/ers/in", + "processed_path": "/tmp/ers/out", + }, + ], +} +}` + + if cfg, err := NewCGRConfigFromJsonStringWithDefaults(cfgJSONStr); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expectedERsCfg, cfg.ersCfg) { + t.Errorf("Expected: %+v ,\n recived: %+v", utils.ToJSON(expectedERsCfg), utils.ToJSON(cfg.ersCfg)) + } + +} diff --git a/config/generalcfg.go b/config/generalcfg.go index b1903cb8e..fed9d7575 100644 --- a/config/generalcfg.go +++ b/config/generalcfg.go @@ -48,7 +48,7 @@ type GeneralCfg struct { LockingTimeout time.Duration // locking mechanism timeout to avoid deadlocks DigestSeparator string // DigestEqual string // - RsrSepatarot string // separator used to split RSRParser (by degault is used ";") + RSRSep string // separator used to split RSRParser (by degault is used ";") } //loadFromJsonCfg loads General config from JsonCfg @@ -127,7 +127,7 @@ func (gencfg *GeneralCfg) loadFromJsonCfg(jsnGeneralCfg *GeneralJsonCfg) (err er gencfg.DigestEqual = *jsnGeneralCfg.Digest_equal } if jsnGeneralCfg.Rsr_separator != nil { - gencfg.RsrSepatarot = *jsnGeneralCfg.Rsr_separator + gencfg.RSRSep = *jsnGeneralCfg.Rsr_separator } return nil diff --git a/config/multifiles_it_test.go b/config/multifiles_it_test.go index 98d7ca45a..892e28f52 100644 --- a/config/multifiles_it_test.go +++ b/config/multifiles_it_test.go @@ -114,7 +114,7 @@ func TestMfEnvReaderITRead(t *testing.T) { LockingTimeout: time.Duration(0), DigestSeparator: ",", DigestEqual: ":", - RsrSepatarot: ";", + RSRSep: ";", } if !reflect.DeepEqual(expected, *mfCgrCfg.generalCfg) { t.Errorf("Expected: %+v\n, recived: %+v", utils.ToJSON(expected), utils.ToJSON(*mfCgrCfg.generalCfg)) diff --git a/engine/action.go b/engine/action.go index 7bbcbd63a..7332fbe6c 100644 --- a/engine/action.go +++ b/engine/action.go @@ -139,7 +139,7 @@ func cdrLogAction(acc *Account, a *Action, acs Actions, extraData interface{}) ( } for field, rsr := range template { if defaultTemplate[field], err = config.NewRSRParsers(rsr, - true, config.CgrConfig().GeneralCfg().RsrSepatarot); err != nil { + true, config.CgrConfig().GeneralCfg().RSRSep); err != nil { return } } @@ -148,7 +148,7 @@ func cdrLogAction(acc *Account, a *Action, acs Actions, extraData interface{}) ( mapExtraData, _ := extraData.(map[string]interface{}) for key, val := range mapExtraData { if defaultTemplate[key], err = config.NewRSRParsers(utils.IfaceAsString(val), - true, config.CgrConfig().GeneralCfg().RsrSepatarot); err != nil { + true, config.CgrConfig().GeneralCfg().RSRSep); err != nil { return } } diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 2b0952f7f..28ae9c0b2 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2160,7 +2160,7 @@ func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (a attrPrf.Contexts[i] = context } for i, reqAttr := range tpAttr.Attributes { - sbstPrsr, err := config.NewRSRParsers(reqAttr.Value, true, config.CgrConfig().GeneralCfg().RsrSepatarot) + sbstPrsr, err := config.NewRSRParsers(reqAttr.Value, true, config.CgrConfig().GeneralCfg().RSRSep) if err != nil { return nil, err } diff --git a/migrator/attributes.go b/migrator/attributes.go index 74e61137e..4613ef2aa 100644 --- a/migrator/attributes.go +++ b/migrator/attributes.go @@ -251,7 +251,7 @@ func (v1AttrPrf v1AttributeProfile) AsAttributeProfile() (attrPrf *engine.Attrib if attr.Initial != utils.META_ANY { filterIDs = append(filterIDs, utils.MetaString+":"+attr.FieldName+":"+attr.Initial) } - sbstPrsr, err := config.NewRSRParsers(attr.Substitute, true, config.CgrConfig().GeneralCfg().RsrSepatarot) + sbstPrsr, err := config.NewRSRParsers(attr.Substitute, true, config.CgrConfig().GeneralCfg().RSRSep) if err != nil { return nil, err }