From d147a0a888df504343b382497624586774515cce Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Tue, 6 Jul 2021 13:04:07 +0300 Subject: [PATCH] Fix sanity checks for ers and add tests --- config/configsanity.go | 15 +++-- config/configsanity_test.go | 109 +++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/config/configsanity.go b/config/configsanity.go index 4c517e476..e577fdf10 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -693,20 +693,19 @@ func (cfg *CGRConfig) checkConfigSanity() error { if !possibleReaderTypes.Has(rdr.Type) { return fmt.Errorf("<%s> unsupported data type: %s for reader with ID: %s", utils.ERs, rdr.Type, rdr.ID) } - var pAct string - if act, has := rdr.Opts[utils.PartialCacheActionOpt]; has { // check the action from opts - if pAct = utils.IfaceAsString(act); pAct != utils.MetaDumpToFile && - pAct != utils.MetaNone && - pAct != utils.MetaPostCDR { - return fmt.Errorf("<%s> wrong partial expiry action for reader with ID: %s", utils.ERs, rdr.ID) - } + pAct := utils.IfaceAsString(rdr.Opts[utils.PartialCacheActionOpt]) + if pAct != utils.MetaDumpToFile && + pAct != utils.MetaNone && + pAct != utils.MetaPostCDR { + return fmt.Errorf("<%s> wrong partial expiry action for reader with ID: %s", utils.ERs, rdr.ID) } if pAct != utils.MetaNone { // if is *none we do not process the evicted events if fldSep, has := rdr.Opts[utils.PartialOrderFieldOpt]; has && // the field we order after must not be empty utils.IfaceAsString(fldSep) == utils.EmptyString { return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.PartialOrderFieldOpt, rdr.ID) } - } else if pAct == utils.MetaDumpToFile { // only if the action is *dump_to_file + } + if pAct == utils.MetaDumpToFile { // only if the action is *dump_to_file path := rdr.ProcessedPath if pathVal, has := rdr.Opts[utils.PartialPathOpt]; has { // the path from options needs to exists if overwriten by reader path = utils.IfaceAsString(pathVal) diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 472185431..669296427 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1232,6 +1232,9 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test2", Type: utils.MetaFileCSV, ProcessedPath: "not/a/path", + Opts: map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaNone, + }, }} expected = " nonexistent folder: not/a/path for reader with ID: test2" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1243,7 +1246,10 @@ func TestConfigSanityEventReader(t *testing.T) { Type: utils.MetaFileCSV, ProcessedPath: "/", SourcePath: "/", - Opts: map[string]interface{}{"csvFieldSeparator": ""}, + Opts: map[string]interface{}{ + "csvFieldSeparator": "", + utils.PartialCacheActionOpt: utils.MetaNone, + }, }} expected = " empty csvFieldSeparator for reader with ID: test3" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1253,7 +1259,10 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test4", Type: utils.MetaKafkajsonMap, RunDelay: 1, - Opts: map[string]interface{}{"csvFieldSeparator": ","}, + Opts: map[string]interface{}{ + "csvFieldSeparator": ",", + utils.PartialCacheActionOpt: utils.MetaNone, + }, } expected = " the RunDelay field can not be bigger than zero for reader with ID: test4" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1265,6 +1274,9 @@ func TestConfigSanityEventReader(t *testing.T) { RunDelay: 0, ProcessedPath: "not/a/path", SourcePath: "not/a/path", + Opts: map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaNone, + }, } expected = " nonexistent folder: not/a/path for reader with ID: test5" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1277,6 +1289,9 @@ func TestConfigSanityEventReader(t *testing.T) { RunDelay: 0, ProcessedPath: "not/a/path", SourcePath: "not/a/path", + Opts: map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaNone, + }, } expected = " nonexistent folder: not/a/path for reader with ID: test5" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1296,6 +1311,9 @@ func TestConfigSanityEventReader(t *testing.T) { {Tag: "SessionId", Path: utils.EmptyString, Type: "*variable", Value: NewRSRParsersMustCompile("~*req.Session-Id", utils.InfieldSep), Mandatory: true}, }, + Opts: map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaNone, + }, }, }, } @@ -1956,3 +1974,90 @@ func TestCheckConfigSanity(t *testing.T) { t.Errorf("Expecting: %+q received: %+q", expected, err) } } + +func TestConfigSanityErs(t *testing.T) { + cfg = NewDefaultCGRConfig() + cfg.ersCfg.Enabled = true + + cfg.ersCfg.SessionSConns = []string{} + cfg.ersCfg.Readers = []*EventReaderCfg{ + { + Type: utils.MetaNone, + ID: "rdrID", + Opts: map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaPostCDR, + }, + }, + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialOrderFieldOpt: utils.EmptyString, + utils.PartialCacheActionOpt: "invalid", + } + expected := " wrong partial expiry action for reader with ID: rdrID" + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialOrderFieldOpt: utils.EmptyString, + utils.PartialCacheActionOpt: utils.MetaPostCDR, + } + expected = " empty partialOrderField for reader with ID: rdrID" + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaDumpToFile, + utils.PartialCSVFieldSepartorOpt: utils.EmptyString, + } + cfg.ersCfg.Readers[0].ProcessedPath = "/tmp" + expected = " empty partialcsvFieldSeparator for reader with ID: rdrID" + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialCacheActionOpt: utils.MetaDumpToFile, + utils.PartialCSVFieldSepartorOpt: utils.FieldsSep, + } + expected = " empty cache_dump_fields for reader with ID: rdrID" + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialOrderFieldOpt: "non_empty", + utils.PartialCacheActionOpt: utils.MetaDumpToFile, + utils.PartialPathOpt: "path", + } + expected = " nonexistent partial folder: path for reader with ID: rdrID" + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.PartialCSVFieldSepartorOpt: utils.EmptyString, + } + + cfg.ersCfg.Readers[0].Type = utils.MetaFileCSV + cfg.ersCfg.Readers[0].ProcessedPath = utils.EmptyString + cfg.ersCfg.Readers[0].SourcePath = "/tmp" + cfg.ersCfg.Readers[0].ID = utils.MetaDefault + cfg.ersCfg.Readers[0].Opts = map[string]interface{}{ + utils.CSVRowLengthOpt: "can't convert", + utils.CSVLazyQuotes: "can't convert", + utils.PartialCacheActionOpt: utils.MetaNone, + } + expected = ` error when converting csvRowLength: for reader with ID: *default` + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } + + cfg.ersCfg.Readers[0].Opts[utils.CSVRowLengthOpt] = "2" + expected = ` error when converting csvLazyQuotes: for reader with ID: *default` + if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { + t.Errorf("expected: <%v>,\n received: <%v>", expected, err) + } +}