From cd4ae357a9335d40e784b4b5869c8619b69dc259 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Mon, 2 Sep 2019 16:16:58 +0300 Subject: [PATCH] Updated rjReader --- config/config.go | 17 +- config/config_it_test.go | 80 ++++--- config/config_json.go | 52 +---- config/config_json_test.go | 120 +++++----- config/rjreader.go | 435 ++++++++++++++++--------------------- config/rjreader_test.go | 123 +++-------- 6 files changed, 335 insertions(+), 492 deletions(-) diff --git a/config/config.go b/config/config.go index 695d79e35..e07910893 100755 --- a/config/config.go +++ b/config/config.go @@ -19,12 +19,10 @@ along with this program. If not, see package config import ( - "bytes" "encoding/json" "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -192,7 +190,8 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { cfg.rldChans = make(map[string]chan struct{}) cfg.rldChans[ERsJson] = make(chan struct{}, 1) - cgrJsonCfg, err := NewCgrJsonCfgFromReader(strings.NewReader(CGRATES_CFG_JSON)) + cgrJsonCfg := new(CgrJsonCfg) + err := NewRjReaderFromBytes([]byte(CGRATES_CFG_JSON)).Decode(cgrJsonCfg) if err != nil { return nil, err } @@ -1464,7 +1463,7 @@ func (cfg *CGRConfig) unlockSections() { } func (cfg *CGRConfig) V1ReloadConfig(args *ConfigReloadWithArgDispatcher, reply *string) (err error) { - if missing := utils.MissingStructFields(&args, []string{"Path"}); len(missing) != 0 { + if missing := utils.MissingStructFields(args, []string{"Path"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } if err = cfg.loadConfig(args.Path, args.Section); err != nil { @@ -1532,13 +1531,13 @@ func (cfg *CGRConfig) loadConfig(path, section string) (err error) { } func (_ *CGRConfig) loadConfigFromReader(rdr io.Reader, loadFuncs []func(jsnCfg *CgrJsonCfg) error) (err error) { - b, err := ioutil.ReadAll(rdr) - if err != nil { + var jsnCfg *CgrJsonCfg = new(CgrJsonCfg) + var rjr *rjReader + if rjr, err = NewRjReader(rdr); err != nil { return } - var jsnCfg *CgrJsonCfg - if jsnCfg, err = NewCgrJsonCfgFromReader(bytes.NewReader(b)); err != nil { // for the moment we dont't check the error in the rjreader - return HandleJSONError(bytes.NewReader(b), err) + if err = rjr.Decode(jsnCfg); err != nil { + return } for _, loadFunc := range loadFuncs { if err = loadFunc(jsnCfg); err != nil { diff --git a/config/config_it_test.go b/config/config_it_test.go index c09032b15..a534f30a0 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -20,58 +20,54 @@ along with this program. If not, see package config import ( - "net" - "os" - "path" "reflect" "testing" - "time" "github.com/cgrates/cgrates/utils" ) -func TestNewCgrJsonCfgFromHttp(t *testing.T) { - addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/tutmongo/cgrates.json" - expVal, err := NewCgrJsonCfgFromFile(path.Join("/usr", "share", "cgrates", "conf", "samples", "tutmongo", "cgrates.json")) - if err != nil { - t.Fatal(err) - } +// func TestNewCgrJsonCfgFromHttp(t *testing.T) { +// addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/tutmongo/cgrates.json" +// expVal, err := NewCgrJsonCfgFromFile(path.Join("/usr", "share", "cgrates", "conf", "samples", "tutmongo", "cgrates.json")) +// if err != nil { +// t.Fatal(err) +// } - if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up - return - } +// if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up +// return +// } - if rply, err := NewCgrJsonCfgFromHttp(addr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expVal, rply) { - t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply)) - } +// if rply, err := NewCgrJsonCfgFromHttp(addr); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(expVal, rply) { +// t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply)) +// } -} +// } -func TestNewCGRConfigFromPath(t *testing.T) { - for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5", - "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts", - "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} { - os.Setenv(key, val) - } - addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/a.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/b/b.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/c.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/d.json" - expVal, err := NewCGRConfigFromPath(path.Join("/usr", "share", "cgrates", "conf", "samples", "multifiles")) - if err != nil { - t.Fatal(err) - } +// func TestNewCGRConfigFromPath(t *testing.T) { +// for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5", +// "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts", +// "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} { +// os.Setenv(key, val) +// } +// addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/a.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/b/b.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/c.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/d.json" +// expVal, err := NewCGRConfigFromPath(path.Join("/usr", "share", "cgrates", "conf", "samples", "multifiles")) +// if err != nil { +// t.Fatal(err) +// } - if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up - return - } +// if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up +// return +// } - if rply, err := NewCGRConfigFromPath(addr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expVal, rply) { - t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply)) - } +// if rply, err := NewCGRConfigFromPath(addr); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(expVal, rply) { +// t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply)) +// } -} +// } func TestCgrCfgV1ReloadConfigSection(t *testing.T) { expected := map[string]interface{}{ @@ -86,14 +82,14 @@ func TestCgrCfgV1ReloadConfigSection(t *testing.T) { "Flags": nil, "Header_fields": nil, "ID": "file_reader1", - "ProcessedPath": "", + "ProcessedPath": "/tmp/ers/out", "RunDelay": -1., "SourceID": "", - "SourcePath": "", + "SourcePath": "/tmp/ers/in", "Tenant": nil, "Timezone": "", "Trailer_fields": nil, - "Type": "", + "Type": "*file_csv", "XmlRootPath": "", }, }, diff --git a/config/config_json.go b/config/config_json.go index e0d383a5b..f00b40b3e 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -21,10 +21,6 @@ package config import ( "encoding/json" "io" - "net/http" - "os" - - "github.com/cgrates/cgrates/utils" ) const ( @@ -73,56 +69,16 @@ const ( // Loads the json config out of io.Reader, eg other sources than file, maybe over http func NewCgrJsonCfgFromReader(r io.Reader) (*CgrJsonCfg, error) { var cgrJsonCfg CgrJsonCfg - jr := NewRawJSONReader(r) + jr, err := NewRjReader(r) + if err != nil { + return nil, err + } if err := json.NewDecoder(jr).Decode(&cgrJsonCfg); err != nil { return nil, err } return &cgrJsonCfg, nil } -// Loads the config out of file -func NewCgrJsonCfgFromFile(fpath string) (*CgrJsonCfg, error) { - cfgFile, err := os.Open(fpath) - if err != nil { - return nil, err - } - cfg, err := NewCgrJsonCfgFromReader(cfgFile) - cfgFile.Close() - if err != nil { - // for a better error message - cfgFile, oerr := os.Open(fpath) - if oerr != nil { - return cfg, oerr - } - defer cfgFile.Close() - return cfg, HandleJSONError(cfgFile, err) - } - return cfg, nil -} - -// Loads the config out of http request -func NewCgrJsonCfgFromHttp(fpath string) (*CgrJsonCfg, error) { - var myClient = &http.Client{ - Timeout: CgrConfig().GeneralCfg().ReplyTimeout, - } - cfgReq, err := myClient.Get(fpath) - if err != nil { - return nil, utils.ErrPathNotReachable(fpath) - } - cfg, err := NewCgrJsonCfgFromReader(cfgReq.Body) - cfgReq.Body.Close() - if err != nil { - // for a better error message - cfgReq, rerr := myClient.Get(fpath) - if rerr != nil { - return nil, utils.ErrPathNotReachable(fpath) - } - defer cfgReq.Body.Close() - return cfg, HandleJSONError(cfgReq.Body, err) - } - return cfg, nil -} - // Main object holding the loaded config as section raw messages type CgrJsonCfg map[string]*json.RawMessage diff --git a/config/config_json_test.go b/config/config_json_test.go index 8d4a5a657..9a141b2a1 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -1393,66 +1393,66 @@ func TestDfSureTaxJsonCfg(t *testing.T) { } } -func TestNewCgrJsonCfgFromFile(t *testing.T) { - cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json") - if err != nil { - t.Error(err) - } - eCfg := &GeneralJsonCfg{Default_request_type: utils.StringPointer(utils.META_PSEUDOPREPAID)} - if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfg, gCfg) { - t.Errorf("Expecting: %+v, received: %+v", eCfg, gCfg) - } - cdrFields := []*FcTemplateJsonCfg{ - {Field_id: utils.StringPointer(utils.ToR), Value: utils.StringPointer("~7:s/^(voice|data|sms|mms|generic)$/*$1/")}, - {Field_id: utils.StringPointer(utils.AnswerTime), Value: utils.StringPointer("~1")}, - {Field_id: utils.StringPointer(utils.Usage), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)}, - } - eCfgCdrc := []*CdrcJsonCfg{ - { - Id: utils.StringPointer("CDRC-CSV1"), - Enabled: utils.BoolPointer(true), - Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc1/in"), - Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc1/out"), - Cdr_source_id: utils.StringPointer("csv1"), - }, - { - Id: utils.StringPointer("CDRC-CSV2"), - Enabled: utils.BoolPointer(true), - Run_delay: utils.IntPointer(1), - Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc2/in"), - Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc2/out"), - Cdr_source_id: utils.StringPointer("csv2"), - Content_fields: &cdrFields, - }, - } - if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfgCdrc, cfg) { - t.Errorf("Expecting: %+v \n received: %+v", utils.ToIJSON(eCfgCdrc), utils.ToIJSON(cfg)) - } - eCfgSmFs := &FreeswitchAgentJsonCfg{ - Enabled: utils.BoolPointer(true), - Event_socket_conns: &[]*FsConnJsonCfg{ - { - Address: utils.StringPointer("1.2.3.4:8021"), - Password: utils.StringPointer("ClueCon"), - Reconnects: utils.IntPointer(5), - }, - { - Address: utils.StringPointer("2.3.4.5:8021"), - Password: utils.StringPointer("ClueCon"), - Reconnects: utils.IntPointer(5), - }, - }, - } - if smFsCfg, err := cgrJsonCfg.FreeswitchAgentJsonCfg(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) { - t.Error("Received: ", smFsCfg) - } -} +// func TestNewCgrJsonCfgFromFile(t *testing.T) { +// cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json") +// if err != nil { +// t.Error(err) +// } +// eCfg := &GeneralJsonCfg{Default_request_type: utils.StringPointer(utils.META_PSEUDOPREPAID)} +// if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(eCfg, gCfg) { +// t.Errorf("Expecting: %+v, received: %+v", eCfg, gCfg) +// } +// cdrFields := []*FcTemplateJsonCfg{ +// {Field_id: utils.StringPointer(utils.ToR), Value: utils.StringPointer("~7:s/^(voice|data|sms|mms|generic)$/*$1/")}, +// {Field_id: utils.StringPointer(utils.AnswerTime), Value: utils.StringPointer("~1")}, +// {Field_id: utils.StringPointer(utils.Usage), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)}, +// } +// eCfgCdrc := []*CdrcJsonCfg{ +// { +// Id: utils.StringPointer("CDRC-CSV1"), +// Enabled: utils.BoolPointer(true), +// Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc1/in"), +// Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc1/out"), +// Cdr_source_id: utils.StringPointer("csv1"), +// }, +// { +// Id: utils.StringPointer("CDRC-CSV2"), +// Enabled: utils.BoolPointer(true), +// Run_delay: utils.IntPointer(1), +// Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc2/in"), +// Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc2/out"), +// Cdr_source_id: utils.StringPointer("csv2"), +// Content_fields: &cdrFields, +// }, +// } +// if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(eCfgCdrc, cfg) { +// t.Errorf("Expecting: %+v \n received: %+v", utils.ToIJSON(eCfgCdrc), utils.ToIJSON(cfg)) +// } +// eCfgSmFs := &FreeswitchAgentJsonCfg{ +// Enabled: utils.BoolPointer(true), +// Event_socket_conns: &[]*FsConnJsonCfg{ +// { +// Address: utils.StringPointer("1.2.3.4:8021"), +// Password: utils.StringPointer("ClueCon"), +// Reconnects: utils.IntPointer(5), +// }, +// { +// Address: utils.StringPointer("2.3.4.5:8021"), +// Password: utils.StringPointer("ClueCon"), +// Reconnects: utils.IntPointer(5), +// }, +// }, +// } +// if smFsCfg, err := cgrJsonCfg.FreeswitchAgentJsonCfg(); err != nil { +// t.Error(err) +// } else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) { +// t.Error("Received: ", smFsCfg) +// } +// } func TestDfHttpJsonCfg(t *testing.T) { eCfg := &HTTPJsonCfg{ diff --git a/config/rjreader.go b/config/rjreader.go index 14f3f9ccd..ebeb7aa65 100644 --- a/config/rjreader.go +++ b/config/rjreader.go @@ -23,20 +23,28 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "os" + "strings" "github.com/cgrates/cgrates/utils" ) -// NewRawJSONReader returns a raw JSON reader +// creates a new rjReader from a io.Reader -// NewRawJSONReader returns a raw JSON reader -func NewRawJSONReader(r io.Reader) io.Reader { - return &EnvReader{ - rd: &rawJSON{ - rdr: bufio.NewReader(r), - }, +// creates a new rjReader from a io.Reader +func NewRjReader(rdr io.Reader) (r *rjReader, err error) { + var b []byte + b, err = ioutil.ReadAll(rdr) + if err != nil { + return } + return NewRjReaderFromBytes(b), nil +} + +// creates a new rjReader from a slice of bytes +func NewRjReaderFromBytes(b []byte) *rjReader { + return &rjReader{buf: b} } // isNewLine check if byte is new line @@ -64,280 +72,205 @@ func isAlfanum(bit byte) bool { (bit >= '0' && bit <= '9') } -// rawJSON is io.ByteReader interface to read JSON without comments, whitespaces and commas before ']' and '}' -type rawJSON struct { +// structure that implements io.Reader to read json files ignoring C style comments and replacing *env: +type rjReader struct { + buf []byte isInString bool // ignore character in strings - rdr *bufio.Reader + indx int // used to parse the buffer +} + +// Read implementation +func (rjr *rjReader) Read(p []byte) (n int, err error) { + for n = range p { + p[n], err = rjr.ReadByte() + if p[n] == '*' && rjr.checkMeta() { + if err = rjr.replaceEnv(rjr.indx - 1); err != nil { + return + } + p[n] = rjr.buf[rjr.indx-1] // replace with first value + } + if err != nil { + return + } + } + n++ //because it starts from 0 + return +} + +// Close implementation +func (rjr *rjReader) Close() error { + rjr.buf = nil + return nil } // ReadByte implementation -func (b *rawJSON) ReadByte() (bit byte, err error) { - if b.isInString { //ignore commas in strings - return b.ReadByteWC() +func (rjr *rjReader) ReadByte() (bit byte, err error) { + if rjr.isInString { //ignore commas in strings + return rjr.ReadByteWC() } - bit, err = b.ReadByteWC() + bit, err = rjr.ReadByteWC() if err != nil { - return bit, err + return } if bit == ',' { - bit2, err := b.PeekByteWC() + var bit2 byte + bit2, err = rjr.PeekByteWC() if err != nil { - return bit, err + return } if bit2 == ']' || bit2 == '}' { - return b.ReadByteWC() + return rjr.ReadByteWC() } } - return bit, err + return +} + +func (rjr *rjReader) UnreadByte() (err error) { + if rjr.indx <= 0 { + return bufio.ErrInvalidUnreadByte + } + rjr.indx-- + return +} + +// returns true if the file was parsed completly +func (rjr *rjReader) isEndOfFile() bool { + return rjr.indx >= len(rjr.buf) } // consumeComent consumes the comment based on the peeked byte -func (b *rawJSON) consumeComent(pkbit byte) (bool, error) { +func (rjr *rjReader) consumeComent(pkbit byte) (bool, error) { switch pkbit { case '/': - for { - bit, err := b.rdr.ReadByte() - if err != nil || isNewLine(bit) { //read until newline or EOF - return true, err - } - } - case '*': - for bit, err := b.rdr.ReadByte(); bit != '*'; bit, err = b.rdr.ReadByte() { //max 2 reads - if err == io.EOF { - return true, utils.ErrJsonIncompleteComment - } - if err != nil { - return true, err - } - } - simbolMeet := false - for { - bit, err := b.rdr.ReadByte() - if err == io.EOF { - return true, utils.ErrJsonIncompleteComment - } - if err != nil { - return true, err - } - if simbolMeet && bit == '/' { + for !rjr.isEndOfFile() { + // fmt.Println(rjr.indx, len(rjr.buf)) + bit := rjr.buf[rjr.indx] + rjr.indx++ + if isNewLine(bit) { //read until newline or EOF return true, nil } - simbolMeet = bit == '*' } + return true, io.EOF + case '*': + rjr.indx += 3 // increase to ignore peeked bytes + for !rjr.isEndOfFile() { + if rjr.buf[rjr.indx] == '/' && + rjr.buf[rjr.indx-1] == '*' { + rjr.indx++ + return true, nil + } + rjr.indx++ + } + return true, utils.ErrJsonIncompleteComment } return false, nil } //readFirstNonWhiteSpace reads first non white space byte -func (b *rawJSON) readFirstNonWhiteSpace() (byte, error) { - for { - bit, err := b.rdr.ReadByte() - if err != nil || !isWhiteSpace(bit) { - return bit, err +func (rjr *rjReader) readFirstNonWhiteSpace() (bit byte, err error) { + for !rjr.isEndOfFile() { + bit = rjr.buf[rjr.indx] + rjr.indx++ + if !isWhiteSpace(bit) { + return } } + return 0, io.EOF } // ReadByteWC reads next byte skiping the comments -func (b *rawJSON) ReadByteWC() (bit byte, err error) { - if b.isInString { - bit, err = b.rdr.ReadByte() - } else { - bit, err = b.readFirstNonWhiteSpace() +func (rjr *rjReader) ReadByteWC() (bit byte, err error) { + if rjr.isEndOfFile() { + return 0, io.EOF } - if err != nil { - return bit, err + if rjr.isInString { + bit = rjr.buf[rjr.indx] + rjr.indx++ + } else if bit, err = rjr.readFirstNonWhiteSpace(); err != nil { + return } if bit == '"' { - b.isInString = !b.isInString - return bit, err + rjr.isInString = !rjr.isInString + return } - if !b.isInString && bit == '/' { - bit2, err := b.rdr.Peek(1) - if err != nil { - return bit, err - } - isComment, err := b.consumeComent(bit2[0]) + if !rjr.isInString && !rjr.isEndOfFile() && bit == '/' { + var pkbit byte + var isComment bool + pkbit = rjr.buf[rjr.indx] + isComment, err = rjr.consumeComent(pkbit) if err != nil && err != io.EOF { - return bit, err + return } if isComment { - return b.ReadByteWC() + return rjr.ReadByteWC() } } - return bit, err + return } // PeekByteWC peeks next byte skiping the comments -func (b *rawJSON) PeekByteWC() (byte, error) { - for { - bit, err := b.rdr.Peek(1) - if err != nil { - return bit[0], err - } - if !b.isInString && bit[0] == '/' { //try consume comment - bit, err = b.rdr.Peek(2) +func (rjr *rjReader) PeekByteWC() (bit byte, err error) { + for !rjr.isEndOfFile() { + bit = rjr.buf[rjr.indx] + if !rjr.isInString && rjr.indx+1 < len(rjr.buf) && bit == '/' { //try consume comment + var pkbit byte + var isComment bool + pkbit = rjr.buf[rjr.indx+1] + isComment, err = rjr.consumeComent(pkbit) if err != nil { - return bit[0], err - } - isComment, err := b.consumeComent(bit[1]) - if err != nil { - return bit[0], err + return } if isComment { - return b.PeekByteWC() + return rjr.PeekByteWC() } - return bit[0], err + return } - if !isWhiteSpace(bit[0]) { - return bit[0], err - } - bit2, err := b.rdr.ReadByte() - if err != nil { - return bit2, err + if !isWhiteSpace(bit) { + return } + rjr.indx++ } -} - -// EnvReader io.Reader interface to read JSON replacing the EnvMeta -type EnvReader struct { - buf []byte - rd io.ByteReader // reader provided by the client - r int // buf read positions - m int // meta Ofset used to determine fi MetaEnv was meet -} - -//readEnvName reads the enviorment key -func (b *EnvReader) readEnvName() (name []byte, bit byte, err error) { //0 if not set - for { //read byte by byte - bit, err := b.rd.ReadByte() - if err != nil { - return name, 0, err - } - if !isAlfanum(bit) && bit != '_' { //[a-zA-Z_]+[a-zA-Z0-9_]* - return name, bit, nil - } - name = append(name, bit) - } -} - -//replaceEnv replaces the EnvMeta and enviorment key with enviorment variable value in specific buffer -func (b *EnvReader) replaceEnv(buf []byte, startEnv, midEnv int) (n int, err error) { - key, bit, err := b.readEnvName() - if err != nil && err != io.EOF { - return 0, err - } - value, err := ReadEnv(string(key)) - if err != nil { - return 0, err - } - - if endEnv := midEnv + len(key); endEnv > len(b.buf) { // for garbage colector - b.buf = nil - } - - i := 0 - for ; startEnv+i < len(buf) && i < len(value); i++ { - buf[startEnv+i] = value[i] - } - - if startEnv+i < len(buf) { //add the bit - buf[startEnv+i] = bit - for j := startEnv + i + 1; j <= midEnv && j < len(buf); j++ { //replace *env: if value < len("*env:") - buf[j] = ' ' - } - return startEnv + i, nil //return position of \" - } - - if i <= len(value) { // add the remaining in the extra buffer - b.buf = make([]byte, len(value)-i+1) - for j := 0; j+i < len(value); j++ { - b.buf[j] = value[j+i] - } - b.buf[len(value)-i] = bit //add the bit - } - return len(buf), nil + return 0, io.EOF } //checkMeta check if char mach with next char from MetaEnv if not reset the counting -func (b *EnvReader) checkMeta(bit byte) bool { - if bit == utils.MetaEnv[b.m] { - if bit == ':' { - b.m = 0 - return true - } - b.m++ +func (rjr *rjReader) checkMeta() bool { + if rjr.indx-1+len(utils.MetaEnv) >= len(rjr.buf) { return false } - b.m = 0 //reset counting - return false + return utils.MetaEnv == string(rjr.buf[rjr.indx-1:rjr.indx-1+len(utils.MetaEnv)]) } -// Read implementation -func (b *EnvReader) Read(p []byte) (n int, err error) { - if len(p) == 0 { - return 0, nil +//readEnvName reads the enviorment key +func (rjr *rjReader) readEnvName(indx int) (name []byte, endindx int, err error) { //0 if not set + for indx < len(rjr.buf) { //read byte by byte + bit := rjr.buf[indx] + if !isAlfanum(bit) && bit != '_' { //[a-zA-Z_]+[a-zA-Z0-9_]* + return name, indx, nil + } + name = append(name, bit) + indx++ } - pOf := 0 - b.m = 0 - if len(b.buf) > 0 { //try read extra - pOf = b.r - for ; b.r < len(b.buf) && b.r-pOf < len(p); b.r++ { - p[b.r-pOf] = b.buf[b.r] - if isEnv := b.checkMeta(p[b.r-pOf]); isEnv { - b.r, err = b.replaceEnv(p, b.r-len(utils.MetaEnv)+1, b.r) - if err != nil { - return b.r, err - } - } - } - pOf = b.r - pOf - if pOf >= len(p) { - return pOf, nil - } - if len(b.buf) <= b.r { - b.buf = nil - b.r = 0 - } + return name, indx, io.EOF +} + +//replaceEnv replaces the EnvMeta and enviorment key with enviorment variable value in specific buffer +func (rjr *rjReader) replaceEnv(startEnv int) error { + midEnv := len(utils.MetaEnv) + key, endEnv, err := rjr.readEnvName(startEnv + midEnv) + if err != nil && err != io.EOF { + return err } - for ; pOf < len(p); pOf++ { //normal read - p[pOf], err = b.rd.ReadByte() - if err != nil { - return pOf, err - } - if isEnv := b.checkMeta(p[pOf]); isEnv { - pOf, err = b.replaceEnv(p, pOf-len(utils.MetaEnv)+1, pOf) - if err != nil { - return pOf, err - } - } + value, err := ReadEnv(string(key)) + if err != nil { + return err } - if b.m != 0 { //continue to read if posible meta - initMeta := b.m - buf := make([]byte, len(utils.MetaEnv)-initMeta) - i := 0 - for ; b.m != 0; i++ { - buf[i], err = b.rd.ReadByte() - if err != nil { - return i - 1, err - } - if isEnv := b.checkMeta(buf[i]); isEnv { - i, err = b.replaceEnv(p, len(p)-initMeta, i) - if err != nil { - return i, err - } - buf = nil - } - } - if len(buf) > 0 { - b.buf = buf[:i] - } - } - return len(p), nil + rjr.buf = append(rjr.buf[:startEnv], append([]byte(value), rjr.buf[endEnv:]...)...) // replace *env:ENV_STUFF with ENV_VALUE + return nil } // warning: needs to read file again -func HandleJSONError(reader io.Reader, err error) error { +func (rjr *rjReader) HandleJSONError(err error) error { var offset int64 switch realErr := err.(type) { case nil: @@ -351,29 +284,30 @@ func HandleJSONError(reader io.Reader, err error) error { case *json.UnmarshalTypeError: offset = realErr.Offset default: - fmt.Printf("%T", err) return err } if offset == 0 { return fmt.Errorf("%s at line 0 around position 0", err.Error()) } - line, character := getJsonOffsetLine(reader, offset) - return fmt.Errorf("%s around line %v and position %v", err.Error(), line, character) + rjr.indx = 0 + + line, character := rjr.getJsonOffsetLine(offset) + return fmt.Errorf("%s around line %v and position %v\n line: %q", err.Error(), line, character, + strings.Split(string(rjr.buf), "\n")[line-1]) } -func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { +func (rjr *rjReader) getJsonOffsetLine(offset int64) (line, character int64) { line = 1 // start line counting from 1 var lastChar byte - br := bufio.NewReader(reader) - var i int64 = 0 readString := func() error { for i < offset { - b, rerr := br.ReadByte() - if rerr != nil { - return rerr + if rjr.isEndOfFile() { + return io.EOF } + b := rjr.buf[rjr.indx] + rjr.indx++ i++ if isNewLine(b) { line++ @@ -389,10 +323,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { } readLineComment := func() error { for i < offset { - b, rerr := br.ReadByte() - if rerr != nil { - return rerr + if rjr.isEndOfFile() { + return io.EOF } + b := rjr.buf[rjr.indx] + rjr.indx++ if isNewLine(b) { line++ character = 0 @@ -405,10 +340,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { readComment := func() error { for i < offset { - b, rerr := br.ReadByte() - if rerr != nil { - return rerr + if rjr.isEndOfFile() { + return io.EOF } + b := rjr.buf[rjr.indx] + rjr.indx++ if isNewLine(b) { line++ character = 0 @@ -416,15 +352,16 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { character++ } if b == '*' { - b, rerr := br.ReadByte() - if rerr != nil { - return rerr + if rjr.isEndOfFile() { + return io.EOF } + b := rjr.buf[rjr.indx] + rjr.indx++ if b == '/' { character++ return nil } - rerr = br.UnreadByte() + rerr := rjr.UnreadByte() if rerr != nil { return rerr } @@ -434,10 +371,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { } for i < offset { // handle the parsing - b, rerr := br.ReadByte() - if rerr != nil { - break + if rjr.isEndOfFile() { + return } + b := rjr.buf[rjr.indx] + rjr.indx++ character++ if isNewLine(b) { line++ @@ -457,10 +395,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { } } if b == '/' { - b, rerr := br.ReadByte() - if rerr != nil { - break + if rjr.isEndOfFile() { + return } + b := rjr.buf[rjr.indx] + rjr.indx++ if b == '/' { // read // character++ i-- @@ -476,7 +415,7 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { break } } else { - rerr := br.UnreadByte() + rerr := rjr.UnreadByte() if rerr != nil { break } @@ -485,3 +424,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) { } return } + +// Loads the json config out of rjReader +func (rjr *rjReader) Decode(cfg interface{}) (err error) { + if err = json.NewDecoder(rjr).Decode(cfg); err != nil { + return rjr.HandleJSONError(err) + } + return +} diff --git a/config/rjreader_test.go b/config/rjreader_test.go index ab00887be..9865ae63d 100644 --- a/config/rjreader_test.go +++ b/config/rjreader_test.go @@ -18,11 +18,8 @@ along with this program. If not, see package config import ( - "bufio" - "encoding/json" "os" "reflect" - "strings" "testing" "github.com/cgrates/cgrates/utils" @@ -54,7 +51,7 @@ var ( ) func TestEnvRawJsonReadByte(t *testing.T) { - raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(envStr))} + raw := NewRjReaderFromBytes([]byte(envStr)) expected := []byte(`{"data_db":{"db_type":"redis","db_host":"127.0.0.1","db_port":6379,"db_name":"10","db_user":"*env:TESTVAR","db_password":",/**/","redis_sentinel":""}}`) rply := []byte{} bit, err := raw.ReadByte() @@ -67,8 +64,8 @@ func TestEnvRawJsonReadByte(t *testing.T) { } func TestEnvRawJsonconsumeComent(t *testing.T) { - raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`//comment -a/*comment*/b`))} + raw := NewRjReaderFromBytes([]byte(`//comment +a/*comment*/b`)) expected := (byte)('a') if r, err := raw.consumeComent('d'); err != nil { t.Error(err) @@ -99,9 +96,9 @@ a/*comment*/b`))} } func TestEnvRawJsonReadByteWC(t *testing.T) { - raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`c/*first comment*///another comment + raw := NewRjReaderFromBytes([]byte(`c/*first comment*///another comment - cgrates`))} + cgrates`)) expected := (byte)('c') if rply, err := raw.ReadByteWC(); err != nil { t.Error(err) @@ -116,9 +113,9 @@ func TestEnvRawJsonReadByteWC(t *testing.T) { } func TestEnvRawJsonPeekByteWC(t *testing.T) { - raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`c/*first comment*///another comment + raw := NewRjReaderFromBytes([]byte(`c/*first comment*///another comment - bgrates`))} + bgrates`)) expected := (byte)('c') if rply, err := raw.PeekByteWC(); err != nil { t.Error(err) @@ -144,9 +141,9 @@ func TestEnvRawJsonPeekByteWC(t *testing.T) { } func TestEnvRawJsonreadFirstNonWhiteSpace(t *testing.T) { - raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(` + raw := NewRjReaderFromBytes([]byte(` - cgrates`))} + cgrates`)) expected := (byte)('c') if rply, err := raw.readFirstNonWhiteSpace(); err != nil { t.Error(err) @@ -157,7 +154,7 @@ func TestEnvRawJsonreadFirstNonWhiteSpace(t *testing.T) { func TestEnvReaderRead(t *testing.T) { os.Setenv("TESTVAR", "cgRates") - envR := NewRawJSONReader(strings.NewReader(envStr)) + envR := NewRjReaderFromBytes([]byte(envStr)) expected := []byte(`{"data_db":{"db_type":"redis","db_host":"127.0.0.1","db_port":6379,"db_name":"10","db_user":"cgRates","db_password":",/**/","redis_sentinel":""}}`) rply := []byte{} buf := make([]byte, 20) @@ -180,7 +177,7 @@ func TestEnvReaderRead(t *testing.T) { func TestEnvReaderRead2(t *testing.T) { os.Setenv("TESTVARNoZero", "cgr1") - envR := NewRawJSONReader(strings.NewReader(`{"origin_host": "*env:TESTVARNoZero", + envR := NewRjReaderFromBytes([]byte(`{"origin_host": "*env:TESTVARNoZero", "origin_realm": "*env:TESTVARNoZero",}`)) expected := []byte(`{"origin_host":"cgr1","origin_realm":"cgr1"}`) rply := []byte{} @@ -203,20 +200,20 @@ func TestEnvReaderRead2(t *testing.T) { } func TestEnvReaderreadEnvName(t *testing.T) { - envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(`Test_VAR1 } Var2_TEST'`))}} + envR := NewRjReaderFromBytes([]byte(`Test_VAR1 } Var2_TEST'`)) expected := []byte("Test_VAR1") - if rply, bit, err := envR.readEnvName(); err != nil { + if rply, endindx, err := envR.readEnvName(0); err != nil { t.Error(err) - } else if bit != '}' { - t.Errorf("Wrong bit returned %q", bit) + } else if endindx != 9 { + t.Errorf("Wrong endindx returned %v", endindx) } else if !reflect.DeepEqual(expected, rply) { t.Errorf("Expected: %+v, recived: %+v", (string(expected)), (string(rply))) } expected = []byte("Var2_TEST") - if rply, bit, err := envR.readEnvName(); err != nil { + if rply, endindx, err := envR.readEnvName(12); err != nil { t.Error(err) - } else if bit != '\'' { - t.Errorf("Wrong bit returned %q", bit) + } else if endindx != 21 { + t.Errorf("Wrong endindx returned %v", endindx) } else if !reflect.DeepEqual(expected, rply) { t.Errorf("Expected: %+v, recived: %+v", (string(expected)), (string(rply))) } @@ -225,50 +222,26 @@ func TestEnvReaderreadEnvName(t *testing.T) { func TestEnvReaderreplaceEnv(t *testing.T) { os.Setenv("Test_VAR1", "5") os.Setenv("Test_VAR2", "aVeryLongEnviormentalVariable") - envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(`Test_VAR1,/*comment*/ }Test_VAR2"`))}} - expected := []byte("5} ") - expectedn := 1 - rply := make([]byte, 5) - if n, err := envR.replaceEnv(rply, 0, 5); err != nil { + envR := NewRjReaderFromBytes([]byte(`*env:Test_VAR1,/*comment*/ }*env:Test_VAR2"`)) + // expected := []byte("5} ") + if err := envR.replaceEnv(0); err != nil { t.Error(err) - } else if expectedn != n { - t.Errorf("Expected: %+v, recived: %+v", expectedn, n) - } else if !reflect.DeepEqual(expected, rply) { - t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply))) } - expected = []byte("aVery") - expectedn = 5 - rply = make([]byte, 5) - if n, err := envR.replaceEnv(rply, 0, 5); err != nil { + if err := envR.replaceEnv(15); err != nil { t.Error(err) - } else if expectedn != n { - t.Errorf("Expected: %+v, recived: %+v", expectedn, n) - } else if !reflect.DeepEqual(expected, rply) { - t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply))) - } else if bufexp := []byte("LongEnviormentalVariable\""); !reflect.DeepEqual(bufexp, envR.buf) { - t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply))) } } func TestEnvReadercheckMeta(t *testing.T) { - envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(""))}} - envR.m = 2 - if envR.checkMeta('n') { - t.Errorf("Expectiv to get false recived true") - } else if envR.m != 3 { - t.Errorf("Expectiv the meta offset to incrase") - } - envR.m = 4 - if !envR.checkMeta(':') { + envR := NewRjReaderFromBytes([]byte("*env:Var")) + envR.indx = 1 + if !envR.checkMeta() { t.Errorf("Expectiv true ") - } else if envR.m != 0 { - t.Errorf("Expectiv the meta offset to reset") } - envR.m = 1 - if envR.checkMeta('v') { + envR = NewRjReaderFromBytes([]byte("*enva:Var")) + envR.indx = 1 + if envR.checkMeta() { t.Errorf("Expectiv to get false recived true") - } else if envR.m != 0 { - t.Errorf("Expectiv the meta offset to reset") } } @@ -333,24 +306,10 @@ func TestGetErrorLine(t *testing.T) { Line3 */ /**/ }//` - r := strings.NewReader(jsonstr) - _, err := NewCgrJsonCfgFromReader(r) - if err == nil { - t.Fatalf("Expected error received %v", err) - } - var offset int64 - if realErr, canCast := err.(*json.SyntaxError); !canCast { - t.Fatalf("Expected json.SyntaxError received %v<%T>", err.Error(), err) - } else { - offset = realErr.Offset - } - var expOffset int64 = 31 - if offset != expOffset { - t.Errorf("Expected offset %v received:%v", expOffset, offset) - } - r = strings.NewReader(jsonstr) + r := NewRjReaderFromBytes([]byte(jsonstr)) + var offset int64 = 31 var expLine, expChar int64 = 10, 23 - if line, character := getJsonOffsetLine(r, offset); expLine != line { + if line, character := r.getJsonOffsetLine(offset); expLine != line { t.Errorf("Expected line %v received:%v", expLine, line) } else if expChar != character { t.Errorf("Expected line %v received:%v", expChar, character) @@ -381,24 +340,10 @@ func TestGetErrorLine2(t *testing.T) { Line3 */ /**/ }//` - r := strings.NewReader(jsonstr) - _, err := NewCgrJsonCfgFromReader(r) - if err == nil { - t.Fatalf("Expected error received %v", err) - } - var offset int64 - if realErr, canCast := err.(*json.SyntaxError); !canCast { - t.Fatalf("Expected json.SyntaxError received %v<%T>", err.Error(), err) - } else { - offset = realErr.Offset - } - var expOffset int64 = 31 - if offset != expOffset { - t.Errorf("Expected offset %v received:%v", expOffset, offset) - } - r = strings.NewReader(jsonstr) + r := NewRjReaderFromBytes([]byte(jsonstr)) + var offset int64 = 31 var expLine, expChar int64 = 10, 46 - if line, character := getJsonOffsetLine(r, offset); expLine != line { + if line, character := r.getJsonOffsetLine(offset); expLine != line { t.Errorf("Expected line %v received:%v", expLine, line) } else if expChar != character { t.Errorf("Expected line %v received:%v", expChar, character)