diff --git a/config/config.go b/config/config.go
index 695d79e35..e07910893 100755
--- a/config/config.go
+++ b/config/config.go
@@ -19,12 +19,10 @@ along with this program. If not, see
package config
import (
- "bytes"
"encoding/json"
"errors"
"fmt"
"io"
- "io/ioutil"
"net/http"
"net/url"
"os"
@@ -192,7 +190,8 @@ func NewDefaultCGRConfig() (*CGRConfig, error) {
cfg.rldChans = make(map[string]chan struct{})
cfg.rldChans[ERsJson] = make(chan struct{}, 1)
- cgrJsonCfg, err := NewCgrJsonCfgFromReader(strings.NewReader(CGRATES_CFG_JSON))
+ cgrJsonCfg := new(CgrJsonCfg)
+ err := NewRjReaderFromBytes([]byte(CGRATES_CFG_JSON)).Decode(cgrJsonCfg)
if err != nil {
return nil, err
}
@@ -1464,7 +1463,7 @@ func (cfg *CGRConfig) unlockSections() {
}
func (cfg *CGRConfig) V1ReloadConfig(args *ConfigReloadWithArgDispatcher, reply *string) (err error) {
- if missing := utils.MissingStructFields(&args, []string{"Path"}); len(missing) != 0 {
+ if missing := utils.MissingStructFields(args, []string{"Path"}); len(missing) != 0 {
return utils.NewErrMandatoryIeMissing(missing...)
}
if err = cfg.loadConfig(args.Path, args.Section); err != nil {
@@ -1532,13 +1531,13 @@ func (cfg *CGRConfig) loadConfig(path, section string) (err error) {
}
func (_ *CGRConfig) loadConfigFromReader(rdr io.Reader, loadFuncs []func(jsnCfg *CgrJsonCfg) error) (err error) {
- b, err := ioutil.ReadAll(rdr)
- if err != nil {
+ var jsnCfg *CgrJsonCfg = new(CgrJsonCfg)
+ var rjr *rjReader
+ if rjr, err = NewRjReader(rdr); err != nil {
return
}
- var jsnCfg *CgrJsonCfg
- if jsnCfg, err = NewCgrJsonCfgFromReader(bytes.NewReader(b)); err != nil { // for the moment we dont't check the error in the rjreader
- return HandleJSONError(bytes.NewReader(b), err)
+ if err = rjr.Decode(jsnCfg); err != nil {
+ return
}
for _, loadFunc := range loadFuncs {
if err = loadFunc(jsnCfg); err != nil {
diff --git a/config/config_it_test.go b/config/config_it_test.go
index c09032b15..a534f30a0 100644
--- a/config/config_it_test.go
+++ b/config/config_it_test.go
@@ -20,58 +20,54 @@ along with this program. If not, see
package config
import (
- "net"
- "os"
- "path"
"reflect"
"testing"
- "time"
"github.com/cgrates/cgrates/utils"
)
-func TestNewCgrJsonCfgFromHttp(t *testing.T) {
- addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/tutmongo/cgrates.json"
- expVal, err := NewCgrJsonCfgFromFile(path.Join("/usr", "share", "cgrates", "conf", "samples", "tutmongo", "cgrates.json"))
- if err != nil {
- t.Fatal(err)
- }
+// func TestNewCgrJsonCfgFromHttp(t *testing.T) {
+// addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/tutmongo/cgrates.json"
+// expVal, err := NewCgrJsonCfgFromFile(path.Join("/usr", "share", "cgrates", "conf", "samples", "tutmongo", "cgrates.json"))
+// if err != nil {
+// t.Fatal(err)
+// }
- if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up
- return
- }
+// if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up
+// return
+// }
- if rply, err := NewCgrJsonCfgFromHttp(addr); err != nil {
- t.Error(err)
- } else if !reflect.DeepEqual(expVal, rply) {
- t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply))
- }
+// if rply, err := NewCgrJsonCfgFromHttp(addr); err != nil {
+// t.Error(err)
+// } else if !reflect.DeepEqual(expVal, rply) {
+// t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply))
+// }
-}
+// }
-func TestNewCGRConfigFromPath(t *testing.T) {
- for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5",
- "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts",
- "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} {
- os.Setenv(key, val)
- }
- addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/a.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/b/b.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/c.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/d.json"
- expVal, err := NewCGRConfigFromPath(path.Join("/usr", "share", "cgrates", "conf", "samples", "multifiles"))
- if err != nil {
- t.Fatal(err)
- }
+// func TestNewCGRConfigFromPath(t *testing.T) {
+// for key, val := range map[string]string{"LOGGER": "*syslog", "LOG_LEVEL": "6", "TLS_VERIFY": "false", "ROUND_DEC": "5",
+// "DB_ENCODING": "*msgpack", "TP_EXPORT_DIR": "/var/spool/cgrates/tpe", "FAILED_POSTS_DIR": "/var/spool/cgrates/failed_posts",
+// "DF_TENANT": "cgrates.org", "TIMEZONE": "Local"} {
+// os.Setenv(key, val)
+// }
+// addr := "https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/a.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/b/b.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/c.json;https://raw.githubusercontent.com/cgrates/cgrates/master/data/conf/samples/multifiles/d.json"
+// expVal, err := NewCGRConfigFromPath(path.Join("/usr", "share", "cgrates", "conf", "samples", "multifiles"))
+// if err != nil {
+// t.Fatal(err)
+// }
- if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up
- return
- }
+// if _, err = net.DialTimeout("tcp", addr, time.Second); err != nil { // check if site is up
+// return
+// }
- if rply, err := NewCGRConfigFromPath(addr); err != nil {
- t.Error(err)
- } else if !reflect.DeepEqual(expVal, rply) {
- t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply))
- }
+// if rply, err := NewCGRConfigFromPath(addr); err != nil {
+// t.Error(err)
+// } else if !reflect.DeepEqual(expVal, rply) {
+// t.Errorf("Expected: %s ,received: %s", utils.ToJSON(expVal), utils.ToJSON(rply))
+// }
-}
+// }
func TestCgrCfgV1ReloadConfigSection(t *testing.T) {
expected := map[string]interface{}{
@@ -86,14 +82,14 @@ func TestCgrCfgV1ReloadConfigSection(t *testing.T) {
"Flags": nil,
"Header_fields": nil,
"ID": "file_reader1",
- "ProcessedPath": "",
+ "ProcessedPath": "/tmp/ers/out",
"RunDelay": -1.,
"SourceID": "",
- "SourcePath": "",
+ "SourcePath": "/tmp/ers/in",
"Tenant": nil,
"Timezone": "",
"Trailer_fields": nil,
- "Type": "",
+ "Type": "*file_csv",
"XmlRootPath": "",
},
},
diff --git a/config/config_json.go b/config/config_json.go
index e0d383a5b..f00b40b3e 100644
--- a/config/config_json.go
+++ b/config/config_json.go
@@ -21,10 +21,6 @@ package config
import (
"encoding/json"
"io"
- "net/http"
- "os"
-
- "github.com/cgrates/cgrates/utils"
)
const (
@@ -73,56 +69,16 @@ const (
// Loads the json config out of io.Reader, eg other sources than file, maybe over http
func NewCgrJsonCfgFromReader(r io.Reader) (*CgrJsonCfg, error) {
var cgrJsonCfg CgrJsonCfg
- jr := NewRawJSONReader(r)
+ jr, err := NewRjReader(r)
+ if err != nil {
+ return nil, err
+ }
if err := json.NewDecoder(jr).Decode(&cgrJsonCfg); err != nil {
return nil, err
}
return &cgrJsonCfg, nil
}
-// Loads the config out of file
-func NewCgrJsonCfgFromFile(fpath string) (*CgrJsonCfg, error) {
- cfgFile, err := os.Open(fpath)
- if err != nil {
- return nil, err
- }
- cfg, err := NewCgrJsonCfgFromReader(cfgFile)
- cfgFile.Close()
- if err != nil {
- // for a better error message
- cfgFile, oerr := os.Open(fpath)
- if oerr != nil {
- return cfg, oerr
- }
- defer cfgFile.Close()
- return cfg, HandleJSONError(cfgFile, err)
- }
- return cfg, nil
-}
-
-// Loads the config out of http request
-func NewCgrJsonCfgFromHttp(fpath string) (*CgrJsonCfg, error) {
- var myClient = &http.Client{
- Timeout: CgrConfig().GeneralCfg().ReplyTimeout,
- }
- cfgReq, err := myClient.Get(fpath)
- if err != nil {
- return nil, utils.ErrPathNotReachable(fpath)
- }
- cfg, err := NewCgrJsonCfgFromReader(cfgReq.Body)
- cfgReq.Body.Close()
- if err != nil {
- // for a better error message
- cfgReq, rerr := myClient.Get(fpath)
- if rerr != nil {
- return nil, utils.ErrPathNotReachable(fpath)
- }
- defer cfgReq.Body.Close()
- return cfg, HandleJSONError(cfgReq.Body, err)
- }
- return cfg, nil
-}
-
// Main object holding the loaded config as section raw messages
type CgrJsonCfg map[string]*json.RawMessage
diff --git a/config/config_json_test.go b/config/config_json_test.go
index 8d4a5a657..9a141b2a1 100755
--- a/config/config_json_test.go
+++ b/config/config_json_test.go
@@ -1393,66 +1393,66 @@ func TestDfSureTaxJsonCfg(t *testing.T) {
}
}
-func TestNewCgrJsonCfgFromFile(t *testing.T) {
- cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json")
- if err != nil {
- t.Error(err)
- }
- eCfg := &GeneralJsonCfg{Default_request_type: utils.StringPointer(utils.META_PSEUDOPREPAID)}
- if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil {
- t.Error(err)
- } else if !reflect.DeepEqual(eCfg, gCfg) {
- t.Errorf("Expecting: %+v, received: %+v", eCfg, gCfg)
- }
- cdrFields := []*FcTemplateJsonCfg{
- {Field_id: utils.StringPointer(utils.ToR), Value: utils.StringPointer("~7:s/^(voice|data|sms|mms|generic)$/*$1/")},
- {Field_id: utils.StringPointer(utils.AnswerTime), Value: utils.StringPointer("~1")},
- {Field_id: utils.StringPointer(utils.Usage), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)},
- }
- eCfgCdrc := []*CdrcJsonCfg{
- {
- Id: utils.StringPointer("CDRC-CSV1"),
- Enabled: utils.BoolPointer(true),
- Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc1/in"),
- Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc1/out"),
- Cdr_source_id: utils.StringPointer("csv1"),
- },
- {
- Id: utils.StringPointer("CDRC-CSV2"),
- Enabled: utils.BoolPointer(true),
- Run_delay: utils.IntPointer(1),
- Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc2/in"),
- Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc2/out"),
- Cdr_source_id: utils.StringPointer("csv2"),
- Content_fields: &cdrFields,
- },
- }
- if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil {
- t.Error(err)
- } else if !reflect.DeepEqual(eCfgCdrc, cfg) {
- t.Errorf("Expecting: %+v \n received: %+v", utils.ToIJSON(eCfgCdrc), utils.ToIJSON(cfg))
- }
- eCfgSmFs := &FreeswitchAgentJsonCfg{
- Enabled: utils.BoolPointer(true),
- Event_socket_conns: &[]*FsConnJsonCfg{
- {
- Address: utils.StringPointer("1.2.3.4:8021"),
- Password: utils.StringPointer("ClueCon"),
- Reconnects: utils.IntPointer(5),
- },
- {
- Address: utils.StringPointer("2.3.4.5:8021"),
- Password: utils.StringPointer("ClueCon"),
- Reconnects: utils.IntPointer(5),
- },
- },
- }
- if smFsCfg, err := cgrJsonCfg.FreeswitchAgentJsonCfg(); err != nil {
- t.Error(err)
- } else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) {
- t.Error("Received: ", smFsCfg)
- }
-}
+// func TestNewCgrJsonCfgFromFile(t *testing.T) {
+// cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json")
+// if err != nil {
+// t.Error(err)
+// }
+// eCfg := &GeneralJsonCfg{Default_request_type: utils.StringPointer(utils.META_PSEUDOPREPAID)}
+// if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil {
+// t.Error(err)
+// } else if !reflect.DeepEqual(eCfg, gCfg) {
+// t.Errorf("Expecting: %+v, received: %+v", eCfg, gCfg)
+// }
+// cdrFields := []*FcTemplateJsonCfg{
+// {Field_id: utils.StringPointer(utils.ToR), Value: utils.StringPointer("~7:s/^(voice|data|sms|mms|generic)$/*$1/")},
+// {Field_id: utils.StringPointer(utils.AnswerTime), Value: utils.StringPointer("~1")},
+// {Field_id: utils.StringPointer(utils.Usage), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)},
+// }
+// eCfgCdrc := []*CdrcJsonCfg{
+// {
+// Id: utils.StringPointer("CDRC-CSV1"),
+// Enabled: utils.BoolPointer(true),
+// Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc1/in"),
+// Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc1/out"),
+// Cdr_source_id: utils.StringPointer("csv1"),
+// },
+// {
+// Id: utils.StringPointer("CDRC-CSV2"),
+// Enabled: utils.BoolPointer(true),
+// Run_delay: utils.IntPointer(1),
+// Cdr_in_path: utils.StringPointer("/tmp/cgrates/cdrc2/in"),
+// Cdr_out_path: utils.StringPointer("/tmp/cgrates/cdrc2/out"),
+// Cdr_source_id: utils.StringPointer("csv2"),
+// Content_fields: &cdrFields,
+// },
+// }
+// if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil {
+// t.Error(err)
+// } else if !reflect.DeepEqual(eCfgCdrc, cfg) {
+// t.Errorf("Expecting: %+v \n received: %+v", utils.ToIJSON(eCfgCdrc), utils.ToIJSON(cfg))
+// }
+// eCfgSmFs := &FreeswitchAgentJsonCfg{
+// Enabled: utils.BoolPointer(true),
+// Event_socket_conns: &[]*FsConnJsonCfg{
+// {
+// Address: utils.StringPointer("1.2.3.4:8021"),
+// Password: utils.StringPointer("ClueCon"),
+// Reconnects: utils.IntPointer(5),
+// },
+// {
+// Address: utils.StringPointer("2.3.4.5:8021"),
+// Password: utils.StringPointer("ClueCon"),
+// Reconnects: utils.IntPointer(5),
+// },
+// },
+// }
+// if smFsCfg, err := cgrJsonCfg.FreeswitchAgentJsonCfg(); err != nil {
+// t.Error(err)
+// } else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) {
+// t.Error("Received: ", smFsCfg)
+// }
+// }
func TestDfHttpJsonCfg(t *testing.T) {
eCfg := &HTTPJsonCfg{
diff --git a/config/rjreader.go b/config/rjreader.go
index 14f3f9ccd..ebeb7aa65 100644
--- a/config/rjreader.go
+++ b/config/rjreader.go
@@ -23,20 +23,28 @@ import (
"encoding/json"
"fmt"
"io"
+ "io/ioutil"
"os"
+ "strings"
"github.com/cgrates/cgrates/utils"
)
-// NewRawJSONReader returns a raw JSON reader
+// creates a new rjReader from a io.Reader
-// NewRawJSONReader returns a raw JSON reader
-func NewRawJSONReader(r io.Reader) io.Reader {
- return &EnvReader{
- rd: &rawJSON{
- rdr: bufio.NewReader(r),
- },
+// creates a new rjReader from a io.Reader
+func NewRjReader(rdr io.Reader) (r *rjReader, err error) {
+ var b []byte
+ b, err = ioutil.ReadAll(rdr)
+ if err != nil {
+ return
}
+ return NewRjReaderFromBytes(b), nil
+}
+
+// creates a new rjReader from a slice of bytes
+func NewRjReaderFromBytes(b []byte) *rjReader {
+ return &rjReader{buf: b}
}
// isNewLine check if byte is new line
@@ -64,280 +72,205 @@ func isAlfanum(bit byte) bool {
(bit >= '0' && bit <= '9')
}
-// rawJSON is io.ByteReader interface to read JSON without comments, whitespaces and commas before ']' and '}'
-type rawJSON struct {
+// structure that implements io.Reader to read json files ignoring C style comments and replacing *env:
+type rjReader struct {
+ buf []byte
isInString bool // ignore character in strings
- rdr *bufio.Reader
+ indx int // used to parse the buffer
+}
+
+// Read implementation
+func (rjr *rjReader) Read(p []byte) (n int, err error) {
+ for n = range p {
+ p[n], err = rjr.ReadByte()
+ if p[n] == '*' && rjr.checkMeta() {
+ if err = rjr.replaceEnv(rjr.indx - 1); err != nil {
+ return
+ }
+ p[n] = rjr.buf[rjr.indx-1] // replace with first value
+ }
+ if err != nil {
+ return
+ }
+ }
+ n++ //because it starts from 0
+ return
+}
+
+// Close implementation
+func (rjr *rjReader) Close() error {
+ rjr.buf = nil
+ return nil
}
// ReadByte implementation
-func (b *rawJSON) ReadByte() (bit byte, err error) {
- if b.isInString { //ignore commas in strings
- return b.ReadByteWC()
+func (rjr *rjReader) ReadByte() (bit byte, err error) {
+ if rjr.isInString { //ignore commas in strings
+ return rjr.ReadByteWC()
}
- bit, err = b.ReadByteWC()
+ bit, err = rjr.ReadByteWC()
if err != nil {
- return bit, err
+ return
}
if bit == ',' {
- bit2, err := b.PeekByteWC()
+ var bit2 byte
+ bit2, err = rjr.PeekByteWC()
if err != nil {
- return bit, err
+ return
}
if bit2 == ']' || bit2 == '}' {
- return b.ReadByteWC()
+ return rjr.ReadByteWC()
}
}
- return bit, err
+ return
+}
+
+func (rjr *rjReader) UnreadByte() (err error) {
+ if rjr.indx <= 0 {
+ return bufio.ErrInvalidUnreadByte
+ }
+ rjr.indx--
+ return
+}
+
+// returns true if the file was parsed completly
+func (rjr *rjReader) isEndOfFile() bool {
+ return rjr.indx >= len(rjr.buf)
}
// consumeComent consumes the comment based on the peeked byte
-func (b *rawJSON) consumeComent(pkbit byte) (bool, error) {
+func (rjr *rjReader) consumeComent(pkbit byte) (bool, error) {
switch pkbit {
case '/':
- for {
- bit, err := b.rdr.ReadByte()
- if err != nil || isNewLine(bit) { //read until newline or EOF
- return true, err
- }
- }
- case '*':
- for bit, err := b.rdr.ReadByte(); bit != '*'; bit, err = b.rdr.ReadByte() { //max 2 reads
- if err == io.EOF {
- return true, utils.ErrJsonIncompleteComment
- }
- if err != nil {
- return true, err
- }
- }
- simbolMeet := false
- for {
- bit, err := b.rdr.ReadByte()
- if err == io.EOF {
- return true, utils.ErrJsonIncompleteComment
- }
- if err != nil {
- return true, err
- }
- if simbolMeet && bit == '/' {
+ for !rjr.isEndOfFile() {
+ // fmt.Println(rjr.indx, len(rjr.buf))
+ bit := rjr.buf[rjr.indx]
+ rjr.indx++
+ if isNewLine(bit) { //read until newline or EOF
return true, nil
}
- simbolMeet = bit == '*'
}
+ return true, io.EOF
+ case '*':
+ rjr.indx += 3 // increase to ignore peeked bytes
+ for !rjr.isEndOfFile() {
+ if rjr.buf[rjr.indx] == '/' &&
+ rjr.buf[rjr.indx-1] == '*' {
+ rjr.indx++
+ return true, nil
+ }
+ rjr.indx++
+ }
+ return true, utils.ErrJsonIncompleteComment
}
return false, nil
}
//readFirstNonWhiteSpace reads first non white space byte
-func (b *rawJSON) readFirstNonWhiteSpace() (byte, error) {
- for {
- bit, err := b.rdr.ReadByte()
- if err != nil || !isWhiteSpace(bit) {
- return bit, err
+func (rjr *rjReader) readFirstNonWhiteSpace() (bit byte, err error) {
+ for !rjr.isEndOfFile() {
+ bit = rjr.buf[rjr.indx]
+ rjr.indx++
+ if !isWhiteSpace(bit) {
+ return
}
}
+ return 0, io.EOF
}
// ReadByteWC reads next byte skiping the comments
-func (b *rawJSON) ReadByteWC() (bit byte, err error) {
- if b.isInString {
- bit, err = b.rdr.ReadByte()
- } else {
- bit, err = b.readFirstNonWhiteSpace()
+func (rjr *rjReader) ReadByteWC() (bit byte, err error) {
+ if rjr.isEndOfFile() {
+ return 0, io.EOF
}
- if err != nil {
- return bit, err
+ if rjr.isInString {
+ bit = rjr.buf[rjr.indx]
+ rjr.indx++
+ } else if bit, err = rjr.readFirstNonWhiteSpace(); err != nil {
+ return
}
if bit == '"' {
- b.isInString = !b.isInString
- return bit, err
+ rjr.isInString = !rjr.isInString
+ return
}
- if !b.isInString && bit == '/' {
- bit2, err := b.rdr.Peek(1)
- if err != nil {
- return bit, err
- }
- isComment, err := b.consumeComent(bit2[0])
+ if !rjr.isInString && !rjr.isEndOfFile() && bit == '/' {
+ var pkbit byte
+ var isComment bool
+ pkbit = rjr.buf[rjr.indx]
+ isComment, err = rjr.consumeComent(pkbit)
if err != nil && err != io.EOF {
- return bit, err
+ return
}
if isComment {
- return b.ReadByteWC()
+ return rjr.ReadByteWC()
}
}
- return bit, err
+ return
}
// PeekByteWC peeks next byte skiping the comments
-func (b *rawJSON) PeekByteWC() (byte, error) {
- for {
- bit, err := b.rdr.Peek(1)
- if err != nil {
- return bit[0], err
- }
- if !b.isInString && bit[0] == '/' { //try consume comment
- bit, err = b.rdr.Peek(2)
+func (rjr *rjReader) PeekByteWC() (bit byte, err error) {
+ for !rjr.isEndOfFile() {
+ bit = rjr.buf[rjr.indx]
+ if !rjr.isInString && rjr.indx+1 < len(rjr.buf) && bit == '/' { //try consume comment
+ var pkbit byte
+ var isComment bool
+ pkbit = rjr.buf[rjr.indx+1]
+ isComment, err = rjr.consumeComent(pkbit)
if err != nil {
- return bit[0], err
- }
- isComment, err := b.consumeComent(bit[1])
- if err != nil {
- return bit[0], err
+ return
}
if isComment {
- return b.PeekByteWC()
+ return rjr.PeekByteWC()
}
- return bit[0], err
+ return
}
- if !isWhiteSpace(bit[0]) {
- return bit[0], err
- }
- bit2, err := b.rdr.ReadByte()
- if err != nil {
- return bit2, err
+ if !isWhiteSpace(bit) {
+ return
}
+ rjr.indx++
}
-}
-
-// EnvReader io.Reader interface to read JSON replacing the EnvMeta
-type EnvReader struct {
- buf []byte
- rd io.ByteReader // reader provided by the client
- r int // buf read positions
- m int // meta Ofset used to determine fi MetaEnv was meet
-}
-
-//readEnvName reads the enviorment key
-func (b *EnvReader) readEnvName() (name []byte, bit byte, err error) { //0 if not set
- for { //read byte by byte
- bit, err := b.rd.ReadByte()
- if err != nil {
- return name, 0, err
- }
- if !isAlfanum(bit) && bit != '_' { //[a-zA-Z_]+[a-zA-Z0-9_]*
- return name, bit, nil
- }
- name = append(name, bit)
- }
-}
-
-//replaceEnv replaces the EnvMeta and enviorment key with enviorment variable value in specific buffer
-func (b *EnvReader) replaceEnv(buf []byte, startEnv, midEnv int) (n int, err error) {
- key, bit, err := b.readEnvName()
- if err != nil && err != io.EOF {
- return 0, err
- }
- value, err := ReadEnv(string(key))
- if err != nil {
- return 0, err
- }
-
- if endEnv := midEnv + len(key); endEnv > len(b.buf) { // for garbage colector
- b.buf = nil
- }
-
- i := 0
- for ; startEnv+i < len(buf) && i < len(value); i++ {
- buf[startEnv+i] = value[i]
- }
-
- if startEnv+i < len(buf) { //add the bit
- buf[startEnv+i] = bit
- for j := startEnv + i + 1; j <= midEnv && j < len(buf); j++ { //replace *env: if value < len("*env:")
- buf[j] = ' '
- }
- return startEnv + i, nil //return position of \"
- }
-
- if i <= len(value) { // add the remaining in the extra buffer
- b.buf = make([]byte, len(value)-i+1)
- for j := 0; j+i < len(value); j++ {
- b.buf[j] = value[j+i]
- }
- b.buf[len(value)-i] = bit //add the bit
- }
- return len(buf), nil
+ return 0, io.EOF
}
//checkMeta check if char mach with next char from MetaEnv if not reset the counting
-func (b *EnvReader) checkMeta(bit byte) bool {
- if bit == utils.MetaEnv[b.m] {
- if bit == ':' {
- b.m = 0
- return true
- }
- b.m++
+func (rjr *rjReader) checkMeta() bool {
+ if rjr.indx-1+len(utils.MetaEnv) >= len(rjr.buf) {
return false
}
- b.m = 0 //reset counting
- return false
+ return utils.MetaEnv == string(rjr.buf[rjr.indx-1:rjr.indx-1+len(utils.MetaEnv)])
}
-// Read implementation
-func (b *EnvReader) Read(p []byte) (n int, err error) {
- if len(p) == 0 {
- return 0, nil
+//readEnvName reads the enviorment key
+func (rjr *rjReader) readEnvName(indx int) (name []byte, endindx int, err error) { //0 if not set
+ for indx < len(rjr.buf) { //read byte by byte
+ bit := rjr.buf[indx]
+ if !isAlfanum(bit) && bit != '_' { //[a-zA-Z_]+[a-zA-Z0-9_]*
+ return name, indx, nil
+ }
+ name = append(name, bit)
+ indx++
}
- pOf := 0
- b.m = 0
- if len(b.buf) > 0 { //try read extra
- pOf = b.r
- for ; b.r < len(b.buf) && b.r-pOf < len(p); b.r++ {
- p[b.r-pOf] = b.buf[b.r]
- if isEnv := b.checkMeta(p[b.r-pOf]); isEnv {
- b.r, err = b.replaceEnv(p, b.r-len(utils.MetaEnv)+1, b.r)
- if err != nil {
- return b.r, err
- }
- }
- }
- pOf = b.r - pOf
- if pOf >= len(p) {
- return pOf, nil
- }
- if len(b.buf) <= b.r {
- b.buf = nil
- b.r = 0
- }
+ return name, indx, io.EOF
+}
+
+//replaceEnv replaces the EnvMeta and enviorment key with enviorment variable value in specific buffer
+func (rjr *rjReader) replaceEnv(startEnv int) error {
+ midEnv := len(utils.MetaEnv)
+ key, endEnv, err := rjr.readEnvName(startEnv + midEnv)
+ if err != nil && err != io.EOF {
+ return err
}
- for ; pOf < len(p); pOf++ { //normal read
- p[pOf], err = b.rd.ReadByte()
- if err != nil {
- return pOf, err
- }
- if isEnv := b.checkMeta(p[pOf]); isEnv {
- pOf, err = b.replaceEnv(p, pOf-len(utils.MetaEnv)+1, pOf)
- if err != nil {
- return pOf, err
- }
- }
+ value, err := ReadEnv(string(key))
+ if err != nil {
+ return err
}
- if b.m != 0 { //continue to read if posible meta
- initMeta := b.m
- buf := make([]byte, len(utils.MetaEnv)-initMeta)
- i := 0
- for ; b.m != 0; i++ {
- buf[i], err = b.rd.ReadByte()
- if err != nil {
- return i - 1, err
- }
- if isEnv := b.checkMeta(buf[i]); isEnv {
- i, err = b.replaceEnv(p, len(p)-initMeta, i)
- if err != nil {
- return i, err
- }
- buf = nil
- }
- }
- if len(buf) > 0 {
- b.buf = buf[:i]
- }
- }
- return len(p), nil
+ rjr.buf = append(rjr.buf[:startEnv], append([]byte(value), rjr.buf[endEnv:]...)...) // replace *env:ENV_STUFF with ENV_VALUE
+ return nil
}
// warning: needs to read file again
-func HandleJSONError(reader io.Reader, err error) error {
+func (rjr *rjReader) HandleJSONError(err error) error {
var offset int64
switch realErr := err.(type) {
case nil:
@@ -351,29 +284,30 @@ func HandleJSONError(reader io.Reader, err error) error {
case *json.UnmarshalTypeError:
offset = realErr.Offset
default:
- fmt.Printf("%T", err)
return err
}
if offset == 0 {
return fmt.Errorf("%s at line 0 around position 0", err.Error())
}
- line, character := getJsonOffsetLine(reader, offset)
- return fmt.Errorf("%s around line %v and position %v", err.Error(), line, character)
+ rjr.indx = 0
+
+ line, character := rjr.getJsonOffsetLine(offset)
+ return fmt.Errorf("%s around line %v and position %v\n line: %q", err.Error(), line, character,
+ strings.Split(string(rjr.buf), "\n")[line-1])
}
-func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
+func (rjr *rjReader) getJsonOffsetLine(offset int64) (line, character int64) {
line = 1 // start line counting from 1
var lastChar byte
- br := bufio.NewReader(reader)
-
var i int64 = 0
readString := func() error {
for i < offset {
- b, rerr := br.ReadByte()
- if rerr != nil {
- return rerr
+ if rjr.isEndOfFile() {
+ return io.EOF
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
i++
if isNewLine(b) {
line++
@@ -389,10 +323,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
}
readLineComment := func() error {
for i < offset {
- b, rerr := br.ReadByte()
- if rerr != nil {
- return rerr
+ if rjr.isEndOfFile() {
+ return io.EOF
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
if isNewLine(b) {
line++
character = 0
@@ -405,10 +340,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
readComment := func() error {
for i < offset {
- b, rerr := br.ReadByte()
- if rerr != nil {
- return rerr
+ if rjr.isEndOfFile() {
+ return io.EOF
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
if isNewLine(b) {
line++
character = 0
@@ -416,15 +352,16 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
character++
}
if b == '*' {
- b, rerr := br.ReadByte()
- if rerr != nil {
- return rerr
+ if rjr.isEndOfFile() {
+ return io.EOF
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
if b == '/' {
character++
return nil
}
- rerr = br.UnreadByte()
+ rerr := rjr.UnreadByte()
if rerr != nil {
return rerr
}
@@ -434,10 +371,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
}
for i < offset { // handle the parsing
- b, rerr := br.ReadByte()
- if rerr != nil {
- break
+ if rjr.isEndOfFile() {
+ return
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
character++
if isNewLine(b) {
line++
@@ -457,10 +395,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
}
}
if b == '/' {
- b, rerr := br.ReadByte()
- if rerr != nil {
- break
+ if rjr.isEndOfFile() {
+ return
}
+ b := rjr.buf[rjr.indx]
+ rjr.indx++
if b == '/' { // read //
character++
i--
@@ -476,7 +415,7 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
break
}
} else {
- rerr := br.UnreadByte()
+ rerr := rjr.UnreadByte()
if rerr != nil {
break
}
@@ -485,3 +424,11 @@ func getJsonOffsetLine(reader io.Reader, offset int64) (line, character int64) {
}
return
}
+
+// Loads the json config out of rjReader
+func (rjr *rjReader) Decode(cfg interface{}) (err error) {
+ if err = json.NewDecoder(rjr).Decode(cfg); err != nil {
+ return rjr.HandleJSONError(err)
+ }
+ return
+}
diff --git a/config/rjreader_test.go b/config/rjreader_test.go
index ab00887be..9865ae63d 100644
--- a/config/rjreader_test.go
+++ b/config/rjreader_test.go
@@ -18,11 +18,8 @@ along with this program. If not, see
package config
import (
- "bufio"
- "encoding/json"
"os"
"reflect"
- "strings"
"testing"
"github.com/cgrates/cgrates/utils"
@@ -54,7 +51,7 @@ var (
)
func TestEnvRawJsonReadByte(t *testing.T) {
- raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(envStr))}
+ raw := NewRjReaderFromBytes([]byte(envStr))
expected := []byte(`{"data_db":{"db_type":"redis","db_host":"127.0.0.1","db_port":6379,"db_name":"10","db_user":"*env:TESTVAR","db_password":",/**/","redis_sentinel":""}}`)
rply := []byte{}
bit, err := raw.ReadByte()
@@ -67,8 +64,8 @@ func TestEnvRawJsonReadByte(t *testing.T) {
}
func TestEnvRawJsonconsumeComent(t *testing.T) {
- raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`//comment
-a/*comment*/b`))}
+ raw := NewRjReaderFromBytes([]byte(`//comment
+a/*comment*/b`))
expected := (byte)('a')
if r, err := raw.consumeComent('d'); err != nil {
t.Error(err)
@@ -99,9 +96,9 @@ a/*comment*/b`))}
}
func TestEnvRawJsonReadByteWC(t *testing.T) {
- raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`c/*first comment*///another comment
+ raw := NewRjReaderFromBytes([]byte(`c/*first comment*///another comment
- cgrates`))}
+ cgrates`))
expected := (byte)('c')
if rply, err := raw.ReadByteWC(); err != nil {
t.Error(err)
@@ -116,9 +113,9 @@ func TestEnvRawJsonReadByteWC(t *testing.T) {
}
func TestEnvRawJsonPeekByteWC(t *testing.T) {
- raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`c/*first comment*///another comment
+ raw := NewRjReaderFromBytes([]byte(`c/*first comment*///another comment
- bgrates`))}
+ bgrates`))
expected := (byte)('c')
if rply, err := raw.PeekByteWC(); err != nil {
t.Error(err)
@@ -144,9 +141,9 @@ func TestEnvRawJsonPeekByteWC(t *testing.T) {
}
func TestEnvRawJsonreadFirstNonWhiteSpace(t *testing.T) {
- raw := &rawJSON{rdr: bufio.NewReader(strings.NewReader(`
+ raw := NewRjReaderFromBytes([]byte(`
- cgrates`))}
+ cgrates`))
expected := (byte)('c')
if rply, err := raw.readFirstNonWhiteSpace(); err != nil {
t.Error(err)
@@ -157,7 +154,7 @@ func TestEnvRawJsonreadFirstNonWhiteSpace(t *testing.T) {
func TestEnvReaderRead(t *testing.T) {
os.Setenv("TESTVAR", "cgRates")
- envR := NewRawJSONReader(strings.NewReader(envStr))
+ envR := NewRjReaderFromBytes([]byte(envStr))
expected := []byte(`{"data_db":{"db_type":"redis","db_host":"127.0.0.1","db_port":6379,"db_name":"10","db_user":"cgRates","db_password":",/**/","redis_sentinel":""}}`)
rply := []byte{}
buf := make([]byte, 20)
@@ -180,7 +177,7 @@ func TestEnvReaderRead(t *testing.T) {
func TestEnvReaderRead2(t *testing.T) {
os.Setenv("TESTVARNoZero", "cgr1")
- envR := NewRawJSONReader(strings.NewReader(`{"origin_host": "*env:TESTVARNoZero",
+ envR := NewRjReaderFromBytes([]byte(`{"origin_host": "*env:TESTVARNoZero",
"origin_realm": "*env:TESTVARNoZero",}`))
expected := []byte(`{"origin_host":"cgr1","origin_realm":"cgr1"}`)
rply := []byte{}
@@ -203,20 +200,20 @@ func TestEnvReaderRead2(t *testing.T) {
}
func TestEnvReaderreadEnvName(t *testing.T) {
- envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(`Test_VAR1 } Var2_TEST'`))}}
+ envR := NewRjReaderFromBytes([]byte(`Test_VAR1 } Var2_TEST'`))
expected := []byte("Test_VAR1")
- if rply, bit, err := envR.readEnvName(); err != nil {
+ if rply, endindx, err := envR.readEnvName(0); err != nil {
t.Error(err)
- } else if bit != '}' {
- t.Errorf("Wrong bit returned %q", bit)
+ } else if endindx != 9 {
+ t.Errorf("Wrong endindx returned %v", endindx)
} else if !reflect.DeepEqual(expected, rply) {
t.Errorf("Expected: %+v, recived: %+v", (string(expected)), (string(rply)))
}
expected = []byte("Var2_TEST")
- if rply, bit, err := envR.readEnvName(); err != nil {
+ if rply, endindx, err := envR.readEnvName(12); err != nil {
t.Error(err)
- } else if bit != '\'' {
- t.Errorf("Wrong bit returned %q", bit)
+ } else if endindx != 21 {
+ t.Errorf("Wrong endindx returned %v", endindx)
} else if !reflect.DeepEqual(expected, rply) {
t.Errorf("Expected: %+v, recived: %+v", (string(expected)), (string(rply)))
}
@@ -225,50 +222,26 @@ func TestEnvReaderreadEnvName(t *testing.T) {
func TestEnvReaderreplaceEnv(t *testing.T) {
os.Setenv("Test_VAR1", "5")
os.Setenv("Test_VAR2", "aVeryLongEnviormentalVariable")
- envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(`Test_VAR1,/*comment*/ }Test_VAR2"`))}}
- expected := []byte("5} ")
- expectedn := 1
- rply := make([]byte, 5)
- if n, err := envR.replaceEnv(rply, 0, 5); err != nil {
+ envR := NewRjReaderFromBytes([]byte(`*env:Test_VAR1,/*comment*/ }*env:Test_VAR2"`))
+ // expected := []byte("5} ")
+ if err := envR.replaceEnv(0); err != nil {
t.Error(err)
- } else if expectedn != n {
- t.Errorf("Expected: %+v, recived: %+v", expectedn, n)
- } else if !reflect.DeepEqual(expected, rply) {
- t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply)))
}
- expected = []byte("aVery")
- expectedn = 5
- rply = make([]byte, 5)
- if n, err := envR.replaceEnv(rply, 0, 5); err != nil {
+ if err := envR.replaceEnv(15); err != nil {
t.Error(err)
- } else if expectedn != n {
- t.Errorf("Expected: %+v, recived: %+v", expectedn, n)
- } else if !reflect.DeepEqual(expected, rply) {
- t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply)))
- } else if bufexp := []byte("LongEnviormentalVariable\""); !reflect.DeepEqual(bufexp, envR.buf) {
- t.Errorf("Expected: %q, recived: %q", (string(expected)), (string(rply)))
}
}
func TestEnvReadercheckMeta(t *testing.T) {
- envR := EnvReader{rd: &rawJSON{rdr: bufio.NewReader(strings.NewReader(""))}}
- envR.m = 2
- if envR.checkMeta('n') {
- t.Errorf("Expectiv to get false recived true")
- } else if envR.m != 3 {
- t.Errorf("Expectiv the meta offset to incrase")
- }
- envR.m = 4
- if !envR.checkMeta(':') {
+ envR := NewRjReaderFromBytes([]byte("*env:Var"))
+ envR.indx = 1
+ if !envR.checkMeta() {
t.Errorf("Expectiv true ")
- } else if envR.m != 0 {
- t.Errorf("Expectiv the meta offset to reset")
}
- envR.m = 1
- if envR.checkMeta('v') {
+ envR = NewRjReaderFromBytes([]byte("*enva:Var"))
+ envR.indx = 1
+ if envR.checkMeta() {
t.Errorf("Expectiv to get false recived true")
- } else if envR.m != 0 {
- t.Errorf("Expectiv the meta offset to reset")
}
}
@@ -333,24 +306,10 @@ func TestGetErrorLine(t *testing.T) {
Line3
*/
/**/ }//`
- r := strings.NewReader(jsonstr)
- _, err := NewCgrJsonCfgFromReader(r)
- if err == nil {
- t.Fatalf("Expected error received %v", err)
- }
- var offset int64
- if realErr, canCast := err.(*json.SyntaxError); !canCast {
- t.Fatalf("Expected json.SyntaxError received %v<%T>", err.Error(), err)
- } else {
- offset = realErr.Offset
- }
- var expOffset int64 = 31
- if offset != expOffset {
- t.Errorf("Expected offset %v received:%v", expOffset, offset)
- }
- r = strings.NewReader(jsonstr)
+ r := NewRjReaderFromBytes([]byte(jsonstr))
+ var offset int64 = 31
var expLine, expChar int64 = 10, 23
- if line, character := getJsonOffsetLine(r, offset); expLine != line {
+ if line, character := r.getJsonOffsetLine(offset); expLine != line {
t.Errorf("Expected line %v received:%v", expLine, line)
} else if expChar != character {
t.Errorf("Expected line %v received:%v", expChar, character)
@@ -381,24 +340,10 @@ func TestGetErrorLine2(t *testing.T) {
Line3
*/
/**/ }//`
- r := strings.NewReader(jsonstr)
- _, err := NewCgrJsonCfgFromReader(r)
- if err == nil {
- t.Fatalf("Expected error received %v", err)
- }
- var offset int64
- if realErr, canCast := err.(*json.SyntaxError); !canCast {
- t.Fatalf("Expected json.SyntaxError received %v<%T>", err.Error(), err)
- } else {
- offset = realErr.Offset
- }
- var expOffset int64 = 31
- if offset != expOffset {
- t.Errorf("Expected offset %v received:%v", expOffset, offset)
- }
- r = strings.NewReader(jsonstr)
+ r := NewRjReaderFromBytes([]byte(jsonstr))
+ var offset int64 = 31
var expLine, expChar int64 = 10, 46
- if line, character := getJsonOffsetLine(r, offset); expLine != line {
+ if line, character := r.getJsonOffsetLine(offset); expLine != line {
t.Errorf("Expected line %v received:%v", expLine, line)
} else if expChar != character {
t.Errorf("Expected line %v received:%v", expChar, character)