From ed87bf245ee4dd09aae878f02af277fb7f752371 Mon Sep 17 00:00:00 2001 From: TeoV Date: Sat, 11 Jan 2020 17:17:47 +0200 Subject: [PATCH] Add a skelet for fwv ers --- cdrc/fwv.go | 84 +--------------- config/configsanity.go | 2 +- config/fwvdp.go | 109 ++++++++++++++++++++ ers/filefwv.go | 219 +++++++++++++++++++++++++++++++++++++++++ ers/reader.go | 2 + 5 files changed, 334 insertions(+), 82 deletions(-) create mode 100644 config/fwvdp.go create mode 100644 ers/filefwv.go diff --git a/cdrc/fwv.go b/cdrc/fwv.go index 9e0f9a06f..c576aa032 100644 --- a/cdrc/fwv.go +++ b/cdrc/fwv.go @@ -23,10 +23,7 @@ import ( "encoding/json" "fmt" "io" - "net" "os" - "strconv" - "strings" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -112,7 +109,7 @@ func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { } self.processedRecordsNr += 1 record := string(buf) - fwvProvider := newfwvProvider(record) + fwvProvider := config.NewFWVProvider(record, utils.MetaReq) for _, cdrcCfg := range self.cdrcCfgs { tenant, err := cdrcCfg.Tenant.ParseDataProvider(fwvProvider, utils.NestingSep) // each profile of cdrc can have different tenant if err != nil { @@ -145,8 +142,8 @@ func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *confi var lazyHttpFields []*config.FCTemplate var cfgFields []*config.FCTemplate var storedCdr *engine.CDR - fwvProvider := newfwvProvider(record) // used for filterS and for RSRParsers - if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there) + fwvProvider := config.NewFWVProvider(record, utils.MetaReq) // used for filterS and for RSRParsers + if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there) storedCdr = self.headerCdr.Clone() } else { storedCdr = &engine.CDR{OriginHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1} @@ -248,78 +245,3 @@ func (self *FwvRecordsProcessor) processTrailer() error { } return nil } - -// newfwvProvider constructs a DataProvider -func newfwvProvider(record string) (dP config.DataProvider) { - dP = &fwvProvider{req: record, cache: config.NewNavigableMap(nil)} - return -} - -// fwvProvider implements engine.DataProvider so we can pass it to filters -type fwvProvider struct { - req string - cache *config.NavigableMap -} - -// String is part of engine.DataProvider interface -// when called, it will display the already parsed values out of cache -func (fP *fwvProvider) String() string { - return utils.ToJSON(fP) -} - -// FieldAsInterface is part of engine.DataProvider interface -func (fP *fwvProvider) FieldAsInterface(fldPath []string) (data interface{}, err error) { - if len(fldPath) == 0 { - return - } - if fldPath[0] != utils.MetaReq || len(fldPath) < 2 { - return "", utils.ErrPrefixNotFound(strings.Join(fldPath, utils.NestingSep)) - } - if data, err = fP.cache.FieldAsInterface(fldPath); err == nil || - err != utils.ErrNotFound { // item found in cache - return - } - err = nil // cancel previous err - indexes := strings.Split(fldPath[1], "-") - if len(indexes) != 2 { - return "", fmt.Errorf("Invalid format for index : %+v", fldPath[1]) - } - startIndex, err := strconv.Atoi(indexes[0]) - if err != nil { - return nil, err - } - if startIndex > len(fP.req) { - return "", fmt.Errorf("StartIndex : %+v is greater than : %+v", startIndex, len(fP.req)) - } - finalIndex, err := strconv.Atoi(indexes[1]) - if err != nil { - return nil, err - } - if finalIndex > len(fP.req) { - return "", fmt.Errorf("FinalIndex : %+v is greater than : %+v", finalIndex, len(fP.req)) - } - data = fP.req[startIndex:finalIndex] - fP.cache.Set(fldPath, data, false, false) - return -} - -// FieldAsString is part of engine.DataProvider interface -func (fP *fwvProvider) FieldAsString(fldPath []string) (data string, err error) { - var valIface interface{} - valIface, err = fP.FieldAsInterface(fldPath) - if err != nil { - return - } - return utils.IfaceAsString(valIface), nil -} - -// AsNavigableMap is part of engine.DataProvider interface -func (fP *fwvProvider) AsNavigableMap([]*config.FCTemplate) ( - nm *config.NavigableMap, err error) { - return nil, utils.ErrNotImplemented -} - -// RemoteHost is part of engine.DataProvider interface -func (fP *fwvProvider) RemoteHost() net.Addr { - return utils.LocalAddr() -} diff --git a/config/configsanity.go b/config/configsanity.go index abf95f00e..64f8fce2f 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -429,7 +429,7 @@ func (cfg *CGRConfig) checkConfigSanity() error { if rdr.RunDelay > 0 { return fmt.Errorf("<%s> RunDelay field can not be bigger than zero for reader with ID: %s", utils.ERs, rdr.ID) } - case utils.MetaFileXML: + case utils.MetaFileXML, utils.MetaFileFWV: for _, dir := range []string{rdr.ProcessedPath, rdr.SourcePath} { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { return fmt.Errorf("<%s> Nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) diff --git a/config/fwvdp.go b/config/fwvdp.go new file mode 100644 index 000000000..9b6a34573 --- /dev/null +++ b/config/fwvdp.go @@ -0,0 +1,109 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package config + +import ( + "fmt" + "net" + "strconv" + "strings" + + "github.com/cgrates/cgrates/utils" +) + +// NewfwvProvider constructs a DataProvider +func NewFWVProvider(record, pathPrfx string) (dP DataProvider) { + dP = &FWVProvider{req: record, cache: NewNavigableMap(nil), pathPrfx: pathPrfx} + return +} + +// fwvProvider implements engine.DataProvider so we can pass it to filters +type FWVProvider struct { + req string + cache *NavigableMap + pathPrfx string // if this comes in path it will be ignored + // pathPrfx should be reviewed once the cdrc is removed +} + +// String is part of engine.DataProvider interface +// when called, it will display the already parsed values out of cache +func (fP *FWVProvider) String() string { + return utils.ToJSON(fP) +} + +// FieldAsInterface is part of engine.DataProvider interface +func (fP *FWVProvider) FieldAsInterface(fldPath []string) (data interface{}, err error) { + if len(fldPath) == 0 { + return + } + fwvIdx := fldPath[0] + if len(fldPath) == 2 { + fwvIdx = fldPath[1] + } + if fP.pathPrfx != utils.EmptyString && (fldPath[0] != fP.pathPrfx || len(fldPath) < 2) { + return "", utils.ErrPrefixNotFound(strings.Join(fldPath, utils.NestingSep)) + } + if data, err = fP.cache.FieldAsInterface(fldPath); err == nil || + err != utils.ErrNotFound { // item found in cache + return + } + err = nil // cancel previous err + indexes := strings.Split(fwvIdx, "-") + if len(indexes) != 2 { + return "", fmt.Errorf("Invalid format for index : %+v", fldPath[1]) + } + startIndex, err := strconv.Atoi(indexes[0]) + if err != nil { + return nil, err + } + if startIndex > len(fP.req) { + return "", fmt.Errorf("StartIndex : %+v is greater than : %+v", startIndex, len(fP.req)) + } + finalIndex, err := strconv.Atoi(indexes[1]) + if err != nil { + return nil, err + } + if finalIndex > len(fP.req) { + return "", fmt.Errorf("FinalIndex : %+v is greater than : %+v", finalIndex, len(fP.req)) + } + data = fP.req[startIndex:finalIndex] + fP.cache.Set(fldPath, data, false, false) + return +} + +// FieldAsString is part of engine.DataProvider interface +func (fP *FWVProvider) FieldAsString(fldPath []string) (data string, err error) { + var valIface interface{} + valIface, err = fP.FieldAsInterface(fldPath) + if err != nil { + return + } + return utils.IfaceAsString(valIface), nil +} + +// AsNavigableMap is part of engine.DataProvider interface +func (fP *FWVProvider) AsNavigableMap([]*FCTemplate) ( + nm *NavigableMap, err error) { + return nil, utils.ErrNotImplemented +} + +// RemoteHost is part of engine.DataProvider interface +func (fP *FWVProvider) RemoteHost() net.Addr { + return utils.LocalAddr() +} diff --git a/ers/filefwv.go b/ers/filefwv.go new file mode 100644 index 000000000..21128e7a1 --- /dev/null +++ b/ers/filefwv.go @@ -0,0 +1,219 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package ers + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/cgrates/cgrates/agents" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +func NewFWVFileERER(cfg *config.CGRConfig, cfgIdx int, + rdrEvents chan *erEvent, rdrErr chan error, + fltrS *engine.FilterS, rdrExit chan struct{}) (er EventReader, err error) { + srcPath := cfg.ERsCfg().Readers[cfgIdx].SourcePath + if strings.HasSuffix(srcPath, utils.Slash) { + srcPath = srcPath[:len(srcPath)-1] + } + return &FWVFileER{ + cgrCfg: cfg, + cfgIdx: cfgIdx, + fltrS: fltrS, + rdrDir: srcPath, + rdrEvents: rdrEvents, + rdrError: rdrErr, + rdrExit: rdrExit}, nil +} + +// XMLFileER implements EventReader interface for .xml files +type FWVFileER struct { + sync.RWMutex + cgrCfg *config.CGRConfig + cfgIdx int // index of config instance within ERsCfg.Readers + fltrS *engine.FilterS + rdrDir string + rdrEvents chan *erEvent // channel to dispatch the events created to + rdrError chan error + rdrExit chan struct{} + conReqs chan struct{} // limit number of opened files + lineLen int64 // Length of the line in the file + offset int64 // Index of the next byte to process + trailerOffset int64 // Index where trailer starts, to be used as boundary when reading cdrs +} + +func (rdr *FWVFileER) Config() *config.EventReaderCfg { + return rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx] +} + +func (rdr *FWVFileER) Serve() (err error) { + switch rdr.Config().RunDelay { + case time.Duration(0): // 0 disables the automatic read, maybe done per API + return + case time.Duration(-1): + return watchDir(rdr.rdrDir, rdr.processFile, + utils.ERs, rdr.rdrExit) + default: + go func() { + for { + // Not automated, process and sleep approach + select { + case <-rdr.rdrExit: + utils.Logger.Info( + fmt.Sprintf("<%s> stop monitoring path <%s>", + utils.ERs, rdr.rdrDir)) + return + default: + } + filesInDir, _ := ioutil.ReadDir(rdr.rdrDir) + for _, file := range filesInDir { + if !strings.HasSuffix(file.Name(), utils.XMLSuffix) { // hardcoded file extension for xml event reader + continue // used in order to filter the files from directory + } + go func(fileName string) { + if err := rdr.processFile(rdr.rdrDir, fileName); err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> processing file %s, error: %s", + utils.ERs, fileName, err.Error())) + } + }(file.Name()) + } + time.Sleep(rdr.Config().RunDelay) + } + }() + } + return +} + +// processFile is called for each file in a directory and dispatches erEvents from it +func (rdr *FWVFileER) processFile(fPath, fName string) (err error) { + if cap(rdr.conReqs) != 0 { // 0 goes for no limit + processFile := <-rdr.conReqs // Queue here for maxOpenFiles + defer func() { rdr.conReqs <- processFile }() + } + absPath := path.Join(fPath, fName) + utils.Logger.Info( + fmt.Sprintf("<%s> parsing <%s>", utils.ERs, absPath)) + var file *os.File + if file, err = os.Open(absPath); err != nil { + return + } + defer file.Close() + + rowNr := 0 // This counts the rows in the file, not really number of CDRs + evsPosted := 0 + timeStart := time.Now() + reqVars := make(map[string]interface{}) + + for { + cntFld := rdr.Config().ContentFields + if rdr.offset == 0 { // First time, set the necessary offsets + if err := rdr.setLineLen(file); err != nil { + utils.Logger.Err(fmt.Sprintf(" Row 0, error: cannot set lineLen: %s", err.Error())) + break + } + if len(rdr.Config().TrailerFields) != 0 { + if fi, err := file.Stat(); err != nil { + utils.Logger.Err(fmt.Sprintf(" Row 0, error: cannot get file stats: %s", err.Error())) + return err + } else { + rdr.trailerOffset = fi.Size() - rdr.lineLen + } + } + if len(rdr.Config().HeaderFields) != 0 { + buf := make([]byte, rdr.lineLen) + if nRead, err := file.Read(buf); err != nil { + return err + } else if nRead != len(buf) { + return fmt.Errorf("In header, line len: %d, have read: %d", rdr.lineLen, nRead) + } + cntFld = rdr.Config().HeaderFields + } + } + rowNr++ // increment the rowNr after checking if it's not the end of file + agReq := agents.NewAgentRequest( + config.NewFWVProvider("a", utils.EmptyString), reqVars, + nil, nil, rdr.Config().Tenant, + rdr.cgrCfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(rdr.Config().Timezone, + rdr.cgrCfg.GeneralCfg().DefaultTimezone), + rdr.fltrS) // create an AgentRequest + if pass, err := rdr.fltrS.Pass(agReq.Tenant, rdr.Config().Filters, + agReq); err != nil || !pass { + continue + } + navMp, err := agReq.AsNavigableMap(cntFld) + if err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> reading file: <%s> row <%d>, ignoring due to error: <%s>", + utils.ERs, absPath, rowNr, err.Error())) + continue + } + rdr.rdrEvents <- &erEvent{cgrEvent: navMp.AsCGREvent( + agReq.Tenant, utils.NestingSep), + rdrCfg: rdr.Config()} + evsPosted++ + } + + if rdr.Config().ProcessedPath != "" { + // Finished with file, move it to processed folder + outPath := path.Join(rdr.Config().ProcessedPath, fName) + if err = os.Rename(absPath, outPath); err != nil { + return + } + } + + utils.Logger.Info( + fmt.Sprintf("%s finished processing file <%s>. Total records processed: %d, events posted: %d, run duration: %s", + utils.ERs, absPath, rowNr, evsPosted, time.Now().Sub(timeStart))) + return +} + +// Sets the line length based on first line, sets offset back to initial after reading +func (rdr *FWVFileER) setLineLen(file *os.File) error { + buff := bufio.NewReader(file) + readBytes, err := buff.ReadBytes('\n') + if err != nil { + return err + } + rdr.lineLen = int64(len(readBytes)) + if _, err := file.Seek(0, 0); err != nil { + return err + } + return nil +} + +func (rdr *FWVFileER) processTrailer(file *os.File) error { + buf := make([]byte, rdr.lineLen) + if nRead, err := file.ReadAt(buf, rdr.trailerOffset); err != nil { + return err + } else if nRead != len(buf) { + return fmt.Errorf("In trailer, line len: %d, have read: %d", rdr.lineLen, nRead) + } + return nil +} diff --git a/ers/reader.go b/ers/reader.go index 185796b87..abd8c72ab 100644 --- a/ers/reader.go +++ b/ers/reader.go @@ -42,6 +42,8 @@ func NewEventReader(cfg *config.CGRConfig, cfgIdx int, return NewCSVFileER(cfg, cfgIdx, rdrEvents, rdrErr, fltrS, rdrExit) case utils.MetaFileXML: return NewXMLFileER(cfg, cfgIdx, rdrEvents, rdrErr, fltrS, rdrExit) + case utils.MetaFileFWV: + return NewFWVFileERER(cfg, cfgIdx, rdrEvents, rdrErr, fltrS, rdrExit) case utils.MetaKafkajsonMap: return NewKafkaER(cfg, cfgIdx, rdrEvents, rdrErr, fltrS, rdrExit) case utils.MetaSQL: