diff --git a/apier/v1/cdrc.go b/apier/v1/cdrc.go deleted file mode 100644 index 6152e071d..000000000 --- a/apier/v1/cdrc.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -type AttrReloadConfig struct { - ConfigDir string -} - -// Retrieves the callCost out of CGR logDb -func (apier *ApierV1) ReloadCdrcConfig(attrs AttrReloadConfig, reply *string) error { - if attrs.ConfigDir == "" { - attrs.ConfigDir = utils.CONFIG_PATH - } - newCfg, err := config.NewCGRConfigFromPath(attrs.ConfigDir) - if err != nil { - return utils.NewErrServerError(err) - } - apier.Config.CdrcProfiles = newCfg.CdrcProfiles // ToDo: Check if there can be any concurency involved here so we need to lock maybe - apier.Config.ConfigReloads[utils.CDRC] <- struct{}{} - *reply = utils.OK - return nil -} diff --git a/apier/v1/cdre.go b/apier/v1/cdre.go index c0d29eda2..3e6a6a578 100644 --- a/apier/v1/cdre.go +++ b/apier/v1/cdre.go @@ -169,6 +169,10 @@ func (api *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.Ex return nil } +type AttrReloadConfig struct { + ConfigDir string +} + // Reloads CDRE configuration out of folder specified func (apier *ApierV1) ReloadCdreConfig(attrs AttrReloadConfig, reply *string) error { if attrs.ConfigDir == "" { diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go deleted file mode 100644 index 42fc82bde..000000000 --- a/cdrc/cdrc.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "bufio" - "encoding/csv" - "fmt" - "io" - "io/ioutil" - "os" - "path" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" - "github.com/fsnotify/fsnotify" -) - -const ( - UNPAIRED_SUFFIX = ".unpaired" -) - -// Understands and processes a specific format of cdr (eg: .csv or .fwv) -type RecordsProcessor interface { - ProcessNextRecord() ([]*engine.CDR, error) // Process a single record in the CDR file, return a slice of CDRs since based on configuration we can have more templates - ProcessedRecordsNr() int64 -} - -/* -One instance of CDRC will act on one folder. -Common parameters within configs processed: - * cdrS, cdrFormat, CDRInPath, CDROutPath, runDelay -Parameters specific per config instance: - * cdrSourceId, cdrFilter, cdrFields -*/ -func NewCdrc(cdrcCfgs []*config.CdrcCfg, httpSkipTlsCheck bool, cdrs rpcclient.ClientConnector, - closeChan chan struct{}, dfltTimezone string, filterS *engine.FilterS, connMgr *engine.ConnManager) (cdrc *Cdrc, err error) { - cdrcCfg := cdrcCfgs[0] - cdrc = &Cdrc{ - httpSkipTlsCheck: httpSkipTlsCheck, - cdrcCfgs: cdrcCfgs, - dfltCdrcCfg: cdrcCfg, - timezone: utils.FirstNonEmpty(cdrcCfg.Timezone, dfltTimezone), - closeChan: closeChan, - maxOpenFiles: make(chan struct{}, cdrcCfg.MaxOpenFiles), - connMgr: connMgr, - } - // Before processing, make sure in and out folders exist - if utils.CDRCFileFormats.Has(cdrcCfg.CdrFormat) { - for _, dir := range []string{cdrcCfg.CDRInPath, cdrcCfg.CDROutPath} { - if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { - return nil, fmt.Errorf(" nonexistent folder: %s", dir) - } - } - - var processFile struct{} - for i := 0; i < cdrcCfg.MaxOpenFiles; i++ { - cdrc.maxOpenFiles <- processFile // Empty initiate so we do not need to wait later when we pop - } - } - cdrc.unpairedRecordsCache = NewUnpairedRecordsCache(cdrcCfg.PartialRecordCache, - cdrcCfg.CDROutPath, cdrcCfg.FieldSeparator) - cdrc.partialRecordsCache = NewPartialRecordsCache(cdrcCfg.PartialRecordCache, - cdrcCfg.PartialCacheExpiryAction, cdrcCfg.CDROutPath, - cdrcCfg.FieldSeparator, cdrc.timezone, httpSkipTlsCheck, filterS, cdrc.dfltCdrcCfg.CdrsConns, cdrc.connMgr) - cdrc.filterS = filterS - return -} - -type Cdrc struct { - httpSkipTlsCheck bool - cdrcCfgs []*config.CdrcCfg // All cdrc config profiles attached to this CDRC (key will be profile instance name) - dfltCdrcCfg *config.CdrcCfg - timezone string - closeChan chan struct{} // Used to signal config reloads when we need to span different CDRC-Client - maxOpenFiles chan struct{} // Maximum number of simultaneous files processed - filterS *engine.FilterS - unpairedRecordsCache *UnpairedRecordsCache // Shared between all files in the folder we process - partialRecordsCache *PartialRecordsCache - connMgr *engine.ConnManager -} - -// When called fires up folder monitoring, either automated via inotify or manual by sleeping between processing -func (self *Cdrc) Run() error { - if self.dfltCdrcCfg.RunDelay == time.Duration(0) { // Automated via inotify - return self.trackCDRFiles() - } - // Not automated, process and sleep approach - for { - select { - case <-self.closeChan: // Exit, reinject closeChan for other CDRCs - utils.Logger.Info(fmt.Sprintf(" Shutting down CDRC on path %s.", self.dfltCdrcCfg.CDRInPath)) - return nil - default: - } - self.processCdrDir() - time.Sleep(self.dfltCdrcCfg.RunDelay) - } -} - -// Watch the specified folder for file moves and parse the files on events -func (self *Cdrc) trackCDRFiles() (err error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return - } - defer watcher.Close() - err = watcher.Add(self.dfltCdrcCfg.CDRInPath) - if err != nil { - return - } - utils.Logger.Info(fmt.Sprintf(" Monitoring %s for file moves.", self.dfltCdrcCfg.CDRInPath)) - for { - select { - case <-self.closeChan: // Exit, reinject closeChan for other CDRCs - utils.Logger.Info(fmt.Sprintf(" Shutting down CDRC on path %s.", self.dfltCdrcCfg.CDRInPath)) - return nil - case ev := <-watcher.Events: - if ev.Op&fsnotify.Create == fsnotify.Create && (self.dfltCdrcCfg.CdrFormat != utils.MetaFScsv || path.Ext(ev.Name) != ".csv") { - go func() { //Enable async processing here - if err = self.processFile(ev.Name); err != nil { - utils.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error())) - } - }() - } - case err := <-watcher.Errors: - utils.Logger.Err(fmt.Sprintf("Inotify error: %s", err.Error())) - } - } -} - -// One run over the CDR folder -func (self *Cdrc) processCdrDir() error { - utils.Logger.Info(fmt.Sprintf(" Parsing folder %s for CDR files.", self.dfltCdrcCfg.CDRInPath)) - filesInDir, _ := ioutil.ReadDir(self.dfltCdrcCfg.CDRInPath) - for _, file := range filesInDir { - if self.dfltCdrcCfg.CdrFormat != utils.MetaFScsv || path.Ext(file.Name()) != ".csv" { - go func(file os.FileInfo) { //Enable async processing here - if err := self.processFile(path.Join(self.dfltCdrcCfg.CDRInPath, file.Name())); err != nil { - utils.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", file, err.Error())) - } - }(file) - } - } - return nil -} - -// Processe file at filePath and posts the valid cdr rows out of it -func (self *Cdrc) processFile(filePath string) error { - if cap(self.maxOpenFiles) != 0 { // 0 goes for no limit - processFile := <-self.maxOpenFiles // Queue here for maxOpenFiles - defer func() { self.maxOpenFiles <- processFile }() - } - _, fn := path.Split(filePath) - utils.Logger.Info(fmt.Sprintf(" Parsing: %s", filePath)) - file, err := os.Open(filePath) - defer file.Close() - if err != nil { - utils.Logger.Crit(err.Error()) - return err - } - var recordsProcessor RecordsProcessor - switch self.dfltCdrcCfg.CdrFormat { - case utils.MetaFileCSV, utils.MetaFScsv, utils.MetaKamFlatstore, utils.MetaOsipsFlatstore, utils.MetaPartialCSV: - csvReader := csv.NewReader(bufio.NewReader(file)) - csvReader.Comma = self.dfltCdrcCfg.FieldSeparator - csvReader.Comment = '#' - recordsProcessor = NewCsvRecordsProcessor(csvReader, self.timezone, fn, self.dfltCdrcCfg, - self.cdrcCfgs, self.httpSkipTlsCheck, - self.dfltCdrcCfg.CacheDumpFields, self.filterS, - self.unpairedRecordsCache, self.partialRecordsCache, self.connMgr) - case utils.MetaFileFWV: - recordsProcessor = NewFwvRecordsProcessor(file, self.dfltCdrcCfg, self.cdrcCfgs, - self.httpSkipTlsCheck, self.timezone, self.filterS) - case utils.MetaFileXML: - if recordsProcessor, err = NewXMLRecordsProcessor(file, self.dfltCdrcCfg.CDRRootPath, - self.timezone, self.httpSkipTlsCheck, self.cdrcCfgs, self.filterS); err != nil { - return err - } - default: - return fmt.Errorf("Unsupported CDR format: %s", self.dfltCdrcCfg.CdrFormat) - } - rowNr := 0 // This counts the rows in the file, not really number of CDRs - cdrsPosted := 0 - timeStart := time.Now() - for { - cdrs, err := recordsProcessor.ProcessNextRecord() - if err != nil && err == io.EOF { - break - } - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Row %d, error: %s", rowNr, err.Error())) - continue - } - for _, storedCdr := range cdrs { // Send CDRs to CDRS - var reply string - if self.dfltCdrcCfg.DryRun { - utils.Logger.Info(fmt.Sprintf(" DryRun CDR: %+v", storedCdr)) - continue - } - if err := self.connMgr.Call(self.dfltCdrcCfg.CdrsConns, nil, utils.CDRsV1ProcessEvent, - &engine.ArgV1ProcessEvent{CGREvent: *storedCdr.AsCGREvent()}, &reply); err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed sending CDR, %+v, error: %s", storedCdr, err.Error())) - } else if reply != utils.OK { - utils.Logger.Err(fmt.Sprintf(" Received unexpected reply for CDR, %+v, reply: %s", storedCdr, reply)) - } - cdrsPosted += 1 - } - } - // Finished with file, move it to processed folder - newPath := path.Join(self.dfltCdrcCfg.CDROutPath, fn) - if err := os.Rename(filePath, newPath); err != nil { - utils.Logger.Err(err.Error()) - return err - } - utils.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, CDRs posted: %d, run duration: %s", - fn, newPath, recordsProcessor.ProcessedRecordsNr(), cdrsPosted, time.Now().Sub(timeStart))) - return nil -} diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go deleted file mode 100644 index f1c85fa3d..000000000 --- a/cdrc/cdrc_test.go +++ /dev/null @@ -1,313 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -/* -func TestNewPartialFlatstoreRecord(t *testing.T) { - ePr := &PartialFlatstoreRecord{Method: "INVITE", AccId: "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC), - Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}} - if pr, err := NewPartialFlatstoreRecord(ePr.Values); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(ePr, pr) { - t.Errorf("Expecting: %+v, received: %+v", ePr, pr) - } - if _, err := NewPartialFlatstoreRecord([]string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK"}); err == nil || err.Error() != "MISSING_IE" { - t.Error(err) - } -} -*/ - -/* -func TestOsipsFlatstoreCdrs(t *testing.T) { - flatstoreCdrs := ` -INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475 -BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475 -INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741 -BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741 -INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533 -BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533 -INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290 -BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290 -` - - eCdrs := []*engine.StoredCdr{ - &engine.StoredCdr{ - CgrId: "e61034c34148a7c4f40623e00ca5e551d1408bf3", - TOR: utils.VOICE, - AccId: "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_PREPAID, - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1002", - SetupTime: time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC), - Usage: time.Duration(2) * time.Second, - DisconnectCause: "200 OK", - ExtraFields: map[string]string{ - "DialogIdentifier": "3401:2069362475", - }, - Cost: -1, - }, - &engine.StoredCdr{ - CgrId: "3ed64a28190e20ac8a6fd8fd48cb23efbfeb7a17", - TOR: utils.VOICE, - AccId: "214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0f9d3d5c3c863a6e3", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_POSTPAID, - Tenant: "cgrates.org", - Category: "call", - Account: "1002", - Subject: "1002", - Destination: "1001", - SetupTime: time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC), - Usage: time.Duration(4) * time.Second, - DisconnectCause: "200 OK", - ExtraFields: map[string]string{ - "DialogIdentifier": "1877:893549741", - }, - Cost: -1, - }, - &engine.StoredCdr{ - CgrId: "f2f8d9341adfbbe1836b22f75182142061ef3d20", - TOR: utils.VOICE, - AccId: "3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:036e39a542d996f9", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_PREPAID, - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1002", - SetupTime: time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC), - Usage: time.Duration(4) * time.Second, - DisconnectCause: "200 OK", - ExtraFields: map[string]string{ - "DialogIdentifier": "2407:1884881533", - }, - Cost: -1, - }, - &engine.StoredCdr{ - CgrId: "ccf05e7e3b9db9d2370bcbe316817447dba7df54", - TOR: utils.VOICE, - AccId: "a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:03111f3c949ca4c42", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_PREPAID, - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1002", - SetupTime: time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC), //2015-07-09T17:11:30+02:00 - AnswerTime: time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC), - Usage: time.Duration(2) * time.Second, - DisconnectCause: "200 OK", - ExtraFields: map[string]string{ - "DialogIdentifier": "3099:1909036290", - }, - Cost: -1, - }, - } - - cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{ - &config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.OriginID, Mandatory: true}, - &config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Duration", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true}, - &config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)}, - }} - cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls", - cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), - guard: engine.Guardian} - cdrsContent := bytes.NewReader([]byte(flatstoreCdrs)) - csvReader := csv.NewReader(cdrsContent) - csvReader.Comma = '|' - cdrs := make([]*engine.StoredCdr, 0) - recNrs := 0 - for { - recNrs++ - cdrCsv, err := csvReader.Read() - if err != nil && err == io.EOF { - break // End of file - } else if err != nil { - t.Error("Unexpected error:", err) - } - record, err := cdrc.processPartialRecord(cdrCsv, "dummyfilename") - if err != nil { - t.Error(err) - } - if record == nil { - continue // Partial record - } - if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil { - t.Error(err) - } else if storedCdr != nil { - cdrs = append(cdrs, storedCdr) - } - } - if !reflect.DeepEqual(eCdrs, cdrs) { - t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs) - } - -} - -func TestOsipsFlatstoreMissedCdrs(t *testing.T) { - flatstoreCdrs := ` -INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783 -INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844 -INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066 -` - eCdrs := []*engine.StoredCdr{ - &engine.StoredCdr{ - CgrId: "1c20aa6543a5a30d26b2354ae79e1f5fb720e8e5", - TOR: utils.VOICE, - AccId: "0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0ef6c6256da501581", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_PREPAID, - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1002", - SetupTime: time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC), - Usage: 0, - DisconnectCause: "487 Request Terminated", - ExtraFields: map[string]string{ - "DialogIdentifier": "1224:339382783", - }, - Cost: -1, - }, - &engine.StoredCdr{ - CgrId: "054ab7c6c7fe6dc4a72f34e270027fa2aa930a58", - TOR: utils.VOICE, - AccId: "81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:07905e511", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_PREPAID, - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1002", - SetupTime: time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC), - Usage: 0, - DisconnectCause: "404 Not Found", - ExtraFields: map[string]string{ - "DialogIdentifier": "1980:1216490844", - }, - Cost: -1, - }, - &engine.StoredCdr{ - CgrId: "d49ea63d1655b15149336004629f1cadd1434b89", - TOR: utils.VOICE, - AccId: "8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0324cb497d4af7023", - CdrHost: "0.0.0.0", - CdrSource: "TEST_CDRC", - ReqType: utils.META_POSTPAID, - Direction: "*out", - Tenant: "cgrates.org", - Category: "call", - Account: "1002", - Subject: "1002", - Destination: "1001", - SetupTime: time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC), - AnswerTime: time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC), - Usage: 0, - DisconnectCause: "486 Busy here", - ExtraFields: map[string]string{ - "DialogIdentifier": "474:130115066", - }, - Cost: -1, - }, - } - - cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{ - &config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.OriginID, Mandatory: true}, - &config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "Usage", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true}, - &config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true}, - &config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)}, - }} - cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls", - cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), - guard: engine.Guardian} - cdrsContent := bytes.NewReader([]byte(flatstoreCdrs)) - csvReader := csv.NewReader(cdrsContent) - csvReader.Comma = '|' - cdrs := make([]*engine.StoredCdr, 0) - recNrs := 0 - for { - recNrs++ - cdrCsv, err := csvReader.Read() - if err != nil && err == io.EOF { - break // End of file - } else if err != nil { - t.Error("Unexpected error:", err) - } - record, err := cdrc.processPartialRecord(cdrCsv, "missed_calls_1.log") - if err != nil { - t.Error(err) - } - if record == nil { - continue // Partial record - } - if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil { - t.Error(err) - } else if storedCdr != nil { - cdrs = append(cdrs, storedCdr) - } - } - if !reflect.DeepEqual(eCdrs, cdrs) { - t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs) - } - -} -*/ diff --git a/cdrc/csv.go b/cdrc/csv.go deleted file mode 100644 index bd3ef5f36..000000000 --- a/cdrc/csv.go +++ /dev/null @@ -1,227 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "encoding/csv" - "encoding/json" - "fmt" - "strconv" - "strings" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func NewCsvRecordsProcessor(csvReader *csv.Reader, timezone, fileName string, - dfltCdrcCfg *config.CdrcCfg, cdrcCfgs []*config.CdrcCfg, - httpSkipTlsCheck bool, cacheDumpFields []*config.FCTemplate, - filterS *engine.FilterS, - unp *UnpairedRecordsCache, prt *PartialRecordsCache, - connMgr *engine.ConnManager) *CsvRecordsProcessor { - return &CsvRecordsProcessor{csvReader: csvReader, - timezone: timezone, fileName: fileName, - dfltCdrcCfg: dfltCdrcCfg, cdrcCfgs: cdrcCfgs, - httpSkipTlsCheck: httpSkipTlsCheck, - unpairedRecordsCache: unp, - partialRecordsCache: prt, - partialCacheDumpFields: cacheDumpFields, filterS: filterS, - connMgr: connMgr} - -} - -type CsvRecordsProcessor struct { - csvReader *csv.Reader - timezone string // Timezone for CDRs which are not clearly specifying it - fileName string - dfltCdrcCfg *config.CdrcCfg - cdrcCfgs []*config.CdrcCfg - processedRecordsNr int64 // Number of content records in file - httpSkipTlsCheck bool - unpairedRecordsCache *UnpairedRecordsCache // Shared by cdrc so we can cache for all files in a folder - partialRecordsCache *PartialRecordsCache // Cache records which are of type "Partial" - partialCacheDumpFields []*config.FCTemplate - filterS *engine.FilterS - connMgr *engine.ConnManager -} - -func (self *CsvRecordsProcessor) ProcessedRecordsNr() int64 { - return self.processedRecordsNr -} - -func (self *CsvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { - record, err := self.csvReader.Read() - if err != nil { - return nil, err - } - self.processedRecordsNr += 1 - if utils.SliceHasMember([]string{utils.MetaKamFlatstore, utils.MetaOsipsFlatstore}, self.dfltCdrcCfg.CdrFormat) { - if record, err = self.processFlatstoreRecord(record); err != nil { - return nil, err - } else if record == nil { - return nil, nil // Due to partial, none returned - } - } - // Record was overwriten with complete information out of cache - return self.processRecord(record) -} - -// Processes a single partial record for flatstore CDRs -func (self *CsvRecordsProcessor) processFlatstoreRecord(record []string) ([]string, error) { - if strings.HasPrefix(self.fileName, self.dfltCdrcCfg.FailedCallsPrefix) { // Use the first index since they should be the same in all configs - record = append(record, "0") // Append duration 0 for failed calls flatstore CDR and do not process it further - return record, nil - } - pr, err := NewUnpairedRecord(record, self.timezone) - if err != nil { - return nil, err - } - // Retrieve and complete the record from cache - cachedFilename, cachedPartial := self.unpairedRecordsCache.GetPartialRecord(pr.OriginID, self.fileName) - if cachedPartial == nil { // Not cached, do it here and stop processing - self.unpairedRecordsCache.CachePartial(self.fileName, pr) - return nil, nil - } - pairedRecord, err := pairToRecord(cachedPartial, pr) - if err != nil { - return nil, err - } - self.unpairedRecordsCache.UncachePartial(cachedFilename, pr) - return pairedRecord, nil -} - -// Takes the record from a slice and turns it into StoredCdrs, posting them to the cdrServer -func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.CDR, error) { - csvProvider := config.NewSliceDP(record, utils.MetaReq) - recordCdrs := make([]*engine.CDR, 0) // More CDRs based on the number of filters and field templates - for _, cdrcCfg := range self.cdrcCfgs { // cdrFields coming from more templates will produce individual storCdr records - tenant, err := cdrcCfg.Tenant.ParseDataProvider(csvProvider, utils.NestingSep) // each profile of cdrc can have different tenant - if err != nil { - return nil, err - } - if tenant == "" { - tenant = config.CgrConfig().GeneralCfg().DefaultTenant - } - // Make sure filters are matching - if len(cdrcCfg.Filters) != 0 { - if pass, err := self.filterS.Pass(tenant, - cdrcCfg.Filters, csvProvider); err != nil || !pass { - continue // Not passes filters, ignore this CDR - } - } - storedCdr, err := self.recordToStoredCdr(record, cdrcCfg, tenant) - if err != nil { - return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error()) - } else if self.dfltCdrcCfg.CdrFormat == utils.MetaPartialCSV { - if storedCdr, err = self.partialRecordsCache.MergePartialCDRRecord(NewPartialCDRRecord(storedCdr, self.partialCacheDumpFields)); err != nil { - return nil, fmt.Errorf("Failed merging PartialCDR, error: %s", err.Error()) - } else if storedCdr == nil { // CDR was absorbed by cache since it was partial - continue - } - } - recordCdrs = append(recordCdrs, storedCdr) - if !cdrcCfg.ContinueOnSuccess { - break - } - } - return recordCdrs, nil -} - -// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS -func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cdrcCfg *config.CdrcCfg, tenant string) (*engine.CDR, error) { - storedCdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} - var err error - csvProvider := config.NewSliceDP(record, utils.MetaReq) // used for filterS and for RSRParsers - var lazyHttpFields []*config.FCTemplate - fldVals := make(map[string]string) - for _, cdrFldCfg := range cdrcCfg.ContentFields { - if len(cdrFldCfg.Filters) != 0 { - if pass, err := self.filterS.Pass(tenant, - cdrFldCfg.Filters, csvProvider); err != nil { - return nil, err - } else if !pass { - continue - } - } - if utils.SliceHasMember([]string{utils.MetaKamFlatstore, utils.MetaOsipsFlatstore}, self.dfltCdrcCfg.CdrFormat) { // Hardcode some values in case of flatstore - switch cdrFldCfg.FieldId { - case utils.OriginID: - cdrFldCfg.Value = config.NewRSRParsersMustCompile("~*req.3;~*req.1;~*req.2", true, utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag - case utils.Usage: - cdrFldCfg.Value = config.NewRSRParsersMustCompile("~*req."+strconv.Itoa(len(record)-1), true, utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us - } - } - switch cdrFldCfg.Type { - case utils.META_COMPOSED: - out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider, utils.NestingSep) - if err != nil { - return nil, err - } - fldVals[cdrFldCfg.FieldId] += out - case utils.MetaUnixTimestamp: - out, err := cdrFldCfg.Value.ParseDataProvider(csvProvider, utils.NestingSep) - if err != nil { - return nil, err - } - t, err := utils.ParseTimeDetectLayout(out, self.timezone) - if err != nil { - return nil, err - } - fldVals[cdrFldCfg.FieldId] += strconv.Itoa(int(t.Unix())) - case utils.META_HTTP_POST: - lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server - default: - return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type) - } - if err := storedCdr.ParseFieldValue(cdrFldCfg.FieldId, fldVals[cdrFldCfg.FieldId], self.timezone); err != nil { - return nil, err - } - } - storedCdr.CGRID = utils.Sha1(storedCdr.OriginID, storedCdr.OriginHost) - for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields - var outValByte []byte - var fieldVal, httpAddr string - for _, rsrFld := range httpFieldCfg.Value { - if parsed, err := rsrFld.ParseValue(utils.EmptyString); err != nil { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract http address for field %+v, err: %s", - record, rsrFld, err.Error()) - } else { - httpAddr += parsed - } - } - var jsn []byte - jsn, err = json.Marshal(storedCdr) - if err != nil { - return nil, err - } - if outValByte, err = engine.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { - return nil, err - } else { - fieldVal = string(outValByte) - if len(fieldVal) == 0 && httpFieldCfg.Mandatory { - return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag) - } - if err := storedCdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, self.timezone); err != nil { - return nil, err - } - } - } - return storedCdr, nil -} diff --git a/cdrc/csv_it_test.go b/cdrc/csv_it_test.go deleted file mode 100644 index fc5ff534e..000000000 --- a/cdrc/csv_it_test.go +++ /dev/null @@ -1,587 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "errors" - "flag" - "io/ioutil" - "net/rpc" - "net/rpc/jsonrpc" - "os" - "path" - "testing" - "time" - - v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -/* -README: - - Enable local tests by passing '-local' to the go test command - It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data or passed via command arguments. - Prior running the tests, create database and users by running: - mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql - What these tests do: - * Flush tables in storDb. - * Start engine with default configuration and give it some time to listen (here caching can slow down). - * -*/ - -var ( - csvCfgPath string - csvCfg *config.CGRConfig - cdrcCfgs []*config.CdrcCfg - cdrcCfg *config.CdrcCfg - cdrcRpc *rpc.Client - - dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") - waitRater = flag.Int("wait_rater", 500, "Number of miliseconds to wait for rater to start and cache") - encoding = flag.String("rpc", utils.MetaJSON, "what encoding whould be uused for rpc comunication") - - fileContent1 = `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,*rated,*out,cgrates.org,call,1001,1001,+4986517174963,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10s,1.0100,val_extra3,"",val_extra1 -dbafe9c8614c785a65aabd116dd3959c3c56f7f7,default,*voice,dsafdsag,*rated,*out,cgrates.org,call,1001,1001,+4986517174964,2013-11-07 09:42:25 +0000 UTC,2013-11-07 09:42:26 +0000 UTC,20s,1.0100,val_extra3,"",val_extra1 -` - - fileContent2 = `accid21;*prepaid;itsyscom.com;1001;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 -accid22;*postpaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 -#accid1;*pseudoprepaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;12;val_extra3;"";val_extra1 -accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;"";val_extra1` -) - -func newRPCClient(cfg *config.ListenCfg) (c *rpc.Client, err error) { - switch *encoding { - case utils.MetaJSON: - return jsonrpc.Dial(utils.TCP, cfg.RPCJSONListen) - case utils.MetaGOB: - return rpc.Dial(utils.TCP, cfg.RPCGOBListen) - default: - return nil, errors.New("UNSUPPORTED_RPC") - } -} - -func TestCsvITInitConfig(t *testing.T) { - var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") - if csvCfg, err = config.NewCGRConfigFromPath(csvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestCsvITInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -// Remove data in both rating and accounting db -func TestCsvITResetDataDb(t *testing.T) { - if err := engine.InitDataDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -func TestCsvITCreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range csvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestCsvITStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestCsvITRpcConn(t *testing.T) { - var err error - cdrcRpc, err = newRPCClient(csvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestCsvITHandleCdr1File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctests/csvit1/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -// Scenario out of first .xml config -func TestCsvITHandleCdr2File(t *testing.T) { - fileName := "file2.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent2), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cdrctests/csvit2/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestCsvITProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile("/tmp/cdrctests/csvit1/out/file1.csv"); err != nil { - t.Error(err) - } else if fileContent1 != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", fileContent1, string(outContent1)) - } - if outContent2, err := ioutil.ReadFile("/tmp/cdrctests/csvit2/out/file2.csv"); err != nil { - t.Error(err) - } else if fileContent2 != string(outContent2) { - t.Errorf("Expecting: %q, received: %q", fileContent1, string(outContent2)) - } -} - -func TestCsvITAnalyseCDRs(t *testing.T) { - time.Sleep(500 * time.Millisecond) - var reply []*engine.ExternalCDR - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 5 { // 1 injected, 1 rated, 1 *raw and it's pair in *default run - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{DestinationPrefixes: []string{"08651"}}, - &reply); err == nil || err.Error() != utils.NotFoundCaps { - t.Error("Unexpected error: ", err) // Original 08651 was converted - } -} - -func TestCsvITKillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc csv with new filters -var fileContent1_2 = `accid21;*prepaid;itsyscom.com;1002;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 -accid22;*postpaid;itsyscom.com;1002;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 -accid23;*rated;cgrates.org;1001;086517174963;2013-02-03 19:54:00;26;val_extra3;"";val_extra1` - -func TestCsvIT2InitConfig(t *testing.T) { - var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") - if csvCfg, err = config.NewCGRConfigFromPath(csvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestCsvIT2InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -func TestCsvIT2CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range csvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestCsvIT2StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestCsvIT2RpcConn(t *testing.T) { - var err error - cdrcRpc, err = newRPCClient(csvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// Scenario out of first .xml config -func TestCsvIT2HandleCdr2File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_2), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter1/csvit1/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestCsvIT2ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter1/csvit1/out/file1.csv"); err != nil { - t.Error(err) - } else if fileContent1_2 != string(outContent2) { - t.Errorf("Expecting: %q, received: %q", fileContent1_2, string(outContent2)) - } -} - -func TestCsvIT2AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{RequestTypes: []string{utils.META_PREPAID}}, &reply); err != nil { - t.Error("Unexpected error: ", err) // Original 08651 was converted - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestCsvIT2KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) -} - -// Begin tests for cdrc csv with new filters -var fileContent1_3 = `accid21;*prepaid;itsyscom.com;1002;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 -accid22;*prepaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 -accid23;*prepaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra3;"";val_extra1` - -func TestCsvIT3InitConfig(t *testing.T) { - var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") - if csvCfg, err = config.NewCGRConfigFromPath(csvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestCsvIT3InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -func TestCsvIT3CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range csvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestCsvIT3StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestCsvIT3RpcConn(t *testing.T) { - var err error - cdrcRpc, err = newRPCClient(csvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// Scenario out of first .xml config -func TestCsvIT3HandleCdr2File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_3), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter2/csvit2/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestCsvIT3ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent2, err := ioutil.ReadFile("/tmp/csvwithfilter2/csvit2/out/file1.csv"); err != nil { - t.Error(err) - } else if fileContent1_3 != string(outContent2) { - t.Errorf("Expecting: %q, received: %q", fileContent1_3, string(outContent2)) - } -} - -func TestCsvIT3AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestCsvIT3KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) -} - -// Begin tests for cdrc csv with new filters -var fileContent1_4 = `accid21;*prepaid;itsyscom.com;1002;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1 -accid22;*postpaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1 -accid23;*postpaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra3;"";val_extra1` - -func TestCsvIT4InitConfig(t *testing.T) { - var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") - if csvCfg, err = config.NewCGRConfigFromPath(csvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestCsvIT4InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -func TestCsvIT4CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range csvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestCsvIT4StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestCsvIT4RpcConn(t *testing.T) { - var err error - cdrcRpc, err = newRPCClient(csvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// Scenario out of first .xml config -func TestCsvIT4HandleCdr2File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_4), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter3/csvit3/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestCsvIT4ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent4, err := ioutil.ReadFile("/tmp/csvwithfilter3/csvit3/out/file1.csv"); err != nil { - t.Error(err) - } else if fileContent1_4 != string(outContent4) { - t.Errorf("Expecting: %q, received: %q", fileContent1_4, string(outContent4)) - } -} - -func TestCsvIT4AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestCsvIT4KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc csv with new filters -var fileContent1_5 = `accid21;*prepaid;itsyscom.com;1002;086517174963;2013-02-03 19:54:00;62;val_extra3;"";val_extra1;10.10.10.10 -accid22;*postpaid;itsyscom.com;1001;+4986517174963;2013-02-03 19:54:00;123;val_extra3;"";val_extra1;11.10.10.10 -accid23;*postpaid;cgrates.org;1002;086517174963;2013-02-03 19:54:00;76;val_extra3;"";val_extra1;12.10.10.10 -accid24;*postpaid;cgrates.org;1001;+4986517174963;2013-02-03 19:54:00;76;val_extra3;"";val_extra1;12.10.10.10` - -func TestCsvIT5InitConfig(t *testing.T) { - var err error - csvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrccsv") - if csvCfg, err = config.NewCGRConfigFromPath(csvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestCsvIT5InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(csvCfg); err != nil { - t.Fatal(err) - } -} - -func TestCsvIT5CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range csvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestCsvIT5StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(csvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestCsvIT5RpcConn(t *testing.T) { - var err error - cdrcRpc, err = newRPCClient(csvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func TestCsvIT5AddFilters(t *testing.T) { - filter := v1.FilterWithCache{ - Filter: &engine.Filter{ - Tenant: "cgrates.org", - ID: "FLTR_CDRC_ACC", - Rules: []*engine.FilterRule{ - { - Type: utils.MetaString, - Element: "~*req.3", - Values: []string{"1002"}, - }, - }, - }, - } - var result string - if err := cdrcRpc.Call(utils.ApierV1SetFilter, filter, &result); err != nil { - t.Error(err) - } else if result != utils.OK { - t.Error("Unexpected reply returned", result) - } - filter2 := v1.FilterWithCache{ - Filter: &engine.Filter{ - Tenant: "itsyscom.com", - ID: "FLTR_CDRC_ACC", - Rules: []*engine.FilterRule{ - { - Type: utils.MetaString, - Element: "~*req.3", - Values: []string{"1001"}, - }, - }, - }, - } - if err := cdrcRpc.Call(utils.ApierV1SetFilter, filter2, &result); err != nil { - t.Error(err) - } else if result != utils.OK { - t.Error("Unexpected reply returned", result) - } -} - -// Scenario out of first .xml config -func TestCsvIT5HandleCdr2File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1_5), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/csvwithfilter4/csvit4/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestCsvIT5ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent4, err := ioutil.ReadFile("/tmp/csvwithfilter4/csvit4/out/file1.csv"); err != nil { - t.Error(err) - } else if fileContent1_5 != string(outContent4) { - t.Errorf("Expecting: %q, received: %q", fileContent1_5, string(outContent4)) - } -} - -func TestCsvIT5AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestCsvIT5KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/cdrc/csv_test.go b/cdrc/csv_test.go deleted file mode 100644 index 9cd2f08a2..000000000 --- a/cdrc/csv_test.go +++ /dev/null @@ -1,189 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func TestCsvRecordToCDR(t *testing.T) { - cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] - cdrcConfig.CdrSourceId = "TEST_CDRC" - cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.FCTemplate{ - Tag: utils.RunID, Type: utils.META_COMPOSED, FieldId: utils.RunID, - Value: config.NewRSRParsersMustCompile(utils.MetaDefault, true, utils.INFIELD_SEP)}) - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcCfg{cdrcConfig}} - cdrRow := []string{"firstField", "secondField"} - _, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig, "cgrates.org") - if err == nil { - t.Error("Failed to corectly detect missing fields from record") - } - cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", utils.META_PREPAID, "*out", "cgrates.org", - "call", "1001", "1001", "+4986517174963", "2013-02-03 19:50:00", "2013-02-03 19:54:00", - "62s", "supplier1", "172.16.1.1", "NORMAL_DISCONNECT"} - rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, cdrcConfig, "cgrates.org") - if err != nil { - t.Error("Failed to parse CDR in rated cdr", err) - } - expectedCdr := &engine.CDR{ - CGRID: utils.Sha1(cdrRow[3], "0.0.0.0"), - RunID: utils.MetaDefault, - ToR: cdrRow[2], - OriginID: cdrRow[3], - OriginHost: "0.0.0.0", // Got it over internal interface - Source: "TEST_CDRC", - RequestType: cdrRow[4], - Tenant: cdrRow[6], - Category: cdrRow[7], - Account: cdrRow[8], - Subject: cdrRow[9], - Destination: cdrRow[10], - SetupTime: time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC), - AnswerTime: time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC), - Usage: time.Duration(62) * time.Second, - ExtraFields: map[string]string{}, - Cost: -1, - } - if !reflect.DeepEqual(expectedCdr, rtCdr) { - t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) - } -} - -func TestCsvDataMultiplyFactor(t *testing.T) { - cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] - data := engine.NewInternalDB(nil, nil, true, cgrConfig.DataDbCfg().Items) - dm := engine.NewDataManager(data, cgrConfig.CacheCfg(), nil) - filterS := engine.NewFilterS(cgrConfig, nil, dm) - cdrcConfig.CdrSourceId = "TEST_CDRC" - cdrcConfig.ContentFields = []*config.FCTemplate{ - {Tag: "TORField", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: config.NewRSRParsersMustCompile("~*req.0", true, utils.INFIELD_SEP)}, - {Tag: "UsageField", Type: utils.META_COMPOSED, FieldId: utils.Usage, Filters: []string{"*notstring:~*req.0:*data"}, - Value: config.NewRSRParsersMustCompile("~*req.1", true, utils.INFIELD_SEP)}, - {Tag: "UsageField", Type: utils.META_COMPOSED, FieldId: utils.Usage, Filters: []string{"*string:~*req.0:*data"}, - Value: config.NewRSRParsersMustCompile("~*req.1{*multiply:1024}", true, utils.INFIELD_SEP)}, - } - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcCfg{cdrcConfig}, filterS: filterS} - cdrRow := []string{"*data", "1"} - expectedCdr := &engine.CDR{ - CGRID: utils.Sha1("", "0.0.0.0"), - ToR: cdrRow[0], - OriginHost: "0.0.0.0", - Source: "TEST_CDRC", - Usage: time.Duration(1024), - ExtraFields: map[string]string{}, - Cost: -1, - } - if rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, - cdrcConfig, "cgrates.org"); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expectedCdr, rtCdr) { - t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) - } - cdrRow = []string{"*voice", "1s"} - expectedCdr = &engine.CDR{ - CGRID: utils.Sha1("", "0.0.0.0"), - ToR: cdrRow[0], - OriginHost: "0.0.0.0", - Source: "TEST_CDRC", - Usage: time.Duration(1) * time.Second, - ExtraFields: map[string]string{}, - Cost: -1, - } - if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, - cdrcConfig, "cgrates.org"); !reflect.DeepEqual(expectedCdr, rtCdr) { - t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) - } -} - -func TestCsvPairToRecord(t *testing.T) { - eRecord := []string{"INVITE", "2daec40c", "548625ac", - "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", utils.OK, "1436454408", - "*prepaid", "1001", "1002", "", "3401:2069362475", "2"} - invPr := &UnpairedRecord{Method: "INVITE", - Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC), - Values: []string{"INVITE", "2daec40c", "548625ac", - "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", utils.OK, - "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}} - byePr := &UnpairedRecord{Method: "BYE", Timestamp: time.Date(2015, 7, 9, 15, 6, 50, 0, time.UTC), - Values: []string{"BYE", "2daec40c", "548625ac", - "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", utils.OK, - "1436454410", "", "", "", "", "3401:2069362475"}} - if rec, err := pairToRecord(invPr, byePr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eRecord, rec) { - t.Errorf("Expected: %+v, received: %+v", eRecord, rec) - } - if rec, err := pairToRecord(byePr, invPr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eRecord, rec) { - t.Errorf("Expected: %+v, received: %+v", eRecord, rec) - } - if _, err := pairToRecord(byePr, byePr); err == nil || err.Error() != "MISSING_INVITE" { - t.Error(err) - } - if _, err := pairToRecord(invPr, invPr); err == nil || err.Error() != "MISSING_BYE" { - t.Error(err) - } - byePr.Values = []string{"BYE", "2daec40c", "548625ac", - "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", utils.OK, - "1436454410", "", "", "", "3401:2069362475"} // Took one value out - if _, err := pairToRecord(invPr, byePr); err == nil || err.Error() != "INCONSISTENT_VALUES_LENGTH" { - t.Error(err) - } -} - -func TestCsvSecondUsage(t *testing.T) { - cgrConfig, _ := config.NewDefaultCGRConfig() - cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] - data := engine.NewInternalDB(nil, nil, true, cgrConfig.DataDbCfg().Items) - dm := engine.NewDataManager(data, cgrConfig.CacheCfg(), nil) - filterS := engine.NewFilterS(cgrConfig, nil, dm) - cdrcConfig.CdrSourceId = "TEST_CDRC" - cdrcConfig.ContentFields = []*config.FCTemplate{ - {Tag: "TORField", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: config.NewRSRParsersMustCompile("~*req.0", true, utils.INFIELD_SEP)}, - - {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, - Value: config.NewRSRParsersMustCompile("~*req.1;s", true, utils.INFIELD_SEP)}, - } - csvProcessor := &CsvRecordsProcessor{dfltCdrcCfg: cdrcConfig, cdrcCfgs: []*config.CdrcCfg{cdrcConfig}, filterS: filterS} - - cdrRow := []string{"*voice", "12"} - expectedCdr := &engine.CDR{ - CGRID: utils.Sha1("", "0.0.0.0"), - ToR: cdrRow[0], - OriginHost: "0.0.0.0", - Source: "TEST_CDRC", - Usage: time.Duration(12) * time.Second, - ExtraFields: map[string]string{}, - Cost: -1, - } - if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, - cdrcConfig, "cgrates.org"); !reflect.DeepEqual(expectedCdr, rtCdr) { - t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) - } -} diff --git a/cdrc/flatstore_it_test.go b/cdrc/flatstore_it_test.go deleted file mode 100644 index be16fafca..000000000 --- a/cdrc/flatstore_it_test.go +++ /dev/null @@ -1,187 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "io/ioutil" - "net/rpc" - "os" - "path" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var flatstoreCfgPath string -var flatstoreCfg *config.CGRConfig -var flatstoreRpc *rpc.Client -var flatstoreCdrcCfg *config.CdrcCfg - -var fullSuccessfull = `INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475 -BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475 -INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741 -BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741 -INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533 -BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533 -INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290 -BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290 -` - -var fullMissed = `INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783 -INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844 -INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066` - -var part1 = `BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549742 -` - -var part2 = `INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1003||1877:893549742 -INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9p@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475` - -func TestFlatstoreitInitCfg(t *testing.T) { - var err error - flatstoreCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcflatstore") - if flatstoreCfg, err = config.NewCGRConfigFromPath(flatstoreCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestFlatstoreitInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(flatstoreCfg); err != nil { - t.Fatal(err) - } -} - -// Remove data in both rating and accounting db -func TestFlatstoreitResetDataDb(t *testing.T) { - if err := engine.InitDataDb(flatstoreCfg); err != nil { - t.Fatal(err) - } -} - -// Creates cdr files and moves them into processing folder -func TestFlatstoreitCreateCdrFiles(t *testing.T) { - if flatstoreCfg == nil { - t.Fatal("Empty default cdrc configuration") - } - for _, cdrcCfg := range flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"] { - if cdrcCfg.ID == "FLATSTORE" { - flatstoreCdrcCfg = cdrcCfg - } - } - if err := os.RemoveAll(flatstoreCdrcCfg.CDRInPath); err != nil { - t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CDRInPath, err) - } - if err := os.MkdirAll(flatstoreCdrcCfg.CDRInPath, 0755); err != nil { - t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CDRInPath, err) - } - if err := os.RemoveAll(flatstoreCdrcCfg.CDROutPath); err != nil { - t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CDROutPath, err) - } - if err := os.MkdirAll(flatstoreCdrcCfg.CDROutPath, 0755); err != nil { - t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CDROutPath, err) - } -} - -func TestFlatstoreitStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(flatstoreCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestFlatstoreitRpcConn(t *testing.T) { - var err error - flatstoreRpc, err = newRPCClient(flatstoreCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func TestFlatstoreitProcessFiles(t *testing.T) { - if err := ioutil.WriteFile(path.Join("/tmp", "acc_1.log"), []byte(fullSuccessfull), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := ioutil.WriteFile(path.Join("/tmp", "missed_calls_1.log"), []byte(fullMissed), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := ioutil.WriteFile(path.Join("/tmp", "acc_2.log"), []byte(part1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := ioutil.WriteFile(path.Join("/tmp", "acc_3.log"), []byte(part2), 0644); err != nil { - t.Fatal(err.Error()) - } - //Rename(oldpath, newpath string) - for _, fileName := range []string{"acc_1.log", "missed_calls_1.log", "acc_2.log", "acc_3.log"} { - if err := os.Rename(path.Join("/tmp", fileName), path.Join(flatstoreCdrcCfg.CDRInPath, fileName)); err != nil { - t.Fatal(err) - } - } - time.Sleep(time.Duration(3) * time.Second) // Give time for processing to happen and the .unparired file to be written - filesInDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CDRInPath) - if len(filesInDir) != 0 { - t.Errorf("Files in cdrcInDir: %+v", filesInDir) - } - filesOutDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CDROutPath) - if len(filesOutDir) != 5 { - f := []string{} - for _, s := range filesOutDir { - f = append(f, s.Name()) - t.Errorf("File %s:", s.Name()) - if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CDROutPath, s.Name())); err != nil { - t.Error(err) - } else { - t.Errorf("%s", partContent) - } - t.Errorf("==============================================================================") - } - t.Errorf("In CdrcOutDir, expecting 5 files, got: %d, for %s", len(filesOutDir), utils.ToJSON(f)) - return - } - ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9p@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n" - if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CDROutPath, "acc_3.log.unpaired")); err != nil { - t.Error(err) - } else if (ePartContent) != (string(partContent)) { - t.Errorf("Expecting:\n%s\nReceived:\n%s", ePartContent, string(partContent)) - } -} - -func TestFlatstoreitAnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := flatstoreRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 8 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := flatstoreRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{MinUsage: "1"}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 5 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestFlatstoreitKillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/cdrc/fwv.go b/cdrc/fwv.go deleted file mode 100644 index c576aa032..000000000 --- a/cdrc/fwv.go +++ /dev/null @@ -1,247 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "os" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func NewFwvRecordsProcessor(file *os.File, dfltCfg *config.CdrcCfg, - cdrcCfgs []*config.CdrcCfg, - httpSkipTlsCheck bool, timezone string, filterS *engine.FilterS) *FwvRecordsProcessor { - return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, - httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone, filterS: filterS} -} - -type FwvRecordsProcessor struct { - file *os.File - dfltCfg *config.CdrcCfg // General parameters - cdrcCfgs []*config.CdrcCfg - httpSkipTlsCheck bool - timezone string - lineLen int64 // Length of the line in the file - offset int64 // Index of the next byte to process - processedRecordsNr int64 // Number of content records in file - trailerOffset int64 // Index where trailer starts, to be used as boundary when reading cdrs - headerCdr *engine.CDR // Cache here the general purpose stored CDR - filterS *engine.FilterS -} - -// Sets the line length based on first line, sets offset back to initial after reading -func (self *FwvRecordsProcessor) setLineLen() error { - rdr := bufio.NewReader(self.file) - readBytes, err := rdr.ReadBytes('\n') - if err != nil { - return err - } - self.lineLen = int64(len(readBytes)) - if _, err := self.file.Seek(0, 0); err != nil { - return err - } - return nil -} - -func (self *FwvRecordsProcessor) ProcessedRecordsNr() int64 { - return self.processedRecordsNr -} - -func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.CDR, error) { - defer func() { self.offset += self.lineLen }() // Schedule increasing the offset once we are out from processing the record - if self.offset == 0 { // First time, set the necessary offsets - if err := self.setLineLen(); err != nil { - utils.Logger.Err(fmt.Sprintf(" Row 0, error: cannot set lineLen: %s", err.Error())) - return nil, io.EOF - } - if len(self.dfltCfg.TrailerFields) != 0 { - if fi, err := self.file.Stat(); err != nil { - utils.Logger.Err(fmt.Sprintf(" Row 0, error: cannot get file stats: %s", err.Error())) - return nil, err - } else { - self.trailerOffset = fi.Size() - self.lineLen - } - } - if len(self.dfltCfg.HeaderFields) != 0 { // ToDo: Process here the header fields - if err := self.processHeader(); err != nil { - utils.Logger.Err(fmt.Sprintf(" Row 0, error reading header: %s", err.Error())) - return nil, io.EOF - } - return nil, nil - } - } - recordCdrs := make([]*engine.CDR, 0) // More CDRs based on the number of filters and field templates - if self.trailerOffset != 0 && self.offset >= self.trailerOffset { - if err := self.processTrailer(); err != nil && err != io.EOF { - utils.Logger.Err(fmt.Sprintf(" Read trailer error: %s ", err.Error())) - } - return nil, io.EOF - } - buf := make([]byte, self.lineLen) - nRead, err := self.file.Read(buf) - if err != nil { - return nil, err - } else if nRead != len(buf) { - utils.Logger.Err(fmt.Sprintf(" Could not read complete line, have instead: %s", string(buf))) - return nil, io.EOF - } - self.processedRecordsNr += 1 - record := string(buf) - fwvProvider := config.NewFWVProvider(record, utils.MetaReq) - for _, cdrcCfg := range self.cdrcCfgs { - tenant, err := cdrcCfg.Tenant.ParseDataProvider(fwvProvider, utils.NestingSep) // each profile of cdrc can have different tenant - if err != nil { - return nil, err - } - if tenant == "" { - tenant = config.CgrConfig().GeneralCfg().DefaultTenant - } - if len(cdrcCfg.Filters) != 0 { - if pass, err := self.filterS.Pass(tenant, - cdrcCfg.Filters, fwvProvider); err != nil || !pass { - continue // Not passes filters, ignore this CDR - } - } - if storedCdr, err := self.recordToStoredCdr(record, cdrcCfg, cdrcCfg.ID); err != nil { - return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error()) - } else { - recordCdrs = append(recordCdrs, storedCdr) - } - if !cdrcCfg.ContinueOnSuccess { // Successfully executed one config, do not continue for next one - break - } - } - return recordCdrs, nil -} - -// Converts a record (header or normal) to CDR -func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cdrcCfg *config.CdrcCfg, cfgKey string) (*engine.CDR, error) { - var err error - var lazyHttpFields []*config.FCTemplate - var cfgFields []*config.FCTemplate - var storedCdr *engine.CDR - fwvProvider := config.NewFWVProvider(record, utils.MetaReq) // used for filterS and for RSRParsers - if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there) - storedCdr = self.headerCdr.Clone() - } else { - storedCdr = &engine.CDR{OriginHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1} - } - if cfgKey == "*header" { - cfgFields = cdrcCfg.HeaderFields - storedCdr.Source = cdrcCfg.CdrSourceId - } else { - cfgFields = cdrcCfg.ContentFields - storedCdr.Source = cdrcCfg.CdrSourceId - } - fldVals := make(map[string]string) - for _, cdrFldCfg := range cfgFields { - if len(cdrFldCfg.Filters) != 0 { - tenant, err := cdrcCfg.Tenant.ParseValue("") - if err != nil { - return nil, err - } - if pass, err := self.filterS.Pass(tenant, - cdrFldCfg.Filters, fwvProvider); err != nil || !pass { - continue // Not passes filters, ignore this CDR - } - } - switch cdrFldCfg.Type { - case utils.META_COMPOSED: - out, err := cdrFldCfg.Value.ParseDataProvider(fwvProvider, utils.NestingSep) - if err != nil { - return nil, err - } - fldVals[cdrFldCfg.FieldId] += out - case utils.META_HTTP_POST: - lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server - default: - //return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type) - continue // Don't do anything for unsupported fields - } - if fldVals[cdrFldCfg.FieldId], err = utils.FmtFieldWidth(cdrFldCfg.Tag, fldVals[cdrFldCfg.FieldId], cdrFldCfg.Width, - cdrFldCfg.Strip, cdrFldCfg.Padding, cdrFldCfg.Mandatory); err != nil { - return nil, err - } - if err := storedCdr.ParseFieldValue(cdrFldCfg.FieldId, fldVals[cdrFldCfg.FieldId], self.timezone); err != nil { - return nil, err - } - } - if storedCdr.CGRID == "" && storedCdr.OriginID != "" && cfgKey != "*header" { - storedCdr.CGRID = utils.Sha1(storedCdr.OriginID, storedCdr.OriginHost) - } - for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields - var outValByte []byte - var fieldVal, httpAddr string - for _, rsrFld := range httpFieldCfg.Value { - if parsed, err := rsrFld.ParseValue(utils.EmptyString); err != nil { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract http address, err: %s", - record, err.Error()) - } else { - httpAddr += parsed - } - } - var jsn []byte - jsn, err = json.Marshal(storedCdr) - if err != nil { - return nil, err - } - if outValByte, err = engine.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { - return nil, err - } else { - fieldVal = string(outValByte) - if len(fieldVal) == 0 && httpFieldCfg.Mandatory { - return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag) - } - if err := storedCdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, self.timezone); err != nil { - return nil, err - } - } - } - return storedCdr, nil -} - -func (self *FwvRecordsProcessor) processHeader() error { - buf := make([]byte, self.lineLen) - if nRead, err := self.file.Read(buf); err != nil { - return err - } else if nRead != len(buf) { - return fmt.Errorf("In header, line len: %d, have read: %d", self.lineLen, nRead) - } - var err error - if self.headerCdr, err = self.recordToStoredCdr(string(buf), self.dfltCfg, "*header"); err != nil { - return err - } - return nil -} - -func (self *FwvRecordsProcessor) processTrailer() error { - buf := make([]byte, self.lineLen) - if nRead, err := self.file.ReadAt(buf, self.trailerOffset); err != nil { - return err - } else if nRead != len(buf) { - return fmt.Errorf("In trailer, line len: %d, have read: %d", self.lineLen, nRead) - } - return nil -} diff --git a/cdrc/fwv_it_test.go b/cdrc/fwv_it_test.go deleted file mode 100644 index 225c4864d..000000000 --- a/cdrc/fwv_it_test.go +++ /dev/null @@ -1,381 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -// -import ( - "io/ioutil" - "net/rpc" - "os" - "path" - "testing" - "time" - - v1 "github.com/cgrates/cgrates/apier/v1" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var fwvCfgPath string -var fwvCfg *config.CGRConfig -var fwvRpc *rpc.Client -var fwvCdrcCfg *config.CdrcCfg - -var FW_CDR_FILE1 = `HDR0001DDB ABC Some Connect A.B. DDB-Some-10022-20120711-309.CDR 00030920120711100255 -CDR0000010 0 20120708181506000123451234 0040123123120 004 000018009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009 -CDR0000020 0 20120708190945000123451234 0040123123120 004 000016009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009 -CDR0000030 0 20120708191009000123451234 0040123123120 004 000020009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009 -CDR0000040 0 20120708231043000123451234 0040123123120 004 000011009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009 -CDR0000050 0 20120709122216000123451235 004212 004 000217009980010001ISDN ABC 10Buiten uw regio HMR 00000000190000000000 -CDR0000060 0 20120709130542000123451236 0012323453 004 000019009980010001ISDN ABC 35Sterdiensten AP 00000000190000000000 -CDR0000070 0 20120709140032000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000080 0 20120709140142000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000090 0 20120709150305000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000100 0 20120709150414000123451237 0040012323453100 001 000057009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000110 0 20120709150531000123451237 0040012323453100 001 000059009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000120 0 20120709150635000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000130 0 20120709151756000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000140 0 20120709154549000123451237 0040012323453100 001 000052009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000150 0 20120709154701000123451237 0040012323453100 001 000121009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000160 0 20120709154842000123451237 0040012323453100 001 000055009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000170 0 20120709154956000123451237 0040012323453100 001 000115009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000180 0 20120709155131000123451237 0040012323453100 001 000059009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000190 0 20120709155236000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000200 0 20120709160309000123451237 0040012323453100 001 000100009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000210 0 20120709160415000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000220 0 20120709161739000123451237 0040012323453100 001 000058009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000230 0 20120709170356000123123459 0040123234531 004 000012002760010001ISDN 276 10Buiten uw regio TB 00000009190000000009 -CDR0000240 0 20120709181036000123123450 0012323453 004 000042009980010001ISDN ABC 05Binnen uw regio AP 00000010190000000010 -CDR0000250 0 20120709191245000123123458 0040123232350 004 000012002760000001PSTN 276 10Buiten uw regio TB 00000009190000000009 -CDR0000260 0 20120709202324000123123459 0040123234531 004 000011002760010001ISDN 276 10Buiten uw regio TB 00000009190000000009 -CDR0000270 0 20120709211756000123451237 0040012323453100 001 000051009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000280 0 20120709211852000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000290 0 20120709212904000123123458 0040123232350 004 000012002760000001PSTN 276 10Buiten uw regio TB 00000009190000000009 -CDR0000300 0 20120709073707000123123459 0040123234531 004 000012002760010001ISDN 276 10Buiten uw regio TB 00000009190000000009 -CDR0000310 0 20120709085451000123451237 0040012323453100 001 000744009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000320 0 20120709091756000123451237 0040012323453100 001 000050009980030001ISDN ABD 20Internationaal NLB 00000000190000000000 -CDR0000330 0 20120710070434000123123458 0040123232350 004 000012002760000001PSTN 276 10Buiten uw regio TB 00000009190000000009 -TRL0001DDB ABC Some Connect A.B. DDB-Some-10022-20120711-309.CDR 0003090000003300000030550000000001000000000100Y -` - -func TestFwvitInitCfg(t *testing.T) { - var err error - fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwv") - if fwvCfg, err = config.NewCGRConfigFromPath(fwvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// Creates cdr files and moves them into processing folder -func TestFwvitCreateCdrFiles(t *testing.T) { - if fwvCfg == nil { - t.Fatal("Empty default cdrc configuration") - } - for _, cdrcCfg := range fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"] { - if cdrcCfg.ID == "FWV1" { - fwvCdrcCfg = cdrcCfg - } - } - for _, cdrcProfiles := range fwvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestFwvitStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestFwvitRpcConn(t *testing.T) { - var err error - fwvRpc, err = newRPCClient(fwvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestFwvitInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(fwvCfg); err != nil { - t.Fatal(err) - } -} - -// Remove data in both rating and accounting db -func TestFwvitResetDataDb(t *testing.T) { - if err := engine.InitDataDb(fwvCfg); err != nil { - t.Fatal(err) - } -} - -func TestFwvitProcessFiles(t *testing.T) { - fileName := "test1.fwv" - if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0755); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CDRInPath, fileName)); err != nil { - t.Fatal(err) - } - time.Sleep(time.Duration(1) * time.Second) - filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDRInPath) - if len(filesInDir) != 0 { - t.Errorf("Files in cdrcInDir: %d", len(filesInDir)) - } - filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDROutPath) - if len(filesOutDir) != 1 { - t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir)) - } -} - -func TestFwvitAnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := fwvRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 4 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := fwvRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{OriginIDs: []string{"CDR0000010"}}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestFwvitKillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc fwv with new filters -func TestFwvit2InitCfg(t *testing.T) { - var err error - fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwvwithfilter") - if fwvCfg, err = config.NewCGRConfigFromPath(fwvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// Creates cdr files and moves them into processing folder -func TestFwvit2CreateCdrFiles(t *testing.T) { - if fwvCfg == nil { - t.Fatal("Empty default cdrc configuration") - } - for _, cdrcCfg := range fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"] { - if cdrcCfg.ID == "FWVWithFilter" { - fwvCdrcCfg = cdrcCfg - } - } - if err := os.RemoveAll(fwvCdrcCfg.CDRInPath); err != nil { - t.Fatal("Error removing folder: ", fwvCdrcCfg.CDRInPath, err) - } - if err := os.MkdirAll(fwvCdrcCfg.CDRInPath, 0755); err != nil { - t.Fatal("Error creating folder: ", fwvCdrcCfg.CDRInPath, err) - } - if err := os.RemoveAll(fwvCdrcCfg.CDROutPath); err != nil { - t.Fatal("Error removing folder: ", fwvCdrcCfg.CDROutPath, err) - } - if err := os.MkdirAll(fwvCdrcCfg.CDROutPath, 0755); err != nil { - t.Fatal("Error creating folder: ", fwvCdrcCfg.CDROutPath, err) - } -} - -func TestFwvit2StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestFwvit2RpcConn(t *testing.T) { - var err error - fwvRpc, err = newRPCClient(fwvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestFwvit2InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(fwvCfg); err != nil { - t.Fatal(err) - } -} - -func TestFwvit2ProcessFiles(t *testing.T) { - fileName := "test1.fwv" - if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CDRInPath, fileName)); err != nil { - t.Fatal(err) - } - time.Sleep(time.Duration(1) * time.Second) - filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDRInPath) - if len(filesInDir) != 0 { - t.Errorf("Files in cdrcInDir: %d", len(filesInDir)) - } - filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDROutPath) - if len(filesOutDir) != 1 { - t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir)) - } -} - -func TestFwvit2AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := fwvRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestFwvit2KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc fwv with new filters -func TestFwvit3InitCfg(t *testing.T) { - var err error - fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwvwithfilter") - if fwvCfg, err = config.NewCGRConfigFromPath(fwvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestFwvit3InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(fwvCfg); err != nil { - t.Fatal(err) - } -} - -// Creates cdr files and moves them into processing folder -func TestFwvit3CreateCdrFiles(t *testing.T) { - if fwvCfg == nil { - t.Fatal("Empty default cdrc configuration") - } - for _, cdrcCfg := range fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"] { - if cdrcCfg.ID == "FWVWithFilterID" { - fwvCdrcCfg = cdrcCfg - } - } - if err := os.RemoveAll(fwvCdrcCfg.CDRInPath); err != nil { - t.Fatal("Error removing folder: ", fwvCdrcCfg.CDRInPath, err) - } - if err := os.MkdirAll(fwvCdrcCfg.CDRInPath, 0755); err != nil { - t.Fatal("Error creating folder: ", fwvCdrcCfg.CDRInPath, err) - } - if err := os.RemoveAll(fwvCdrcCfg.CDROutPath); err != nil { - t.Fatal("Error removing folder: ", fwvCdrcCfg.CDROutPath, err) - } - if err := os.MkdirAll(fwvCdrcCfg.CDROutPath, 0755); err != nil { - t.Fatal("Error creating folder: ", fwvCdrcCfg.CDROutPath, err) - } -} - -func TestFwvit3StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestFwvit3RpcConn(t *testing.T) { - var err error - fwvRpc, err = newRPCClient(fwvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func TestFwvit3AddFilters(t *testing.T) { - filter := v1.FilterWithCache{ - Filter: &engine.Filter{ - Tenant: "cgrates.org", - ID: "FLTR_FWV", - Rules: []*engine.FilterRule{ - { - Type: utils.MetaString, - Element: "0-10", - Values: []string{"CDR0000010"}, - }, - }, - }, - } - var result string - if err := fwvRpc.Call(utils.ApierV1SetFilter, filter, &result); err != nil { - t.Error(err) - } else if result != utils.OK { - t.Error("Unexpected reply returned", result) - } -} - -func TestFwvit3ProcessFiles(t *testing.T) { - fileName := "test1.fwv" - if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CDRInPath, fileName)); err != nil { - t.Fatal(err) - } - time.Sleep(time.Duration(1) * time.Second) - filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDRInPath) - if len(filesInDir) != 0 { - t.Errorf("Files in cdrcInDir: %d", len(filesInDir)) - } - filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CDROutPath) - if len(filesOutDir) != 1 { - t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir)) - } -} - -func TestFwvit3AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := fwvRpc.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestFwvit3KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/cdrc/fwv_test.go b/cdrc/fwv_test.go deleted file mode 100644 index aca6d2989..000000000 --- a/cdrc/fwv_test.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc diff --git a/cdrc/partial_cdr.go b/cdrc/partial_cdr.go deleted file mode 100644 index 23afbce29..000000000 --- a/cdrc/partial_cdr.go +++ /dev/null @@ -1,271 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "encoding/csv" - "fmt" - "os" - "path" - "reflect" - "sort" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/guardian" - "github.com/cgrates/cgrates/utils" -) - -const ( - PartialRecordsSuffix = "partial" -) - -func NewPartialRecordsCache(ttl time.Duration, expiryAction string, cdrOutDir string, csvSep rune, - timezone string, httpSkipTlsCheck bool, - filterS *engine.FilterS, cdrsConnIDs []string, connMgr *engine.ConnManager) *PartialRecordsCache { - return &PartialRecordsCache{ttl: ttl, expiryAction: expiryAction, cdrOutDir: cdrOutDir, - csvSep: csvSep, timezone: timezone, - httpSkipTlsCheck: httpSkipTlsCheck, - partialRecords: make(map[string]*PartialCDRRecord), - dumpTimers: make(map[string]*time.Timer), - guard: guardian.Guardian, filterS: filterS, - connMgr: connMgr, - cdrsConnIDs: cdrsConnIDs} -} - -type PartialRecordsCache struct { - ttl time.Duration - expiryAction string - cdrOutDir string - csvSep rune - timezone string - httpSkipTlsCheck bool - partialRecords map[string]*PartialCDRRecord // [OriginID]*PartialRecord - dumpTimers map[string]*time.Timer // [OriginID]*time.Timer which can be canceled or reset - guard *guardian.GuardianLocker - filterS *engine.FilterS - connMgr *engine.ConnManager - cdrsConnIDs []string -} - -// Dumps the cache into a .unpaired file in the outdir and cleans cache after -func (prc *PartialRecordsCache) dumpPartialRecords(originID string) { - _, err := prc.guard.Guard(func() (interface{}, error) { - if prc.partialRecords[originID].Len() != 0 { // Only write the file if there are records in the cache - dumpFilePath := path.Join(prc.cdrOutDir, fmt.Sprintf("%s.%s.%d", originID, PartialRecordsSuffix, time.Now().Unix())) - fileOut, err := os.Create(dumpFilePath) - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed creating %s, error: %s", dumpFilePath, err.Error())) - return nil, err - } - csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = prc.csvSep - for _, cdr := range prc.partialRecords[originID].cdrs { - expRec, err := cdr.AsExportRecord(prc.partialRecords[originID].cacheDumpFields, - prc.httpSkipTlsCheck, nil, prc.filterS) - if err != nil { - return nil, err - } - if err := csvWriter.Write(expRec); err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed writing partial CDR %v to file: %s, error: %s", cdr, dumpFilePath, err.Error())) - return nil, err - } - } - csvWriter.Flush() - } - delete(prc.partialRecords, originID) - return nil, nil - }, 0, originID) - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed dumping CDR with originID: %s, error: %s", originID, err.Error())) - } -} - -// Called when record expires in cache, will send the CDR merged (forcing it's completion) to the CDRS -func (prc *PartialRecordsCache) postCDR(originID string) { - _, err := prc.guard.Guard(func() (interface{}, error) { - if prc.partialRecords[originID].Len() != 0 { // Only write the file if there are records in the cache - cdr := prc.partialRecords[originID].MergeCDRs() - cdr.Partial = false // force completion - var reply string - if err := prc.connMgr.Call(prc.cdrsConnIDs, nil, utils.CDRsV1ProcessEvent, - &engine.ArgV1ProcessEvent{CGREvent: *cdr.AsCGREvent()}, &reply); err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed sending CDR %+v from partial cache, error: %s", cdr, err.Error())) - } else if reply != utils.OK { - utils.Logger.Err(fmt.Sprintf(" Received unexpected reply for CDR, %+v, reply: %s", cdr, reply)) - } - } - delete(prc.partialRecords, originID) - return nil, nil - }, 0, originID) - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed posting from cache CDR with originID: %s, error: %s", originID, err.Error())) - } -} - -// Called to cache a partial record. -// If exists in cache, CDRs will be updated -// Locking should be handled at higher layer -func (prc *PartialRecordsCache) cachePartialCDR(pCDR *PartialCDRRecord) (*PartialCDRRecord, error) { - originID := pCDR.cdrs[0].OriginID - if tmr, hasIt := prc.dumpTimers[originID]; hasIt { // Update existing timer - tmr.Reset(prc.ttl) - } else { - switch prc.expiryAction { - case utils.MetaDumpToFile: - prc.dumpTimers[originID] = time.AfterFunc(prc.ttl, func() { prc.dumpPartialRecords(originID) }) // Schedule dumping of the partial CDR - case utils.MetaPostCDR: - prc.dumpTimers[originID] = time.AfterFunc(prc.ttl, func() { prc.postCDR(originID) }) // Schedule dumping of the partial CDR - default: - return nil, fmt.Errorf("Unsupported PartialCacheExpiryAction: %s", prc.expiryAction) - } - } - if _, hasIt := prc.partialRecords[originID]; !hasIt { - prc.partialRecords[originID] = pCDR - } else { // Exists, update it's records - prc.partialRecords[originID].cdrs = append(prc.partialRecords[originID].cdrs, pCDR.cdrs...) - } - return prc.partialRecords[originID], nil -} - -// Called to uncache partialCDR and remove automatic dumping of the cached records -func (prc *PartialRecordsCache) uncachePartialCDR(pCDR *PartialCDRRecord) { - originID := pCDR.cdrs[0].OriginID - if tmr, hasIt := prc.dumpTimers[originID]; hasIt { - tmr.Stop() - } - delete(prc.partialRecords, originID) -} - -// Returns PartialCDR only if merge was possible -func (prc *PartialRecordsCache) MergePartialCDRRecord(pCDR *PartialCDRRecord) (*engine.CDR, error) { - if pCDR.Len() == 0 || pCDR.cdrs[0].OriginID == "" { // Sanity check - return nil, nil - } - originID := pCDR.cdrs[0].OriginID - pCDRIf, err := prc.guard.Guard(func() (interface{}, error) { - if _, hasIt := prc.partialRecords[originID]; !hasIt && pCDR.Len() == 1 && !pCDR.cdrs[0].Partial { - return pCDR.cdrs[0], nil // Special case when not a partial CDR and not having cached CDRs on same OriginID - } - cachedPartialCDR, err := prc.cachePartialCDR(pCDR) - if err != nil { - return nil, err - } - var final bool - for _, cdr := range pCDR.cdrs { - if !cdr.Partial { - final = true - break - } - } - if !final { - return nil, nil - } - prc.uncachePartialCDR(cachedPartialCDR) - return cachedPartialCDR.MergeCDRs(), nil - }, 0, originID) - if pCDRIf == nil { - return nil, err - } - return pCDRIf.(*engine.CDR), err -} - -func NewPartialCDRRecord(cdr *engine.CDR, cacheDumpFlds []*config.FCTemplate) *PartialCDRRecord { - return &PartialCDRRecord{cdrs: []*engine.CDR{cdr}, cacheDumpFields: cacheDumpFlds} -} - -// PartialCDRRecord is a record which can be updated later -// different from PartialFlatstoreRecordsCache which is incomplete (eg: need to calculate duration out of 2 records) -type PartialCDRRecord struct { - cdrs []*engine.CDR // Number of CDRs - cacheDumpFields []*config.FCTemplate // Fields template to use when dumping from cache on disk -} - -// Part of sort interface -func (partCDR *PartialCDRRecord) Len() int { - return len(partCDR.cdrs) -} - -// Part of sort interface -func (partCDR *PartialCDRRecord) Less(i, j int) bool { - return partCDR.cdrs[i].OrderID < partCDR.cdrs[j].OrderID -} - -// Part of sort interface -func (partCDR *PartialCDRRecord) Swap(i, j int) { - partCDR.cdrs[i], partCDR.cdrs[j] = partCDR.cdrs[j], partCDR.cdrs[i] -} - -// Orders CDRs and merge them into one final -func (partCDR *PartialCDRRecord) MergeCDRs() *engine.CDR { - sort.Sort(partCDR) - if len(partCDR.cdrs) == 0 { - return nil - } - retCdr := partCDR.cdrs[0].Clone() // Make sure we don't work on original data - retCdrRVal := reflect.ValueOf(retCdr).Elem() // So we can set it's fields using reflect - for idx, cdr := range partCDR.cdrs { - if idx == 0 { // First CDR is not merged - continue - } - cdrRVal := reflect.ValueOf(cdr).Elem() - for i := 0; i < cdrRVal.NumField(); i++ { // Find out fields which were modified from previous CDR - fld := cdrRVal.Field(i) - var updated bool - switch v := fld.Interface().(type) { - case string: - if v != "" { - updated = true - } - case int64: - if v != 0 { - updated = true - } - case float64: - if v != 0.0 { - updated = true - } - case bool: - if v || cdrRVal.Type().Field(i).Name == utils.Partial { // Partial field is always updated, even if false - updated = true - } - case time.Time: - nilTime := time.Time{} - if v != nilTime { - updated = true - } - case time.Duration: - if v != time.Duration(0) { - updated = true - } - case map[string]string: - for fldName, fldVal := range v { - if origVal, hasIt := retCdr.ExtraFields[fldName]; !hasIt || origVal != fldVal { - retCdr.ExtraFields[fldName] = fldVal - } - } - } - if updated { - retCdrRVal.Field(i).Set(fld) - } - } - } - return retCdr -} diff --git a/cdrc/partial_cdr_test.go b/cdrc/partial_cdr_test.go deleted file mode 100644 index e6d7b8b01..000000000 --- a/cdrc/partial_cdr_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "reflect" - "sort" - "testing" - "time" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func TestPartialCDRRecordSort(t *testing.T) { - cdrsRaw := []*engine.CDR{&engine.CDR{OrderID: 3}, &engine.CDR{OrderID: 1}, &engine.CDR{OrderID: 2}} - pCdr := &PartialCDRRecord{cdrs: cdrsRaw} - sort.Sort(pCdr) - cdrsO := []*engine.CDR{&engine.CDR{OrderID: 1}, &engine.CDR{OrderID: 2}, &engine.CDR{OrderID: 3}} - if !reflect.DeepEqual(cdrsO, pCdr.cdrs) { - t.Errorf("Expecting: %+v, received: %+v", cdrsO, pCdr.cdrs) - } -} - -func TestPartialCDRRecordMergeCDRs(t *testing.T) { - cdr1 := &engine.CDR{OrderID: 1, ToR: utils.VOICE, - OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - Source: "TestPartialCDRRecordMergeCDRs", RequestType: utils.META_RATED, - Tenant: "cgrates.org", Category: "call", Account: "1001", - Subject: "1001", Destination: "1002", - SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), Partial: true, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, - } - cdr2 := &engine.CDR{OrderID: 3, Partial: false, - ExtraFields: map[string]string{"disconnect_direction": "upstream"}, - Usage: time.Duration(62 * time.Second), - } - cdr3 := &engine.CDR{OrderID: 2, Partial: true, - ExtraFields: map[string]string{"field_extr1": "val_extr11"}, - AnswerTime: time.Date(2013, 11, 7, 8, 43, 0, 0, time.UTC), - Usage: time.Duration(30 * time.Second), - } - pCdr := &PartialCDRRecord{cdrs: []*engine.CDR{cdr1, cdr2, cdr3}} - eCDR := &engine.CDR{OrderID: 3, ToR: utils.VOICE, - OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - Source: "TestPartialCDRRecordMergeCDRs", RequestType: utils.META_RATED, - Tenant: "cgrates.org", Category: "call", Account: "1001", - Subject: "1001", Destination: "1002", - SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 8, 43, 0, 0, time.UTC), Partial: false, - Usage: time.Duration(62 * time.Second), - ExtraFields: map[string]string{"field_extr1": "val_extr11", "fieldextr2": "valextr2", "disconnect_direction": "upstream"}, - } - if mCdr := pCdr.MergeCDRs(); !reflect.DeepEqual(eCDR, mCdr) { - t.Errorf("Expecting: %+v, received: %+v", eCDR, mCdr) - } -} diff --git a/cdrc/partialcsv_it_test.go b/cdrc/partialcsv_it_test.go deleted file mode 100644 index 901d3a7c4..000000000 --- a/cdrc/partialcsv_it_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "io/ioutil" - "net/rpc" - "os" - "path" - "strings" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var partpartcsvCfgPath string -var partcsvCfg *config.CGRConfig -var partcsvRPC *rpc.Client -var partcsvCDRCDirIn1, partcsvCDRCDirOut1, partcsvCDRCDirIn2, partcsvCDRCDirOut2 string - -var partCsvFileContent1 = `4986517174963,004986517174964,DE-National,04.07.2016 18:58:55,04.07.2016 18:58:55,1,65,Peak,0.014560,498651,partial -4986517174964,004986517174963,DE-National,04.07.2016 20:58:55,04.07.2016 20:58:55,0,74,Offpeak,0.003360,498651,complete -` - -var partCsvFileContent2 = `4986517174963,004986517174964,DE-National,04.07.2016 19:00:00,04.07.2016 18:58:55,0,15,Offpeak,0.003360,498651,partial` -var partCsvFileContent3 = `4986517174964,004986517174960,DE-National,04.07.2016 19:05:55,04.07.2016 19:05:55,0,23,Offpeak,0.003360,498651,partial` - -var eCacheDumpFile1 = `4986517174963_004986517174964_04.07.2016 18:58:55,1467651535,*rated,086517174963,+4986517174964,2016-07-04T18:58:55+02:00,2016-07-04T18:58:55+02:00,1m5s,-1.00000 -4986517174963_004986517174964_04.07.2016 18:58:55,1467651600,*rated,086517174963,+4986517174964,2016-07-04T18:58:55+02:00,2016-07-04T18:58:55+02:00,15s,-1.00000 -` - -func TestPartcsvITInitConfig(t *testing.T) { - var err error - partpartcsvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrc_partcsv") - if partcsvCfg, err = config.NewCGRConfigFromPath(partpartcsvCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestPartcsvITInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(partcsvCfg); err != nil { - t.Fatal(err) - } -} - -// Remove data in both rating and accounting db -func TestPartcsvITResetDataDb(t *testing.T) { - if err := engine.InitDataDb(partcsvCfg); err != nil { - t.Fatal(err) - } -} - -func TestPartcsvITCreateCdrDirs(t *testing.T) { - for path, cdrcProfiles := range partcsvCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - if path == "/tmp/cdrctests/partcsv1/in" { - partcsvCDRCDirIn1, partcsvCDRCDirOut1 = cdrcInst.CDRInPath, cdrcInst.CDROutPath - } else if path == "/tmp/cdrctests/partcsv2/in" { - partcsvCDRCDirIn2, partcsvCDRCDirOut2 = cdrcInst.CDRInPath, cdrcInst.CDROutPath - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} - -func TestPartcsvITStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(partpartcsvCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestPartcsvITRpcConn(t *testing.T) { - time.Sleep(5 * time.Second) - var err error - partcsvRPC, err = newRPCClient(partcsvCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestPartcsvITHandleCdr1File(t *testing.T) { - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -// Scenario out of first .xml config -func TestPartcsvITHandleCdr2File(t *testing.T) { - fileName := "file2.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent2), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -// Scenario out of first .xml config -func TestPartcsvITHandleCdr3File(t *testing.T) { - fileName := "file3.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(partCsvFileContent3), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(partcsvCDRCDirIn2, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestPartcsvITProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(3 * time.Second)) - if outContent1, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut1, "file1.csv")); err != nil { - t.Error(err) - } else if partCsvFileContent1 != string(outContent1) { - t.Errorf("Expecting: %q, \n received: %q", partCsvFileContent1, string(outContent1)) - } - if outContent2, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut1, "file2.csv")); err != nil { - t.Error(err) - } else if len(partCsvFileContent2) != len(string(outContent2)) { - t.Errorf("Expecting: %q, received: %q", partCsvFileContent2, string(outContent2)) - } - filesInDir, _ := ioutil.ReadDir(partcsvCDRCDirOut1) - if len(filesInDir) == 0 { - t.Errorf("No files found in folder: <%s>", partcsvCDRCDirOut1) - } - var fileName string - for _, file := range filesInDir { // First file in directory is the one we need, harder to find it's name out of config - if strings.HasPrefix(file.Name(), "4986517174963_004986517174964") { - fileName = file.Name() - break - } - } - if contentCacheDump, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut1, fileName)); err != nil { - t.Error(err) - } else if len(eCacheDumpFile1) != len(string(contentCacheDump)) { - t.Errorf("Expecting: %q, \n received: %q", eCacheDumpFile1, string(contentCacheDump)) - } - if outContent3, err := ioutil.ReadFile(path.Join(partcsvCDRCDirOut2, "file3.csv")); err != nil { - t.Error(err) - } else if partCsvFileContent3 != string(outContent3) { - t.Errorf("Expecting: %q, received: %q", partCsvFileContent3, string(outContent3)) - } -} - -func TestPartcsvITAnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := partcsvRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := partcsvRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{DestinationPrefixes: []string{"+4986517174963"}}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := partcsvRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{DestinationPrefixes: []string{"+4986517174960"}}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - -} - -func TestPartcsvITKillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/cdrc/reader.go b/cdrc/reader.go deleted file mode 100644 index b0cd6eaee..000000000 --- a/cdrc/reader.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "github.com/cgrates/cgrates/utils" -) - -type CDRCReader interface { - Read() (*utils.CGREvent, error) // Process a single record in the CDR file, return a slice of CDRs since based on configuration we can have more templates - Processed() int64 // number of records processed -} diff --git a/cdrc/unpairedrecords.go b/cdrc/unpairedrecords.go deleted file mode 100644 index d964c3662..000000000 --- a/cdrc/unpairedrecords.go +++ /dev/null @@ -1,178 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "encoding/csv" - "errors" - "fmt" - "os" - "path" - "strconv" - "time" - - "github.com/cgrates/cgrates/guardian" - "github.com/cgrates/cgrates/utils" -) - -func NewUnpairedRecordsCache(ttl time.Duration, cdrOutDir string, csvSep rune) *UnpairedRecordsCache { - return &UnpairedRecordsCache{ttl: ttl, cdrOutDir: cdrOutDir, csvSep: csvSep, - partialRecords: make(map[string]map[string]*UnpairedRecord), guard: guardian.Guardian} -} - -type UnpairedRecordsCache struct { - ttl time.Duration - cdrOutDir string - csvSep rune - partialRecords map[string]map[string]*UnpairedRecord // [FileName"][OriginID]*PartialRecord - guard *guardian.GuardianLocker -} - -// Dumps the cache into a .unpaired file in the outdir and cleans cache after -func (self *UnpairedRecordsCache) dumpUnpairedRecords(fileName string) error { - _, err := self.guard.Guard(func() (interface{}, error) { - if len(self.partialRecords[fileName]) != 0 { // Only write the file if there are records in the cache - unpairedFilePath := path.Join(self.cdrOutDir, fileName+UNPAIRED_SUFFIX) - fileOut, err := os.Create(unpairedFilePath) - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed creating %s, error: %s", unpairedFilePath, err.Error())) - return nil, err - } - csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = self.csvSep - for _, pr := range self.partialRecords[fileName] { - if err := csvWriter.Write(pr.Values); err != nil { - utils.Logger.Err(fmt.Sprintf(" Failed writing unpaired record %v to file: %s, error: %s", pr, unpairedFilePath, err.Error())) - return nil, err - } - } - csvWriter.Flush() - } - delete(self.partialRecords, fileName) - return nil, nil - }, 0, fileName) - return err -} - -// Search in cache and return the partial record with accountind id defined, prefFilename is searched at beginning because of better match probability -func (self *UnpairedRecordsCache) GetPartialRecord(OriginID, prefFileName string) (string, *UnpairedRecord) { - var cachedFilename string - var cachedPartial *UnpairedRecord - checkCachedFNames := []string{prefFileName} // Higher probability to match as firstFileName - for fName := range self.partialRecords { - if fName != prefFileName { - checkCachedFNames = append(checkCachedFNames, fName) - } - } - for _, fName := range checkCachedFNames { // Need to lock them individually - self.guard.Guard(func() (interface{}, error) { - var hasPartial bool - if cachedPartial, hasPartial = self.partialRecords[fName][OriginID]; hasPartial { - cachedFilename = fName - } - return nil, nil - }, 0, fName) - if cachedPartial != nil { - break - } - } - return cachedFilename, cachedPartial -} - -func (self *UnpairedRecordsCache) CachePartial(fileName string, pr *UnpairedRecord) { - self.guard.Guard(func() (interface{}, error) { - if fileMp, hasFile := self.partialRecords[fileName]; !hasFile { - self.partialRecords[fileName] = map[string]*UnpairedRecord{pr.OriginID: pr} - if self.ttl != 0 { // Schedule expiry/dump of the just created entry in cache - go func() { - time.Sleep(self.ttl) - self.dumpUnpairedRecords(fileName) - }() - } - } else if _, hasOriginID := fileMp[pr.OriginID]; !hasOriginID { - self.partialRecords[fileName][pr.OriginID] = pr - } - return nil, nil - }, 0, fileName) -} - -func (self *UnpairedRecordsCache) UncachePartial(fileName string, pr *UnpairedRecord) { - self.guard.Guard(func() (interface{}, error) { - delete(self.partialRecords[fileName], pr.OriginID) // Remove the record out of cache - return nil, nil - }, 0, fileName) -} - -func NewUnpairedRecord(record []string, timezone string) (*UnpairedRecord, error) { - if len(record) < 7 { - return nil, errors.New("MISSING_IE") - } - pr := &UnpairedRecord{Method: record[0], OriginID: record[3] + record[1] + record[2], Values: record} - var err error - if pr.Timestamp, err = utils.ParseTimeDetectLayout(record[6], timezone); err != nil { - return nil, err - } - return pr, nil -} - -// This is a partial record received from Flatstore, can be INVITE or BYE and it needs to be paired in order to produce duration -type UnpairedRecord struct { - Method string // INVITE or BYE - OriginID string // Copute here the OriginID - Timestamp time.Time // Timestamp of the event, as written by db_flastore module - Values []string // Can contain original values or updated via UpdateValues -} - -// Pairs INVITE and BYE into final record containing as last element the duration -func pairToRecord(part1, part2 *UnpairedRecord) ([]string, error) { - var invite, bye *UnpairedRecord - if part1.Method == "INVITE" { - invite = part1 - } else if part2.Method == "INVITE" { - invite = part2 - } else { - return nil, errors.New("MISSING_INVITE") - } - if part1.Method == "BYE" { - bye = part1 - } else if part2.Method == "BYE" { - bye = part2 - } else { - return nil, errors.New("MISSING_BYE") - } - if len(invite.Values) != len(bye.Values) { - return nil, errors.New("INCONSISTENT_VALUES_LENGTH") - } - record := invite.Values - for idx := range record { - switch idx { - case 0, 1, 2, 3, 6: // Leave these values as they are - case 4, 5: - record[idx] = bye.Values[idx] // Update record with status from bye - default: - if bye.Values[idx] != "" { // Any value higher than 6 is dynamically inserted, overwrite if non empty - record[idx] = bye.Values[idx] - } - - } - } - callDur := bye.Timestamp.Sub(invite.Timestamp) - record = append(record, strconv.FormatFloat(callDur.Seconds(), 'f', -1, 64)) - return record, nil -} diff --git a/cdrc/xml.go b/cdrc/xml.go deleted file mode 100644 index c011cad44..000000000 --- a/cdrc/xml.go +++ /dev/null @@ -1,196 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package cdrc - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "strings" - "time" - - "github.com/antchfx/xmlquery" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -// handlerUsageDiff will calculate the usage as difference between timeEnd and timeStart -// Expects the 2 arguments in template separated by | -func handlerSubstractUsage(xmlElement *xmlquery.Node, argsTpl config.RSRParsers, - cdrPath utils.HierarchyPath, timezone string) (time.Duration, error) { - var argsStr string - for _, rsrArg := range argsTpl { - if rsrArg.Rules == utils.HandlerArgSep { - argsStr += rsrArg.Rules - continue - } - absolutePath := utils.ParseHierarchyPath(rsrArg.Rules, "") - relPath := utils.HierarchyPath(absolutePath[len(cdrPath)+1:]) // Need relative path to the xmlElmnt - argStr, _ := config.ElementText(xmlElement, relPath.AsString("/", false)) - argsStr += argStr - } - handlerArgs := strings.Split(argsStr, utils.HandlerArgSep) - if len(handlerArgs) != 2 { - return time.Duration(0), errors.New("Unexpected number of arguments") - } - tEnd, err := utils.ParseTimeDetectLayout(handlerArgs[0], timezone) - if err != nil { - return time.Duration(0), err - } - if tEnd.IsZero() { - return time.Duration(0), fmt.Errorf("EndTime is 0") - } - tStart, err := utils.ParseTimeDetectLayout(handlerArgs[1], timezone) - if err != nil { - return time.Duration(0), err - } - if tStart.IsZero() { - return time.Duration(0), fmt.Errorf("StartTime is 0") - } - return tEnd.Sub(tStart), nil -} - -func NewXMLRecordsProcessor(recordsReader io.Reader, cdrPath utils.HierarchyPath, timezone string, - httpSkipTlsCheck bool, cdrcCfgs []*config.CdrcCfg, filterS *engine.FilterS) (*XMLRecordsProcessor, error) { - //create doc - doc, err := xmlquery.Parse(recordsReader) - if err != nil { - return nil, err - } - xmlProc := &XMLRecordsProcessor{cdrPath: cdrPath, timezone: timezone, - httpSkipTlsCheck: httpSkipTlsCheck, cdrcCfgs: cdrcCfgs, filterS: filterS} - - xmlProc.cdrXmlElmts = xmlquery.Find(doc, cdrPath.AsString("/", true)) - return xmlProc, nil -} - -type XMLRecordsProcessor struct { - cdrXmlElmts []*xmlquery.Node // result of splitting the XML doc into CDR elements - procItems int // current number of processed records from file - cdrPath utils.HierarchyPath // path towards one CDR element - timezone string - httpSkipTlsCheck bool - cdrcCfgs []*config.CdrcCfg // individual configs for the folder CDRC is monitoring - filterS *engine.FilterS -} - -func (xmlProc *XMLRecordsProcessor) ProcessedRecordsNr() int64 { - return int64(xmlProc.procItems) -} - -func (xmlProc *XMLRecordsProcessor) ProcessNextRecord() (cdrs []*engine.CDR, err error) { - if len(xmlProc.cdrXmlElmts) <= xmlProc.procItems { - return nil, io.EOF // have processed all items - } - cdrs = make([]*engine.CDR, 0) - cdrXML := xmlProc.cdrXmlElmts[xmlProc.procItems] - xmlProc.procItems += 1 - xmlProvider := config.NewXmlProvider(cdrXML, xmlProc.cdrPath, utils.MetaReq) - for _, cdrcCfg := range xmlProc.cdrcCfgs { - tenant, err := cdrcCfg.Tenant.ParseDataProvider(xmlProvider, utils.NestingSep) - if err != nil { - return nil, err - } - if tenant == "" { - tenant = config.CgrConfig().GeneralCfg().DefaultTenant - } - if len(cdrcCfg.Filters) != 0 { - if pass, err := xmlProc.filterS.Pass(tenant, - cdrcCfg.Filters, xmlProvider); err != nil || !pass { - continue // Not passes filters, ignore this CDR - } - } - if cdr, err := xmlProc.recordToCDR(cdrXML, cdrcCfg, tenant); err != nil { - return nil, fmt.Errorf(" Failed converting to CDR, error: %s", err.Error()) - } else { - cdrs = append(cdrs, cdr) - } - if !cdrcCfg.ContinueOnSuccess { - break - } - } - return cdrs, nil -} - -func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity *xmlquery.Node, cdrcCfg *config.CdrcCfg, tenant string) (*engine.CDR, error) { - cdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} - var lazyHttpFields []*config.FCTemplate - var err error - fldVals := make(map[string]string) - xmlProvider := config.NewXmlProvider(xmlEntity, xmlProc.cdrPath, utils.MetaReq) - for _, cdrFldCfg := range cdrcCfg.ContentFields { - if len(cdrFldCfg.Filters) != 0 { - if pass, err := xmlProc.filterS.Pass(tenant, - cdrFldCfg.Filters, xmlProvider); err != nil || !pass { - continue // Not passes filters, ignore this CDR - } - } - if cdrFldCfg.Type == utils.META_COMPOSED { - out, err := cdrFldCfg.Value.ParseDataProvider(xmlProvider, utils.NestingSep) - if err != nil { - return nil, err - } - fldVals[cdrFldCfg.FieldId] += out - } else if cdrFldCfg.Type == utils.META_HTTP_POST { - lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of cdr to http server - } else if cdrFldCfg.Type == utils.META_HANDLER && cdrFldCfg.HandlerId == utils.HandlerSubstractUsage { - usage, err := handlerSubstractUsage(xmlEntity, cdrFldCfg.Value, xmlProc.cdrPath, xmlProc.timezone) - if err != nil { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s, err: %s", xmlEntity, cdrFldCfg.Tag, err.Error()) - } - fldVals[cdrFldCfg.FieldId] += usage.String() - } else { - return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type) - } - if err := cdr.ParseFieldValue(cdrFldCfg.FieldId, fldVals[cdrFldCfg.FieldId], xmlProc.timezone); err != nil { - return nil, err - } - } - cdr.CGRID = utils.Sha1(cdr.OriginID, cdr.OriginHost) - for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields - var outValByte []byte - var fieldVal, httpAddr string - for _, rsrFld := range httpFieldCfg.Value { - if parsed, err := rsrFld.ParseValue(utils.EmptyString); err != nil { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract http address, err: %s", xmlEntity, err.Error()) - } else { - httpAddr += parsed - } - } - var jsn []byte - jsn, err = json.Marshal(cdr) - if err != nil { - return nil, err - } - if outValByte, err = engine.HttpJsonPost(httpAddr, xmlProc.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { - return nil, err - } else { - fieldVal = string(outValByte) - if len(fieldVal) == 0 && httpFieldCfg.Mandatory { - return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag) - } - if err := cdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, xmlProc.timezone); err != nil { - return nil, err - } - } - } - return cdr, nil -} diff --git a/cdrc/xml_it_test.go b/cdrc/xml_it_test.go deleted file mode 100644 index d2be26e5f..000000000 --- a/cdrc/xml_it_test.go +++ /dev/null @@ -1,613 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "io/ioutil" - "net/rpc" - "os" - "path" - "testing" - "time" - - v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var xmlCfgPath string -var xmlCfg *config.CGRConfig -var cdrcXmlCfgs []*config.CdrcCfg -var cdrcXmlCfg *config.CdrcCfg -var cdrcXmlRPC *rpc.Client -var xmlPathIn1, xmlPathOut1 string - -func TestXmlITInitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxml") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlITInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -// Remove data in both rating and accounting db -func TestXmlITResetDataDb(t *testing.T) { - if err := engine.InitDataDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlITCreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "XMLit1" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlITStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlITRpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlITHandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(cdrXmlBroadsoft), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlITProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if cdrXmlBroadsoft != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", cdrXmlBroadsoft, string(outContent1)) - } -} - -func TestXmlITAnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{DestinationPrefixes: []string{"+4986517174963"}}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlITKillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc xml with new filters -func TestXmlIT2InitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxmlwithfilter") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlIT2InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlIT2CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "XMLWithFilter" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlIT2StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlIT2RpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlIT2HandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(cdrXmlBroadsoft), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlIT2ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if cdrXmlBroadsoft != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", cdrXmlBroadsoft, string(outContent1)) - } -} - -func TestXmlIT2AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlIT2KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc xml with new filters -func TestXmlIT3InitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxmlwithfilter") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlIT3InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlIT3CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if !cdrcInst.Enabled { - continue - } - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "msw_xml" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlIT3StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlIT3RpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlIT3HandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(xmlContent), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlIT3ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if xmlContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", xmlContent, string(outContent1)) - } -} - -func TestXmlIT3AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlIT3KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc xml with new filters -func TestXmlIT4InitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxmlwithfilter") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlIT4InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlIT4CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "msw_xml2" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlIT4StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlIT4RpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlIT4HandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(xmlContent), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlIT4ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if xmlContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", xmlContent, string(outContent1)) - } -} - -func TestXmlIT4AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 2 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlIT4KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc xml with new filters -func TestXmlIT5InitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxmlwithfilter") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlIT5InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlIT5CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "XMLWithFilterID" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlIT5StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlIT5RpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func TestXmlIT5AddFilters(t *testing.T) { - filter := v1.FilterWithCache{ - Filter: &engine.Filter{ - Tenant: "cgrates.org", - ID: "FLTR_XML", - Rules: []*engine.FilterRule{ - { - Type: utils.MetaString, - Element: "~*req.broadWorksCDR.cdrData.basicModule.userNumber", - Values: []string{"1002"}, - }, - { - Type: utils.MetaString, - Element: "~*req.broadWorksCDR.cdrData.headerModule.type", - Values: []string{"Normal"}, - }, - }, - }, - } - var result string - if err := cdrcXmlRPC.Call(utils.ApierV1SetFilter, filter, &result); err != nil { - t.Error(err) - } else if result != utils.OK { - t.Error("Unexpected reply returned", result) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlIT5HandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(cdrXmlBroadsoft), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlIT5ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if cdrXmlBroadsoft != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", cdrXmlBroadsoft, string(outContent1)) - } -} - -func TestXmlIT5AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlIT5KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} - -// Begin tests for cdrc xml with index -func TestXmlIT6InitConfig(t *testing.T) { - var err error - xmlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcxmlwithfilter") - if xmlCfg, err = config.NewCGRConfigFromPath(xmlCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -// InitDb so we can rely on count -func TestXmlIT6InitCdrDb(t *testing.T) { - if err := engine.InitStorDb(xmlCfg); err != nil { - t.Fatal(err) - } -} - -func TestXmlIT6CreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range xmlCfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - if !cdrcInst.Enabled { - continue - } - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if cdrcInst.ID == "XMLWithIndex" { // Initialize the folders to check later - xmlPathIn1 = cdrcInst.CDRInPath - xmlPathOut1 = cdrcInst.CDROutPath - } - } - } -} - -func TestXmlIT6StartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(xmlCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func TestXmlIT6RpcConn(t *testing.T) { - var err error - cdrcXmlRPC, err = newRPCClient(xmlCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// The default scenario, out of cdrc defined in .cfg file -func TestXmlIT6HandleCdr1File(t *testing.T) { - fileName := "file1.xml" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(xmlMultipleIndex), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join(xmlPathIn1, fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func TestXmlIT6ProcessedFiles(t *testing.T) { - time.Sleep(time.Duration(2**waitRater) * time.Millisecond) - if outContent1, err := ioutil.ReadFile(path.Join(xmlPathOut1, "file1.xml")); err != nil { - t.Error(err) - } else if xmlMultipleIndex != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", xmlMultipleIndex, string(outContent1)) - } -} - -func TestXmlIT6AnalyseCDRs(t *testing.T) { - var reply []*engine.ExternalCDR - if err := cdrcXmlRPC.Call(utils.ApierV2GetCDRs, utils.RPCCDRsFilter{}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(reply) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(reply)) - } -} - -func TestXmlIT6KillEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/cdrc/xml_test.go b/cdrc/xml_test.go deleted file mode 100644 index 06b38d61f..000000000 --- a/cdrc/xml_test.go +++ /dev/null @@ -1,611 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package cdrc - -import ( - "bytes" - "path" - "reflect" - "strings" - "testing" - "time" - - "github.com/antchfx/xmlquery" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var cdrXmlBroadsoft = ` - - - - - - 0002183384 - CGRateSaabb - 20160419210000.104 - 1+020000 - - Start - - - - - - 0002183385 - CGRateSaabb - 20160419210005.247 - 1+020000 - - MBC - Normal - - - 1001 - 2001 - Network - 1001 - Public - +4986517174963 - 20160419210005.247 - 1+020000 - 25160047719:0 - Yes - 20160419210006.813 - 20160419210020.296 - 016 - y - local - 1001@cgrates.org - Yes - Yes - - - CGR_GROUP - CGR_GROUP/CGR_GROUP_TRUNK30 - Normal - - - 1001@cgrates.org - Primary Device - - - 31.882 - - - gw04.cgrates.org - 74122796919420162305@172.16.1.2 - PCMA/8000 - 172.16.1.4 - BW2300052501904161738474465@172.16.1.10 - 31.882 - OmniPCX Enterprise R11.0.1 k1.520.22.b - - - - - - 0002183386 - CGRateSaabb - 20160419210006.909 - 1+020000 - - MBC - Normal - - - 1002 - 2001 - Network - +4986517174964 - Public - 1002 - 20160419210006.909 - 1+020000 - 27280048121:0 - Yes - 20160419210007.037 - 20160419210030.322 - 016 - y - local - 314028947650@cgrates.org - Yes - Yes - - - CGR_GROUP - CGR_GROUP/CGR_GROUP_TRUNK65 - Normal - - - 31403456100@cgrates.org - Primary Device - - - 26.244 - - - gw01.cgrates.org - 108352493719420162306@172.31.250.150 - PCMA/8000 - 172.16.1.4 - 2345300069121904161716512907@172.16.1.10 - 26.244 - Altitude vBox - - - - - - 0002183486 - CGRateSaabb - 20160419211500.104 - 1+020000 - - End - - -` - -var xmlMultipleIndex = ` - 2005-08-26T14:16:42 - 2005-08-26T14:16:56 - 2005-08-26T14:17:34 - My Call Reference - 386 - sampleusername - 1 - Conecto LLC - US$0.21 - yes - no - US$0.13 - 44 - - +441624828505 - Isle of Man - 38 - US$0.0200 - US$0.0140 - US$0.0130 - US$0.0082 - - - +44 7624 494075 - Isle of Man - 37 - US$0.2700 - US$0.1890 - US$0.1880 - US$0.1159 - - -` - -func TestXMLHandlerSubstractUsage(t *testing.T) { - doc, err := xmlquery.Parse(strings.NewReader(cdrXmlBroadsoft)) - if err != nil { - t.Error(err) - } - - cdrs := xmlquery.Find(doc, path.Join("/broadWorksCDR/cdrData/")) - cdrWithUsage := cdrs[1] - if usage, err := handlerSubstractUsage(cdrWithUsage, - config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.releaseTime;|;~*req.broadWorksCDR.cdrData.basicModule.answerTime", true, utils.INFIELD_SEP), - utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), "UTC"); err != nil { - t.Error(err) - } else if usage != time.Duration(13483000000) { - t.Errorf("Expected: 13.483s, received: %v", usage) - } -} - -func TestXMLRPProcess(t *testing.T) { - cdrcCfgs := []*config.CdrcCfg{ - { - ID: "TestXML", - Enabled: true, - CdrFormat: "xml", - CDRRootPath: utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), - CdrSourceId: "TestXML", - ContentFields: []*config.FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.localCallId", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: config.NewRSRParsersMustCompile("*rated", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: config.NewRSRParsersMustCompile("call", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.userNumber", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.calledNumber", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.startTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.answerTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_HANDLER, - FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.releaseTime;|;~*req.broadWorksCDR.cdrData.basicModule.answerTime", - true, utils.INFIELD_SEP), Mandatory: true}, - }, - }, - } - xmlRP, err := NewXMLRecordsProcessor(bytes.NewBufferString(cdrXmlBroadsoft), - utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), "UTC", true, cdrcCfgs, nil) - if err != nil { - t.Error(err) - } - var cdrs []*engine.CDR - for i := 0; i < 4; i++ { - cdrs, err = xmlRP.ProcessNextRecord() - if i == 1 { // Take second CDR since the first one cannot be processed - break - } - } - if err != nil { - t.Error(err) - } - expectedCDRs := []*engine.CDR{ - { - CGRID: utils.Sha1("25160047719:0", "0.0.0.0"), - OriginHost: "0.0.0.0", - Source: "TestXML", - OriginID: "25160047719:0", - ToR: "*voice", - RequestType: "*rated", - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Destination: "+4986517174963", - SetupTime: time.Date(2016, 4, 19, 21, 0, 5, 247000000, time.UTC), - AnswerTime: time.Date(2016, 4, 19, 21, 0, 6, 813000000, time.UTC), - Usage: time.Duration(13483000000), - ExtraFields: map[string]string{}, - Cost: -1, - }, - } - if !reflect.DeepEqual(expectedCDRs, cdrs) { - t.Errorf("Expecting: %+v\n, received: %+v\n", expectedCDRs, cdrs) - } -} - -func TestXMLRPProcessWithNewFilters(t *testing.T) { - cdrcCfgs := []*config.CdrcCfg{ - { - ID: "XMLWithFilters", - Enabled: true, - CdrFormat: "xml", - CDRRootPath: utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), - CdrSourceId: "XMLWithFilters", - Filters: []string{"*string:~*req.broadWorksCDR.cdrData.headerModule.type:Normal"}, - ContentFields: []*config.FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.localCallId", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: config.NewRSRParsersMustCompile("*rated", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: config.NewRSRParsersMustCompile("call", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.userNumber", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.calledNumber", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.startTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.answerTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_HANDLER, - FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~*req.broadWorksCDR.cdrData.basicModule.releaseTime;|;~*req.broadWorksCDR.cdrData.basicModule.answerTime", - true, utils.INFIELD_SEP), Mandatory: true}, - }, - }, - } - defaultCfg, _ := config.NewDefaultCGRConfig() - data := engine.NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) - xmlRP, err := NewXMLRecordsProcessor(bytes.NewBufferString(cdrXmlBroadsoft), - utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), "UTC", true, - cdrcCfgs, engine.NewFilterS(defaultCfg, nil, - engine.NewDataManager(data, defaultCfg.CacheCfg(), nil))) - if err != nil { - t.Error(err) - } - var cdrs []*engine.CDR - for i := 0; i < 4; i++ { - cdrs, err = xmlRP.ProcessNextRecord() - if i == 1 { // Take second CDR since the first one cannot be processed - break - } - } - if err != nil { - t.Error(err) - } - expectedCDRs := []*engine.CDR{ - { - CGRID: utils.Sha1("25160047719:0", "0.0.0.0"), - OriginHost: "0.0.0.0", - Source: "XMLWithFilters", - OriginID: "25160047719:0", - ToR: "*voice", - RequestType: "*rated", - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Destination: "+4986517174963", - SetupTime: time.Date(2016, 4, 19, 21, 0, 5, 247000000, time.UTC), - AnswerTime: time.Date(2016, 4, 19, 21, 0, 6, 813000000, time.UTC), - Usage: time.Duration(13483000000), - ExtraFields: map[string]string{}, - Cost: -1, - }, - } - if !reflect.DeepEqual(expectedCDRs, cdrs) { - t.Errorf("Expecting: %+v\n, received: %+v\n", expectedCDRs, cdrs) - } -} - -var xmlContent = ` - - - Metaswitch CFS - - 1510225200002 - - - - National - - Orig - 19 - 480 - No answer - 223007622 - - +27110493421 - +27110493421 - +27110493421 - Ro_test - 0gQAAC8WAAACBAAALxYAAD57SAEV7ekv/OSKkO7qmD82OmbfHO+Z7wIZJkXdCv8z@10.170.248.200 - - - - 1 - 0 - - 8824071D@10.170.248.140 - 19 - 480 - - - 0763371551 - +270763371551 - 0763371551 - 110493421 - 110493421 - - - False - NetworkDefault - 0 - - - Speech - - 13442698e525ad2c21251f76479ab2b4 - voice.lab.liquidtelecom.net - - - 1510225513055 - 1510225513304 - 1510225514836 - - 1510225516981 - 1510225516981 - 1510225516981 - - - Premium - - Orig - 16 - No error - 223007623 - - +27110493421 - +27110493421 - +27110493421 - Ro_test - 0gQAAC8WAAACBAAALxYAAPkyWDO29Do1SyxNi5UV71mJYEIEkfNa9wCFCCjY2asU@10.170.248.200 - - - - 1 - 0 - - 8E450FA1@10.170.248.140 - - - 0843073451 - +270843073451 - 0843073451 - 110493421 - 110493421 - - - False - NetworkDefault - 0 - - - Speech - - 46d7974398c2671016afccc3f2c428c7 - voice.lab.liquidtelecom.net - - - 1510225531933 - 1510225532183 - 1510225534973 - 1510225539364 - 1510225593101 - 1510225593101 - 1510225593101 - - - International - - Orig - 16 - No error - 223007624 - - +27110493421 - +27110493421 - +27110493421 - Ro_test - 0gQAAC8WAAACBAAALxYAAJrUscTicyU5GtjPyQnpAeuNmz9p/bdOoR/Mk9RXciOI@10.170.248.200 - - - - 1 - 0 - - BC8B2801@10.170.248.140 - - - 263772822306 - +263772822306 - 263772822306 - 110493421 - 110493421 - - - False - NetworkDefault - 0 - - - Speech - - 750b8b73e41ba7b59b21240758522268 - voice.lab.liquidtelecom.net - - - 1510225865894 - 1510225866144 - 1510225866756 - 1510225876243 - 1510225916144 - 1510225916144 - 1510225916144 - - - - 1510227591467 - 3 - 0 - 0 - - -` - -func TestXMLRPNestingSeparator(t *testing.T) { - cdrcCfgs := []*config.CdrcCfg{ - { - ID: "msw_xml", - Enabled: true, - CdrFormat: "xml", - CDRRootPath: utils.HierarchyPath([]string{"File", "CDRs", "Call"}), - CdrSourceId: "zw_cfs1", - Filters: []string{}, - ContentFields: []*config.FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.SignalingInfo.PChargingVector.icidvalue", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: config.NewRSRParsersMustCompile("*rated", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: config.NewRSRParsersMustCompile("XX.liquid.tel", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: config.NewRSRParsersMustCompile("call", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.OrigParty.SubscriberAddr", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.RoutingInfo.DestAddr", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.RingingTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.ConnectTime", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_HANDLER, - FieldId: utils.Usage, HandlerId: utils.HandlerSubstractUsage, - Value: config.NewRSRParsersMustCompile("~*req.File.CDRs.Call.ReleaseTime;|;~*req.File.CDRs.Call.ConnectTime", - true, utils.INFIELD_SEP), Mandatory: true}, - }, - }, - } - defaultCfg, _ := config.NewDefaultCGRConfig() - data := engine.NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) - xmlRP, err := NewXMLRecordsProcessor(bytes.NewBufferString(xmlContent), - utils.HierarchyPath([]string{"File", "CDRs", "Call"}), "UTC", true, - cdrcCfgs, engine.NewFilterS(defaultCfg, nil, - engine.NewDataManager(data, defaultCfg.CacheCfg(), nil))) - if err != nil { - t.Error(err) - } - var cdrs []*engine.CDR - for i := 0; i < 4; i++ { - cdrs, err = xmlRP.ProcessNextRecord() - if i == 1 { // Take second CDR since the first one cannot be processed - break - } - } - if err != nil { - t.Error(err) - } - expectedCDRs := []*engine.CDR{ - { - CGRID: utils.Sha1("46d7974398c2671016afccc3f2c428c7", "0.0.0.0"), - OriginHost: "0.0.0.0", - Source: "zw_cfs1", - OriginID: "46d7974398c2671016afccc3f2c428c7", - ToR: "*voice", - RequestType: "*rated", - Tenant: "XX.liquid.tel", - Category: "call", - Account: "+27110493421", - Destination: "+270843073451", - SetupTime: time.Date(2017, 11, 9, 11, 5, 34, 973000000, time.UTC), - AnswerTime: time.Date(2017, 11, 9, 11, 5, 39, 364000000, time.UTC), - Usage: time.Duration(53737000000), - ExtraFields: map[string]string{}, - Cost: -1, - }, - } - if !reflect.DeepEqual(expectedCDRs, cdrs) { - t.Errorf("Expecting: %+v\n, received: %+v\n", expectedCDRs, cdrs) - } -} diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index cb70f5df8..4360ae2c5 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -32,7 +32,6 @@ import ( "time" v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/cdrc" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/services" @@ -60,64 +59,6 @@ var ( cfg *config.CGRConfig ) -func startCdrcs(filterSChan chan *engine.FilterS, exitChan chan bool, connMgr *engine.ConnManager) { - filterS := <-filterSChan - filterSChan <- filterS - cdrcInitialized := false // Control whether the cdrc was already initialized (so we don't reload in that case) - var cdrcChildrenChan chan struct{} // Will use it to communicate with the children of one fork - for { - select { - case <-exitChan: // Stop forking CDRCs - break - case <-cfg.ConfigReloads[utils.CDRC]: // Consume the load request and wait for a new one - if cdrcInitialized { - utils.Logger.Info(" Configuration reload") - close(cdrcChildrenChan) // Stop all the children of the previous run - } - cdrcChildrenChan = make(chan struct{}) - } - // Start CDRCs - for _, cdrcCfgs := range cfg.CdrcProfiles { - var enabledCfgs []*config.CdrcCfg - for _, cdrcCfg := range cdrcCfgs { // Take a random config out since they should be the same - if cdrcCfg.Enabled { - enabledCfgs = append(enabledCfgs, cdrcCfg) - } - } - if len(enabledCfgs) != 0 { - go startCdrc(enabledCfgs, - cfg.GeneralCfg().HttpSkipTlsVerify, filterSChan, - cdrcChildrenChan, exitChan, connMgr) - } else { - utils.Logger.Info(" No enabled CDRC clients") - } - } - cdrcInitialized = true // Initialized - } -} - -// Fires up a cdrc instance -func startCdrc(cdrcCfgs []*config.CdrcCfg, httpSkipTlsCheck bool, - filterSChan chan *engine.FilterS, closeChan chan struct{}, exitChan chan bool, connMgr *engine.ConnManager) { - filterS := <-filterSChan - filterSChan <- filterS - var err error - var cdrsConn rpcclient.ClientConnector - - cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan, - cfg.GeneralCfg().DefaultTimezone, filterS, connMgr) - if err != nil { - utils.Logger.Crit(fmt.Sprintf("Cdrc config parsing error: %s", err.Error())) - exitChan <- true - return - } - if err := cdrc.Run(); err != nil { - utils.Logger.Crit(fmt.Sprintf("Cdrc run error: %s", err.Error())) - exitChan <- true // If run stopped, something is bad, stop the application - return - } -} - // startFilterService fires up the FilterS func startFilterService(filterSChan chan *engine.FilterS, cacheS *engine.CacheS, connMgr *engine.ConnManager, cfg *config.CGRConfig, dm *engine.DataManager, exitChan chan bool) { @@ -597,9 +538,6 @@ func main() { initConfigSv1(internalConfigChan, server) - // Start CDRC components if necessary - go startCdrcs(filterSChan, exitChan, connManager) - // Serve rpc connections go startRpc(server, rals.GetResponder().GetIntenternalChan(), cdrS.GetIntenternalChan(), reS.GetIntenternalChan(), stS.GetIntenternalChan(), diff --git a/config/cdrccfg.go b/config/cdrccfg.go deleted file mode 100644 index 245c055e4..000000000 --- a/config/cdrccfg.go +++ /dev/null @@ -1,202 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package config - -import ( - "time" - - "github.com/cgrates/cgrates/utils" -) - -type CdrcCfg struct { - ID string // free-form text identifying this CDRC instance - Enabled bool // Enable/Disable the profile - DryRun bool // Do not post CDRs to the server - CdrsConns []string // The address where CDRs can be reached - CdrFormat string // The type of CDR file to process <*csv|*opensips_flatstore> - FieldSeparator rune // The separator to use when reading csvs - Timezone string // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> - RunDelay time.Duration // Delay between runs, 0 for inotify driven requests - MaxOpenFiles int // Maximum number of files opened simultaneously - CDRInPath string // Folder to process CDRs from - CDROutPath string // Folder to move processed CDRs to - FailedCallsPrefix string // Used in case of flatstore CDRs to avoid searching for BYE records - CDRRootPath utils.HierarchyPath // used for XML CDRs to specify the path towards CDR elements - CdrSourceId string // Source identifier for the processed CDRs - Filters []string - Tenant RSRParsers - ContinueOnSuccess bool // Continue after execution - PartialRecordCache time.Duration // Duration to cache partial records when not pairing - PartialCacheExpiryAction string - HeaderFields []*FCTemplate - ContentFields []*FCTemplate - TrailerFields []*FCTemplate - CacheDumpFields []*FCTemplate -} - -func (self *CdrcCfg) loadFromJsonCfg(jsnCfg *CdrcJsonCfg, separator string) error { - if jsnCfg == nil { - return nil - } - var err error - if jsnCfg.Id != nil { - self.ID = *jsnCfg.Id - } - if jsnCfg.Enabled != nil { - self.Enabled = *jsnCfg.Enabled - } - if jsnCfg.Dry_run != nil { - self.DryRun = *jsnCfg.Dry_run - } - if jsnCfg.Cdrs_conns != nil { - self.CdrsConns = make([]string, len(*jsnCfg.Cdrs_conns)) - for idx, connID := range *jsnCfg.Cdrs_conns { - // if we have the connection internal we change the name so we can have internal rpc for each subsystem - if connID == utils.MetaInternal { - self.CdrsConns[idx] = utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs) - } else { - self.CdrsConns[idx] = connID - } - } - } - if jsnCfg.Cdr_format != nil { - self.CdrFormat = *jsnCfg.Cdr_format - } - if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 { - sepStr := *jsnCfg.Field_separator - self.FieldSeparator = rune(sepStr[0]) - } - if jsnCfg.Timezone != nil { - self.Timezone = *jsnCfg.Timezone - } - if jsnCfg.Run_delay != nil { - self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second - } - if jsnCfg.Max_open_files != nil { - self.MaxOpenFiles = *jsnCfg.Max_open_files - } - if jsnCfg.Cdr_in_path != nil { - self.CDRInPath = *jsnCfg.Cdr_in_path - } - if jsnCfg.Cdr_out_path != nil { - self.CDROutPath = *jsnCfg.Cdr_out_path - } - if jsnCfg.Failed_calls_prefix != nil { - self.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix - } - if jsnCfg.Cdr_root_path != nil { - self.CDRRootPath = utils.ParseHierarchyPath(*jsnCfg.Cdr_root_path, utils.EmptyString) - } - if jsnCfg.Cdr_source_id != nil { - self.CdrSourceId = *jsnCfg.Cdr_source_id - } - if jsnCfg.Filters != nil { - self.Filters = make([]string, len(*jsnCfg.Filters)) - for i, fltr := range *jsnCfg.Filters { - self.Filters[i] = fltr - } - } - if jsnCfg.Tenant != nil { - if self.Tenant, err = NewRSRParsers(*jsnCfg.Tenant, true, separator); err != nil { - return err - } - } - if jsnCfg.Continue_on_success != nil { - self.ContinueOnSuccess = *jsnCfg.Continue_on_success - } - if jsnCfg.Partial_record_cache != nil { - if self.PartialRecordCache, err = utils.ParseDurationWithNanosecs(*jsnCfg.Partial_record_cache); err != nil { - return err - } - } - if jsnCfg.Partial_cache_expiry_action != nil { - self.PartialCacheExpiryAction = *jsnCfg.Partial_cache_expiry_action - } - if jsnCfg.Header_fields != nil { - if self.HeaderFields, err = FCTemplatesFromFCTemplatesJsonCfg(*jsnCfg.Header_fields, separator); err != nil { - return err - } - } - if jsnCfg.Content_fields != nil { - if self.ContentFields, err = FCTemplatesFromFCTemplatesJsonCfg(*jsnCfg.Content_fields, separator); err != nil { - return err - } - } - if jsnCfg.Trailer_fields != nil { - if self.TrailerFields, err = FCTemplatesFromFCTemplatesJsonCfg(*jsnCfg.Trailer_fields, separator); err != nil { - return err - } - } - if jsnCfg.Cache_dump_fields != nil { - if self.CacheDumpFields, err = FCTemplatesFromFCTemplatesJsonCfg(*jsnCfg.Cache_dump_fields, separator); err != nil { - return err - } - } - return nil -} - -// Clone itself into a new CdrcCfg -func (self *CdrcCfg) Clone() *CdrcCfg { - clnCdrc := new(CdrcCfg) - clnCdrc.ID = self.ID - clnCdrc.Enabled = self.Enabled - clnCdrc.DryRun = self.DryRun - clnCdrc.CdrsConns = make([]string, len(self.CdrsConns)) - for idx, connID := range self.CdrsConns { - clnCdrc.CdrsConns[idx] = connID - } - clnCdrc.CdrFormat = self.CdrFormat - clnCdrc.FieldSeparator = self.FieldSeparator - clnCdrc.Timezone = self.Timezone - clnCdrc.RunDelay = self.RunDelay - clnCdrc.MaxOpenFiles = self.MaxOpenFiles - clnCdrc.CDRInPath = self.CDRInPath - clnCdrc.CDROutPath = self.CDROutPath - clnCdrc.CDRRootPath = make(utils.HierarchyPath, len(self.CDRRootPath)) - for i, path := range self.CDRRootPath { - clnCdrc.CDRRootPath[i] = path - } - clnCdrc.FailedCallsPrefix = self.FailedCallsPrefix - clnCdrc.Filters = make([]string, len(self.Filters)) - for i, fltr := range self.Filters { - clnCdrc.Filters[i] = fltr - } - clnCdrc.Tenant = self.Tenant - clnCdrc.CdrSourceId = self.CdrSourceId - clnCdrc.ContinueOnSuccess = self.ContinueOnSuccess - clnCdrc.PartialRecordCache = self.PartialRecordCache - clnCdrc.PartialCacheExpiryAction = self.PartialCacheExpiryAction - clnCdrc.HeaderFields = make([]*FCTemplate, len(self.HeaderFields)) - clnCdrc.ContentFields = make([]*FCTemplate, len(self.ContentFields)) - clnCdrc.TrailerFields = make([]*FCTemplate, len(self.TrailerFields)) - clnCdrc.CacheDumpFields = make([]*FCTemplate, len(self.CacheDumpFields)) - for idx, fld := range self.HeaderFields { - clnCdrc.HeaderFields[idx] = fld.Clone() - } - for idx, fld := range self.ContentFields { - clnCdrc.ContentFields[idx] = fld.Clone() - } - for idx, fld := range self.TrailerFields { - clnCdrc.TrailerFields[idx] = fld.Clone() - } - for idx, fld := range self.CacheDumpFields { - clnCdrc.CacheDumpFields[idx] = fld.Clone() - } - return clnCdrc -} diff --git a/config/cdrccfg_test.go b/config/cdrccfg_test.go deleted file mode 100644 index 0f6e4a714..000000000 --- a/config/cdrccfg_test.go +++ /dev/null @@ -1,133 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package config - -import ( - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/utils" -) - -var cdrcCfg = CdrcCfg{ - ID: utils.MetaDefault, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: "csv", - FieldSeparator: utils.CSV_SEP, - MaxOpenFiles: 1024, - CDRInPath: "/var/spool/cgrates/cdrc/in", - CDROutPath: "/var/spool/cgrates/cdrc/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: []string{""}, - CdrSourceId: "cdrc_csv", - Filters: []string{}, - Tenant: NewRSRParsersMustCompile("cgrates.org", true, utils.INFIELD_SEP), - PartialRecordCache: time.Duration(10 * time.Second), - PartialCacheExpiryAction: "*dump_to_file", - HeaderFields: []*FCTemplate{}, - ContentFields: []*FCTemplate{ - { - Tag: "TOR", - FieldId: "ToR", - Type: "*composed", - Value: NewRSRParsersMustCompile("~2", true, utils.INFIELD_SEP), - Mandatory: true, - }, - }, - TrailerFields: []*FCTemplate{}, - CacheDumpFields: []*FCTemplate{ - { - Tag: "CGRID", - Type: "*composed", - Value: NewRSRParsersMustCompile("~CGRID", true, utils.INFIELD_SEP), - }, - }, -} - -func TestCdrcCfgloadFromJsonCfg(t *testing.T) { - var cdrccfg, expected CdrcCfg - if err := cdrccfg.loadFromJsonCfg(nil, utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(cdrccfg, expected) { - t.Errorf("Expected: %+v ,recived: %+v", expected, cdrccfg) - } - if err := cdrccfg.loadFromJsonCfg(new(CdrcJsonCfg), utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(cdrccfg, expected) { - t.Errorf("Expected: %+v ,recived: %+v", expected, cdrccfg) - } - cfgJSONStr := `{ -"cdrc": [ - { - "id": "*default", // identifier of the CDRC runner - "enabled": false, // enable CDR client functionality - "dry_run": false, // do not send the CDRs to CDRS, just parse them - "cdrs_conns": ["*internal"], - "cdr_format": "csv", // CDR file format - "field_separator": ",", // separator used in case of csv files - "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> - "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify - "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited - "cdr_in_path": "/var/spool/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/var/spool/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records - "cdr_root_path": "", // path towards one CDR element in case of XML CDRs - "cdr_source_id": "cdrc_csv", // free form field, tag identifying the source of the CDRs within CDRS database - "filters" :[], // new filters used in FilterS subsystem - "tenant": "cgrates.org", // default tenant - "continue_on_success": false, // continue to the next template if executed - "partial_record_cache": "10s", // duration to cache partial records when not pairing - "partial_cache_expiry_action": "*dump_to_file", // action taken when cache when records in cache are timed-out <*dump_to_file|*post_cdr> - "header_fields": [], // template of the import header fields - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "~2", "mandatory": true}, - ], - "trailer_fields": [], // template of the import trailer fields - "cache_dump_fields": [ // template used when dumping cached CDR, eg: partial CDRs - {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, - ], - }, -], -}` - expected = cdrcCfg - if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { - t.Error(err) - } else if jsnCdrcCfg, err := jsnCfg.CdrcJsonCfg(); err != nil { - t.Error(err) - } else if err = cdrccfg.loadFromJsonCfg(jsnCdrcCfg[0], utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expected, cdrccfg) { - t.Errorf("Expected: %+v , recived: %+v", utils.ToJSON(expected), utils.ToJSON(cdrccfg)) - } -} - -func TestCdrcCfgClone(t *testing.T) { - clnCdrcCfg := *cdrcCfg.Clone() - if !reflect.DeepEqual(cdrcCfg, clnCdrcCfg) { - t.Errorf("Expected: %+v ,\n recived: %+v", utils.ToJSON(cdrcCfg), utils.ToJSON(clnCdrcCfg)) - } - cdrcCfg.ContentFields[0].Tag = "CGRID" - if reflect.DeepEqual(cdrcCfg, clnCdrcCfg) { // MOdifying a field after clone should not affect cloned instance - t.Errorf("Cloned result: %+v", utils.ToJSON(clnCdrcCfg)) - } - clnCdrcCfg.ContentFields[0].FieldId = "destination" - if cdrcCfg.ContentFields[0].FieldId != "ToR" { - t.Error("Unexpected change of FieldId: ", cdrcCfg.ContentFields[0].FieldId) - } -} diff --git a/config/cfg_data.json b/config/cfg_data.json index f047e5bc4..2ac7666a8 100644 --- a/config/cfg_data.json +++ b/config/cfg_data.json @@ -15,29 +15,6 @@ "enabled": true, // enable Rater service: }, -"cdrc": [ - { - "id": "CDRC-CSV1", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database - }, - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "data_usage_multiply_factor": 0.000976563, - "run_delay": 1, - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"field_id": "ToR", "value": "~*req.7:s/^(voice|data|sms|mms|generic)$/*$1/"}, - {"field_id": "AnswerTime", "value": "~*req.1"}, - {"field_id": "Usage", "value": "~*req.9:s/^(\\d+)$/${1}s/"}, - ], - }, -], "sessions": { "enabled": true, // enable Rater service: diff --git a/config/cfg_data2.json b/config/cfg_data2.json deleted file mode 100644 index 70ea242c1..000000000 --- a/config/cfg_data2.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - -// Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH - -"cdrc": [ - { - "id": "CDRC-CSV3", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database - }, -], - -} diff --git a/config/config.go b/config/config.go index e6c8fb83d..0e859428e 100755 --- a/config/config.go +++ b/config/config.go @@ -155,7 +155,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.schedulerCfg = new(SchedulerCfg) cfg.cdrsCfg = new(CdrsCfg) cfg.CdreProfiles = make(map[string]*CdreCfg) - cfg.CdrcProfiles = make(map[string][]*CdrcCfg) cfg.analyzerSCfg = new(AnalyzerSCfg) cfg.sessionSCfg = new(SessionSCfg) cfg.fsAgentCfg = new(FsAgentCfg) @@ -180,8 +179,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.ersCfg = new(ERsCfg) cfg.ConfigReloads = make(map[string]chan struct{}) - cfg.ConfigReloads[utils.CDRC] = make(chan struct{}, 1) - cfg.ConfigReloads[utils.CDRC] <- struct{}{} // Unlock the channel cfg.ConfigReloads[utils.CDRE] = make(chan struct{}, 1) cfg.ConfigReloads[utils.CDRE] <- struct{}{} // Unlock the channel @@ -194,7 +191,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { } cfg.dfltCdreProfile = cfg.CdreProfiles[utils.MetaDefault].Clone() // So default will stay unique, will have nil pointer in case of no defaults loaded which is an extra check - cfg.dfltCdrcProfile = cfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0].Clone() // populate default ERs reader for _, ersRdr := range cfg.ersCfg.Readers { if ersRdr.ID == utils.MetaDefault { @@ -251,13 +247,11 @@ type CGRConfig struct { // Cache defaults loaded from json and needing clones dfltCdreProfile *CdreCfg // Default cdreConfig profile - dfltCdrcProfile *CdrcCfg // Default cdrcConfig profile dfltEvRdr *EventReaderCfg // default event reader - CdreProfiles map[string]*CdreCfg // Cdre config profiles - CdrcProfiles map[string][]*CdrcCfg // Number of CDRC instances running imports, format map[dirPath][]{Configs} - loaderCfg LoaderSCfgs // LoaderS configs - httpAgentCfg HttpAgentCfgs // HttpAgent configs + CdreProfiles map[string]*CdreCfg // Cdre config profiles + loaderCfg LoaderSCfgs // LoaderS configs + httpAgentCfg HttpAgentCfgs // HttpAgent configs ConfigReloads map[string]chan struct{} // Signals to specific entities that a config reload should occur rldChans map[string]chan struct{} // index here the channels used for reloads @@ -332,8 +326,8 @@ func (cfg *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { cfg.loadGeneralCfg, cfg.loadCacheCfg, cfg.loadListenCfg, cfg.loadHTTPCfg, cfg.loadDataDBCfg, cfg.loadStorDBCfg, cfg.loadFilterSCfg, cfg.loadRalSCfg, cfg.loadSchedulerCfg, - cfg.loadCdrsCfg, cfg.loadCdreCfg, cfg.loadCdrcCfg, - cfg.loadSessionSCfg, cfg.loadFreeswitchAgentCfg, cfg.loadKamAgentCfg, + cfg.loadCdrsCfg, cfg.loadCdreCfg, cfg.loadSessionSCfg, + cfg.loadFreeswitchAgentCfg, cfg.loadKamAgentCfg, cfg.loadAsteriskAgentCfg, cfg.loadDiameterAgentCfg, cfg.loadRadiusAgentCfg, cfg.loadDNSAgentCfg, cfg.loadHttpAgentCfg, cfg.loadAttributeSCfg, cfg.loadChargerSCfg, cfg.loadResourceSCfg, cfg.loadStatSCfg, @@ -537,57 +531,6 @@ func (cfg *CGRConfig) loadCdreCfg(jsnCfg *CgrJsonCfg) (err error) { return } -// loadCdrcCfg loads the Cdrc section of the configuration -func (cfg *CGRConfig) loadCdrcCfg(jsnCfg *CgrJsonCfg) (err error) { - var jsnCdrcCfg []*CdrcJsonCfg - if jsnCdrcCfg, err = jsnCfg.CdrcJsonCfg(); err != nil { - return - } - if jsnCdrcCfg != nil { - for _, jsnCrc1Cfg := range jsnCdrcCfg { - if jsnCrc1Cfg.Id == nil || *jsnCrc1Cfg.Id == "" { - return utils.ErrCDRCNoProfileID - } - if *jsnCrc1Cfg.Id == utils.MetaDefault { - if cfg.dfltCdrcProfile == nil { - cfg.dfltCdrcProfile = new(CdrcCfg) - } - } - indxFound := -1 // Will be different than -1 if an instance with same id will be found - pathFound := "" // Will be populated with the path where slice of cfgs was found - var cdrcInstCfg *CdrcCfg - for path := range cfg.CdrcProfiles { - for i := range cfg.CdrcProfiles[path] { - if cfg.CdrcProfiles[path][i].ID == *jsnCrc1Cfg.Id { - indxFound = i - pathFound = path - cdrcInstCfg = cfg.CdrcProfiles[path][i] - break - } - } - } - if cdrcInstCfg == nil { - cdrcInstCfg = cfg.dfltCdrcProfile.Clone() - } - if err := cdrcInstCfg.loadFromJsonCfg(jsnCrc1Cfg, cfg.generalCfg.RSRSep); err != nil { - return err - } - if cdrcInstCfg.CDRInPath == "" { - return utils.ErrCDRCNoInPath - } - if _, hasDir := cfg.CdrcProfiles[cdrcInstCfg.CDRInPath]; !hasDir { - cfg.CdrcProfiles[cdrcInstCfg.CDRInPath] = make([]*CdrcCfg, 0) - } - if indxFound != -1 { // Replace previous config so we have inheritance - cfg.CdrcProfiles[pathFound][indxFound] = cdrcInstCfg - } else { - cfg.CdrcProfiles[cdrcInstCfg.CDRInPath] = append(cfg.CdrcProfiles[cdrcInstCfg.CDRInPath], cdrcInstCfg) - } - } - } - return -} - // loadSessionSCfg loads the SessionS section of the configuration func (cfg *CGRConfig) loadSessionSCfg(jsnCfg *CgrJsonCfg) (err error) { var jsnSessionSCfg *SessionSJsonCfg @@ -1139,8 +1082,6 @@ func (cfg *CGRConfig) V1GetConfigSection(args *StringWithArgDispatcher, reply *m jsonString = utils.ToJSON(cfg.MigratorCgrCfg()) case Apier: jsonString = utils.ToJSON(cfg.ApierCfg()) - case CDRC_JSN: - jsonString = utils.ToJSON(cfg.CdrcProfiles) case CDRE_JSN: jsonString = utils.ToJSON(cfg.CdreProfiles) case ERsJson: @@ -1230,7 +1171,6 @@ func (cfg *CGRConfig) getLoadFunctions() map[string]func(*CgrJsonCfg) error { RALS_JSN: cfg.loadRalSCfg, CDRS_JSN: cfg.loadCdrsCfg, CDRE_JSN: cfg.loadCdreCfg, - CDRC_JSN: cfg.loadCdrcCfg, ERsJson: cfg.loadErsCfg, SessionSJson: cfg.loadSessionSCfg, AsteriskAgentJSN: cfg.loadAsteriskAgentCfg, @@ -1482,7 +1422,6 @@ func (cfg *CGRConfig) reloadSections(sections ...string) (err error) { cfg.rldChans[RALS_JSN] <- struct{}{} case CDRS_JSN: cfg.rldChans[CDRS_JSN] <- struct{}{} - case CDRC_JSN: case ERsJson: cfg.rldChans[ERsJson] <- struct{}{} case SessionSJson: diff --git a/config/config_defaults.go b/config/config_defaults.go index f16ad0684..997c4374b 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -300,62 +300,6 @@ const CGRATES_CFG_JSON = ` }, -"cdrc": [ // CDRc config - { - "id": "*default", // identifier of the CDRC runner - "enabled": false, // enable CDR client functionality - "dry_run": false, // do not send the CDRs to CDRS, just parse them - "cdrs_conns": ["*internal"], - "cdr_format": "*file_csv", // CDR file format <*file_csv|*freeswitch_csv|*file_fwv|*opensips_flatstore|*partial_csv|*file_xml> - "field_separator": ",", // separator used in case of csv files - "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> - "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify - "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited - "cdr_in_path": "/var/spool/cgrates/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/var/spool/cgrates/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records - "cdr_root_path": "", // path towards one CDR element in case of XML CDRs - "cdr_source_id": "cdrc_csv", // free form field, tag identifying the source of the CDRs within CDRS database - "filters" :[], // limit parsing based on the filters - "tenant": "", // tenant used by import - "continue_on_success": false, // continue to the next template if executed - "partial_record_cache": "10s", // duration to cache partial records when not pairing - "partial_cache_expiry_action": "*dump_to_file", // action taken when cache when records in cache are timed-out <*dump_to_file|*post_cdr> - "header_fields": [], // template of the import header fields - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.4", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "~*req.7", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.8", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.9", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.10", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.11", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.12", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.13", "mandatory": true}, - ], - "trailer_fields": [], // template of the import trailer fields - "cache_dump_fields": [ // template used when dumping cached CDR, eg: partial CDRs - {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, - {"tag": "RunID", "type": "*composed", "value": "~RunID"}, - {"tag": "TOR", "type": "*composed", "value": "~ToR"}, - {"tag": "OriginID", "type": "*composed", "value": "~OriginID"}, - {"tag": "RequestType", "type": "*composed", "value": "~RequestType"}, - {"tag": "Tenant", "type": "*composed", "value": "~Tenant"}, - {"tag": "Category", "type": "*composed", "value": "~Category"}, - {"tag": "Account", "type": "*composed", "value": "~Account"}, - {"tag": "Subject", "type": "*composed", "value": "~Subject"}, - {"tag": "Destination", "type": "*composed", "value": "~Destination"}, - {"tag": "SetupTime", "type": "*composed", "value": "~SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "Usage", "type": "*composed", "value": "~Usage"}, - {"tag": "Cost", "type": "*composed", "value": "~Cost"}, - ], - }, -], - - "ers": { // EventReaderService "enabled": false, // starts the EventReader service: "sessions_conns":["*internal"], // RPC Connections IDs diff --git a/config/config_json.go b/config/config_json.go index e0322cafa..5bee771b4 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -34,7 +34,6 @@ const ( SCHEDULER_JSN = "schedulers" CDRS_JSN = "cdrs" CDRE_JSN = "cdre" - CDRC_JSN = "cdrc" SessionSJson = "sessions" FreeSWITCHAgentJSN = "freeswitch_agent" KamailioAgentJSN = "kamailio_agent" @@ -64,7 +63,7 @@ const ( var ( sortedCfgSections = []string{GENERAL_JSN, RPCConnsJsonName, DATADB_JSN, STORDB_JSN, LISTEN_JSN, TlsCfgJson, HTTP_JSN, SCHEDULER_JSN, CACHE_JSN, FilterSjsn, RALS_JSN, - CDRS_JSN, CDRE_JSN, CDRC_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN, + CDRS_JSN, CDRE_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN, DA_JSN, RA_JSN, HttpAgentJson, DNSAgentJson, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON, SupplierSJson, LoaderJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, AnalyzerCfgJson, Apier} ) @@ -211,18 +210,6 @@ func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) { return cfg, nil } -func (self CgrJsonCfg) CdrcJsonCfg() ([]*CdrcJsonCfg, error) { - rawCfg, hasKey := self[CDRC_JSN] - if !hasKey { - return nil, nil - } - cfg := make([]*CdrcJsonCfg, 0) - if err := json.Unmarshal(*rawCfg, &cfg); err != nil { - return nil, err - } - return cfg, nil -} - func (self CgrJsonCfg) ERsJsonCfg() (erSCfg *ERsJsonCfg, err error) { rawCfg, hasKey := self[ERsJson] if !hasKey { diff --git a/config/config_json_test.go b/config/config_json_test.go index 88c192450..d980ba441 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -520,112 +520,6 @@ func TestDfCdreJsonCfgs(t *testing.T) { } } -func TestDfCdrcJsonCfg(t *testing.T) { - eFields := []*FcTemplateJsonCfg{} - cdrFields := []*FcTemplateJsonCfg{ - {Tag: utils.StringPointer("TOR"), Field_id: utils.StringPointer(utils.ToR), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.2"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("OriginID"), Field_id: utils.StringPointer(utils.OriginID), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.3"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("RequestType"), Field_id: utils.StringPointer(utils.RequestType), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.4"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Tenant"), Field_id: utils.StringPointer(utils.Tenant), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.6"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Category"), Field_id: utils.StringPointer(utils.Category), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.7"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Account"), Field_id: utils.StringPointer(utils.Account), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.8"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Subject"), Field_id: utils.StringPointer(utils.Subject), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.9"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Destination"), Field_id: utils.StringPointer(utils.Destination), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.10"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("SetupTime"), Field_id: utils.StringPointer(utils.SetupTime), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.11"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("AnswerTime"), Field_id: utils.StringPointer(utils.AnswerTime), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.12"), Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Usage"), Field_id: utils.StringPointer(utils.Usage), Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~*req.13"), Mandatory: utils.BoolPointer(true)}, - } - cacheDumpFields := []*FcTemplateJsonCfg{ - {Tag: utils.StringPointer("CGRID"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.CGRID)}, - {Tag: utils.StringPointer("RunID"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.RunID)}, - {Tag: utils.StringPointer("TOR"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.ToR)}, - {Tag: utils.StringPointer("OriginID"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.OriginID)}, - {Tag: utils.StringPointer("RequestType"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.RequestType)}, - {Tag: utils.StringPointer("Tenant"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Tenant)}, - {Tag: utils.StringPointer("Category"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Category)}, - {Tag: utils.StringPointer("Account"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Account)}, - {Tag: utils.StringPointer("Subject"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Subject)}, - {Tag: utils.StringPointer("Destination"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Destination)}, - {Tag: utils.StringPointer("SetupTime"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.SetupTime), - Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, - {Tag: utils.StringPointer("AnswerTime"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.AnswerTime), - Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00")}, - {Tag: utils.StringPointer("Usage"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.Usage)}, - {Tag: utils.StringPointer("Cost"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.COST)}, - } - eCfg := []*CdrcJsonCfg{ - { - Id: utils.StringPointer(utils.MetaDefault), - Enabled: utils.BoolPointer(false), - Dry_run: utils.BoolPointer(false), - Cdrs_conns: &[]string{utils.MetaInternal}, - Cdr_format: utils.StringPointer("*file_csv"), - Field_separator: utils.StringPointer(","), - Timezone: utils.StringPointer(""), - Run_delay: utils.IntPointer(0), - Max_open_files: utils.IntPointer(1024), - Cdr_in_path: utils.StringPointer("/var/spool/cgrates/cdrc/in"), - Cdr_out_path: utils.StringPointer("/var/spool/cgrates/cdrc/out"), - Failed_calls_prefix: utils.StringPointer("missed_calls"), - Cdr_root_path: utils.StringPointer(""), - Cdr_source_id: utils.StringPointer("cdrc_csv"), - Filters: &[]string{}, - Tenant: utils.StringPointer(""), - Continue_on_success: utils.BoolPointer(false), - Partial_record_cache: utils.StringPointer("10s"), - Partial_cache_expiry_action: utils.StringPointer(utils.MetaDumpToFile), - Header_fields: &eFields, - Content_fields: &cdrFields, - Trailer_fields: &eFields, - Cache_dump_fields: &cacheDumpFields, - }, - } - if cfg, err := dfCgrJsonCfg.CdrcJsonCfg(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfg, cfg) { - t.Errorf("Expecting: %s \n, received: %s: ", utils.ToIJSON(eCfg), utils.ToIJSON(cfg)) - } -} - func TestSmgJsonCfg(t *testing.T) { eCfg := &SessionSJsonCfg{ Enabled: utils.BoolPointer(false), diff --git a/config/config_test.go b/config/config_test.go index cce2fc96d..874c6334b 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -186,95 +186,6 @@ func TestCgrCfgListener(t *testing.T) { } } -func TestCgrCfgCDRC(t *testing.T) { - JSN_RAW_CFG := ` -{ -"cdrc": [ - { - "id": "*default", - "enabled": true, // enable CDR client functionality - "Content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"field_id": "ToR", "type": "*composed", "value": "~7:s/^(voice|data|sms|mms|generic)$/*$1/"}, - {"field_id": "AnswerTime", "type": "*composed", "value": "~1"}, - {"field_id": "Usage", "type": "*composed", "value": "~9:s/^(\\d+)$/${1}s/"}, - ], - }, -], -}` - eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ - { - ID: utils.MetaDefault, - Enabled: true, - DryRun: false, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: utils.MetaFileCSV, - FieldSeparator: rune(utils.CSV_SEP), - Timezone: "", - RunDelay: 0, - MaxOpenFiles: 1024, - CDRInPath: "/var/spool/cgrates/cdrc/in", - CDROutPath: "/var/spool/cgrates/cdrc/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "cdrc_csv", - Filters: []string{}, - ContinueOnSuccess: false, - PartialRecordCache: time.Duration(10 * time.Second), - PartialCacheExpiryAction: "*dump_to_file", - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: "ToR", FieldId: "ToR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~7:s/^(voice|data|sms|mms|generic)$/*$1/", true, utils.INFIELD_SEP)}, - {Tag: "AnswerTime", FieldId: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~1", true, utils.INFIELD_SEP)}, - {Tag: "Usage", FieldId: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~9:s/^(\\d+)$/${1}s/", true, utils.INFIELD_SEP)}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - if cgrCfg, err := NewCGRConfigFromJsonStringWithDefaults(JSN_RAW_CFG); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) { - t.Errorf("Expected: %+v,\n received: %+v", - utils.ToJSON(eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0]), - utils.ToJSON(cgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0])) - } -} - func TestHttpAgentCfg(t *testing.T) { JSN_RAW_CFG := ` { @@ -1569,95 +1480,6 @@ func TestCgrMigratorCfgDefault(t *testing.T) { } } -func TestCDRCWithDefault(t *testing.T) { - eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ - { - ID: utils.MetaDefault, - Enabled: false, - DryRun: false, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: utils.MetaFileCSV, - FieldSeparator: rune(utils.CSV_SEP), - Timezone: "", - RunDelay: 0, - MaxOpenFiles: 1024, - CDRInPath: "/var/spool/cgrates/cdrc/in", - CDROutPath: "/var/spool/cgrates/cdrc/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "cdrc_csv", - Filters: []string{}, - ContinueOnSuccess: false, - PartialRecordCache: time.Duration(10 * time.Second), - PartialCacheExpiryAction: "*dump_to_file", - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: "TOR", FieldId: "ToR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", FieldId: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.3", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", FieldId: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.4", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", FieldId: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.6", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", FieldId: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.7", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", FieldId: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.8", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Subject", FieldId: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.9", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", FieldId: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.10", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", FieldId: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.11", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", FieldId: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.12", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", FieldId: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile("~*req.13", true, utils.INFIELD_SEP), Mandatory: true}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) { - t.Errorf("Expected: %+v,\n received: %+v", - utils.ToJSON(eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0]), - utils.ToJSON(cgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0])) - } -} - func TestCgrMigratorCfg2(t *testing.T) { JSN_CFG := ` { @@ -1980,49 +1802,4 @@ func TestCheckConfigSanity(t *testing.T) { if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } - // CDRC sanity checks - cfg, _ = NewDefaultCGRConfig() - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - }, - }, - } - expected = " Instance: , cdrc enabled but no CDRs defined!" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.dispatcherSCfg.Enabled = false - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - ID: "test", - CdrsConns: []string{utils.MetaInternal}, - }, - }, - } - expected = " not enabled but requested by cdrcProfile" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - CdrsConns: []string{utils.MetaInternal}, - ContentFields: []*FCTemplate{}, - }, - }, - } - cfg.cdrsCfg = &CdrsCfg{ - Enabled: true, - } - - expected = " enabled but no fields to be processed defined!" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } } diff --git a/config/configcdrc_test.go b/config/configcdrc_test.go deleted file mode 100644 index db166ee05..000000000 --- a/config/configcdrc_test.go +++ /dev/null @@ -1,331 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package config - -import ( - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/utils" -) - -func TestLoadCdrcConfigMultipleFiles(t *testing.T) { - cgrCfg, err := NewCGRConfigFromPath(".") - if err != nil { - t.Error(err) - } - eCgrCfg, _ := NewDefaultCGRConfig() - eCgrCfg.CdrcProfiles = make(map[string][]*CdrcCfg) - // Default instance first - eCgrCfg.CdrcProfiles["/var/spool/cgrates/cdrc/in"] = []*CdrcCfg{ - { - ID: utils.MetaDefault, - Enabled: false, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: "*file_csv", - FieldSeparator: utils.CSV_SEP, - RunDelay: 0, - MaxOpenFiles: 1024, - CDRInPath: "/var/spool/cgrates/cdrc/in", - CDROutPath: "/var/spool/cgrates/cdrc/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "cdrc_csv", - Filters: []string{}, - PartialRecordCache: time.Duration(10) * time.Second, - PartialCacheExpiryAction: utils.MetaDumpToFile, - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: NewRSRParsersMustCompile("~*req.3", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: NewRSRParsersMustCompile("~*req.4", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: NewRSRParsersMustCompile("~*req.6", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: NewRSRParsersMustCompile("~*req.7", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: NewRSRParsersMustCompile("~*req.8", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, - Value: NewRSRParsersMustCompile("~*req.9", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: NewRSRParsersMustCompile("~*req.10", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: NewRSRParsersMustCompile("~*req.11", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: NewRSRParsersMustCompile("~*req.12", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, - Value: NewRSRParsersMustCompile("~*req.13", true, utils.INFIELD_SEP), Mandatory: true}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = []*CdrcCfg{ - { - ID: "CDRC-CSV1", - Enabled: true, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: "*file_csv", - FieldSeparator: utils.CSV_SEP, - RunDelay: 0, - MaxOpenFiles: 1024, - CDRInPath: "/tmp/cgrates/cdrc1/in", - CDROutPath: "/tmp/cgrates/cdrc1/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "csv1", - Filters: []string{}, - PartialRecordCache: time.Duration(10) * time.Second, - PartialCacheExpiryAction: utils.MetaDumpToFile, - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: NewRSRParsersMustCompile("~*req.3", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: NewRSRParsersMustCompile("~*req.4", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: NewRSRParsersMustCompile("~*req.6", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: NewRSRParsersMustCompile("~*req.7", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: NewRSRParsersMustCompile("~*req.8", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, - Value: NewRSRParsersMustCompile("~*req.9", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: NewRSRParsersMustCompile("~*req.10", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: NewRSRParsersMustCompile("~*req.11", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: NewRSRParsersMustCompile("~*req.12", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, - Value: NewRSRParsersMustCompile("~*req.13", true, utils.INFIELD_SEP), Mandatory: true}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = []*CdrcCfg{ - { - ID: "CDRC-CSV2", - Enabled: true, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: "*file_csv", - FieldSeparator: utils.CSV_SEP, - RunDelay: 1000000000, - MaxOpenFiles: 1024, - CDRInPath: "/tmp/cgrates/cdrc2/in", - CDROutPath: "/tmp/cgrates/cdrc2/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "csv2", - Filters: []string{}, - PartialRecordCache: time.Duration(10) * time.Second, - PartialCacheExpiryAction: utils.MetaDumpToFile, - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: utils.ToR, FieldId: utils.ToR, - Value: NewRSRParsersMustCompile("~*req.7:s/^(voice|data|sms|mms|generic)$/*$1/", true, utils.INFIELD_SEP)}, - {Tag: utils.AnswerTime, Type: "", FieldId: utils.AnswerTime, - Value: NewRSRParsersMustCompile("~*req.1", true, utils.INFIELD_SEP)}, - {Tag: utils.Usage, FieldId: utils.Usage, - Value: NewRSRParsersMustCompile("~*req.9:s/^(\\d+)$/${1}s/", true, utils.INFIELD_SEP)}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = []*CdrcCfg{ - { - ID: "CDRC-CSV3", - Enabled: true, - CdrsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs)}, - CdrFormat: utils.MetaFileCSV, - FieldSeparator: utils.CSV_SEP, - RunDelay: 0, - MaxOpenFiles: 1024, - CDRInPath: "/tmp/cgrates/cdrc3/in", - CDROutPath: "/tmp/cgrates/cdrc3/out", - FailedCallsPrefix: "missed_calls", - CDRRootPath: utils.HierarchyPath([]string{""}), - CdrSourceId: "csv3", - Filters: []string{}, - PartialRecordCache: time.Duration(10) * time.Second, - PartialCacheExpiryAction: utils.MetaDumpToFile, - HeaderFields: make([]*FCTemplate, 0), - ContentFields: []*FCTemplate{ - {Tag: "TOR", Type: utils.META_COMPOSED, FieldId: utils.ToR, - Value: NewRSRParsersMustCompile("~*req.2", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "OriginID", Type: utils.META_COMPOSED, FieldId: utils.OriginID, - Value: NewRSRParsersMustCompile("~*req.3", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "RequestType", Type: utils.META_COMPOSED, FieldId: utils.RequestType, - Value: NewRSRParsersMustCompile("~*req.4", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Tenant", Type: utils.META_COMPOSED, FieldId: utils.Tenant, - Value: NewRSRParsersMustCompile("~*req.6", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Category", Type: utils.META_COMPOSED, FieldId: utils.Category, - Value: NewRSRParsersMustCompile("~*req.7", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Account", Type: utils.META_COMPOSED, FieldId: utils.Account, - Value: NewRSRParsersMustCompile("~*req.8", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Subject", Type: utils.META_COMPOSED, FieldId: utils.Subject, - Value: NewRSRParsersMustCompile("~*req.9", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, - Value: NewRSRParsersMustCompile("~*req.10", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, FieldId: utils.SetupTime, - Value: NewRSRParsersMustCompile("~*req.11", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, FieldId: utils.AnswerTime, - Value: NewRSRParsersMustCompile("~*req.12", true, utils.INFIELD_SEP), Mandatory: true}, - {Tag: "Usage", Type: utils.META_COMPOSED, FieldId: utils.Usage, - Value: NewRSRParsersMustCompile("~*req.13", true, utils.INFIELD_SEP), Mandatory: true}, - }, - TrailerFields: make([]*FCTemplate, 0), - CacheDumpFields: []*FCTemplate{ - {Tag: "CGRID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.CGRID, true, utils.INFIELD_SEP)}, - {Tag: "RunID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RunID, true, utils.INFIELD_SEP)}, - {Tag: "TOR", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.ToR, true, utils.INFIELD_SEP)}, - {Tag: "OriginID", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.OriginID, true, utils.INFIELD_SEP)}, - {Tag: "RequestType", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.RequestType, true, utils.INFIELD_SEP)}, - {Tag: "Tenant", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Tenant, true, utils.INFIELD_SEP)}, - {Tag: "Category", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Category, true, utils.INFIELD_SEP)}, - {Tag: "Account", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Account, true, utils.INFIELD_SEP)}, - {Tag: "Subject", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Subject, true, utils.INFIELD_SEP)}, - {Tag: "Destination", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Destination, true, utils.INFIELD_SEP)}, - {Tag: "SetupTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.SetupTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "AnswerTime", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.AnswerTime, true, utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - {Tag: "Usage", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.Usage, true, utils.INFIELD_SEP)}, - {Tag: "Cost", Type: utils.META_COMPOSED, - Value: NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.COST, true, utils.INFIELD_SEP)}, - }, - }, - } - if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) { - t.Errorf("Expected: %s \n, received: %s", utils.ToJSON(eCgrCfg.CdrcProfiles), utils.ToJSON(cgrCfg.CdrcProfiles)) - } -} diff --git a/config/configsanity.go b/config/configsanity.go index b59dd7eea..3e36c5c74 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -94,28 +94,6 @@ func (cfg *CGRConfig) checkConfigSanity() error { } } } - // CDRC sanity checks - for _, cdrcCfgs := range cfg.CdrcProfiles { - for _, cdrcInst := range cdrcCfgs { - if !cdrcInst.Enabled { - continue - } - if len(cdrcInst.CdrsConns) == 0 { - return fmt.Errorf("<%s> Instance: %s, %s enabled but no %s defined!", utils.CDRC, cdrcInst.ID, utils.CDRC, utils.CDRs) - } - for _, connID := range cdrcInst.CdrsConns { - if strings.HasPrefix(connID, utils.MetaInternal) && !cfg.cdrsCfg.Enabled { - return fmt.Errorf("<%s> not enabled but requested by <%s> cdrcProfile", utils.CDRs, cdrcInst.ID) - } - if _, has := cfg.rpcConns[connID]; !has && !strings.HasPrefix(connID, utils.MetaInternal) { - return fmt.Errorf("<%s> Connection with id: <%s> not defined", utils.CDRs, connID) - } - } - if len(cdrcInst.ContentFields) == 0 { - return fmt.Errorf("<%s> enabled but no fields to be processed defined!", utils.CDRC) - } - } - } // Loaders sanity checks for _, ldrSCfg := range cfg.loaderCfg { if !ldrSCfg.Enabled { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index eab2afd81..4707e9cec 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -93,50 +93,6 @@ func TestConfigSanityCDRServer(t *testing.T) { } } -func TestConfigSanityCDRC(t *testing.T) { - cfg, _ = NewDefaultCGRConfig() - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - }, - }, - } - expected := " Instance: , cdrc enabled but no CDRs defined!" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - ID: "test", - CdrsConns: []string{utils.MetaInternal}, - }, - }, - } - expected = " not enabled but requested by cdrcProfile" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.CdrcProfiles = map[string][]*CdrcCfg{ - "test": []*CdrcCfg{ - &CdrcCfg{ - Enabled: true, - ID: "test", - CdrsConns: []string{utils.MetaInternal}, - ContentFields: []*FCTemplate{}, - }, - }, - } - cfg.cdrsCfg.Enabled = true - expected = " enabled but no fields to be processed defined!" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } -} - func TestConfigSanityLoaders(t *testing.T) { cfg, _ = NewDefaultCGRConfig() cfg.loaderCfg = LoaderSCfgs{ diff --git a/config/libconfig_json.go b/config/libconfig_json.go index e8d057bf8..eb0a5d070 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -157,33 +157,6 @@ type CdreJsonCfg struct { Trailer_fields *[]*FcTemplateJsonCfg } -// Cdrc config section -type CdrcJsonCfg struct { - Id *string - Enabled *bool - Dry_run *bool - Cdrs_conns *[]string - Cdr_format *string - Field_separator *string - Timezone *string - Run_delay *int - Cdr_in_path *string - Cdr_out_path *string - Failed_calls_prefix *string - Cdr_root_path *string - Cdr_source_id *string - Filters *[]string - Tenant *string - Continue_on_success *bool - Max_open_files *int - Partial_record_cache *string - Partial_cache_expiry_action *string - Header_fields *[]*FcTemplateJsonCfg - Content_fields *[]*FcTemplateJsonCfg - Trailer_fields *[]*FcTemplateJsonCfg - Cache_dump_fields *[]*FcTemplateJsonCfg -} - // EventReaderSJsonCfg contains the configuration of EventReaderService type ERsJsonCfg struct { Enabled *bool diff --git a/data/conf/samples/cdrc_partcsv/cgrates.json b/data/conf/samples/cdrc_partcsv/cgrates.json deleted file mode 100644 index 3a2eeddac..000000000 --- a/data/conf/samples/cdrc_partcsv/cgrates.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - "rals": { - "enabled": true // so we can query CDRs - }, - - "cdrs": { - "enabled": true, - "rals_conns": [], // no rating support, just *raw CDR testing -}, - - -"chargers": { - "enabled": true, -}, - - - "cdrc": [ - { - "id": "part1", - "enabled": true, - "cdr_format": "*partial_csv", - "cdr_in_path": "/tmp/cdrctests/partcsv1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cdrctests/partcsv1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "partial_csv_test", // free form field, tag identifying the source of the CDRs within CDRS database - "partial_record_cache": "2s", // duration to cache partial records when not pairing - "partial_cache_expiry_action": "*dump_to_file", - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "ToR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "AccId1", "field_id": "OriginID", "type": "*composed", "value": "~*req.0"}, - {"tag": "AccId2", "field_id": "OriginID", "type": "*composed", "value": "_"}, - {"tag": "AccId3", "field_id": "OriginID", "type": "*composed", "value": "~*req.1"}, - {"tag": "AccId4", "field_id": "OriginID", "type": "*composed", "value": "_"}, - {"tag": "AccId5", "field_id": "OriginID", "type": "*composed", "value": "~*req.4"}, - {"tag": "OrderID", "field_id": "OrderID", "type": "*unix_timestamp", "value": "~*req.3"}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.0:s/^49([1-9]\\d+)$/0$1/", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.1:s/^00(\\d+)$/+$1/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.4", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.4", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, - {"tag": "Partial", "field_id": "Partial", "type": "*composed", "value": "true", "filters":["*string:~*req.10:partial"]}, - ], - "cache_dump_fields": [ - {"tag": "OriginID", "type": "*composed", "value": "~OriginID"}, - {"tag": "OrderID", "type": "*composed", "value": "~OrderID"}, - {"tag": "RequestType", "type": "*composed", "value": "~RequestType"}, - {"tag": "Account", "type": "*composed", "value": "~Account"}, - {"tag": "Destination", "type": "*composed", "value": "~Destination"}, - {"tag": "SetupTime", "type": "*composed", "value": "~SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag": "Usage", "type": "*composed", "value": "~Usage"}, - {"tag": "Cost", "type": "*composed", "value": "~Cost","rounding_decimals":5}, - ], - }, - { - "id": "post_on_expiry", - "enabled": true, - "cdr_format": "*partial_csv", - "cdr_in_path": "/tmp/cdrctests/partcsv2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cdrctests/partcsv2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "partial_csv_test2", // free form field, tag identifying the source of the CDRs within CDRS database - "partial_record_cache": "1s", // duration to cache partial records when not pairing - "partial_cache_expiry_action": "*post_cdr", - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "ToR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "AccId1", "field_id": "OriginID", "type": "*composed", "value": "~*req.0"}, - {"tag": "AccId2", "field_id": "OriginID", "type": "*composed", "value": "_"}, - {"tag": "AccId3", "field_id": "OriginID", "type": "*composed", "value": "~*req.1"}, - {"tag": "AccId4", "field_id": "OriginID", "type": "*composed", "value": "_"}, - {"tag": "AccId5", "field_id": "OriginID", "type": "*composed", "value": "~*req.4"}, - {"tag": "OrderID", "field_id": "OrderID", "type": "*unix_timestamp", "value": "~*req.3"}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.0:s/^49([1-9]\\d+)$/0$1/", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.1:s/^00(\\d+)$/+$1/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.4", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.4", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, - {"tag": "Partial", "field_id": "Partial", "type": "*composed", "value": "true", "filters":["*string:~*req.10:partial"]}, - ], - }, -], - - -} diff --git a/data/conf/samples/cdrccsv/cgrates.json b/data/conf/samples/cdrccsv/cgrates.json deleted file mode 100644 index c77817049..000000000 --- a/data/conf/samples/cdrccsv/cgrates.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. -"general": { - "log_level": 7, -}, - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - - "rals": { - "enabled": true // so we can query CDRs - }, - - - "cdrs": { - "enabled": true, - "rals_conns": [], // no rating support, just *raw CDR testing -}, - -"chargers": { - "enabled": true, -}, - - "cdrc": [ - { - "id": "*default", - "enabled": true, - "cdr_in_path": "/tmp/cdrctests/csvit1/in", - "cdr_out_path": "/tmp/cdrctests/csvit1/out", - "cdr_source_id": "csvit1", - }, - { - "id": "*CSVit2", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_path": "/tmp/cdrctests/csvit2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cdrctests/csvit2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, - ], - }, - { - "id": "*CSVWithFilter1", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_path": "/tmp/csvwithfilter1/csvit1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/csvwithfilter1/csvit1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit1", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:~*req.3:1002"], //filter Account to be 1002 - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, - ], - }, - { - "id": "*CSVWithFilter2", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_path": "/tmp/csvwithfilter2/csvit2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/csvwithfilter2/csvit2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit2", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:~*req.3:1002","*string:~*req.1:*prepaid","*gte:~*req.6:70"], //filter Account to be 1002 and RequestType *prepaid - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, - ], - }, - { - "id": "*CSVit4", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_path": "/tmp/csvwithfilter3/csvit3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/csvwithfilter3/csvit3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csvit4", // free form field, tag identifying the source of the CDRs within CDRS database - "filters":["*string:~*req.1:*postpaid"], //filter Account to be 1002 - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:~*req.3:1001"]}, - ], - }, - { - "id": "DifferentTenant", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "field_separator": ";", - "cdr_in_path": "/tmp/csvwithfilter4/csvit4/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/csvwithfilter4/csvit4/out", // absolute path towards the directory where processed CDRs will be moved - "tenant": "~*req.2", // tenant used in filterS.Pass - "filters":["FLTR_CDRC_ACC"], // if tenant is itsyscom.com will check if Account is 1001 and if tenant is cgrates.org will check if Account is 1002 - "cdr_source_id": "diffTenant", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0", "mandatory": true}, - {"tag": "OriginHost", "field_id": "OriginHost", "type": "*composed", "value": "~*req.10", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.1", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.2", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.3", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.5", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra3", "field_id": "HDRExtra3", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra2", "field_id": "HDRExtra2", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "HDRExtra1", "field_id": "HDRExtra1", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "RandomVal", "field_id": "RandomVal", "type": "*composed", "value": "*randomValue","filters":["*string:~*req.3:1001"]}, - ], - }, -], - - -} \ No newline at end of file diff --git a/data/conf/samples/cdrcflatstore/cgrates.json b/data/conf/samples/cdrcflatstore/cgrates.json deleted file mode 100644 index 827446284..000000000 --- a/data/conf/samples/cdrcflatstore/cgrates.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. - -"general": { - "log_level": 7, -}, - - -"rals": { - "enabled": true, // enable Rater service: -}, - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "FLATSTORE", - "enabled": true, // enable CDR client functionality - "cdrs_conns": ["*internal"], - "cdr_format": "*opensips_flatstore", // CDR file format - "field_separator": "|", // separator used in case of csv files - "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify - "max_open_files": 1024, // maximum simultaneous files to process - "cdr_in_path": "/tmp/cgr_flatstore/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgr_flatstore/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "failed_calls_prefix": "missed_calls", // used in case of flatstore CDRs to avoid searching for BYE records - "cdr_source_id": "flatstore", // free form field, id identifying the source of the CDRs within CDRS database - "partial_record_cache": "1s", // duration to cache partial records when not pairing - "content_fields":[ // import template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "Tor", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "~*req.7", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.8", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.8", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.9", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.6", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "mandatory": true}, - {"tag": "DisconnectCause", "field_id": "DisconnectCause", "type": "*composed", "value": "~*req.4; ;~*req.5", "mandatory": true}, - {"tag": "DialogId", "field_id": "DialogId", "type": "*composed", "value": "~*req.11"}, - ], - }, -], - -"apier": { - "scheduler_conns": ["*internal"], -}, - -} diff --git a/data/conf/samples/cdrcfwv/cgrates.json b/data/conf/samples/cdrcfwv/cgrates.json deleted file mode 100644 index e53ef5d19..000000000 --- a/data/conf/samples/cdrcfwv/cgrates.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. - -"general": { - "log_level": 7, -}, - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - -"rals": { - "enabled": true, // enable Rater service: -}, - - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "FWV1", - "enabled": true, // enable CDR client functionality - "dry_run": false, - "cdrs_conns": ["*internal"], - "cdr_format": "*file_fwv", // CDR file format - "cdr_in_path": "/tmp/cgr_fwv/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgr_fwv/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "cdrc", // free form field, tag identifying the source of the CDRs within CDRS database - "header_fields": [ - {"tag": "FileName", "field_id": "CdrFileName", "type": "*composed", "value": "~*req.95-135", "padding":"right"}, - {"tag": "FileSeqNr", "field_id": "FileSeqNr", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - ], - "content_fields": [ // import template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "Tor", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "rated", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0-10", "padding":"right", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.52-80", "padding":"right", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.127-135:s/(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}h${2}m${3}s/", "mandatory": true}, - {"tag": "DisconnectCause", "field_id": "DisconnectCause", "type": "*composed", "value": "~*req.138-139", "mandatory": true}, - {"tag": "RetailAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.103-111", "padding":"zeroleft"}, - {"tag": "WholesaleAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.115-123", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.3-6", "padding":"zeroleft", "mandatory": true}, - {"tag": "AccId2", "field_id": "AccId2", "type": "*composed", "value": "~*req.14-30", "padding":"right", "mandatory": true}, - ], - "trailer_fields": [ - {"tag": "NrOfCdrs", "type": "metatag", "metatag_id":"total_cdrs", "value": "~*req.142-150"}, - {"tag": "TotalDuration", "type": "metatag", "metatag_id":"total_duration", "value": "~*req.150-162"}, - ], - }, -], - -"apier": { - "scheduler_conns": ["*internal"], -}, - -} diff --git a/data/conf/samples/cdrcfwvwithfilter/cgrates.json b/data/conf/samples/cdrcfwvwithfilter/cgrates.json deleted file mode 100755 index d0adf1e42..000000000 --- a/data/conf/samples/cdrcfwvwithfilter/cgrates.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH -// -// This file contains the default configuration hardcoded into CGRateS. -// This is what you get when you load CGRateS with an empty configuration file. - -"general": { - "log_level": 7, -}, - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - -"rals": { - "enabled": true, // enable Rater service: -}, - - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - - -"cdrc": [ - { - "id": "FWVWithFilter", - "enabled": true, // enable CDR client functionality - "dry_run": false, - "cdrs_conns": ["*internal"], - "cdr_format": "*file_fwv", // CDR file format - "cdr_in_path": "/tmp/cgr_fwv/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgr_fwv/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "cdrc", // free form field, id identifying the source of the CDRs within CDRS database - "filters":["*string:~*req.0-10:CDR0000010"], - "header_fields": [ - {"tag": "FileName", "field_id": "CdrFileName", "type": "*composed", "value": "~*req.95-135", "padding":"right"}, - {"tag": "FileSeqNr", "field_id": "FileSeqNr", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - ], - "content_fields": [ // import template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "Tor", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "rated", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0-10", "padding":"right", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.52-80", "padding":"right", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.127-135:s/(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}h${2}m${3}s/", "mandatory": true}, - {"tag": "DisconnectCause", "field_id": "DisconnectCause", "type": "*composed", "value": "~*req.138-139", "mandatory": true}, - {"tag": "RetailAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.103-111", "padding":"zeroleft"}, - {"tag": "WholesaleAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.115-123", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.3-6", "padding":"zeroleft", "mandatory": true}, - {"tag": "AccId2", "field_id": "AccId2", "type": "*composed", "value": "~*req.14-30", "padding":"right", "mandatory": true}, - ], - "trailer_fields": [ - {"tag": "NrOfCdrs", "type": "metatag", "metatag_id":"total_cdrs", "value": "~*req.142-150"}, - {"tag": "TotalDuration", "type": "metatag", "metatag_id":"total_duration", "value": "~*req.150-162"}, - ], - }, - - { - "id": "FWVWithFilterID", - "enabled": true, // enable CDR client functionality - "dry_run": false, - "cdrs_conns": ["*internal"], - "cdr_format": "fwv", // CDR file format - "cdr_in_path": "/tmp/cgr_fwv/cdrc/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgr_fwv/cdrc/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "cdrc", // free form field, id identifying the source of the CDRs within CDRS database - "filters":["FLTR_FWV"], - "header_fields": [ - {"tag": "FileName", "field_id": "CdrFileName", "type": "*composed", "value": "~*req.95-135", "padding":"right"}, - {"tag": "FileSeqNr", "field_id": "FileSeqNr", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.135-141", "padding":"zeroleft"}, - ], - "content_fields": [ // import template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "Tor", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "rated", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.0-10", "padding":"right", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Subject", "field_id": "Subject", "type": "*composed", "value": "~*req.30-49", "padding":"right", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.52-80", "padding":"right", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.14-30:s/(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}-${2}-${3} ${4}:${5}:${6}/", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*composed", "value": "~*req.127-135:s/(\\d{2})(\\d{2})(\\d{2})(\\d{2})/${1}h${2}m${3}s/", "mandatory": true}, - {"tag": "DisconnectCause", "field_id": "DisconnectCause", "type": "*composed", "value": "~*req.138-139", "mandatory": true}, - {"tag": "RetailAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.103-111", "padding":"zeroleft"}, - {"tag": "WholesaleAmount", "field_id": "RetailAmount", "type": "*composed", "value": "~*req.115-123", "padding":"zeroleft"}, - {"tag": "AccId1", "field_id": "AccId1", "type": "*composed", "value": "~*req.3-6", "padding":"zeroleft", "mandatory": true}, - {"tag": "AccId2", "field_id": "AccId2", "type": "*composed", "value": "~*req.14-30", "padding":"right", "mandatory": true}, - ], - "trailer_fields": [ - {"tag": "NrOfCdrs", "type": "metatag", "metatag_id":"total_cdrs", "value": "~*req.142-150"}, - {"tag": "TotalDuration", "type": "metatag", "metatag_id":"total_duration", "value": "~*req.150-162"}, - ], - }, -], - -"apier": { - "scheduler_conns": ["*internal"], -}, - -} diff --git a/data/conf/samples/cdrcxml/cgrates.json b/data/conf/samples/cdrcxml/cgrates.json deleted file mode 100644 index b15a47379..000000000 --- a/data/conf/samples/cdrcxml/cgrates.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - "rals": { - "enabled": true - }, - - "cdrs": { - "enabled": true, - "rals_conns": [], -}, - -"chargers": { - "enabled": true, -}, - - - "cdrc": [ - { - "id": "XMLit1", - "enabled": true, - "cdr_format": "*file_xml", - "cdr_in_path": "/tmp/cdrctests/xmlit1/in", - "cdr_out_path": "/tmp/cdrctests/xmlit1/out", - "cdr_root_path": "broadWorksCDR.cdrData", - "cdr_source_id": "xmlit1", - "content_fields":[ // import content_fields template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.userNumber", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.calledNumber", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.startTime", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~*req.broadWorksCDR.cdrData.basicModule.releaseTime;|;~*req.broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, - ], - }, -], - - -} \ No newline at end of file diff --git a/data/conf/samples/cdrcxmlwithfilter/cgrates.json b/data/conf/samples/cdrcxmlwithfilter/cgrates.json deleted file mode 100755 index 8bb57eaaf..000000000 --- a/data/conf/samples/cdrcxmlwithfilter/cgrates.json +++ /dev/null @@ -1,139 +0,0 @@ -{ - -// Real-time Charging System for Telecom & ISP environments -// Copyright (C) ITsysCOM GmbH - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - - "rals": { - "enabled": true - }, - - "cdrs": { - "enabled": true, - "rals_conns": [], -}, - -"chargers": { - "enabled": true, -}, - - - "cdrc": [ - { - "id": "XMLWithFilter", - "enabled": true, - "cdr_format": "*file_xml", - "cdr_in_path": "/tmp/cdrcxmlwithfilters/xmlit1/in", - "cdr_out_path": "/tmp/cdrcxmlwithfilters/xmlit1/out", - "cdr_root_path": "broadWorksCDR.cdrData", - "cdr_source_id": "xmlit1", - "filters": ["*string:~*req.broadWorksCDR.cdrData.basicModule.userNumber:1002","*string:~*req.broadWorksCDR.cdrData.headerModule.type:Normal"], - "content_fields":[ // import content_fields template, id will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "TOR", "field_id": "ToR", "type": "*composed", "value": "*voice", "mandatory": true}, - {"tag": "OriginID", "field_id": "OriginID", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, - {"tag": "RequestType", "field_id": "RequestType", "type": "*composed", "value": "*rated", "mandatory": true}, - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.userId:s/.*@(.*)/${1}/", "mandatory": true}, - {"tag": "Category", "field_id": "Category", "type": "*composed", "value": "call", "mandatory": true}, - {"tag": "Account", "field_id": "Account", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.userNumber", "mandatory": true}, - {"tag": "Destination", "field_id": "Destination", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.calledNumber", "mandatory": true}, - {"tag": "SetupTime", "field_id": "SetupTime", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.startTime", "mandatory": true}, - {"tag": "AnswerTime", "field_id": "AnswerTime", "type": "*composed", "value": "~*req.broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, - {"tag": "Usage", "field_id": "Usage", "type": "*handler", "handler_id": "*substract_usage", "value": "~*req.broadWorksCDR.cdrData.basicModule.releaseTime;|;~*req.broadWorksCDR.cdrData.basicModule.answerTime", "mandatory": true}, - ], - }, - { - "id": "msw_xml", // identifier of the CDRC runner - "enabled": true, // enable CDR client functionality - "cdr_format": "*file_xml", // CDR file format -}, - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"apier": { - "scheduler_conns": ["*internal"], -}, - -} diff --git a/data/conf/samples/fscsv/freeswitch_csvcdr.json b/data/conf/samples/fscsv/freeswitch_csvcdr.json deleted file mode 100644 index 704718970..000000000 --- a/data/conf/samples/fscsv/freeswitch_csvcdr.json +++ /dev/null @@ -1,27 +0,0 @@ -{ -// Contains CDRC template for FreeSWITCH CDR - -"cdrc": [ - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc_fs/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc_fs/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "fs_csv", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"tag": "tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "^*voice", "mandatory": true}, - {"tag": "accid", "cdr_field_id": "accid", "type": "cdrfield", "value": "10", "mandatory": true}, - {"tag": "reqtype", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "^rated", "mandatory": true}, - {"tag": "tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "^cgrates.org", "mandatory": true}, - {"tag": "category", "cdr_field_id": "category", "type": "cdrfield", "value": "^call", "mandatory": true}, - {"tag": "account", "cdr_field_id": "account", "type": "cdrfield", "value": "12", "mandatory": true}, - {"tag": "subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "12", "mandatory": true}, - {"tag": "destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "2", "mandatory": true}, - {"tag": "setup_time", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "4", "mandatory": true}, - {"tag": "answer_time", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "5", "mandatory": true}, - {"tag": "usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "~8:s/^(\\d+)$/${1}s/", "mandatory": true}, - ], - }, -], - -} diff --git a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json deleted file mode 100644 index b8996c811..000000000 --- a/data/conf/samples/multiplecdrc/multiplecdrc_fwexport.json +++ /dev/null @@ -1,135 +0,0 @@ -{ -// CGRateS Configuration file -// -// Used in mediator_local_test -// Starts rater, cdrs and mediator connecting over internal channel - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - -"rals": { - "enabled": true, // enable Rater service: -}, - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "CDRC-CSV1", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database - }, - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"}, - {"cdr_field_id": "accid", "value": "0"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "~7:s/^voice$/call/"}, - {"cdr_field_id": "account", "value": "3"}, - {"cdr_field_id": "subject", "value": "3"}, - {"cdr_field_id": "destination", "value": "~5:s/^0([1-9]\\d+)$/+49$1/"}, - {"cdr_field_id": "setup_time", "value": "1"}, - {"cdr_field_id": "answer_time", "value": "1"}, - {"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"}, - ], - }, - { - "id": "CDRC-CSV3", - "enabled": true, // enable CDR client functionality - "field_separator": ";", // separator used in case of csv files - "cdr_in_path": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "^*voice"}, - {"cdr_field_id": "accid", "value": "~3:s/^(\\d{2})\\.(\\d{2})\\.(\\d{4})\\s{2}(\\d{2}):(\\d{2}):(\\d{2})$/$1$2$3$4$5$6/"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "^call"}, - {"cdr_field_id": "account", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "subject", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "destination", "value": "~1:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "setup_time", "value": "4"}, - {"cdr_field_id": "answer_time", "value": "4"}, - {"cdr_field_id": "usage", "value": "~6:s/^(\\d+)$/${1}s/"}, - ], - } -], - -"cdre": { - "CDRE-FW1": { - "export_format": "*file_fwv", - "field_separator": "", - "header_fields": [ - {"tag": "ToR", "type": "constant", "value": "10", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "LastCdr", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "FileCreationfTime", "type": "metatag", "value": "time_now", "layout": "020106150405", "width": 12}, - {"tag": "FileVersion", "type": "constant", "value": "01", "width": 2}, - {"tag": "Filler2", "type": "filler", "width": 105}, - ], // template of the exported header fields - "content_fields": [ // template of the exported content fields - {"tag": "ToR", "type": "constant", "value": "20", "width": 2}, - {"tag": "Subject", "type": "cdrfield", "value": "subject", "width": 12, "padding": "right", "mandatory": true}, - {"tag": "ConnectionNumber", "type": "constant", "value": "00000", "width": 5}, - {"tag": "CallerId", "type": "cdrfield", "value": "~callerid:s/\\+(\\d+)/00$1/", "strip": "xright", "width": 15, "padding": "right"}, - {"tag": "Destination", "type": "cdrfield", "value": "~destination:s/^\\+311400(\\d+)/$1/:s/^\\+311412\\d\\d112/112/:s/^\\+31(\\d+)/0$1/:s/^\\+(\\d+)/00$1/", - "strip": "xright", "width": 24, "padding": "right", "mandatory": true}, - {"tag": "TypeOfService", "type": "constant", "value": "00", "width": 2}, - {"tag": "ServiceId", "type": "constant", "value": "11", "width": 4, "padding": "right"}, - {"tag": "AnswerTime", "type": "cdrfield", "value": "answer_time", "layout": "020106150405", "width": 12, "mandatory": true}, - {"tag": "Usage", "type": "cdrfield", "value": "usage", "layout": "seconds", "width": 6, "padding": "right", "mandatory": true}, - {"tag": "DataCounter", "type": "filler", "width": 6}, - {"tag": "VatCode", "type": "constant", "value": "1", "width": 1}, - {"tag": "NetworkId", "type": "constant", "value": "S1", "width": 2}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "NetworkSubtype", "type": "constant", "value": "3", "width": 1, "padding": "left"}, - {"tag": "CgrId", "type": "cdrfield", "value": "cgrid", "strip": "xleft", "width": 16, "padding": "right", "mandatory": true}, - {"tag": "FillerVolume1", "type": "filler", "width": 8}, - {"tag": "FillerVolume2", "type": "filler", "width": 8}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "Cost", "type": "cdrfield", "value": "cost", "padding": "zeroleft", "width": 9}, - {"tag": "MaskDestination", "type": "metatag", "value": "mask_destination", "width": 1}, - ], - "trailer_fields": [ - {"tag": "ToR", "type": "constant", "value": "90", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "TotalRecords", "type": "metatag", "value": "cdrs_number", "padding": "zeroleft", "width": 6}, - {"tag": "TotalDuration", "type": "metatag", "value": "cdrs_duration", "padding": "zeroleft", "width": 8}, - {"tag": "FirstCdrTime", "type": "metatag", "value": "first_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "LastCdrTime", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "Filler1", "type": "filler", "width": 93}, - ], // template of the exported trailer fields - } -}, - - -"apier": { - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/multiplecdrc_internal/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc_internal/multiplecdrc_fwexport.json deleted file mode 100644 index 70b615b13..000000000 --- a/data/conf/samples/multiplecdrc_internal/multiplecdrc_fwexport.json +++ /dev/null @@ -1,139 +0,0 @@ -{ -// CGRateS Configuration file -// -// Used in mediator_local_test -// Starts rater, cdrs and mediator connecting over internal channel - -"data_db": { - "db_type": "*internal", -}, - - -"stor_db": { - "db_type": "*internal", -}, -"rals": { - "enabled": true, // enable Rater service: -}, - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "CDRC-CSV1", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database - }, - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"}, - {"cdr_field_id": "accid", "value": "0"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "~7:s/^voice$/call/"}, - {"cdr_field_id": "account", "value": "3"}, - {"cdr_field_id": "subject", "value": "3"}, - {"cdr_field_id": "destination", "value": "~5:s/^0([1-9]\\d+)$/+49$1/"}, - {"cdr_field_id": "setup_time", "value": "1"}, - {"cdr_field_id": "answer_time", "value": "1"}, - {"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"}, - ], - }, - { - "id": "CDRC-CSV3", - "enabled": true, // enable CDR client functionality - "field_separator": ";", // separator used in case of csv files - "cdr_in_path": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "^*voice"}, - {"cdr_field_id": "accid", "value": "~3:s/^(\\d{2})\\.(\\d{2})\\.(\\d{4})\\s{2}(\\d{2}):(\\d{2}):(\\d{2})$/$1$2$3$4$5$6/"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "^call"}, - {"cdr_field_id": "account", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "subject", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "destination", "value": "~1:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "setup_time", "value": "4"}, - {"cdr_field_id": "answer_time", "value": "4"}, - {"cdr_field_id": "usage", "value": "~6:s/^(\\d+)$/${1}s/"}, - ], - } -], - -"cdre": { - "CDRE-FW1": { - "export_format": "*file_fwv", - "field_separator": "", - "header_fields": [ - {"tag": "ToR", "type": "constant", "value": "10", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "LastCdr", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "FileCreationfTime", "type": "metatag", "value": "time_now", "layout": "020106150405", "width": 12}, - {"tag": "FileVersion", "type": "constant", "value": "01", "width": 2}, - {"tag": "Filler2", "type": "filler", "width": 105}, - ], // template of the exported header fields - "content_fields": [ // template of the exported content fields - {"tag": "ToR", "type": "constant", "value": "20", "width": 2}, - {"tag": "Subject", "type": "cdrfield", "value": "subject", "width": 12, "padding": "right", "mandatory": true}, - {"tag": "ConnectionNumber", "type": "constant", "value": "00000", "width": 5}, - {"tag": "CallerId", "type": "cdrfield", "value": "~callerid:s/\\+(\\d+)/00$1/", "strip": "xright", "width": 15, "padding": "right"}, - {"tag": "Destination", "type": "cdrfield", "value": "~destination:s/^\\+311400(\\d+)/$1/:s/^\\+311412\\d\\d112/112/:s/^\\+31(\\d+)/0$1/:s/^\\+(\\d+)/00$1/", - "strip": "xright", "width": 24, "padding": "right", "mandatory": true}, - {"tag": "TypeOfService", "type": "constant", "value": "00", "width": 2}, - {"tag": "ServiceId", "type": "constant", "value": "11", "width": 4, "padding": "right"}, - {"tag": "AnswerTime", "type": "cdrfield", "value": "answer_time", "layout": "020106150405", "width": 12, "mandatory": true}, - {"tag": "Usage", "type": "cdrfield", "value": "usage", "layout": "seconds", "width": 6, "padding": "right", "mandatory": true}, - {"tag": "DataCounter", "type": "filler", "width": 6}, - {"tag": "VatCode", "type": "constant", "value": "1", "width": 1}, - {"tag": "NetworkId", "type": "constant", "value": "S1", "width": 2}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "NetworkSubtype", "type": "constant", "value": "3", "width": 1, "padding": "left"}, - {"tag": "CgrId", "type": "cdrfield", "value": "cgrid", "strip": "xleft", "width": 16, "padding": "right", "mandatory": true}, - {"tag": "FillerVolume1", "type": "filler", "width": 8}, - {"tag": "FillerVolume2", "type": "filler", "width": 8}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "Cost", "type": "cdrfield", "value": "cost", "padding": "zeroleft", "width": 9}, - {"tag": "MaskDestination", "type": "metatag", "value": "mask_destination", "width": 1}, - ], - "trailer_fields": [ - {"tag": "ToR", "type": "constant", "value": "90", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "TotalRecords", "type": "metatag", "value": "cdrs_number", "padding": "zeroleft", "width": 6}, - {"tag": "TotalDuration", "type": "metatag", "value": "cdrs_duration", "padding": "zeroleft", "width": 8}, - {"tag": "FirstCdrTime", "type": "metatag", "value": "first_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "LastCdrTime", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "Filler1", "type": "filler", "width": 93}, - ], // template of the exported trailer fields - } -}, - - -"apier": { - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/multiplecdrc_mongo/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc_mongo/multiplecdrc_fwexport.json deleted file mode 100644 index 8609014ea..000000000 --- a/data/conf/samples/multiplecdrc_mongo/multiplecdrc_fwexport.json +++ /dev/null @@ -1,144 +0,0 @@ -{ -// CGRateS Configuration file -// -// Used in mediator_local_test -// Starts rater, cdrs and mediator connecting over internal channel - -"data_db": { - "db_type": "mongo", - "db_name": "10", - "db_port": 27017, -}, - - -"stor_db": { - "db_type": "mongo", - "db_name": "cgrates", - "db_port": 27017, -}, - -"rals": { - "enabled": true, // enable Rater service: -}, - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "CDRC-CSV1", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database - }, - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"}, - {"cdr_field_id": "accid", "value": "0"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "~7:s/^voice$/call/"}, - {"cdr_field_id": "account", "value": "3"}, - {"cdr_field_id": "subject", "value": "3"}, - {"cdr_field_id": "destination", "value": "~5:s/^0([1-9]\\d+)$/+49$1/"}, - {"cdr_field_id": "setup_time", "value": "1"}, - {"cdr_field_id": "answer_time", "value": "1"}, - {"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"}, - ], - }, - { - "id": "CDRC-CSV3", - "enabled": true, // enable CDR client functionality - "field_separator": ";", // separator used in case of csv files - "cdr_in_path": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "^*voice"}, - {"cdr_field_id": "accid", "value": "~3:s/^(\\d{2})\\.(\\d{2})\\.(\\d{4})\\s{2}(\\d{2}):(\\d{2}):(\\d{2})$/$1$2$3$4$5$6/"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "^call"}, - {"cdr_field_id": "account", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "subject", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "destination", "value": "~1:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "setup_time", "value": "4"}, - {"cdr_field_id": "answer_time", "value": "4"}, - {"cdr_field_id": "usage", "value": "~6:s/^(\\d+)$/${1}s/"}, - ], - } -], - -"cdre": { - "CDRE-FW1": { - "export_format": "*file_fwv", - "field_separator": "", - "header_fields": [ - {"tag": "ToR", "type": "constant", "value": "10", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "LastCdr", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "FileCreationfTime", "type": "metatag", "value": "time_now", "layout": "020106150405", "width": 12}, - {"tag": "FileVersion", "type": "constant", "value": "01", "width": 2}, - {"tag": "Filler2", "type": "filler", "width": 105}, - ], // template of the exported header fields - "content_fields": [ // template of the exported content fields - {"tag": "ToR", "type": "constant", "value": "20", "width": 2}, - {"tag": "Subject", "type": "cdrfield", "value": "subject", "width": 12, "padding": "right", "mandatory": true}, - {"tag": "ConnectionNumber", "type": "constant", "value": "00000", "width": 5}, - {"tag": "CallerId", "type": "cdrfield", "value": "~callerid:s/\\+(\\d+)/00$1/", "strip": "xright", "width": 15, "padding": "right"}, - {"tag": "Destination", "type": "cdrfield", "value": "~destination:s/^\\+311400(\\d+)/$1/:s/^\\+311412\\d\\d112/112/:s/^\\+31(\\d+)/0$1/:s/^\\+(\\d+)/00$1/", - "strip": "xright", "width": 24, "padding": "right", "mandatory": true}, - {"tag": "TypeOfService", "type": "constant", "value": "00", "width": 2}, - {"tag": "ServiceId", "type": "constant", "value": "11", "width": 4, "padding": "right"}, - {"tag": "AnswerTime", "type": "cdrfield", "value": "answer_time", "layout": "020106150405", "width": 12, "mandatory": true}, - {"tag": "Usage", "type": "cdrfield", "value": "usage", "layout": "seconds", "width": 6, "padding": "right", "mandatory": true}, - {"tag": "DataCounter", "type": "filler", "width": 6}, - {"tag": "VatCode", "type": "constant", "value": "1", "width": 1}, - {"tag": "NetworkId", "type": "constant", "value": "S1", "width": 2}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "NetworkSubtype", "type": "constant", "value": "3", "width": 1, "padding": "left"}, - {"tag": "CgrId", "type": "cdrfield", "value": "cgrid", "strip": "xleft", "width": 16, "padding": "right", "mandatory": true}, - {"tag": "FillerVolume1", "type": "filler", "width": 8}, - {"tag": "FillerVolume2", "type": "filler", "width": 8}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "Cost", "type": "cdrfield", "value": "cost", "padding": "zeroleft", "width": 9}, - {"tag": "MaskDestination", "type": "metatag", "value": "mask_destination", "width": 1}, - ], - "trailer_fields": [ - {"tag": "ToR", "type": "constant", "value": "90", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "TotalRecords", "type": "metatag", "value": "cdrs_number", "padding": "zeroleft", "width": 6}, - {"tag": "TotalDuration", "type": "metatag", "value": "cdrs_duration", "padding": "zeroleft", "width": 8}, - {"tag": "FirstCdrTime", "type": "metatag", "value": "first_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "LastCdrTime", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "Filler1", "type": "filler", "width": 93}, - ], // template of the exported trailer fields - } -}, - - -"apier": { - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/multiplecdrc_mysql/multiplecdrc_fwexport.json b/data/conf/samples/multiplecdrc_mysql/multiplecdrc_fwexport.json deleted file mode 100644 index b8996c811..000000000 --- a/data/conf/samples/multiplecdrc_mysql/multiplecdrc_fwexport.json +++ /dev/null @@ -1,135 +0,0 @@ -{ -// CGRateS Configuration file -// -// Used in mediator_local_test -// Starts rater, cdrs and mediator connecting over internal channel - -"stor_db": { // database used to store offline tariff plans and CDRs - "db_password": "CGRateS.org", // password to use when connecting to stordb -}, - -"rals": { - "enabled": true, // enable Rater service: -}, - -"schedulers": { - "enabled": true, // start Scheduler service: -}, - -"cdrs": { - "enabled": true, // start the CDR Server service: -}, - -"chargers": { - "enabled": true, -}, - -"cdrc": [ - { - "id": "CDRC-CSV1", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc1/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc1/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv1", // free form field, tag identifying the source of the CDRs within CDRS database - }, - { - "id": "CDRC-CSV2", - "enabled": true, // enable CDR client functionality - "cdr_in_path": "/tmp/cgrates/cdrc2/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc2/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv2", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"}, - {"cdr_field_id": "accid", "value": "0"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "~7:s/^voice$/call/"}, - {"cdr_field_id": "account", "value": "3"}, - {"cdr_field_id": "subject", "value": "3"}, - {"cdr_field_id": "destination", "value": "~5:s/^0([1-9]\\d+)$/+49$1/"}, - {"cdr_field_id": "setup_time", "value": "1"}, - {"cdr_field_id": "answer_time", "value": "1"}, - {"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"}, - ], - }, - { - "id": "CDRC-CSV3", - "enabled": true, // enable CDR client functionality - "field_separator": ";", // separator used in case of csv files - "cdr_in_path": "/tmp/cgrates/cdrc3/in", // absolute path towards the directory where the CDRs are stored - "cdr_out_path": "/tmp/cgrates/cdrc3/out", // absolute path towards the directory where processed CDRs will be moved - "cdr_source_id": "csv3", // free form field, tag identifying the source of the CDRs within CDRS database - "content_fields":[ // import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value - {"cdr_field_id": "tor", "value": "^*voice"}, - {"cdr_field_id": "accid", "value": "~3:s/^(\\d{2})\\.(\\d{2})\\.(\\d{4})\\s{2}(\\d{2}):(\\d{2}):(\\d{2})$/$1$2$3$4$5$6/"}, - {"cdr_field_id": "reqtype", "value": "^rated"}, - {"cdr_field_id": "tenant", "value": "^cgrates.org"}, - {"cdr_field_id": "category", "value": "^call"}, - {"cdr_field_id": "account", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "subject", "value": "~0:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "destination", "value": "~1:s/^([1-9]\\d+)$/+$1/"}, - {"cdr_field_id": "setup_time", "value": "4"}, - {"cdr_field_id": "answer_time", "value": "4"}, - {"cdr_field_id": "usage", "value": "~6:s/^(\\d+)$/${1}s/"}, - ], - } -], - -"cdre": { - "CDRE-FW1": { - "export_format": "*file_fwv", - "field_separator": "", - "header_fields": [ - {"tag": "ToR", "type": "constant", "value": "10", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "LastCdr", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "FileCreationfTime", "type": "metatag", "value": "time_now", "layout": "020106150405", "width": 12}, - {"tag": "FileVersion", "type": "constant", "value": "01", "width": 2}, - {"tag": "Filler2", "type": "filler", "width": 105}, - ], // template of the exported header fields - "content_fields": [ // template of the exported content fields - {"tag": "ToR", "type": "constant", "value": "20", "width": 2}, - {"tag": "Subject", "type": "cdrfield", "value": "subject", "width": 12, "padding": "right", "mandatory": true}, - {"tag": "ConnectionNumber", "type": "constant", "value": "00000", "width": 5}, - {"tag": "CallerId", "type": "cdrfield", "value": "~callerid:s/\\+(\\d+)/00$1/", "strip": "xright", "width": 15, "padding": "right"}, - {"tag": "Destination", "type": "cdrfield", "value": "~destination:s/^\\+311400(\\d+)/$1/:s/^\\+311412\\d\\d112/112/:s/^\\+31(\\d+)/0$1/:s/^\\+(\\d+)/00$1/", - "strip": "xright", "width": 24, "padding": "right", "mandatory": true}, - {"tag": "TypeOfService", "type": "constant", "value": "00", "width": 2}, - {"tag": "ServiceId", "type": "constant", "value": "11", "width": 4, "padding": "right"}, - {"tag": "AnswerTime", "type": "cdrfield", "value": "answer_time", "layout": "020106150405", "width": 12, "mandatory": true}, - {"tag": "Usage", "type": "cdrfield", "value": "usage", "layout": "seconds", "width": 6, "padding": "right", "mandatory": true}, - {"tag": "DataCounter", "type": "filler", "width": 6}, - {"tag": "VatCode", "type": "constant", "value": "1", "width": 1}, - {"tag": "NetworkId", "type": "constant", "value": "S1", "width": 2}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "NetworkSubtype", "type": "constant", "value": "3", "width": 1, "padding": "left"}, - {"tag": "CgrId", "type": "cdrfield", "value": "cgrid", "strip": "xleft", "width": 16, "padding": "right", "mandatory": true}, - {"tag": "FillerVolume1", "type": "filler", "width": 8}, - {"tag": "FillerVolume2", "type": "filler", "width": 8}, - {"tag": "DestinationSubId", "type": "cdrfield", "value": "~cost_details:s/MatchedDestId:.+_(\\w{5})/$1/:s/(\\w{6})/$1/", "width": 5}, - {"tag": "Cost", "type": "cdrfield", "value": "cost", "padding": "zeroleft", "width": 9}, - {"tag": "MaskDestination", "type": "metatag", "value": "mask_destination", "width": 1}, - ], - "trailer_fields": [ - {"tag": "ToR", "type": "constant", "value": "90", "width": 2}, - {"tag": "Filler1", "type": "filler", "width": 3}, - {"tag": "FileType", "type": "constant", "value": "SIP", "width": 3}, - {"tag": "FileSeqNr", "type": "metatag", "value": "export_id", "padding": "zeroleft", "width": 5}, - {"tag": "TotalRecords", "type": "metatag", "value": "cdrs_number", "padding": "zeroleft", "width": 6}, - {"tag": "TotalDuration", "type": "metatag", "value": "cdrs_duration", "padding": "zeroleft", "width": 8}, - {"tag": "FirstCdrTime", "type": "metatag", "value": "first_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "LastCdrTime", "type": "metatag", "value": "last_cdr_atime", "layout": "020106150405", "width": 12}, - {"tag": "Filler1", "type": "filler", "width": 93}, - ], // template of the exported trailer fields - } -}, - - -"apier": { - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/tutmongo2/cgrates.json b/data/conf/samples/tutmongo2/cgrates.json index e6bf5417e..9b5e37820 100644 --- a/data/conf/samples/tutmongo2/cgrates.json +++ b/data/conf/samples/tutmongo2/cgrates.json @@ -78,23 +78,6 @@ }, -"cdrc": [ - { - "id": "tutorial_csv_cdr", - "enabled": true, - "cdr_source_id": "cgr_tutorial", - "content_fields":[ - {"field_id": "OriginID", "type": "*composed", "value": "~3", "mandatory": true}, - {"field_id": "Account", "type": "*composed", "value": "~8", "mandatory": true}, - {"field_id": "Destination", "type": "*composed", "value": "~10", "mandatory": true}, - {"field_id": "SetupTime", "type": "*composed", "value": "~11", "mandatory": true}, - {"field_id": "AnswerTime", "type": "*composed", "value": "~12", "mandatory": true}, - {"field_id": "Usage", "type": "*composed", "value": "~13", "mandatory": true}, - ], - }, -], - - "sessions": { "enabled": true, "resources_conns": ["*localhost"], diff --git a/data/conf/samples/tutmongo2_gob/cgrates.json b/data/conf/samples/tutmongo2_gob/cgrates.json index 05a874d40..a3d1ce0ee 100644 --- a/data/conf/samples/tutmongo2_gob/cgrates.json +++ b/data/conf/samples/tutmongo2_gob/cgrates.json @@ -85,23 +85,6 @@ }, -"cdrc": [ - { - "id": "tutorial_csv_cdr", - "enabled": true, - "cdr_source_id": "cgr_tutorial", - "content_fields":[ - {"field_id": "OriginID", "type": "*composed", "value": "~3", "mandatory": true}, - {"field_id": "Account", "type": "*composed", "value": "~8", "mandatory": true}, - {"field_id": "Destination", "type": "*composed", "value": "~10", "mandatory": true}, - {"field_id": "SetupTime", "type": "*composed", "value": "~11", "mandatory": true}, - {"field_id": "AnswerTime", "type": "*composed", "value": "~12", "mandatory": true}, - {"field_id": "Usage", "type": "*composed", "value": "~13", "mandatory": true}, - ], - }, -], - - "sessions": { "enabled": true, "resources_conns": ["conn1"], diff --git a/data/conf/samples/tutmysql2/cgrates.json b/data/conf/samples/tutmysql2/cgrates.json index 1bcf144bd..327fa0482 100644 --- a/data/conf/samples/tutmysql2/cgrates.json +++ b/data/conf/samples/tutmysql2/cgrates.json @@ -43,23 +43,6 @@ }, -"cdrc": [ - { - "id": "tutorial_csv_cdr", - "enabled": true, - "cdr_source_id": "cgr_tutorial", - "content_fields":[ - {"field_id": "OriginID", "type": "*composed", "value": "~3", "mandatory": true}, - {"field_id": "Account", "type": "*composed", "value": "~8", "mandatory": true}, - {"field_id": "Destination", "type": "*composed", "value": "~10", "mandatory": true}, - {"field_id": "SetupTime", "type": "*composed", "value": "~11", "mandatory": true}, - {"field_id": "AnswerTime", "type": "*composed", "value": "~12", "mandatory": true}, - {"field_id": "Usage", "type": "*composed", "value": "~13", "mandatory": true}, - ], - }, -], - - "sessions": { "enabled": true, "resources_conns": ["*localhost"], diff --git a/data/conf/samples/tutmysql2_gob/cgrates.json b/data/conf/samples/tutmysql2_gob/cgrates.json index d1cabb844..43634561f 100644 --- a/data/conf/samples/tutmysql2_gob/cgrates.json +++ b/data/conf/samples/tutmysql2_gob/cgrates.json @@ -51,22 +51,6 @@ }, -"cdrc": [ - { - "id": "tutorial_csv_cdr", - "enabled": true, - "cdr_source_id": "cgr_tutorial", - "content_fields":[ - {"field_id": "OriginID", "type": "*composed", "value": "~3", "mandatory": true}, - {"field_id": "Account", "type": "*composed", "value": "~8", "mandatory": true}, - {"field_id": "Destination", "type": "*composed", "value": "~10", "mandatory": true}, - {"field_id": "SetupTime", "type": "*composed", "value": "~11", "mandatory": true}, - {"field_id": "AnswerTime", "type": "*composed", "value": "~12", "mandatory": true}, - {"field_id": "Usage", "type": "*composed", "value": "~13", "mandatory": true}, - ], - }, -], - "sessions": { "enabled": true, diff --git a/data/conf/samples/v/cgrates.json b/data/conf/samples/v/cgrates.json deleted file mode 100644 index 2130f26f8..000000000 --- a/data/conf/samples/v/cgrates.json +++ /dev/null @@ -1,128 +0,0 @@ -{ -// CGRateS Configuration file - - -"general": { - "log_level": 7, - "reply_timeout": "30s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - - -"data_db": { - "db_type": "mongo", - "db_name": "10", - "db_port": 27017, -}, - - -"stor_db": { - "db_type": "mongo", - "db_name": "cgrates", - "db_port": 27017, -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], - "remove_expired":false, -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*localhost"], -}, - - -"cdrs": { - "enabled": true, - "chargers_conns":["*localhost"], -}, - - -"cdre": { - "TestTutITExportCDR": { - "content_fields": [ - {"tag": "CGRID", "type": "*composed", "value": "~CGRID"}, - {"tag": "RunID", "type": "*composed", "value": "~RunID"}, - {"tag":"OriginID", "type": "*composed", "value": "~OriginID"}, - {"tag":"RequestType", "type": "*composed", "value": "~RequestType"}, - {"tag":"Tenant", "type": "*composed", "value": "~Tenant"}, - {"tag":"Category", "type": "*composed", "value": "~Category"}, - {"tag":"Account", "type": "*composed", "value": "~Account"}, - {"tag":"Destination", "type": "*composed", "value": "~Destination"}, - {"tag":"AnswerTime", "type": "*composed", "value": "~AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"tag":"Usage", "type": "*composed", "value": "~Usage"}, - {"tag":"Cost", "type": "*composed", "value": "~Cost", "rounding_decimals": 4}, - {"tag":"MatchedDestinationID", "type": "*composed", "value": "~CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"suppliers": { - "enabled": true, - "stats_conns": ["*internal"], -}, - - -"attributes": { // Attribute service - "enabled": true, // starts Attribute service: . -}, - - -"sessions": { - "enabled": true, -}, - - -"migrator": { - "out_datadb_type": "mongo", - "out_datadb_port": "27017", - "out_datadb_name": "10", - "out_stordb_type": "mongo", - "out_stordb_port": "27017", - "out_stordb_name": "cgrates", -}, - - -"apier": { - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/scripts/freeswitch_cdr_csv_rotate.sh b/data/scripts/freeswitch_cdr_csv_rotate.sh deleted file mode 100755 index 09bcbd3f5..000000000 --- a/data/scripts/freeswitch_cdr_csv_rotate.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /usr/bin/env sh - -FS_CDR_CSV_DIR=/var/log/freeswitch/cdr-csv -CGR_CDRC_IN_DIR=/var/log/cgrates/cdr/cdrc/in - -/usr/bin/fs_cli -x "cdr_csv rotate" - -find $FS_CDR_CSV_DIR -maxdepth 1 -mindepth 1 -not -name *.csv -exec chown cgrates:cgrates '{}' \; -exec mv '{}' $CGR_CDRC_IN_DIR \; - -exit 0 - diff --git a/general_tests/multiplecdrc_it_test.go b/general_tests/multiplecdrc_it_test.go deleted file mode 100644 index 6638bfc7c..000000000 --- a/general_tests/multiplecdrc_it_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package general_tests - -import ( - "flag" - "io/ioutil" - "net/rpc" - "os" - "path" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - cfgPath string - cfgDIR string - cfg *config.CGRConfig - rater *rpc.Client - - testCalls = flag.Bool("calls", false, "Run test calls simulation, not by default.") - - sTestMCDRC = []func(t *testing.T){ - testMCDRCLoadConfig, - testMCDRCResetDataDb, - testMCDRCEmptyTables, - testMCDRCCreateCdrDirs, - testMCDRCStartEngine, - testMCDRCRpcConn, - testMCDRCApierLoadTariffPlanFromFolder, - testMCDRCHandleCdr1File, - testMCDRCHandleCdr2File, - testMCDRCHandleCdr3File, - testMCDRCStopEngine, - } -) - -func TestMCDRC(t *testing.T) { - switch *dbType { - case utils.MetaInternal: - cfgDIR = "multiplecdrc_internal" - case utils.MetaSQL: - cfgDIR = "multiplecdrc_mysql" - case utils.MetaMongo: - cfgDIR = "multiplecdrc_mongo" - case utils.MetaPostgres: - t.SkipNow() - default: - t.Fatal("Unknown Database type") - } - for _, stest := range sTestMCDRC { - t.Run(cfgDIR, stest) - } -} - -func testMCDRCLoadConfig(t *testing.T) { - var err error - cfgPath = path.Join(*dataDir, "conf", "samples", cfgDIR) - if cfg, err = config.NewCGRConfigFromPath(cfgPath); err != nil { - t.Error(err) - } -} - -// Remove data in both rating and accounting db -func testMCDRCResetDataDb(t *testing.T) { - if err := engine.InitDataDb(cfg); err != nil { - t.Fatal(err) - } -} - -func testMCDRCEmptyTables(t *testing.T) { - if err := engine.InitStorDb(cfg); err != nil { - t.Fatal(err) - } -} - -func testMCDRCCreateCdrDirs(t *testing.T) { - for _, cdrcProfiles := range cfg.CdrcProfiles { - for _, cdrcInst := range cdrcProfiles { - for _, dir := range []string{cdrcInst.CDRInPath, cdrcInst.CDROutPath} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - } - } -} -func testMCDRCStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(cfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func testMCDRCRpcConn(t *testing.T) { - var err error - rater, err = newRPCClient(cfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -// Test here LoadTariffPlanFromFolder -func testMCDRCApierLoadTariffPlanFromFolder(t *testing.T) { - reply := "" - // Simple test that command is executed without errors - attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "testtp")} - if err := rater.Call(utils.ApierV1LoadTariffPlanFromFolder, attrs, &reply); err != nil { - t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error()) - } else if reply != utils.OK { - t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply) - } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups -} - -// The default scenario, out of cdrc defined in .cfg file -func testMCDRCHandleCdr1File(t *testing.T) { - var fileContent1 = `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,+4986517174963,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10000000000,1.0100,val_extra3,"",val_extra1 -dbafe9c8614c785a65aabd116dd3959c3c56f7f7,default,*voice,dsafdsag,rated,*out,cgrates.org,call,1001,1001,+4986517174964,2013-11-07 09:42:25 +0000 UTC,2013-11-07 09:42:26 +0000 UTC,20000000000,1.0100,val_extra3,"",val_extra1 -` - fileName := "file1.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent1), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cgrates/cdrc1/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -// Scenario out of first .xml config -func testMCDRCHandleCdr2File(t *testing.T) { - var fileContent = `616350843,20131022145011,20131022172857,3656,1001,,,data,mo,640113,0.000000,1.222656,1.222660 -616199016,20131022154924,20131022154955,3656,1001,086517174963,,voice,mo,31,0.000000,0.000000,0.000000 -800873243,20140516063739,20140516063739,9774,1001,+49621621391,,sms,mo,1,0.00000,0.00000,0.00000` - fileName := "file2.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cgrates/cdrc2/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -// Scenario out of second .xml config -func testMCDRCHandleCdr3File(t *testing.T) { - var fileContent = `4986517174960;4986517174963;Sample Mobile;08.04.2014 22:14:29;08.04.2014 22:14:29;1;193;Offeak;0,072728833;31619 -4986517174960;4986517174964;National;08.04.2014 20:34:55;08.04.2014 20:34:55;1;21;Offeak;0,0079135;311` - fileName := "file3.csv" - tmpFilePath := path.Join("/tmp", fileName) - if err := ioutil.WriteFile(tmpFilePath, []byte(fileContent), 0644); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/cgrates/cdrc3/in", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testMCDRCStopEngine(t *testing.T) { - if err := engine.KillEngine(100); err != nil { - t.Error(err) - } -} diff --git a/gob_integration_test.sh b/gob_integration_test.sh index 2f513de34..5738139c0 100755 --- a/gob_integration_test.sh +++ b/gob_integration_test.sh @@ -11,9 +11,6 @@ ap2=$? echo 'go test github.com/cgrates/cgrates/engine -tags=integration -rpc=*gob' go test github.com/cgrates/cgrates/engine -tags=integration -rpc=*gob en=$? -echo 'go test github.com/cgrates/cgrates/cdrc -tags=integration -rpc=*gob' -go test github.com/cgrates/cgrates/cdrc -tags=integration -rpc=*gob -cdrc=$? echo 'go test github.com/cgrates/cgrates/ers -tags=integration -rpc=*gob' go test github.com/cgrates/cgrates/ers -tags=integration -rpc=*gob ers=$? @@ -33,4 +30,4 @@ echo 'go test github.com/cgrates/cgrates/loaders -tags=integration -rpc=*gob' go test github.com/cgrates/cgrates/loaders -tags=integration -rpc=*gob lds=$? -exit $gen && $ap1 && $ap2 && $en && $cdrc && $gnr && $agts && $smg && $dis && $lds && $ers +exit $gen && $ap1 && $ap2 && $en && $gnr && $agts && $smg && $dis && $lds && $ers diff --git a/integration_test.sh b/integration_test.sh index 7bb2aec09..35bfef774 100755 --- a/integration_test.sh +++ b/integration_test.sh @@ -92,17 +92,11 @@ echo 'go test github.com/cgrates/cgrates/agents -tags=integration -dbtype=*postg go test github.com/cgrates/cgrates/agents -tags=integration -dbtype=*postgres agts_postgres=$? - -echo 'go test github.com/cgrates/cgrates/cdrc -tags=integration' -go test github.com/cgrates/cgrates/cdrc -tags=integration -cdrc=$? echo 'go test github.com/cgrates/cgrates/config -tags=integration' go test github.com/cgrates/cgrates/config -tags=integration cfg=$? #All - - echo 'go test github.com/cgrates/cgrates/sessions -tags=integration' go test github.com/cgrates/cgrates/sessions -tags=integration smg=$? @@ -119,4 +113,4 @@ echo 'go test github.com/cgrates/cgrates/apier/v1 -tags=offline' go test github.com/cgrates/cgrates/apier/v1 -tags=offline offline=$? # to do: add '&& $ap1_internal' -exit $gen && $ap1_sql && $ap1_mongo && $ap2 && $en && $cdrc && $cfg && $utl && $gnr && $agts && $smg && $mgr && $dis && $lds && $ers && $srv && $offline +exit $gen && $ap1_sql && $ap1_mongo && $ap2 && $en && $cfg && $utl && $gnr && $agts && $smg && $mgr && $dis && $lds && $ers && $srv && $offline diff --git a/utils/consts.go b/utils/consts.go index ced040552..c19ad2a27 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -223,7 +223,6 @@ const ( OK = "OK" MetaFileXML = "*file_xml" CDRE = "cdre" - CDRC = "cdrc" MASK_CHAR = "*" CONCATENATED_KEY_SEP = ":" UNIT_TEST = "UNIT_TEST"