diff --git a/apier/v1/cdre.go b/apier/v1/cdre.go deleted file mode 100644 index bc64f8141..000000000 --- a/apier/v1/cdre.go +++ /dev/null @@ -1,312 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "archive/zip" - "bytes" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/url" - "os" - "path" - "strconv" - "strings" - "time" - "unicode/utf8" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func (apierSv1 *APIerSv1) ExportCdrsToZipString(attr *utils.AttrExpFileCdrs, reply *string) error { - tmpDir := "/tmp" - attr.ExportDir = &tmpDir // Enforce exporting to tmp always so we avoid cleanup issues - efc := utils.ExportedFileCdrs{} - if err := apierSv1.ExportCdrsToFile(attr, &efc); err != nil { - return err - } else if efc.TotalRecords == 0 || len(efc.ExportedFilePath) == 0 { - return errors.New("No CDR records to export") - } - // Create a buffer to write our archive to. - buf := new(bytes.Buffer) - // Create a new zip archive. - w := zip.NewWriter(buf) - // read generated file - content, err := ioutil.ReadFile(efc.ExportedFilePath) - if err != nil { - return err - } - exportFileName := path.Base(efc.ExportedFilePath) - f, err := w.Create(exportFileName) - if err != nil { - return err - } - _, err = f.Write(content) - if err != nil { - return err - } - // Write metadata into a separate file with extension .cgr - medaData, err := json.MarshalIndent(efc, "", " ") - if err != nil { - errors.New("Failed creating metadata content") - } - medatadaFileName := exportFileName[:len(path.Ext(exportFileName))] + ".cgr" - mf, err := w.Create(medatadaFileName) - if err != nil { - return err - } - _, err = mf.Write(medaData) - if err != nil { - return err - } - // Make sure to check the error on Close. - if err := w.Close(); err != nil { - return err - } - if err := os.Remove(efc.ExportedFilePath); err != nil { - fmt.Errorf("Failed removing exported file at path: %s", efc.ExportedFilePath) - } - *reply = base64.StdEncoding.EncodeToString(buf.Bytes()) - return nil -} - -// Deprecated by AttrExportCDRsToFile -func (apierSv1 *APIerSv1) ExportCdrsToFile(attr *utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) (err error) { - exportTemplate := apierSv1.Config.CdreProfiles[utils.MetaDefault] - if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it - var hasIt bool - if exportTemplate, hasIt = apierSv1.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt { - return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound.Error()) - } - } - if exportTemplate == nil { - return fmt.Errorf("%s:ExportTemplate", utils.ErrMandatoryIeMissing.Error()) - } - exportFormat := exportTemplate.ExportFormat - if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 { - exportFormat = strings.ToLower(*attr.CdrFormat) - } - if !utils.CDRExportFormats.Has(exportFormat) { - return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "CdrFormat") - } - fieldSep := exportTemplate.FieldSeparator - if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 { - fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator) - if fieldSep == utf8.RuneError { - return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError.Error(), "Invalid") - } - } - exportPath := exportTemplate.ExportPath - if attr.ExportDir != nil && len(*attr.ExportDir) != 0 { - exportPath = *attr.ExportDir - } - exportID := strconv.FormatInt(time.Now().Unix(), 10) - if attr.ExportId != nil && len(*attr.ExportId) != 0 { - exportID = *attr.ExportId - } - fileName := fmt.Sprintf("cdre_%s.%s", exportID, exportFormat) - if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 { - fileName = *attr.ExportFileName - } - filePath := path.Join(exportPath, fileName) - if exportFormat == utils.DRYRUN { - filePath = utils.DRYRUN - } - cdrsFltr, err := attr.AsCDRsFilter(apierSv1.Config.GeneralCfg().DefaultTimezone) - if err != nil { - return utils.NewErrServerError(err) - } - cdrs, _, err := apierSv1.CdrDb.GetCDRs(cdrsFltr, false) - if err != nil { - return err - } else if len(cdrs) == 0 { - *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} - return nil - } - cdrexp, err := engine.NewCDRExporter(cdrs, exportTemplate, exportFormat, - filePath, utils.META_NONE, exportID, exportTemplate.Synchronous, - exportTemplate.Attempts, fieldSep, - apierSv1.Config.GeneralCfg().HttpSkipTlsVerify, - apierSv1.Config.ApierCfg().AttributeSConns, apierSv1.FilterS) - if err != nil { - return utils.NewErrServerError(err) - } - if err := cdrexp.ExportCDRs(); err != nil { - return utils.NewErrServerError(err) - } - if cdrexp.TotalExportedCdrs() == 0 { - *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} - return nil - } - *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, - TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), - FirstOrderId: cdrexp.FirstOrderID(), LastOrderId: cdrexp.LastOrderID()} - if !attr.SuppressCgrIds { - reply.ExportedCgrIds = cdrexp.PositiveExports() - reply.UnexportedCgrIds = cdrexp.NegativeExports() - } - return nil -} - -type ConfigPathArg struct { - ConfigPath string -} - -// Reloads CDRE configuration out of folder specified -func (apierSv1 *APIerSv1) ReloadCdreConfig(attrs *ConfigPathArg, reply *string) error { - if attrs.ConfigPath == utils.EmptyString { - attrs.ConfigPath = utils.CONFIG_PATH - } - newCfg, err := config.NewCGRConfigFromPath(attrs.ConfigPath) - if err != nil { - return utils.NewErrServerError(err) - } - cdreReloadStruct := <-apierSv1.Config.ConfigReloads[utils.CDRE] // Get the CDRE reload channel // Read the content of the channel, locking it - apierSv1.Config.CdreProfiles = newCfg.CdreProfiles - apierSv1.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct // Unlock reloads - utils.Logger.Info(" Configuration reloaded") - *reply = utils.OK - return nil -} - -// ArgExportCDRs are the arguments passed to ExportCDRs method -type ArgExportCDRs struct { - ExportArgs map[string]interface{} - Verbose bool // Disable CgrIds reporting in reply/ExportedCgrIds and reply/UnexportedCgrIds - utils.RPCCDRsFilter // Inherit the CDR filter attributes -} - -// RplExportedCDRs contain the reply of the ExportCDRs API -type RplExportedCDRs struct { - ExportedPath string // Full path to the newly generated export file - TotalRecords int // Number of CDRs to be exported - TotalCost float64 // Sum of all costs in exported CDRs - FirstOrderID, LastOrderID int64 // The order id of the last exported CDR - ExportedCGRIDs []string // List of successfuly exported cgrids in the file - UnexportedCGRIDs map[string]string // Map of errored CDRs, map key is cgrid, value will be the error string -} - -// ExportCDRs exports CDRs on a path (file or remote) -func (apierSv1 *APIerSv1) ExportCDRs(arg *ArgExportCDRs, reply *RplExportedCDRs) (err error) { - cdreReloadStruct := <-apierSv1.Config.ConfigReloads[utils.CDRE] // Read the content of the channel, locking it - defer func() { apierSv1.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct }() // Unlock reloads at exit - exportTemplate := apierSv1.Config.CdreProfiles[utils.MetaDefault] - if expTemplate, has := arg.ExportArgs[utils.ExportTemplate]; has { - var hasIt bool - if exportTemplate, hasIt = apierSv1.Config.CdreProfiles[utils.IfaceAsString(expTemplate)]; !hasIt { - return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound) - } - } - exportFormat := exportTemplate.ExportFormat - if expformat, has := arg.ExportArgs[utils.ExportFormat]; has { - exportFormat = strings.ToLower(utils.IfaceAsString(expformat)) - } - if !utils.CDRExportFormats.Has(exportFormat) { - return utils.NewErrMandatoryIeMissing("CdrFormat") - } - synchronous := exportTemplate.Synchronous - if sync, has := arg.ExportArgs[utils.Synchronous]; has { - if synchronous, err = utils.IfaceAsBool(sync); err != nil { - return - } - } - attempts := exportTemplate.Attempts - if ate, has := arg.ExportArgs[utils.Attempts]; has { - var atte int64 - if atte, err = utils.IfaceAsTInt64(ate); err != nil { - return - } - attempts = int(atte) - } - fieldSep := exportTemplate.FieldSeparator - if fieldS, has := arg.ExportArgs[utils.FieldSeparator]; has { - fieldSep, _ = utf8.DecodeRuneInString(utils.IfaceAsString(fieldS)) - if fieldSep == utf8.RuneError { - return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError, "Invalid") - } - } - eDir := exportTemplate.ExportPath - if expPath, has := arg.ExportArgs[utils.ExportPath]; has { - eDir = utils.IfaceAsString(expPath) - } - exportID := strconv.FormatInt(time.Now().Unix(), 10) - if expID, has := arg.ExportArgs[utils.ExportID]; has { - exportID = utils.IfaceAsString(expID) - } - var expFormat string - switch exportFormat { - case utils.MetaFileFWV: - expFormat = utils.FWV - case utils.MetaFileCSV: - expFormat = utils.CSV - default: - expFormat = exportFormat - } - fileName := fmt.Sprintf("cdre_%s.%s", exportID, expFormat) - if expFn, has := arg.ExportArgs[utils.ExportFileName]; has { - fileName = utils.IfaceAsString(expFn) - } - var filePath string - switch exportFormat { - case utils.MetaFileFWV, utils.MetaFileCSV: - filePath = path.Join(eDir, fileName) - case utils.DRYRUN: - filePath = utils.DRYRUN - default: - u, _ := url.Parse(eDir) - u.Path = path.Join(u.Path, fileName) - filePath = u.String() - } - cdrsFltr, err := arg.RPCCDRsFilter.AsCDRsFilter(apierSv1.Config.GeneralCfg().DefaultTimezone) - if err != nil { - return utils.NewErrServerError(err) - } - cdrs, _, err := apierSv1.CdrDb.GetCDRs(cdrsFltr, false) - if err != nil { - return err - } else if len(cdrs) == 0 { - return - } - cdrexp, err := engine.NewCDRExporter(cdrs, exportTemplate, exportFormat, - filePath, utils.META_NONE, exportID, - synchronous, attempts, fieldSep, - apierSv1.Config.GeneralCfg().HttpSkipTlsVerify, - apierSv1.Config.ApierCfg().AttributeSConns, apierSv1.FilterS) - if err != nil { - return utils.NewErrServerError(err) - } - if err := cdrexp.ExportCDRs(); err != nil { - return utils.NewErrServerError(err) - } - if cdrexp.TotalExportedCdrs() == 0 { - return - } - *reply = RplExportedCDRs{ExportedPath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), - FirstOrderID: cdrexp.FirstOrderID(), LastOrderID: cdrexp.LastOrderID()} - if arg.Verbose { - reply.ExportedCGRIDs = cdrexp.PositiveExports() - reply.UnexportedCGRIDs = cdrexp.NegativeExports() - } - return nil -} diff --git a/apier/v1/cdre_it_test.go b/apier/v1/cdre_it_test.go deleted file mode 100755 index 8839629bb..000000000 --- a/apier/v1/cdre_it_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "io/ioutil" - "net/rpc" - "path" - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - cdreCfgPath string - cdreCfg *config.CGRConfig - cdreRPC *rpc.Client - cdreDataDir = "/usr/share/cgrates" - cdreConfigDIR string //run tests for specific configuration - - sTestsCDRE = []func(t *testing.T){ - testCDReInitCfg, - testCDReInitDataDb, - testCDReResetStorDb, - testCDReStartEngine, - testCDReRPCConn, - testCDReAddCDRs, - testCDReExportCDRs, - testCDReFromFolder, - testCDReProcessExternalCdr, - testCDReKillEngine, - } -) - -//Test start here -func TestCDRExport(t *testing.T) { - switch *dbType { - case utils.MetaInternal: - cdreConfigDIR = "cdrewithfilter_internal" - case utils.MetaMySQL: - cdreConfigDIR = "cdrewithfilter_mysql" - case utils.MetaMongo: - cdreConfigDIR = "cdrewithfilter_mongo" - case utils.MetaPostgres: - t.SkipNow() - default: - t.Fatal("Unknown Database type") - } - for _, stest := range sTestsCDRE { - t.Run(cdreConfigDIR, stest) - } -} - -func TestCDRExportWithAttributeS(t *testing.T) { - cdreConfigDIR = "cdrewithattributes" - tests := sTestsCDRE[0:6] - tests = append(tests, testCDReAddAttributes) - tests = append(tests, testCDReExportCDRsWithAttributes) - tests = append(tests, testCDReKillEngine) - for _, stest := range tests { - t.Run(cdreConfigDIR, stest) - } -} - -func testCDReInitCfg(t *testing.T) { - var err error - cdreCfgPath = path.Join(alsPrfDataDir, "conf", "samples", cdreConfigDIR) - cdreCfg, err = config.NewCGRConfigFromPath(cdreCfgPath) - if err != nil { - t.Fatal(err) - } - cdreCfg.DataFolderPath = alsPrfDataDir // Share DataFolderPath through config towards StoreDb for Flush() - config.SetCgrConfig(cdreCfg) -} - -func testCDReInitDataDb(t *testing.T) { - if err := engine.InitDataDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -// Wipe out the cdr database -func testCDReResetStorDb(t *testing.T) { - if err := engine.InitStorDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -// Start CGR Engine -func testCDReStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(cdreCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func testCDReRPCConn(t *testing.T) { - var err error - cdreRPC, err = newRPCClient(cdreCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal(err) - } -} - -func testCDReAddCDRs(t *testing.T) { - storedCdrs := []*engine.CDR{ - {CGRID: "Cdr1", - OrderID: 123, ToR: utils.VOICE, OriginID: "OriginCDR1", OriginHost: "192.168.1.1", Source: "test", - RequestType: utils.META_RATED, Tenant: "cgrates.org", - Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), - AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(10) * time.Second, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - }, - {CGRID: "Cdr2", - OrderID: 123, ToR: utils.VOICE, OriginID: "OriginCDR2", OriginHost: "192.168.1.1", Source: "test2", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), - AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(5) * time.Second, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - }, - {CGRID: "Cdr3", - OrderID: 123, ToR: utils.VOICE, OriginID: "OriginCDR3", OriginHost: "192.168.1.1", Source: "test2", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), - AnswerTime: time.Now(), RunID: utils.MetaDefault, Usage: time.Duration(30) * time.Second, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - }, - {CGRID: "Cdr4", - OrderID: 123, ToR: utils.VOICE, OriginID: "OriginCDR4", OriginHost: "192.168.1.1", Source: "test3", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), - AnswerTime: time.Time{}, RunID: utils.MetaDefault, Usage: time.Duration(0) * time.Second, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - }, - } - for _, cdr := range storedCdrs { - var reply string - if err := cdreRPC.Call(utils.CDRsV1ProcessCDR, &engine.CDRWithArgDispatcher{CDR: cdr}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply != utils.OK { - t.Error("Unexpected reply received: ", reply) - } - } - time.Sleep(100 * time.Millisecond) -} - -func testCDReExportCDRs(t *testing.T) { - attr := ArgExportCDRs{ - ExportArgs: map[string]interface{}{ - utils.ExportTemplate: "TemplateWithFilter", - }, - Verbose: true, - } - var rply *RplExportedCDRs - if err := cdreRPC.Call(utils.APIerSv1ExportCDRs, &attr, &rply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(rply.ExportedCGRIDs) != 2 { - t.Errorf("Unexpected number of CDR exported: %s ", utils.ToJSON(rply)) - } -} - -func testCDReFromFolder(t *testing.T) { - var reply string - attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")} - if err := cdreRPC.Call(utils.APIerSv1LoadTariffPlanFromFolder, attrs, &reply); err != nil { - t.Error(err) - } - time.Sleep(500 * time.Millisecond) -} - -// Test CDR from external sources -func testCDReProcessExternalCdr(t *testing.T) { - cdr := &engine.ExternalCDR{ - ToR: utils.VOICE, - OriginID: "testextcdr1", - OriginHost: "127.0.0.1", - Source: utils.UNIT_TEST, - RequestType: utils.META_RATED, - Tenant: "cgrates.org", - Category: "call", - Account: "1003", - Subject: "1003", - Destination: "1001", - SetupTime: "2014-08-04T13:00:00Z", - AnswerTime: "2014-08-04T13:00:07Z", - Usage: "1s", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, - } - var reply string - if err := cdreRPC.Call(utils.CDRsV1ProcessExternalCDR, engine.ExternalCDRWithArgDispatcher{ExternalCDR: cdr}, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply != utils.OK { - t.Error("Unexpected reply received: ", reply) - } - time.Sleep(50 * time.Millisecond) - var cdrs []*engine.ExternalCDR - args := utils.RPCCDRsFilter{OriginIDs: []string{"testextcdr1"}} - if err := cdreRPC.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Errorf("Unexpected number of CDRs returned: %v, cdrs=%s ", len(cdrs), utils.ToJSON(cdrs)) - return - } else { - for _, c := range cdrs { - if c.RunID == utils.MetaRaw && c.Cost != -1 { - t.Errorf("Expected for %s cdr cost to be -1, recived: %v", utils.MetaRaw, c.Cost) - } - if c.RunID == utils.MetaDefault && c.Cost != 0.3 { - t.Errorf("Expected for *default cdr cost to be 0.3, recived: %v", c.Cost) - } - if c.RunID == utils.MetaDefault { - acdr, err := engine.NewCDRFromExternalCDR(c, "") - if err != nil { - t.Error(err) - return - } - if acdr.CostDetails == nil { - t.Errorf("CostDetails should not be nil") - return - } - if acdr.CostDetails.Usage == nil { - t.Errorf("CostDetails for procesed cdr has usage nil") - } - if acdr.CostDetails.Cost == nil { - t.Errorf("CostDetails for procesed cdr has cost nil") - } - } - if c.Usage != "1s" { - t.Errorf("Expected 1s,recived %s", c.Usage) - } - if c.Source != utils.UNIT_TEST { - t.Errorf("Expected %s,recived %s", utils.UNIT_TEST, c.Source) - } - if c.ToR != utils.VOICE { - t.Errorf("Expected %s,recived %s", utils.VOICE, c.ToR) - } - if c.RequestType != utils.META_RATED { - t.Errorf("Expected %s,recived %s", utils.META_RATED, c.RequestType) - } - if c.Category != "call" { - t.Errorf("Expected call,recived %s", c.Category) - } - if c.Account != "1003" { - t.Errorf("Expected 1003,recived %s", c.Account) - } - if c.Subject != "1003" { - t.Errorf("Expected 1003,recived %s", c.Subject) - } - if c.Destination != "1001" { - t.Errorf("Expected 1001,recived %s", c.Destination) - } - if !reflect.DeepEqual(c.ExtraFields, cdr.ExtraFields) { - t.Errorf("Expected %s,recived %s", utils.ToJSON(c.ExtraFields), utils.ToJSON(cdr.ExtraFields)) - } - } - } -} - -func testCDReAddAttributes(t *testing.T) { - alsPrf := &AttributeWithCache{ - AttributeProfile: &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ATTR_CDRE", - Contexts: []string{"*cdre"}, - FilterIDs: []string{"*string:~*req.Subject:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - ExpiryTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), - }, - Attributes: []*engine.Attribute{ - { - Path: utils.MetaReq + utils.NestingSep + utils.Subject, - Value: config.NewRSRParsersMustCompile("ATTR_SUBJECT", utils.INFIELD_SEP), - }, - { - Path: utils.MetaReq + utils.NestingSep + utils.Category, - Value: config.NewRSRParsersMustCompile("ATTR_CATEGORY", utils.INFIELD_SEP), - }, - }, - Weight: 20, - }, - } - alsPrf.Compile() - var result string - if err := cdreRPC.Call(utils.APIerSv1SetAttributeProfile, alsPrf, &result); err != nil { - t.Error(err) - } else if result != utils.OK { - t.Error("Unexpected reply returned", result) - } - var reply *engine.AttributeProfile - if err := cdreRPC.Call(utils.APIerSv1GetAttributeProfile, - utils.TenantIDWithArgDispatcher{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_CDRE"}}, &reply); err != nil { - t.Fatal(err) - } - reply.Compile() - if !reflect.DeepEqual(alsPrf.AttributeProfile, reply) { - t.Errorf("Expecting : %+v, received: %+v", alsPrf.AttributeProfile, reply) - } -} - -func testCDReExportCDRsWithAttributes(t *testing.T) { - attr := ArgExportCDRs{ - ExportArgs: map[string]interface{}{ - utils.ExportTemplate: "TemplateWithAttributeS", - }, - Verbose: true, - } - var rply *RplExportedCDRs - if err := cdreRPC.Call(utils.APIerSv1ExportCDRs, &attr, &rply); err != nil { - t.Fatal("Unexpected error: ", err.Error()) - } else if len(rply.ExportedCGRIDs) != 2 { - t.Errorf("Unexpected number of CDR exported: %s ", utils.ToJSON(rply)) - } - fileContent1 := `Cdr3,*default,test2,OriginCDR3,cgrates.org,ATTR_CATEGORY,1001,ATTR_SUBJECT,+4986517174963,30s,-1.0000 -Cdr2,*default,test2,OriginCDR2,cgrates.org,ATTR_CATEGORY,1001,ATTR_SUBJECT,+4986517174963,5s,-1.0000 -` - fileContent2 := `Cdr2,*default,test2,OriginCDR2,cgrates.org,ATTR_CATEGORY,1001,ATTR_SUBJECT,+4986517174963,5s,-1.0000 -Cdr3,*default,test2,OriginCDR3,cgrates.org,ATTR_CATEGORY,1001,ATTR_SUBJECT,+4986517174963,30s,-1.0000 -` - if outContent1, err := ioutil.ReadFile(rply.ExportedPath); err != nil { - t.Error(err) - } else if fileContent1 != string(outContent1) && fileContent2 != string(outContent1) { - t.Errorf("Expecting: \n<%q>, \nreceived: \n<%q>", fileContent1, string(outContent1)) - } -} - -func testCDReKillEngine(t *testing.T) { - if err := engine.KillEngine(100); err != nil { - t.Error(err) - } -} diff --git a/apier/v1/ees.go b/apier/v1/ees.go index cf77e73ed..32f9e523b 100644 --- a/apier/v1/ees.go +++ b/apier/v1/ees.go @@ -37,7 +37,7 @@ func (eSv1 *EventExporterSv1) Ping(ign *utils.CGREventWithArgDispatcher, reply * } // ProcessEvent triggers exports on EEs side -func (eSv1 *EventExporterSv1) ProcessEvent(args *utils.CGREventWithOpts, +func (eSv1 *EventExporterSv1) ProcessEvent(args *utils.CGREventWithIDs, reply *string) error { return eSv1.eeS.V1ProcessEvent(args, reply) } diff --git a/apier/v2/cdre.go b/apier/v2/cdre.go deleted file mode 100644 index 7a5a9dc71..000000000 --- a/apier/v2/cdre.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package v2 - -import ( - "fmt" - "path" - "strconv" - "strings" - "time" - "unicode/utf8" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type AttrExportCdrsToFile struct { - CdrFormat *string // Cdr output file format - FieldSeparator *string // Separator used between fields - ExportID *string // Optional exportid - ExportDirectory *string // If provided it overwrites the configured export directory - ExportFileName *string // If provided the output filename will be set to this - ExportTemplate *string // Exported fields template <""|fld1,fld2|> - Verbose bool // Disable CgrIds reporting in reply/ExportedCgrIds and reply/UnexportedCgrIds - utils.RPCCDRsFilter // Inherit the CDR filter attributes -} - -// Deprecated, please use APIerSv1.ExportCDRs instead -func (apiv2 *APIerSv2) ExportCdrsToFile(attr *AttrExportCdrsToFile, reply *utils.ExportedFileCdrs) (err error) { - cdreReloadStruct := <-apiv2.Config.ConfigReloads[utils.CDRE] // Read the content of the channel, locking it - defer func() { apiv2.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct }() // Unlock reloads at exit - exportTemplate := apiv2.Config.CdreProfiles[utils.MetaDefault] - if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it - var hasIt bool - if exportTemplate, hasIt = apiv2.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt { - return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound) - } - } - exportFormat := exportTemplate.ExportFormat - if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 { - exportFormat = strings.ToLower(*attr.CdrFormat) - } - if !utils.CDRExportFormats.Has(exportFormat) { - return utils.NewErrMandatoryIeMissing("CdrFormat") - } - fieldSep := exportTemplate.FieldSeparator - if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 { - fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator) - if fieldSep == utf8.RuneError { - return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError, "Invalid") - } - } - eDir := exportTemplate.ExportPath - if attr.ExportDirectory != nil && len(*attr.ExportDirectory) != 0 { - eDir = *attr.ExportDirectory - } - exportID := strconv.FormatInt(time.Now().Unix(), 10) - if attr.ExportID != nil && len(*attr.ExportID) != 0 { - exportID = *attr.ExportID - } - var expFormat string - switch exportFormat { - case utils.MetaFileFWV: - expFormat = "fwv" - case utils.MetaFileCSV: - expFormat = "csv" - default: - expFormat = exportFormat - } - fileName := fmt.Sprintf("cdre_%s.%s", exportID, expFormat) - if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 { - fileName = *attr.ExportFileName - } - filePath := path.Join(eDir, fileName) - if exportFormat == utils.DRYRUN { - filePath = utils.DRYRUN - } - cdrsFltr, err := attr.RPCCDRsFilter.AsCDRsFilter(apiv2.Config.GeneralCfg().DefaultTimezone) - if err != nil { - return utils.NewErrServerError(err) - } - cdrs, _, err := apiv2.CdrDb.GetCDRs(cdrsFltr, false) - if err != nil { - return err - } else if len(cdrs) == 0 { - *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} - return nil - } - cdrexp, err := engine.NewCDRExporter(cdrs, exportTemplate, exportFormat, - filePath, utils.META_NONE, exportID, exportTemplate.Synchronous, - exportTemplate.Attempts, fieldSep, apiv2.Config.GeneralCfg().HttpSkipTlsVerify, - apiv2.Config.ApierCfg().AttributeSConns, apiv2.FilterS) - if err != nil { - return utils.NewErrServerError(err) - } - if err := cdrexp.ExportCDRs(); err != nil { - return utils.NewErrServerError(err) - } - if cdrexp.TotalExportedCdrs() == 0 { - *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} - return nil - } - *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), - TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderID(), LastOrderId: cdrexp.LastOrderID()} - if attr.Verbose { - reply.ExportedCgrIds = cdrexp.PositiveExports() - reply.UnexportedCgrIds = cdrexp.NegativeExports() - } - return nil -} diff --git a/apier/v2/cdre_it_test.go b/apier/v2/cdre_it_test.go deleted file mode 100644 index cf7a80fa7..000000000 --- a/apier/v2/cdre_it_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package v2 - -import ( - "net/rpc" - "path" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - cdreCfgPath string - cdreCfg *config.CGRConfig - cdreRpc *rpc.Client - cdreConfDIR string - - sTestsCDReIT = []func(t *testing.T){ - testV2CDReInitConfig, - testV2CDReInitDataDb, - testV2CDReInitCdrDb, - testV2CDReStartEngine, - testV2CDReRpcConn, - testV2CDReLoadTariffPlanFromFolder, - testV2CDReProcessCDR, - testV2CDReGetCdrs, - testV2CDReExportCdrs, - } -) - -func TestCDReIT(t *testing.T) { - switch *dbType { - case utils.MetaInternal: - cdreConfDIR = "cdrsv2internal" - case utils.MetaMySQL: - cdreConfDIR = "cdrsv2mysql" - case utils.MetaMongo: - cdreConfDIR = "cdrsv2mongo" - case utils.MetaPostgres: - cdreConfDIR = "cdrsv2psql" - default: - t.Fatal("Unknown Database type") - } - - for _, stest := range sTestsCDReIT { - t.Run(cdreConfDIR, stest) - } -} - -func testV2CDReInitConfig(t *testing.T) { - var err error - cdreCfgPath = path.Join(*dataDir, "conf", "samples", cdreConfDIR) - if cdreCfg, err = config.NewCGRConfigFromPath(cdreCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -func testV2CDReInitDataDb(t *testing.T) { - if err := engine.InitDataDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -func testV2CDReInitCdrDb(t *testing.T) { - if err := engine.InitStorDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -func testV2CDReStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(cdreCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -func testV2CDReRpcConn(t *testing.T) { - cdreRpc, err = newRPCClient(cdreCfg.ListenCfg()) - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func testV2CDReLoadTariffPlanFromFolder(t *testing.T) { - var loadInst utils.LoadInstance - if err := cdreRpc.Call(utils.APIerSv2LoadTariffPlanFromFolder, - &utils.AttrLoadTpFromFolder{FolderPath: path.Join( - *dataDir, "tariffplans", "testit")}, &loadInst); err != nil { - t.Error(err) - } - time.Sleep(time.Duration(*waitRater) * time.Millisecond) -} - -func testV2CDReProcessCDR(t *testing.T) { - args := &engine.ArgV1ProcessEvent{ - Flags: []string{utils.MetaRALs}, - CGREvent: utils.CGREvent{ - Tenant: "cgrates.org", - Event: map[string]interface{}{ - utils.OriginID: "testV2CDReProcessCDR1", - utils.OriginHost: "192.168.1.1", - utils.Source: "testV2CDReProcessCDR", - utils.RequestType: utils.META_RATED, - utils.Account: "testV2CDRsProcessCDR", - utils.Subject: "ANY2CNT", - utils.Destination: "+4986517174963", - utils.AnswerTime: time.Date(2018, 8, 24, 16, 00, 26, 0, time.UTC), - utils.Usage: time.Minute, - "field_extr1": "val_extr1", - "fieldextr2": "valextr2", - }, - }, - } - - var reply string - if err := cdreRpc.Call(utils.CDRsV1ProcessEvent, args, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply != utils.OK { - t.Error("Unexpected reply received: ", reply) - } -} - -func testV2CDReGetCdrs(t *testing.T) { - var cdrCnt int64 - req := utils.AttrGetCdrs{} - if err := cdreRpc.Call(utils.APIerSv2CountCDRs, &req, &cdrCnt); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if cdrCnt != 3 { - t.Error("Unexpected number of CDRs returned: ", cdrCnt) - } - var cdrs []*engine.ExternalCDR - args := utils.RPCCDRsFilter{RunIDs: []string{utils.MetaRaw}} - if err := cdreRpc.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Cost != -1.0 { - t.Errorf("Unexpected cost for CDR: %f", cdrs[0].Cost) - } - if cdrs[0].ExtraFields["PayPalAccount"] != "paypal@cgrates.org" { - t.Errorf("PayPalAccount should be added by AttributeS, have: %s", - cdrs[0].ExtraFields["PayPalAccount"]) - } - } - args = utils.RPCCDRsFilter{RunIDs: []string{"CustomerCharges"}} - if err := cdreRpc.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Cost != 0.0198 { - t.Errorf("Unexpected cost for CDR: %f", cdrs[0].Cost) - } - if cdrs[0].ExtraFields["PayPalAccount"] != "paypal@cgrates.org" { - t.Errorf("PayPalAccount should be added by AttributeS, have: %s", - cdrs[0].ExtraFields["PayPalAccount"]) - } - } - args = utils.RPCCDRsFilter{RunIDs: []string{"SupplierCharges"}} - if err := cdreRpc.Call(utils.APIerSv2GetCDRs, &args, &cdrs); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if len(cdrs) != 1 { - t.Error("Unexpected number of CDRs returned: ", len(cdrs)) - } else { - if cdrs[0].Cost != 0.0102 { - t.Errorf("Unexpected cost for CDR: %f", cdrs[0].Cost) - } - if cdrs[0].ExtraFields["PayPalAccount"] != "paypal@cgrates.org" { - t.Errorf("PayPalAccount should be added by AttributeS, have: %s", - cdrs[0].ExtraFields["PayPalAccount"]) - } - } -} - -func testV2CDReExportCdrs(t *testing.T) { - attrExportCdrsToFile := &AttrExportCdrsToFile{ - Verbose: false, - ExportDirectory: utils.StringPointer("/tmp/"), - } - reply := &utils.ExportedFileCdrs{} - - if err := cdreRpc.Call(utils.APIerSv2ExportCdrsToFile, attrExportCdrsToFile, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply.ExportedCgrIds != nil { - t.Error("Expecting ExportedCgrIds to be nil when verbose is false") - } else if reply.UnexportedCgrIds != nil { - t.Error("Expecting UnexportedCgrIds to be nil when verbose is false") - } - - attrExportCdrsToFile = &AttrExportCdrsToFile{ - Verbose: true, - ExportDirectory: utils.StringPointer("/tmp/"), - } - reply = &utils.ExportedFileCdrs{} - - if err := cdreRpc.Call(utils.APIerSv2ExportCdrsToFile, attrExportCdrsToFile, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply.ExportedCgrIds == nil { - t.Error("Expecting ExportedCgrIds to be different than nil when verbose is true") - } else if reply.UnexportedCgrIds == nil { - t.Error("Expecting UnexportedCgrIds to be different than nil when verbose is true") - } - -} diff --git a/config/cdrecfg.go b/config/cdrecfg.go deleted file mode 100644 index f4b5a556f..000000000 --- a/config/cdrecfg.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package config - -import ( - "github.com/cgrates/cgrates/utils" -) - -// One instance of CdrExporter -type CdreCfg struct { - ExportFormat string - ExportPath string - Filters []string - Tenant string - AttributeSContext string - Synchronous bool - Attempts int - FieldSeparator rune - Fields []*FCTemplate -} - -func (self *CdreCfg) loadFromJsonCfg(jsnCfg *CdreJsonCfg, separator string) (err error) { - if jsnCfg == nil { - return nil - } - if jsnCfg.Export_format != nil { - self.ExportFormat = *jsnCfg.Export_format - } - if jsnCfg.Export_path != nil { - self.ExportPath = *jsnCfg.Export_path - } - if jsnCfg.Filters != nil { - self.Filters = make([]string, len(*jsnCfg.Filters)) - for i, fltr := range *jsnCfg.Filters { - self.Filters[i] = fltr - } - } - if jsnCfg.Tenant != nil { - self.Tenant = *jsnCfg.Tenant - } - if jsnCfg.Synchronous != nil { - self.Synchronous = *jsnCfg.Synchronous - } - if jsnCfg.Attempts != nil { - self.Attempts = *jsnCfg.Attempts - } - if jsnCfg.Attributes_context != nil { - self.AttributeSContext = *jsnCfg.Attributes_context - } - if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 { // Make sure we got at least one character so we don't get panic here - sepStr := *jsnCfg.Field_separator - self.FieldSeparator = rune(sepStr[0]) - } - if jsnCfg.Fields != nil { - if self.Fields, err = FCTemplatesFromFCTemplatesJsonCfg(*jsnCfg.Fields, separator); err != nil { - return err - } - } - return nil -} - -// Clone itself into a new CdreCfg -func (self *CdreCfg) Clone() *CdreCfg { - clnCdre := new(CdreCfg) - clnCdre.ExportFormat = self.ExportFormat - clnCdre.ExportPath = self.ExportPath - clnCdre.Synchronous = self.Synchronous - clnCdre.Attempts = self.Attempts - clnCdre.FieldSeparator = self.FieldSeparator - clnCdre.Tenant = self.Tenant - clnCdre.Filters = make([]string, len(self.Filters)) - for i, fltr := range self.Filters { - clnCdre.Filters[i] = fltr - } - clnCdre.Fields = make([]*FCTemplate, len(self.Fields)) - for idx, fld := range self.Fields { - clnCdre.Fields[idx] = fld.Clone() - } - return clnCdre -} - -func (cdre *CdreCfg) AsMapInterface(separator string) map[string]interface{} { - fields := make([]map[string]interface{}, len(cdre.Fields)) - for i, item := range cdre.Fields { - fields[i] = item.AsMapInterface(separator) - } - - return map[string]interface{}{ - utils.ExportFormatCfg: cdre.ExportFormat, - utils.ExportPathCfg: cdre.ExportPath, - utils.FiltersCfg: cdre.Filters, - utils.TenantCfg: cdre.Tenant, - utils.AttributeSContextCfg: cdre.AttributeSContext, - utils.SynchronousCfg: cdre.Synchronous, - utils.AttemptsCfg: cdre.Attempts, - utils.FieldSeparatorCfg: string(cdre.FieldSeparator), - utils.FieldsCfg: fields, - } -} diff --git a/config/cdrecfg_test.go b/config/cdrecfg_test.go deleted file mode 100644 index 3ff289d1a..000000000 --- a/config/cdrecfg_test.go +++ /dev/null @@ -1,196 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package config - -import ( - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/utils" -) - -func TestCdreCfgClone(t *testing.T) { - cgrIDRsrs := NewRSRParsersMustCompile("cgrid", utils.INFIELD_SEP) - runIDRsrs := NewRSRParsersMustCompile("runid", utils.INFIELD_SEP) - initContentFlds := []*FCTemplate{ - {Tag: "CgrId", - Type: utils.META_COMPOSED, - Path: "cgrid", - Value: cgrIDRsrs}, - {Tag: "RunId", - Type: utils.META_COMPOSED, - Path: "runid", - Value: runIDRsrs}, - } - for _, v := range initContentFlds { - v.ComputePath() - } - initCdreCfg := &CdreCfg{ - ExportFormat: utils.MetaFileCSV, - ExportPath: "/var/spool/cgrates/cdre", - Synchronous: true, - Attempts: 2, - FieldSeparator: rune(utils.CSV_SEP), - Fields: initContentFlds, - } - eClnContentFlds := []*FCTemplate{ - {Tag: "CgrId", - Type: utils.META_COMPOSED, - Path: "cgrid", - Value: cgrIDRsrs}, - {Tag: "RunId", - Type: utils.META_COMPOSED, - Path: "runid", - Value: runIDRsrs}, - } - for _, v := range eClnContentFlds { - v.ComputePath() - } - eClnCdreCfg := &CdreCfg{ - ExportFormat: utils.MetaFileCSV, - ExportPath: "/var/spool/cgrates/cdre", - Synchronous: true, - Attempts: 2, - Filters: []string{}, - FieldSeparator: rune(utils.CSV_SEP), - Fields: eClnContentFlds, - } - clnCdreCfg := initCdreCfg.Clone() - if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { - t.Errorf("Cloned result: %+v", clnCdreCfg) - } - initContentFlds[0].Tag = "Destination" - if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance - t.Errorf("Cloned result: %+v", clnCdreCfg) - } - clnCdreCfg.Fields[0].Path = "destination" - if initCdreCfg.Fields[0].Path != "cgrid" { - t.Error("Unexpected change of Path: ", initCdreCfg.Fields[0].Path) - } - -} - -func TestCdreCfgloadFromJsonCfg(t *testing.T) { - var lstcfg, expected CdreCfg - if err := lstcfg.loadFromJsonCfg(nil, utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(lstcfg, expected) { - t.Errorf("Expected: %+v ,recived: %+v", expected, lstcfg) - } - if err := lstcfg.loadFromJsonCfg(new(CdreJsonCfg), utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(lstcfg, expected) { - t.Errorf("Expected: %+v ,recived: %+v", expected, lstcfg) - } - cfgJSONStr := `{ -"cdre": { - "*default": { - "export_format": "*file_csv", // exported CDRs format <*file_csv|*file_fwv|*http_post|*http_json_cdr|*http_json_map|*amqp_json_cdr|*amqp_json_map> - "export_path": "/var/spool/cgrates/cdre", // path where the exported CDRs will be placed - "filters" :[], // new filters for cdre - "tenant": "cgrates.org", // tenant used in filterS.Pass - "synchronous": false, // block processing until export has a result - "attempts": 1, // Number of attempts if not success - "field_separator": ",", // used field separator in some export formats, eg: *file_csv - "fields": [ // template of the exported content fields - {"path": "*exp.CGRID", "type": "*composed", "value": "~*req.CGRID"}, - ], - }, -}, -}` - val, err := NewRSRParsers("~*req.CGRID", utils.INFIELD_SEP) - if err != nil { - t.Error(err) - } - expected = CdreCfg{ - ExportFormat: "*file_csv", - ExportPath: "/var/spool/cgrates/cdre", - Filters: []string{}, - Tenant: "cgrates.org", - Attempts: 1, - FieldSeparator: utils.CSV_SEP, - Fields: []*FCTemplate{{ - Path: "*exp.CGRID", - Tag: "*exp.CGRID", - Type: "*composed", - Value: val, - Layout: time.RFC3339, - }}, - } - expected.Fields[0].ComputePath() - if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { - t.Error(err) - } else if jsnCdreCfg, err := jsnCfg.CdreJsonCfgs(); err != nil { - t.Error(err) - } else if err = lstcfg.loadFromJsonCfg(jsnCdreCfg[utils.MetaDefault], utils.INFIELD_SEP); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expected, lstcfg) { - t.Errorf("Expected: %+v , recived: %+v", utils.ToJSON(expected), utils.ToJSON(lstcfg)) - } -} - -func TestCdreCfgAsMapInterface(t *testing.T) { - var cdre CdreCfg - cfgJSONStr := `{ - "cdre": { - "*default": { - "export_format": "*file_csv", - "export_path": "/var/spool/cgrates/cdre", - "filters" :[], - "tenant": "", - "synchronous": false, - "attempts": 1, - "field_separator": ",", - "attributes_context": "", - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - ], - }, - }, -}` - eMap := map[string]interface{}{ - "export_format": "*file_csv", - "export_path": "/var/spool/cgrates/cdre", - "filters": []string{}, - "tenant": "", - "synchronous": false, - "attempts": 1, - "field_separator": ",", - "attributes_context": "", - "fields": []map[string]interface{}{ - { - "path": "*exp.CGRID", - "tag": "*exp.CGRID", - "type": "*variable", - "value": "~*req.CGRID", - }, - }, - } - - if jsnCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { - t.Error(err) - } else if cdreCfg, err := jsnCfg.CdreJsonCfgs(); err != nil { - t.Error(err) - } else if err = cdre.loadFromJsonCfg(cdreCfg["*default"], utils.EmptyString); err != nil { - t.Error(err) - } else if rcv := cdre.AsMapInterface(""); !reflect.DeepEqual(eMap, rcv) { - t.Errorf("\nExpected: %+v\nRecived: %+v", utils.ToJSON(eMap), utils.ToJSON(rcv)) - } - -} diff --git a/config/config.go b/config/config.go index cd9489e81..7311c1feb 100755 --- a/config/config.go +++ b/config/config.go @@ -155,7 +155,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.ralsCfg.BalanceRatingSubject = make(map[string]string) cfg.schedulerCfg = new(SchedulerCfg) cfg.cdrsCfg = new(CdrsCfg) - cfg.CdreProfiles = make(map[string]*CdreCfg) cfg.analyzerSCfg = new(AnalyzerSCfg) cfg.sessionSCfg = new(SessionSCfg) cfg.sessionSCfg.STIRCfg = new(STIRcfg) @@ -185,8 +184,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.sipAgentCfg = new(SIPAgentCfg) cfg.ConfigReloads = make(map[string]chan struct{}) - cfg.ConfigReloads[utils.CDRE] = make(chan struct{}, 1) - cfg.ConfigReloads[utils.CDRE] <- struct{}{} // Unlock the channel var cgrJsonCfg *CgrJsonCfg if cgrJsonCfg, err = NewCgrJsonCfgFromBytes([]byte(CGRATES_CFG_JSON)); err != nil { @@ -196,7 +193,6 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { return } - cfg.dfltCdreProfile = cfg.CdreProfiles[utils.MetaDefault].Clone() // So default will stay unique, will have nil pointer in case of no defaults loaded which is an extra check // populate default ERs reader for _, ersRdr := range cfg.ersCfg.Readers { if ersRdr.ID == utils.MetaDefault { @@ -259,13 +255,11 @@ type CGRConfig struct { ConfigPath string // Path towards config // Cache defaults loaded from json and needing clones - dfltCdreProfile *CdreCfg // Default cdreConfig profile - dfltEvRdr *EventReaderCfg // default event reader - dfltEvExp *EventExporterCfg // default event exporter + dfltEvRdr *EventReaderCfg // default event reader + dfltEvExp *EventExporterCfg // default event exporter - CdreProfiles map[string]*CdreCfg // Cdre config profiles - loaderCfg LoaderSCfgs // LoaderS configs - httpAgentCfg HttpAgentCfgs // HttpAgent configs + loaderCfg LoaderSCfgs // LoaderS configs + httpAgentCfg HttpAgentCfgs // HttpAgent configs ConfigReloads map[string]chan struct{} // Signals to specific entities that a config reload should occur rldChans map[string]chan struct{} // index here the channels used for reloads @@ -323,16 +317,18 @@ var possibleExporterTypes = utils.NewStringSet([]string{utils.MetaFileCSV, utils utils.MetaKafkajsonMap, utils.MetaS3jsonMap, utils.MetaVirt}) func (cfg *CGRConfig) LazySanityCheck() { - for _, cdrePrfl := range cfg.cdrsCfg.OnlineCDRExports { - if cdreProfile, hasIt := cfg.CdreProfiles[cdrePrfl]; hasIt && (cdreProfile.ExportFormat == utils.MetaS3jsonMap || cdreProfile.ExportFormat == utils.MetaSQSjsonMap) { - poster := utils.SQSPoster - if cdreProfile.ExportFormat == utils.MetaS3jsonMap { - poster = utils.S3Poster - } - argsMap := utils.GetUrlRawArguments(cdreProfile.ExportPath) - for _, arg := range []string{utils.AWSRegion, utils.AWSKey, utils.AWSSecret} { - if _, has := argsMap[arg]; !has { - utils.Logger.Warning(fmt.Sprintf("<%s> No %s present for AWS for cdre: <%s>.", poster, arg, cdrePrfl)) + for _, expID := range cfg.cdrsCfg.OnlineCDRExports { + for _, ee := range cfg.eesCfg.Exporters { + if ee.ID == expID && ee.Type == utils.MetaS3jsonMap || ee.Type == utils.MetaSQSjsonMap { + poster := utils.SQSPoster + if ee.Type == utils.MetaS3jsonMap { + poster = utils.S3Poster + } + argsMap := utils.GetUrlRawArguments(ee.ExportPath) + for _, arg := range []string{utils.AWSRegion, utils.AWSKey, utils.AWSSecret} { + if _, has := argsMap[arg]; !has { + utils.Logger.Warning(fmt.Sprintf("<%s> No %s present for AWS for exporter with ID : <%s>.", poster, arg, ee.ID)) + } } } } @@ -361,7 +357,7 @@ func (cfg *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { cfg.loadGeneralCfg, cfg.loadCacheCfg, cfg.loadListenCfg, cfg.loadHTTPCfg, cfg.loadDataDBCfg, cfg.loadStorDBCfg, cfg.loadFilterSCfg, cfg.loadRalSCfg, cfg.loadSchedulerCfg, - cfg.loadCdrsCfg, cfg.loadCdreCfg, cfg.loadSessionSCfg, + cfg.loadCdrsCfg, cfg.loadSessionSCfg, cfg.loadFreeswitchAgentCfg, cfg.loadKamAgentCfg, cfg.loadAsteriskAgentCfg, cfg.loadDiameterAgentCfg, cfg.loadRadiusAgentCfg, cfg.loadDNSAgentCfg, cfg.loadHttpAgentCfg, cfg.loadAttributeSCfg, @@ -495,28 +491,6 @@ func (cfg *CGRConfig) loadCdrsCfg(jsnCfg *CgrJsonCfg) (err error) { return cfg.cdrsCfg.loadFromJsonCfg(jsnCdrsCfg) } -// loadCdreCfg loads the Cdre section of the configuration -func (cfg *CGRConfig) loadCdreCfg(jsnCfg *CgrJsonCfg) (err error) { - var jsnCdreCfg map[string]*CdreJsonCfg - if jsnCdreCfg, err = jsnCfg.CdreJsonCfgs(); err != nil { - return - } - if jsnCdreCfg != nil { - for profileName, jsnCdre1Cfg := range jsnCdreCfg { - if _, hasProfile := cfg.CdreProfiles[profileName]; !hasProfile { // New profile, create before loading from json - cfg.CdreProfiles[profileName] = new(CdreCfg) - if profileName != utils.MetaDefault { - cfg.CdreProfiles[profileName] = cfg.dfltCdreProfile.Clone() // Clone default so we do not inherit pointers - } - } - if err = cfg.CdreProfiles[profileName].loadFromJsonCfg(jsnCdre1Cfg, cfg.generalCfg.RSRSep); err != nil { // Update the existing profile with content from json config - return - } - } - } - return -} - // loadSessionSCfg loads the SessionS section of the configuration func (cfg *CGRConfig) loadSessionSCfg(jsnCfg *CgrJsonCfg) (err error) { var jsnSessionSCfg *SessionSJsonCfg @@ -1120,8 +1094,6 @@ func (cfg *CGRConfig) V1GetConfigSection(args *StringWithArgDispatcher, reply *m jsonString = utils.ToJSON(cfg.MigratorCgrCfg()) case ApierS: jsonString = utils.ToJSON(cfg.ApierCfg()) - case CDRE_JSN: - jsonString = utils.ToJSON(cfg.CdreProfiles) case EEsJson: jsonString = utils.ToJSON(cfg.EEsCfg()) case ERsJson: @@ -1228,7 +1200,6 @@ func (cfg *CGRConfig) getLoadFunctions() map[string]func(*CgrJsonCfg) error { FilterSjsn: cfg.loadFilterSCfg, RALS_JSN: cfg.loadRalSCfg, CDRS_JSN: cfg.loadCdrsCfg, - CDRE_JSN: cfg.loadCdreCfg, ERsJson: cfg.loadErsCfg, EEsJson: cfg.loadEesCfg, SessionSJson: cfg.loadSessionSCfg, @@ -1541,11 +1512,6 @@ func (cfg *CGRConfig) AsMapInterface(separator string) map[string]interface{} { rpcConns[key] = val.AsMapInterface() } - cdreProfiles := make(map[string]map[string]interface{}) - for key, val := range cfg.CdreProfiles { - cdreProfiles[key] = val.AsMapInterface(separator) - } - loaderCfg := make([]map[string]interface{}, len(cfg.loaderCfg)) for i, item := range cfg.loaderCfg { loaderCfg[i] = item.AsMapInterface(separator) @@ -1557,7 +1523,6 @@ func (cfg *CGRConfig) AsMapInterface(separator string) map[string]interface{} { } return map[string]interface{}{ - utils.CdreProfiles: cdreProfiles, utils.LoaderCfg: loaderCfg, utils.HttpAgentCfg: httpAgentCfg, utils.RpcConns: rpcConns, diff --git a/config/config_defaults.go b/config/config_defaults.go index 9461a1308..a894a5193 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -313,36 +313,6 @@ const CGRATES_CFG_JSON = ` }, -"cdre": { // CDRe config - "*default": { - "export_format": "*file_csv", // exported CDRs format <*file_csv|*file_fwv|*http_post|*http_json_cdr|*http_json_map|*amqp_json_cdr|*amqp_json_map|*sqs_json_map> - "export_path": "/var/spool/cgrates/cdre", // path where the exported CDRs will be placed - "filters" :[], // filters for this export - "tenant": "", // tenant used in filterS.Pass - "synchronous": false, // block processing until export has a result - "attempts": 1, // export attempts - "field_separator": ",", // used field separator in some export formats, eg: *file_csv - "attributes_context": "", // attributes context - empty disables attributes processing - "fields": [ // template of the exported content fields - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - ], - }, -}, - - "ers": { // EventReaderService "enabled": false, // starts the EventReader service: "sessions_conns":["*internal"], // RPC Connections IDs diff --git a/config/config_json.go b/config/config_json.go index d8b11cfba..9341d7489 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -33,7 +33,6 @@ const ( RALS_JSN = "rals" SCHEDULER_JSN = "schedulers" CDRS_JSN = "cdrs" - CDRE_JSN = "cdre" SessionSJson = "sessions" FreeSWITCHAgentJSN = "freeswitch_agent" KamailioAgentJSN = "kamailio_agent" @@ -66,7 +65,7 @@ const ( var ( sortedCfgSections = []string{GENERAL_JSN, RPCConnsJsonName, DATADB_JSN, STORDB_JSN, LISTEN_JSN, TlsCfgJson, HTTP_JSN, SCHEDULER_JSN, - CACHE_JSN, FilterSjsn, RALS_JSN, CDRS_JSN, CDRE_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, + CACHE_JSN, FilterSjsn, RALS_JSN, CDRS_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN, DA_JSN, RA_JSN, HttpAgentJson, DNSAgentJson, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON, RouteSJson, LoaderJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, AnalyzerCfgJson, ApierS, EEsJson, RateSJson, SIPAgentJson} @@ -202,18 +201,6 @@ func (self CgrJsonCfg) CdrsJsonCfg() (*CdrsJsonCfg, error) { return cfg, nil } -func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) { - rawCfg, hasKey := self[CDRE_JSN] - if !hasKey { - return nil, nil - } - cfg := make(map[string]*CdreJsonCfg) - if err := json.Unmarshal(*rawCfg, &cfg); err != nil { - return nil, err - } - return cfg, nil -} - func (self CgrJsonCfg) ERsJsonCfg() (erSCfg *ERsJsonCfg, err error) { rawCfg, hasKey := self[ERsJson] if !hasKey { diff --git a/config/config_json_test.go b/config/config_json_test.go index afa4b3f8a..20a5009e0 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -626,104 +626,6 @@ func TestDfCdrsJsonCfg(t *testing.T) { } } -func TestDfCdreJsonCfgs(t *testing.T) { - eContentFlds := []*FcTemplateJsonCfg{ - { - Path: utils.StringPointer("*exp.CGRID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.CGRID), - }, - { - Path: utils.StringPointer("*exp.RunID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.RunID), - }, - { - Path: utils.StringPointer("*exp.ToR"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.ToR), - }, - { - Path: utils.StringPointer("*exp.OriginID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.OriginID), - }, - { - Path: utils.StringPointer("*exp.RequestType"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.RequestType), - }, - { - Path: utils.StringPointer("*exp.Tenant"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Tenant), - }, - { - Path: utils.StringPointer("*exp.Category"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Category), - }, - { - Path: utils.StringPointer("*exp.Account"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Account), - }, - { - Path: utils.StringPointer("*exp.Subject"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Subject), - }, - { - Path: utils.StringPointer("*exp.Destination"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Destination), - }, - { - Path: utils.StringPointer("*exp.SetupTime"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.SetupTime), - Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00"), - }, - { - Path: utils.StringPointer("*exp.AnswerTime"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.AnswerTime), - Layout: utils.StringPointer("2006-01-02T15:04:05Z07:00"), - }, - { - Path: utils.StringPointer("*exp.Usage"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Usage), - }, - { - Path: utils.StringPointer("*exp.Cost"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer(utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.COST), - Rounding_decimals: utils.IntPointer(4), - }, - } - eCfg := map[string]*CdreJsonCfg{ - utils.MetaDefault: { - Export_format: utils.StringPointer(utils.MetaFileCSV), - Export_path: utils.StringPointer("/var/spool/cgrates/cdre"), - Synchronous: utils.BoolPointer(false), - Attempts: utils.IntPointer(1), - Tenant: utils.StringPointer(""), - Attributes_context: utils.StringPointer(""), - Field_separator: utils.StringPointer(","), - Fields: &eContentFlds, - Filters: &[]string{}, - }, - } - if cfg, err := dfCgrJSONCfg.CdreJsonCfgs(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfg, cfg) { - expect, _ := json.Marshal(eCfg) - received, _ := json.Marshal(cfg) - t.Errorf("Expecting:\n%s\nReceived:\n%s", string(expect), string(received)) - } -} - func TestSmgJsonCfg(t *testing.T) { eCfg := &SessionSJsonCfg{ Enabled: utils.BoolPointer(false), diff --git a/config/config_test.go b/config/config_test.go index ea38fd759..d8a4245db 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -462,128 +462,6 @@ func TestCgrCfgJSONLoadCDRS(t *testing.T) { } } -func TestCgrCfgJSONDefaultsCdreProfiles(t *testing.T) { - eContentFlds := []*FCTemplate{ - { - Tag: "*exp.CGRID", - Path: "*exp.CGRID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.CGRID", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.RunID", - Path: "*exp.RunID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.RunID", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.ToR", - Path: "*exp.ToR", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.ToR", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.OriginID", - Path: "*exp.OriginID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.OriginID", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.RequestType", - Path: "*exp.RequestType", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.RequestType", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Tenant", - Path: "*exp.Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Tenant", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Category", - Path: "*exp.Category", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Category", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Account", - Path: "*exp.Account", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Account", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Subject", - Path: "*exp.Subject", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Subject", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Destination", - Path: "*exp.Destination", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Destination", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.SetupTime", - Path: "*exp.SetupTime", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.SetupTime", utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00", - }, - { - Tag: "*exp.AnswerTime", - Path: "*exp.AnswerTime", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.AnswerTime", utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00", - }, - { - Tag: "*exp.Usage", - Path: "*exp.Usage", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Usage", utils.INFIELD_SEP), - Layout: time.RFC3339, - }, - { - Tag: "*exp.Cost", - Path: "*exp.Cost", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.Cost", utils.INFIELD_SEP), - Layout: time.RFC3339, - RoundingDecimals: utils.IntPointer(4), - }, - } - for _, v := range eContentFlds { - v.ComputePath() - } - eCdreCfg := map[string]*CdreCfg{ - utils.MetaDefault: { - ExportFormat: utils.MetaFileCSV, - ExportPath: "/var/spool/cgrates/cdre", - Filters: []string{}, - Synchronous: false, - Attempts: 1, - AttributeSContext: "", - FieldSeparator: utils.CSV_SEP, - Fields: eContentFlds, - }, - } - if !reflect.DeepEqual(cgrCfg.CdreProfiles, eCdreCfg) { - t.Errorf("Expecting: %+v,\nReceived %+v", utils.ToJSON(eCdreCfg), utils.ToJSON(cgrCfg.CdreProfiles)) - } -} - func TestCgrCfgJSONDefaultsSMGenericCfg(t *testing.T) { eSessionSCfg := &SessionSCfg{ Enabled: false, @@ -2654,8 +2532,7 @@ func TestCheckConfigSanity(t *testing.T) { } cfg.statsCfg.Enabled = true cfg.cdrsCfg.OnlineCDRExports = []string{"stringy"} - cfg.CdreProfiles = map[string]*CdreCfg{"stringx": &CdreCfg{}} - expected = " cannot find CDR export template with ID: " + expected = " cannot find exporter with ID: " if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } diff --git a/config/configsanity.go b/config/configsanity.go index 15556fbd4..33c97725d 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -93,9 +93,16 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> connection with id: <%s> not defined", utils.CDRs, connID) } } - for _, cdrePrfl := range cfg.cdrsCfg.OnlineCDRExports { - if _, hasIt := cfg.CdreProfiles[cdrePrfl]; !hasIt { - return fmt.Errorf("<%s> cannot find CDR export template with ID: <%s>", utils.CDRs, cdrePrfl) + for _, expID := range cfg.cdrsCfg.OnlineCDRExports { + has := false + for _, ee := range cfg.eesCfg.Exporters { + if ee.ID == expID { + has = true + break + } + } + if !has { + return fmt.Errorf("<%s> cannot find exporter with ID: <%s>", utils.CDRs, expID) } } for _, connID := range cfg.cdrsCfg.EEsConns { @@ -106,13 +113,6 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> connection with id: <%s> not defined", utils.CDRs, connID) } } - for prfl, cdre := range cfg.CdreProfiles { - for _, field := range cdre.Fields { - if field.Type != utils.META_NONE && field.Path == utils.EmptyString { - return fmt.Errorf("<%s> %s for %s at %s", utils.CDRs, utils.NewErrMandatoryIeMissing(utils.Path), prfl, field.Tag) - } - } - } } // Loaders sanity checks for _, ldrSCfg := range cfg.loaderCfg { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 407150569..0391c8d3d 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -108,8 +108,7 @@ func TestConfigSanityCDRServer(t *testing.T) { cfg.cdrsCfg.StatSConns = []string{} cfg.cdrsCfg.OnlineCDRExports = []string{"stringy"} - cfg.CdreProfiles = map[string]*CdreCfg{"stringx": &CdreCfg{}} - expected = " cannot find CDR export template with ID: " + expected = " cannot find exporter with ID: " if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } diff --git a/console/cdre_config_reload.go b/console/cdre_config_reload.go deleted file mode 100644 index 40638a194..000000000 --- a/console/cdre_config_reload.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package console - -import ( - v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdCdreConfigReload{ - name: "cdre_config_reload", - rpcMethod: utils.APIerSv1ReloadCdreConfig, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -// Commander implementation -type CmdCdreConfigReload struct { - name string - rpcMethod string - rpcParams *v1.ConfigPathArg - *CommandExecuter -} - -func (self *CmdCdreConfigReload) Name() string { - return self.name -} - -func (self *CmdCdreConfigReload) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdCdreConfigReload) RpcParams(reset bool) interface{} { - if reset || self.rpcParams == nil { - self.rpcParams = new(v1.ConfigPathArg) - } - return self.rpcParams -} - -func (self *CmdCdreConfigReload) PostprocessRpcParams() error { - return nil -} - -func (self *CmdCdreConfigReload) RpcResult() interface{} { - var s string - return &s -} diff --git a/console/cdrs_export.go b/console/cdrs_export.go deleted file mode 100644 index 12870cf78..000000000 --- a/console/cdrs_export.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdExportCdrs{ - name: "cdrs_export", - rpcMethod: utils.APIerSv1ExportCDRs, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -// Commander implementation -type CmdExportCdrs struct { - name string - rpcMethod string - rpcParams *v1.ArgExportCDRs - *CommandExecuter -} - -func (self *CmdExportCdrs) Name() string { - return self.name -} - -func (self *CmdExportCdrs) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdExportCdrs) RpcParams(reset bool) interface{} { - if reset || self.rpcParams == nil { - self.rpcParams = new(v1.ArgExportCDRs) - } - return self.rpcParams -} - -func (self *CmdExportCdrs) PostprocessRpcParams() error { - return nil -} - -func (self *CmdExportCdrs) RpcResult() interface{} { - var reply *v1.RplExportedCDRs - return &reply -} diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 4197fb8db..36f876b93 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -263,36 +263,6 @@ // }, -// "cdre": { // CDRe config -// "*default": { -// "export_format": "*file_csv", // exported CDRs format <*file_csv|*file_fwv|*http_post|*http_json_cdr|*http_json_map|*amqp_json_cdr|*amqp_json_map|*sqs_json_map> -// "export_path": "/var/spool/cgrates/cdre", // path where the exported CDRs will be placed -// "filters" :[], // filters for this export -// "tenant": "", // tenant used in filterS.Pass -// "synchronous": false, // block processing until export has a result -// "attempts": 1, // export attempts -// "field_separator": ",", // used field separator in some export formats, eg: *file_csv -// "attributes_context": "", // attributes context - empty disables attributes processing -// "fields": [ // template of the exported content fields -// {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, -// {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, -// {"path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, -// {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, -// {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, -// {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, -// {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, -// {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, -// {"path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, -// {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, -// {"path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, -// {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, -// {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, -// ], -// }, -// }, - - // "ers": { // EventReaderService // "enabled": false, // starts the EventReader service: // "sessions_conns":["*internal"], // RPC Connections IDs diff --git a/data/conf/samples/acc_balance_keep_internal/cgrates.json b/data/conf/samples/acc_balance_keep_internal/cgrates.json index 5fda682e6..2db92048c 100644 --- a/data/conf/samples/acc_balance_keep_internal/cgrates.json +++ b/data/conf/samples/acc_balance_keep_internal/cgrates.json @@ -45,26 +45,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/acc_balance_keep_internal_gob/cgrates.json b/data/conf/samples/acc_balance_keep_internal_gob/cgrates.json index f53e8228b..fa36884aa 100644 --- a/data/conf/samples/acc_balance_keep_internal_gob/cgrates.json +++ b/data/conf/samples/acc_balance_keep_internal_gob/cgrates.json @@ -51,26 +51,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/acc_balance_keep_mongo/cgrates.json b/data/conf/samples/acc_balance_keep_mongo/cgrates.json index bd0b4f7b8..92765c08d 100644 --- a/data/conf/samples/acc_balance_keep_mongo/cgrates.json +++ b/data/conf/samples/acc_balance_keep_mongo/cgrates.json @@ -46,26 +46,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/acc_balance_keep_mongo_gob/cgrates.json b/data/conf/samples/acc_balance_keep_mongo_gob/cgrates.json index 23da760f0..45355c687 100644 --- a/data/conf/samples/acc_balance_keep_mongo_gob/cgrates.json +++ b/data/conf/samples/acc_balance_keep_mongo_gob/cgrates.json @@ -55,26 +55,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/acc_balance_keep_mysql/cgrates.json b/data/conf/samples/acc_balance_keep_mysql/cgrates.json index e9f7f59d3..85ddaaa72 100644 --- a/data/conf/samples/acc_balance_keep_mysql/cgrates.json +++ b/data/conf/samples/acc_balance_keep_mysql/cgrates.json @@ -45,26 +45,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/acc_balance_keep_mysql_gob/cgrates.json b/data/conf/samples/acc_balance_keep_mysql_gob/cgrates.json index 4361a5728..f3c243dd3 100644 --- a/data/conf/samples/acc_balance_keep_mysql_gob/cgrates.json +++ b/data/conf/samples/acc_balance_keep_mysql_gob/cgrates.json @@ -52,26 +52,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/apier_mongo/apier.json b/data/conf/samples/apier_mongo/apier.json index 2ec231b8a..bf0d3326d 100644 --- a/data/conf/samples/apier_mongo/apier.json +++ b/data/conf/samples/apier_mongo/apier.json @@ -79,13 +79,6 @@ }, -"cdre": { - "*default": { - "export_path": "/tmp/cgrates/cdr/cdrexport/csv", // path where the exported CDRs will be placed - } -}, - - "apiers": { "enabled": true, "scheduler_conns": ["*internal"], diff --git a/data/conf/samples/apier_mysql/apier.json b/data/conf/samples/apier_mysql/apier.json index e0c6d52ed..abd7d9c0b 100644 --- a/data/conf/samples/apier_mysql/apier.json +++ b/data/conf/samples/apier_mysql/apier.json @@ -76,13 +76,6 @@ }, -"cdre": { - "*default": { - "export_path": "/tmp/cgrates/cdr/cdrexport/csv", // path where the exported CDRs will be placed - } -}, - - "apiers": { "enabled": true, "scheduler_conns": ["*internal"], diff --git a/data/conf/samples/cdrewithattributes/cgrates.json b/data/conf/samples/cdrewithattributes/cgrates.json deleted file mode 100755 index 2b83d94b9..000000000 --- a/data/conf/samples/cdrewithattributes/cgrates.json +++ /dev/null @@ -1,79 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: - "db_port": 6379, // data_db port to reach the database - "db_name": "10", // data_db database name to connect to - -}, - -"stor_db": { - "db_password": "CGRateS.org", -}, - - -"rals": { - "enabled": true, -}, - - -"schedulers": { - "enabled": true, -}, - - -"cdrs": { - "enabled": true, -}, - - -"cdre": { - "TemplateWithAttributeS": { - "export_format": "*file_csv", - "export_path": "/tmp/", - "attributes_context":"*cdre", - "filters" :["*string:~*req.Source:test2"], - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.Source", "type": "*variable", "value": "~*req.Source"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - ], - }, -}, - - -"attributes": { - "enabled": true, -}, - - -"apiers": { - "enabled": true, - "caches_conns":["*internal"], - "scheduler_conns": ["*internal"], - "attributes_conns": ["*internal"], // connections to AttributeS for CDRExporter -}, - -} diff --git a/data/conf/samples/cdrewithfilter_internal/cgrates.json b/data/conf/samples/cdrewithfilter_internal/cgrates.json deleted file mode 100755 index 3c5111240..000000000 --- a/data/conf/samples/cdrewithfilter_internal/cgrates.json +++ /dev/null @@ -1,123 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { - "db_type": "*internal", -}, - - -"stor_db": { - "db_type": "*internal", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, -}, - - -"cdrs": { - "enabled": true, - "rals_conns": ["*internal"], -}, - - -"cdre": { - "TemplateWithFilter": { - "export_format": "*file_csv", - "export_path": "/tmp/", - "filters" :["*string:~*req.Source:test2"], - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.Source", "type": "*variable", "value": "~*req.Source"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - ], - }, -}, - - -"attributes": { - "enabled": true, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "-1", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "-1", - "thresholds_conns": ["*internal"], -}, - -"thresholds": { - "enabled": true, - "store_interval": "-1", -}, - - -"routes": { - "enabled": true, - "stats_conns": ["*localhost"], -}, - - -"sessions": { - "enabled": true, - "routes_conns": ["*internal"], - "resources_conns": ["*internal"], - "attributes_conns": ["*internal"], - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator":{ - "out_stordb_password": "CGRateS.org", -}, - - -"apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/cdrewithfilter_mongo/cgrates.json b/data/conf/samples/cdrewithfilter_mongo/cgrates.json deleted file mode 100755 index c931d63f0..000000000 --- a/data/conf/samples/cdrewithfilter_mongo/cgrates.json +++ /dev/null @@ -1,127 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { - "db_type": "mongo", - "db_name": "10", - "db_port": 27017, -}, - - -"stor_db": { - "db_type": "mongo", - "db_name": "cgrates", - "db_port": 27017, -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, -}, - - -"cdrs": { - "enabled": true, - "rals_conns": ["*internal"], -}, - - -"cdre": { - "TemplateWithFilter": { - "export_format": "*file_csv", - "export_path": "/tmp/", - "filters" :["*string:~*req.Source:test2"], - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.Source", "type": "*variable", "value": "~*req.Source"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - ], - }, -}, - - -"attributes": { - "enabled": true, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "stats_conns": ["*localhost"], -}, - - -"sessions": { - "enabled": true, - "routes_conns": ["*internal"], - "resources_conns": ["*internal"], - "attributes_conns": ["*internal"], - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator":{ - "out_stordb_password": "CGRateS.org", -}, - - -"apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/cdrewithfilter_mysql/cgrates.json b/data/conf/samples/cdrewithfilter_mysql/cgrates.json deleted file mode 100755 index 0f2cec7db..000000000 --- a/data/conf/samples/cdrewithfilter_mysql/cgrates.json +++ /dev/null @@ -1,125 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: - "db_port": 6379, // data_db port to reach the database - "db_name": "10", // data_db database name to connect to - -}, - -"stor_db": { - "db_password": "CGRateS.org", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, -}, - - -"cdrs": { - "enabled": true, - "rals_conns": ["*internal"], -}, - - -"cdre": { - "TemplateWithFilter": { - "export_format": "*file_csv", - "export_path": "/tmp/", - "filters" :["*string:~*req.Source:test2"], - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.Source", "type": "*variable", "value": "~*req.Source"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - ], - }, -}, - - -"attributes": { - "enabled": true, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "stats_conns": ["*localhost"], -}, - - -"sessions": { - "enabled": true, - "routes_conns": ["*internal"], - "resources_conns": ["*internal"], - "attributes_conns": ["*internal"], - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator":{ - "out_stordb_password": "CGRateS.org", -}, - - -"apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/loader_mongo/cgrates.json b/data/conf/samples/loader_mongo/cgrates.json index a63bb7dda..ffcf3eec1 100644 --- a/data/conf/samples/loader_mongo/cgrates.json +++ b/data/conf/samples/loader_mongo/cgrates.json @@ -47,26 +47,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/loader_mysql/cgrates.json b/data/conf/samples/loader_mysql/cgrates.json index db6f37376..98935ff76 100644 --- a/data/conf/samples/loader_mysql/cgrates.json +++ b/data/conf/samples/loader_mysql/cgrates.json @@ -45,26 +45,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "attributes": { "enabled": true, }, diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json index 00c7fe56a..6015be8d7 100644 --- a/data/conf/samples/loaders/tutmongo/cgrates.json +++ b/data/conf/samples/loaders/tutmongo/cgrates.json @@ -46,26 +46,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json index 1d4c9674d..650c222a6 100644 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ b/data/conf/samples/loaders/tutmysql/cgrates.json @@ -44,25 +44,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - "loaders": [ { "id": "CustomLoader", // identifier of the Loader diff --git a/data/conf/samples/mongoatlas/cgrates.json b/data/conf/samples/mongoatlas/cgrates.json index 40a8f825e..0dda2107a 100755 --- a/data/conf/samples/mongoatlas/cgrates.json +++ b/data/conf/samples/mongoatlas/cgrates.json @@ -81,26 +81,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "resources": { "enabled": true, "store_interval": "1s", diff --git a/data/conf/samples/mongoreplica/cgrates.json b/data/conf/samples/mongoreplica/cgrates.json index ce1bbd3dc..c8ec93954 100755 --- a/data/conf/samples/mongoreplica/cgrates.json +++ b/data/conf/samples/mongoreplica/cgrates.json @@ -51,26 +51,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "resources": { "enabled": true, "store_interval": "1s", diff --git a/data/conf/samples/multifiles/a.json b/data/conf/samples/multifiles/a.json index 11621e5ee..0290ee091 100644 --- a/data/conf/samples/multifiles/a.json +++ b/data/conf/samples/multifiles/a.json @@ -7,23 +7,6 @@ "default_request_type": "*postpaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> }, -"cdre": { - "*default": { - "fields": [ // template of the exported content fields - {"path": "*exp.AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"}, - {"path": "*exp.ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"}, - {"path": "*exp.Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, - {"path": "*exp.Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, - {"path": "*exp.Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, - {"path": "*exp.Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, - {"path": "*exp.Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, - {"path": "*exp.SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, - {"path": "*exp.Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, - ], - } -}, "http_agent": [ { diff --git a/data/conf/samples/multifiles/b/b.json b/data/conf/samples/multifiles/b/b.json index e6282ceff..48d38cc0a 100644 --- a/data/conf/samples/multifiles/b/b.json +++ b/data/conf/samples/multifiles/b/b.json @@ -7,11 +7,6 @@ "default_request_type": "*pseudoprepaid", // default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated> }, -"cdre": { - "*default": { - "export_path": "/tmp/cgrates/cdre", // path where the exported CDRs will be placed - }, -}, "http_agent": [ { diff --git a/data/conf/samples/multifiles/c.json b/data/conf/samples/multifiles/c.json index 756af5aa0..5a71426df 100644 --- a/data/conf/samples/multifiles/c.json +++ b/data/conf/samples/multifiles/c.json @@ -3,23 +3,6 @@ // Used in multifile configuration tests // Should be the third file loaded -"cdre": { - "export1": { - "cost_rounding_decimals": 3, // rounding decimals for Cost values. -1 to disable rounding - "fields": [ // template of the exported content fields - {"path": "*exp.Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"}, - {"path": "*exp.Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"}, - {"path": "*exp.Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"}, - {"path": "*exp.Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"}, - {"path": "*exp.Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"}, - {"path": "*exp.SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"}, - {"path": "*exp.Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"}, - ], // template of the exported header fields - } -}, - "http_agent": [ { "id": "conecto_xml", diff --git a/data/conf/samples/tutinternal_new/cgrates.json b/data/conf/samples/tutinternal_new/cgrates.json index 94d53f5c5..045c4e7b5 100644 --- a/data/conf/samples/tutinternal_new/cgrates.json +++ b/data/conf/samples/tutinternal_new/cgrates.json @@ -39,24 +39,6 @@ "cdrs_conns": ["*internal"], }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, "cdrs": { "enabled": true, diff --git a/data/conf/samples/tutmongo/cgrates.json b/data/conf/samples/tutmongo/cgrates.json index 5c9a7f347..f8a445364 100644 --- a/data/conf/samples/tutmongo/cgrates.json +++ b/data/conf/samples/tutmongo/cgrates.json @@ -47,26 +47,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/tutmongo_gob/cgrates.json b/data/conf/samples/tutmongo_gob/cgrates.json index a6f6b7d2b..80a547bce 100644 --- a/data/conf/samples/tutmongo_gob/cgrates.json +++ b/data/conf/samples/tutmongo_gob/cgrates.json @@ -54,26 +54,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/tutmongonew/cgrates.json b/data/conf/samples/tutmongonew/cgrates.json index 73f26d351..7873eac34 100644 --- a/data/conf/samples/tutmongonew/cgrates.json +++ b/data/conf/samples/tutmongonew/cgrates.json @@ -46,26 +46,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "chargers": { "enabled": true, "attributes_conns": ["*internal"], diff --git a/data/conf/samples/tutmysql/cgrates.json b/data/conf/samples/tutmysql/cgrates.json index 7360fc402..b19fbb774 100644 --- a/data/conf/samples/tutmysql/cgrates.json +++ b/data/conf/samples/tutmysql/cgrates.json @@ -45,26 +45,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - - "attributes": { "enabled": true, }, diff --git a/data/conf/samples/tutmysql_internal/cgrates.json b/data/conf/samples/tutmysql_internal/cgrates.json index a13bf16e7..4259a1c97 100644 --- a/data/conf/samples/tutmysql_internal/cgrates.json +++ b/data/conf/samples/tutmysql_internal/cgrates.json @@ -43,25 +43,6 @@ }, -"cdre": { - "TestTutITExportCDR": { - "fields": [ - {"path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, - {"path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, - {"path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, - {"path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, - {"path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, - {"path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, - {"path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, - {"path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, - {"path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, - {"path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost", "rounding_decimals": 4}, - {"path": "*exp.MatchedDestinationID", "type": "*variable", "value": "~*req.CostDetails:s/\"MatchedDestId\":.*_(\\w{4})/${1}/:s/\"MatchedDestId\":\"INTERNAL\"/ON010/"}, - ], - }, -}, - "loaders": [ { "id": "TeoLoader", // identifier of the Loader diff --git a/ees/ees.go b/ees/ees.go index fb9cf861c..810a9d0fc 100644 --- a/ees/ees.go +++ b/ees/ees.go @@ -128,7 +128,7 @@ func (eeS *EventExporterS) attrSProcessEvent(cgrEv *utils.CGREventWithOpts, attr } // V1ProcessEvent will be called each time a new event is received from readers -func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithOpts, rply *string) (err error) { +func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithIDs, rply *string) (err error) { eeS.cfg.RLocks(config.EEsJson) defer eeS.cfg.RUnlocks(config.EEsJson) @@ -139,6 +139,12 @@ func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithOpts, rply *s continue } + if len(cgrEv.IDs) != 0 { + if !utils.IsSliceMember(cgrEv.IDs, eeCfg.ID) { + continue + } + } + if len(eeCfg.Filters) != 0 { cgrDp := utils.MapStorage{utils.MetaReq: cgrEv.Event} tnt := cgrEv.Tenant @@ -155,7 +161,7 @@ func (eeS *EventExporterS) V1ProcessEvent(cgrEv *utils.CGREventWithOpts, rply *s if eeCfg.Flags.GetBool(utils.MetaAttributes) { if err = eeS.attrSProcessEvent( - cgrEv, + cgrEv.CGREventWithOpts, eeCfg.AttributeSIDs, utils.FirstNonEmpty( eeCfg.AttributeSCtx, diff --git a/ees/filecsv_it_test.go b/ees/filecsv_it_test.go index 4598f1523..90f0c0ee7 100644 --- a/ees/filecsv_it_test.go +++ b/ees/filecsv_it_test.go @@ -103,86 +103,92 @@ func testCsvRPCConn(t *testing.T) { } func testCsvExportEvent(t *testing.T) { - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 1.01, - "ExporterUsed": "CSVExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 1.01, + "ExporterUsed": "CSVExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventData := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "dataEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.DATA, - utils.OriginID: "abcdef", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "AnotherTenant", - utils.Category: "call", //for data CDR use different Tenant - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Nanosecond, - utils.RunID: utils.MetaDefault, - utils.Cost: 0.012, - "ExporterUsed": "CSVExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventData := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "dataEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.DATA, + utils.OriginID: "abcdef", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "AnotherTenant", + utils.Category: "call", //for data CDR use different Tenant + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Nanosecond, + utils.RunID: utils.MetaDefault, + utils.Cost: 0.012, + "ExporterUsed": "CSVExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventSMS := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "SMSEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.SMS, - utils.OriginID: "sdfwer", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(1), - utils.RunID: utils.MetaDefault, - utils.Cost: 0.15, - "ExporterUsed": "CSVExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventSMS := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "SMSEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + utils.OriginID: "sdfwer", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(1), + utils.RunID: utils.MetaDefault, + utils.Cost: 0.15, + "ExporterUsed": "CSVExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } @@ -233,60 +239,64 @@ func testCsvVerifyExports(t *testing.T) { } func testCsvExportComposedEvent(t *testing.T) { - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - "ComposedOriginID1": "dsaf", - "ComposedOriginID2": "dsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 1.016374, - "ExporterUsed": "CSVExporterComposed", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + "ComposedOriginID1": "dsaf", + "ComposedOriginID2": "dsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 1.016374, + "ExporterUsed": "CSVExporterComposed", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventSMS := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "SMSEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.SMS, - "ComposedOriginID1": "sdf", - "ComposedOriginID2": "wer", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(1), - utils.RunID: utils.MetaDefault, - utils.Cost: 0.155462, - "ExporterUsed": "CSVExporterComposed", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventSMS := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "SMSEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + "ComposedOriginID1": "sdf", + "ComposedOriginID2": "wer", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(1), + utils.RunID: utils.MetaDefault, + utils.Cost: 0.155462, + "ExporterUsed": "CSVExporterComposed", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } @@ -339,28 +349,30 @@ func testCsvExportMaskedDestination(t *testing.T) { t.Error("Unexpected reply returned", reply) } - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "+4986517174963", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 1.01, - "ExporterUsed": "CSVMaskedDestination", + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "+4986517174963", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 1.01, + "ExporterUsed": "CSVMaskedDestination", + }, }, }, } diff --git a/ees/filefwv_it_test.go b/ees/filefwv_it_test.go index d1c27d9af..8b48ca0c7 100644 --- a/ees/filefwv_it_test.go +++ b/ees/filefwv_it_test.go @@ -97,30 +97,32 @@ func testFwvRPCConn(t *testing.T) { } func testFwvExportEvent(t *testing.T) { - event := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "Event", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.OrderID: 1, - utils.CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), - utils.AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 2.34567, - "ExporterUsed": "FWVExporter", - "ExtraFields": map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + event := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "Event", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.OrderID: 1, + utils.CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), + utils.AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 2.34567, + "ExporterUsed": "FWVExporter", + "ExtraFields": map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + }, }, }, } diff --git a/ees/httpjsonmap_it_test.go b/ees/httpjsonmap_it_test.go index 90800132d..d8edecb67 100644 --- a/ees/httpjsonmap_it_test.go +++ b/ees/httpjsonmap_it_test.go @@ -113,86 +113,92 @@ func testHTTPJsonMapStartHTTPServer(t *testing.T) { } func testHTTPJsonMapExportEvent(t *testing.T) { - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 1.01, - "ExporterUsed": "HTTPJsonMapExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 1.01, + "ExporterUsed": "HTTPJsonMapExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventData := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "dataEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.DATA, - utils.OriginID: "abcdef", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "AnotherTenant", - utils.Category: "call", //for data CDR use different Tenant - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Nanosecond, - utils.RunID: utils.MetaDefault, - utils.Cost: 0.012, - "ExporterUsed": "HTTPJsonMapExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventData := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "dataEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.DATA, + utils.OriginID: "abcdef", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "AnotherTenant", + utils.Category: "call", //for data CDR use different Tenant + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Nanosecond, + utils.RunID: utils.MetaDefault, + utils.Cost: 0.012, + "ExporterUsed": "HTTPJsonMapExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventSMS := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "SMSEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.SMS, - utils.OriginID: "sdfwer", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(1), - utils.RunID: utils.MetaDefault, - utils.Cost: 0.15, - "ExporterUsed": "HTTPJsonMapExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventSMS := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "SMSEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.SMS, + utils.OriginID: "sdfwer", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(1), + utils.RunID: utils.MetaDefault, + utils.Cost: 0.15, + "ExporterUsed": "HTTPJsonMapExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } diff --git a/ees/httppost_it_test.go b/ees/httppost_it_test.go index 92fabdd96..ca637a497 100644 --- a/ees/httppost_it_test.go +++ b/ees/httppost_it_test.go @@ -113,58 +113,62 @@ func testHTTPStartHTTPServer(t *testing.T) { } func testHTTPExportEvent(t *testing.T) { - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: utils.MetaDefault, - utils.Cost: 1.01, - "ExporterUsed": "HTTPPostExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: utils.MetaDefault, + utils.Cost: 1.01, + "ExporterUsed": "HTTPPostExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } - eventData := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "dataEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.DATA, - utils.OriginID: "abcdef", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "AnotherTenant", - utils.Category: "call", //for data CDR use different Tenant - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Nanosecond, - utils.RunID: utils.MetaDefault, - utils.Cost: 0.012, - "ExporterUsed": "HTTPPostExporter", - "ExtraFields": map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, + eventData := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "dataEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.DATA, + utils.OriginID: "abcdef", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "AnotherTenant", + utils.Category: "call", //for data CDR use different Tenant + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Nanosecond, + utils.RunID: utils.MetaDefault, + utils.Cost: 0.012, + "ExporterUsed": "HTTPPostExporter", + "ExtraFields": map[string]string{"extra1": "val_extra1", + "extra2": "val_extra2", "extra3": "val_extra3"}, + }, }, }, } diff --git a/ees/virtual_ee_it_test.go b/ees/virtual_ee_it_test.go index c8d27200b..0d17037a8 100644 --- a/ees/virtual_ee_it_test.go +++ b/ees/virtual_ee_it_test.go @@ -99,28 +99,30 @@ func testVirtRPCConn(t *testing.T) { } func testVirtExportSupplierEvent(t *testing.T) { - supplierEvent := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "supplierEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: "SupplierRun", - utils.Cost: 1.23, - "ExporterUsed": "RouteExporter", + supplierEvent := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "supplierEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: "SupplierRun", + utils.Cost: 1.23, + "ExporterUsed": "RouteExporter", + }, }, }, } @@ -135,28 +137,30 @@ func testVirtExportSupplierEvent(t *testing.T) { } func testVirtExportEvents(t *testing.T) { - eventVoice := &utils.CGREventWithOpts{ - CGREvent: &utils.CGREvent{ - Tenant: "cgrates.org", - ID: "voiceEvent", - Time: utils.TimePointer(time.Now()), - Event: map[string]interface{}{ - utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - utils.ToR: utils.VOICE, - utils.OriginID: "dsafdsaf", - utils.OriginHost: "192.168.1.1", - utils.RequestType: utils.META_RATED, - utils.Tenant: "cgrates.org", - utils.Category: "call", - utils.Account: "1001", - utils.Subject: "1001", - utils.Destination: "1002", - utils.SetupTime: time.Unix(1383813745, 0).UTC(), - utils.AnswerTime: time.Unix(1383813746, 0).UTC(), - utils.Usage: time.Duration(10) * time.Second, - utils.RunID: "SupplierRun", - utils.Cost: 1.01, - "ExporterUsed": "CSVExporterFromVirt", + eventVoice := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: &utils.CGREvent{ + Tenant: "cgrates.org", + ID: "voiceEvent", + Time: utils.TimePointer(time.Now()), + Event: map[string]interface{}{ + utils.CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), + utils.ToR: utils.VOICE, + utils.OriginID: "dsafdsaf", + utils.OriginHost: "192.168.1.1", + utils.RequestType: utils.META_RATED, + utils.Tenant: "cgrates.org", + utils.Category: "call", + utils.Account: "1001", + utils.Subject: "1001", + utils.Destination: "1002", + utils.SetupTime: time.Unix(1383813745, 0).UTC(), + utils.AnswerTime: time.Unix(1383813746, 0).UTC(), + utils.Usage: time.Duration(10) * time.Second, + utils.RunID: "SupplierRun", + utils.Cost: 1.01, + "ExporterUsed": "CSVExporterFromVirt", + }, }, }, } diff --git a/engine/cdre.go b/engine/cdre.go deleted file mode 100644 index 47be575ed..000000000 --- a/engine/cdre.go +++ /dev/null @@ -1,555 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package engine - -import ( - "encoding/csv" - "encoding/json" - "errors" - "fmt" - "io" - "net/url" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -const ( - metaExportID = "*export_id" - metaTimeNow = "*time_now" - metaFirstCDRAtime = "*first_cdr_atime" - metaLastCDRAtime = "*last_cdr_atime" - metaNrCDRs = "*cdrs_number" - metaDurCDRs = "*cdrs_duration" - metaSMSUsage = "*sms_usage" - metaMMSUsage = "*mms_usage" - metaGenericUsage = "*generic_usage" - metaDataUsage = "*data_usage" - metaCostCDRs = "*cdrs_cost" -) - -// NewCDRExporter returns a new CDRExporter -func NewCDRExporter(cdrs []*CDR, exportTemplate *config.CdreCfg, exportFormat, exportPath, fallbackPath, exportID string, - synchronous bool, attempts int, fieldSeparator rune, - httpSkipTLSCheck bool, attrsConns []string, filterS *FilterS) (*CDRExporter, error) { - if len(cdrs) == 0 { // Nothing to export - return nil, nil - } - cdre := &CDRExporter{ - cdrs: cdrs, - exportTemplate: exportTemplate, - exportFormat: exportFormat, - exportPath: exportPath, - fallbackPath: fallbackPath, - exportID: exportID, - synchronous: synchronous, - attempts: attempts, - fieldSeparator: fieldSeparator, - httpSkipTLSCheck: httpSkipTLSCheck, - negativeExports: make(map[string]string), - attrsConns: attrsConns, - filterS: filterS, - } - return cdre, nil -} - -// CDRExporter used to export the CDRs -type CDRExporter struct { - sync.RWMutex - cdrs []*CDR - exportTemplate *config.CdreCfg - exportFormat string - exportPath string - fallbackPath string // folder where we save failed CDRs - exportID string // Unique identifier or this export - synchronous bool - attempts int - fieldSeparator rune - httpSkipTLSCheck bool - - header, trailer []string // Header and Trailer fields - content [][]string // Rows of cdr fields - - firstCdrATime, lastCdrATime time.Time - numberOfRecords int - totalDuration, totalDataUsage, totalSmsUsage, - totalMmsUsage, totalGenericUsage time.Duration - totalCost float64 - firstExpOrderID, lastExpOrderID int64 - positiveExports []string // CGRIDs of successfully exported CDRs - negativeExports map[string]string // CGRIDs of failed exports - - attrsConns []string - filterS *FilterS -} - -// Handle various meta functions used in header/trailer -func (cdre *CDRExporter) metaHandler(tag, arg string) (string, error) { - switch tag { - case metaExportID: - return cdre.exportID, nil - case metaTimeNow: - return time.Now().Format(arg), nil - case metaFirstCDRAtime: - return cdre.firstCdrATime.Format(arg), nil - case metaLastCDRAtime: - return cdre.lastCdrATime.Format(arg), nil - case metaNrCDRs: - return strconv.Itoa(cdre.numberOfRecords), nil - case metaDurCDRs: // ToDo: remove this because they are useless - cdr := &CDR{ToR: utils.VOICE, Usage: cdre.totalDuration} - return cdr.FieldAsString(&config.RSRParser{Rules: utils.DynamicDataPrefix + utils.Usage}) - case metaSMSUsage: - cdr := &CDR{ToR: utils.SMS, Usage: cdre.totalDuration} - return cdr.FieldAsString(&config.RSRParser{Rules: utils.DynamicDataPrefix + utils.Usage}) - case metaMMSUsage: - cdr := &CDR{ToR: utils.MMS, Usage: cdre.totalDuration} - return cdr.FieldAsString(&config.RSRParser{Rules: utils.DynamicDataPrefix + utils.Usage}) - case metaGenericUsage: - cdr := &CDR{ToR: utils.GENERIC, Usage: cdre.totalDuration} - return cdr.FieldAsString(&config.RSRParser{Rules: utils.DynamicDataPrefix + utils.Usage}) - case metaDataUsage: - cdr := &CDR{ToR: utils.DATA, Usage: cdre.totalDuration} - return cdr.FieldAsString(&config.RSRParser{Rules: utils.DynamicDataPrefix + utils.Usage}) - case metaCostCDRs: - return strconv.FormatFloat(utils.Round(cdre.totalCost, - globalRoundingDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil - default: - return "", fmt.Errorf("Unsupported METATAG: %s", tag) - } -} - -// Compose and cache the header -func (cdre *CDRExporter) composeHeader() (err error) { - for _, cfgFld := range cdre.exportTemplate.Fields { - if !strings.HasPrefix(cfgFld.Path, utils.MetaHdr) { - continue - } - if len(cfgFld.Filters) != 0 { - //check filter if pass - } - var outVal string - switch cfgFld.Type { - case utils.META_FILLER: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal = out - cfgFld.Padding = utils.MetaRight - case utils.META_CONSTANT: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal = out - case utils.META_HANDLER: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal, err = cdre.metaHandler(out, cfgFld.Layout) - default: - return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) - } - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error())) - return err - } - fmtOut := outVal - if fmtOut, err = utils.FmtFieldWidth(cfgFld.Tag, outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - utils.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error())) - return err - } - cdre.Lock() - cdre.header = append(cdre.header, fmtOut) - cdre.Unlock() - } - return nil -} - -// Compose and cache the trailer -func (cdre *CDRExporter) composeTrailer() (err error) { - for _, cfgFld := range cdre.exportTemplate.Fields { - if !strings.HasPrefix(cfgFld.Path, utils.MetaTrl) { - continue - } - if len(cfgFld.Filters) != 0 { - //check filter if pass - } - var outVal string - switch cfgFld.Type { - case utils.META_FILLER: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal = out - cfgFld.Padding = utils.MetaRight - case utils.META_CONSTANT: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal = out - case utils.META_HANDLER: - out, err := cfgFld.Value.ParseValue(utils.EmptyString) - if err != nil { - return err - } - outVal, err = cdre.metaHandler(out, cfgFld.Layout) - default: - return fmt.Errorf("Unsupported field type: %s", cfgFld.Type) - } - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error())) - return err - } - fmtOut := outVal - if fmtOut, err = utils.FmtFieldWidth(cfgFld.Tag, outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - utils.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error())) - return err - } - cdre.Lock() - cdre.trailer = append(cdre.trailer, fmtOut) - cdre.Unlock() - } - return nil -} - -func (cdre *CDRExporter) postCdr(cdr *CDR) (err error) { - var body interface{} - switch cdre.exportFormat { - case utils.MetaHTTPjsonCDR, utils.MetaAMQPjsonCDR: - if body, err = json.Marshal(cdr); err != nil { - return - } - case utils.MetaHTTPjsonMap, utils.MetaAMQPjsonMap, utils.MetaAMQPV1jsonMap, utils.MetaSQSjsonMap, utils.MetaKafkajsonMap, utils.MetaS3jsonMap: - var expMp map[string]string - if expMp, err = cdr.AsExportMap(cdre.exportTemplate.Fields, cdre.httpSkipTLSCheck, nil, cdre.filterS); err != nil { - return - } - if body, err = json.Marshal(expMp); err != nil { - return - } - case utils.MetaHTTPPost: - var expMp map[string]string - if expMp, err = cdr.AsExportMap(cdre.exportTemplate.Fields, cdre.httpSkipTLSCheck, nil, cdre.filterS); err != nil { - return - } - vals := url.Values{} - for fld, val := range expMp { - vals.Set(fld, val) - } - body = vals - default: - return fmt.Errorf("unsupported exportFormat: <%s>", cdre.exportFormat) - } - switch cdre.exportFormat { - case utils.MetaHTTPjsonCDR, utils.MetaHTTPjsonMap, utils.MetaHTTPjson, utils.MetaHTTPPost: - var pstr *HTTPPoster - pstr, err = NewHTTPPoster(config.CgrConfig().GeneralCfg().HttpSkipTlsVerify, - config.CgrConfig().GeneralCfg().ReplyTimeout, cdre.exportPath, - utils.PosterTransportContentTypes[cdre.exportFormat], cdre.attempts) - if err != nil { - return err - } - err = pstr.Post(body, utils.EmptyString) - case utils.MetaAMQPjsonCDR, utils.MetaAMQPjsonMap: - err = PostersCache.PostAMQP(cdre.exportPath, cdre.attempts, body.([]byte)) - case utils.MetaAMQPV1jsonMap: - err = PostersCache.PostAMQPv1(cdre.exportPath, cdre.attempts, body.([]byte)) - case utils.MetaSQSjsonMap: - err = PostersCache.PostSQS(cdre.exportPath, cdre.attempts, body.([]byte)) - case utils.MetaKafkajsonMap: - err = PostersCache.PostKafka(cdre.exportPath, cdre.attempts, body.([]byte), utils.ConcatenatedKey(cdr.CGRID, cdr.RunID)) - case utils.MetaS3jsonMap: - err = PostersCache.PostS3(cdre.exportPath, cdre.attempts, body.([]byte), utils.ConcatenatedKey(cdr.CGRID, cdr.RunID)) - } - if err != nil && cdre.fallbackPath != utils.META_NONE { - AddFailedPost(cdre.exportPath, cdre.exportFormat, utils.CDRPoster, body) - } - return -} - -// Write individual cdr into content buffer, build stats -func (cdre *CDRExporter) processCDR(cdr *CDR) (err error) { - if cdr.ExtraFields == nil { // Avoid assignment in nil map if not initialized - cdr.ExtraFields = make(map[string]string) - } - // send the cdr to be processed by attributeS - if cdre.exportTemplate.AttributeSContext != utils.EmptyString { - if len(cdre.attrsConns) == 0 { - return errors.New("no connection to AttributeS") - } - cdrEv := cdr.AsCGREvent() - args := &AttrArgsProcessEvent{ - Context: utils.StringPointer(utils.FirstNonEmpty( - utils.IfaceAsString(cdrEv.Event[utils.OptsContext]), - cdre.exportTemplate.AttributeSContext)), - CGREvent: cdrEv, - } - var evReply AttrSProcessEventReply - if err = connMgr.Call(cdre.attrsConns, nil, - utils.AttributeSv1ProcessEvent, - args, &evReply); err != nil { - return - } - if len(evReply.AlteredFields) != 0 { - if err = cdr.UpdateFromCGREvent(evReply.CGREvent, evReply.AlteredFields); err != nil { - return - } - } - } - - switch cdre.exportFormat { - case utils.MetaFileFWV, utils.MetaFileCSV: - var cdrRow []string - cdrRow, err = cdr.AsExportRecord(cdre.exportTemplate.Fields, cdre.httpSkipTLSCheck, cdre.cdrs, cdre.filterS) - if len(cdrRow) == 0 && err == nil { // No CDR data, most likely no configuration fields defined - return - } - cdre.Lock() - cdre.content = append(cdre.content, cdrRow) - cdre.Unlock() - default: // attempt posting CDR - err = cdre.postCdr(cdr) - } - if err != nil { - utils.Logger.Err(fmt.Sprintf(" Cannot export CDR with CGRID: %s and runid: %s, error: %s", cdr.CGRID, cdr.RunID, err.Error())) - return - } - // Done with writing content, compute stats here - cdre.Lock() - defer cdre.Unlock() - if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) { - cdre.firstCdrATime = cdr.AnswerTime - } - if cdr.AnswerTime.After(cdre.lastCdrATime) { - cdre.lastCdrATime = cdr.AnswerTime - } - cdre.numberOfRecords++ - if cdr.ToR == utils.VOICE { // Only count duration for non data cdrs - cdre.totalDuration += cdr.Usage - } - if cdr.ToR == utils.SMS { // Count usage for SMS - cdre.totalSmsUsage += cdr.Usage - } - if cdr.ToR == utils.MMS { // Count usage for MMS - cdre.totalMmsUsage += cdr.Usage - } - if cdr.ToR == utils.GENERIC { // Count usage for GENERIC - cdre.totalGenericUsage += cdr.Usage - } - if cdr.ToR == utils.DATA { // Count usage for DATA - cdre.totalDataUsage += cdr.Usage - } - if cdr.Cost != -1 { - cdre.totalCost += cdr.Cost - cdre.totalCost = utils.Round(cdre.totalCost, globalRoundingDecimals, utils.ROUNDING_MIDDLE) - } - if cdre.firstExpOrderID > cdr.OrderID || cdre.firstExpOrderID == 0 { - cdre.firstExpOrderID = cdr.OrderID - } - if cdre.lastExpOrderID < cdr.OrderID { - cdre.lastExpOrderID = cdr.OrderID - } - return -} - -// processCDRs proccess every cdr -func (cdre *CDRExporter) processCDRs() (err error) { - var wg sync.WaitGroup - isSync := cdre.exportTemplate.Synchronous || - utils.SliceHasMember([]string{utils.MetaFileCSV, utils.MetaFileFWV}, cdre.exportTemplate.ExportFormat) - for _, cdr := range cdre.cdrs { - if cdr == nil || len(cdr.CGRID) == 0 { // CDR needs to exist and it's CGRID needs to be populated - continue - } - if len(cdre.exportTemplate.Filters) != 0 { - cgrDp := utils.MapStorage{ - utils.MetaReq: cdr.AsMapStringIface(), - utils.MetaEC: cdr.CostDetails, - } - if pass, err := cdre.filterS.Pass(cdre.exportTemplate.Tenant, - cdre.exportTemplate.Filters, cgrDp); err != nil { - return err - } else if !pass { - continue // Not passes filters, ignore this CDR - } - } - if isSync { - wg.Add(1) // wait for synchronous or file ones since these need to be done before continuing - } - go func(cdre *CDRExporter, cdr *CDR) { - if err := cdre.processCDR(cdr); err != nil { - cdre.Lock() - cdre.negativeExports[cdr.CGRID] = err.Error() - cdre.Unlock() - } else { - cdre.Lock() - cdre.positiveExports = append(cdre.positiveExports, cdr.CGRID) - cdre.Unlock() - } - if isSync { - wg.Done() - } - }(cdre, cdr) - } - wg.Wait() - return -} - -// Simple write method -func (cdre *CDRExporter) writeOut(ioWriter io.Writer) (err error) { - cdre.Lock() - defer cdre.Unlock() - if len(cdre.header) != 0 { - for _, fld := range append(cdre.header, "\n") { - if _, err = io.WriteString(ioWriter, fld); err != nil { - return - } - } - } - for _, cdrContent := range cdre.content { - for _, cdrFld := range append(cdrContent, "\n") { - if _, err = io.WriteString(ioWriter, cdrFld); err != nil { - return - } - } - } - if len(cdre.trailer) != 0 { - for _, fld := range append(cdre.trailer, "\n") { - if _, err = io.WriteString(ioWriter, fld); err != nil { - return - } - } - } - return -} - -// csvWriter specific method -func (cdre *CDRExporter) writeCsv(csvWriter *csv.Writer) (err error) { - csvWriter.Comma = cdre.fieldSeparator - cdre.RLock() - defer cdre.RUnlock() - if len(cdre.header) != 0 { - if err = csvWriter.Write(cdre.header); err != nil { - return - } - } - for _, cdrContent := range cdre.content { - if err = csvWriter.Write(cdrContent); err != nil { - return - } - } - if len(cdre.trailer) != 0 { - if err = csvWriter.Write(cdre.trailer); err != nil { - return - } - } - csvWriter.Flush() - return -} - -// ExportCDRs exports the given CDRs -func (cdre *CDRExporter) ExportCDRs() (err error) { - if err = cdre.processCDRs(); err != nil { - return - } - if utils.SliceHasMember([]string{utils.MetaFileCSV, utils.MetaFileFWV}, cdre.exportFormat) { // files are written after processing all CDRs - cdre.RLock() - contLen := len(cdre.content) - cdre.RUnlock() - if contLen == 0 { - return - } - if err = cdre.composeHeader(); err != nil { - return - } - if err = cdre.composeTrailer(); err != nil { - return - } - var expFormat string - switch cdre.exportFormat { - case utils.MetaFileFWV: - expFormat = "fwv" - case utils.MetaFileCSV: - expFormat = "csv" - default: - expFormat = cdre.exportFormat - } - expPath := cdre.exportPath - if len(filepath.Ext(expPath)) == 0 { // verify extension from exportPath (if have extension is file else is directory) - fileName := fmt.Sprintf("cdre_%s.%s", utils.UUIDSha1Prefix(), expFormat) - expPath = path.Join(expPath, fileName) - } - var fileOut *os.File - if fileOut, err = os.Create(expPath); err != nil { - return - } - defer fileOut.Close() - if cdre.exportFormat == utils.MetaFileCSV { - return cdre.writeCsv(csv.NewWriter(fileOut)) - } - return cdre.writeOut(fileOut) - } - return -} - -// FirstOrderID returns the first exported Cdr OrderId -func (cdre *CDRExporter) FirstOrderID() int64 { - return cdre.firstExpOrderID -} - -// LastOrderID return the last exported Cdr OrderId -func (cdre *CDRExporter) LastOrderID() int64 { - return cdre.lastExpOrderID -} - -// TotalCost returns the total cost in the exported cdrs -func (cdre *CDRExporter) TotalCost() float64 { - return cdre.totalCost -} - -// TotalExportedCdrs returns the number of exported CDRs -func (cdre *CDRExporter) TotalExportedCdrs() int { - return cdre.numberOfRecords -} - -// PositiveExports returns the successfully exported CGRIDs -func (cdre *CDRExporter) PositiveExports() []string { - cdre.RLock() - defer cdre.RUnlock() - return cdre.positiveExports -} - -// NegativeExports returns the failed exported CGRIDs together with the reason -func (cdre *CDRExporter) NegativeExports() map[string]string { - cdre.RLock() - defer cdre.RUnlock() - return cdre.negativeExports -} diff --git a/engine/cdrecsv_test.go b/engine/cdrecsv_test.go deleted file mode 100644 index b6ff7e0b9..000000000 --- a/engine/cdrecsv_test.go +++ /dev/null @@ -1,530 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package engine - -import ( - "bytes" - "encoding/csv" - "strings" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -func TestCsvCdrWriter(t *testing.T) { - writer := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - storedCdr1 := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 1.01, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdre, err := NewCDRExporter([]*CDR{storedCdr1}, - cfg.CdreProfiles[utils.MetaDefault], utils.MetaFileCSV, "", "", "firstexport", - true, 1, utils.CSV_SEP, cfg.GeneralCfg().HttpSkipTlsVerify, nil, nil) - if err != nil { - t.Error("Unexpected error received: ", err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - csvWriter := csv.NewWriter(writer) - if err := cdre.writeCsv(csvWriter); err != nil { - t.Error("Unexpected error: ", err) - } - expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,*default,*voice,dsafdsaf,*rated,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10s,1.0100` - result := strings.TrimSpace(writer.String()) - if result != expected { - t.Errorf("Expected: \n%s \n received: \n%s.", expected, result) - } - if cdre.TotalCost() != 1.01 { - t.Error("Unexpected TotalCost: ", cdre.TotalCost()) - } -} - -func TestAlternativeFieldSeparator(t *testing.T) { - writer := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - storedCdr1 := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 1.01, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdre, err := NewCDRExporter([]*CDR{storedCdr1}, cfg.CdreProfiles[utils.MetaDefault], - utils.MetaFileCSV, "", "", "firstexport", true, 1, '|', - cfg.GeneralCfg().HttpSkipTlsVerify, nil, nil) - if err != nil { - t.Error("Unexpected error received: ", err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - csvWriter := csv.NewWriter(writer) - if err := cdre.writeCsv(csvWriter); err != nil { - t.Error("Unexpected error: ", err) - } - expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6|*default|*voice|dsafdsaf|*rated|cgrates.org|call|1001|1001|1002|2013-11-07T08:42:25Z|2013-11-07T08:42:26Z|10s|1.0100` - result := strings.TrimSpace(writer.String()) - if result != expected { - t.Errorf("Expected: \n%s received: \n%s.", expected, result) - } - if cdre.TotalCost() != 1.01 { - t.Error("Unexpected TotalCost: ", cdre.TotalCost()) - } -} - -func TestExportVoiceWithConvert(t *testing.T) { - writer := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - cdreCfg := cfg.CdreProfiles[utils.MetaDefault] - cdreCfg.Fields = []*config.FCTemplate{ - { - Tag: "*exp.ToR", - Path: "*exp.ToR", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"ToR", utils.INFIELD_SEP)}, - { - Tag: "*exp.OriginID", - Path: "*exp.OriginID", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"OriginID", utils.INFIELD_SEP)}, - { - Tag: "*exp.RequestType", - Path: "*exp.RequestType", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"RequestType", utils.INFIELD_SEP)}, - { - Tag: "*exp.Tenant", - Path: "*exp.Tenant", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Tenant", utils.INFIELD_SEP)}, - { - Tag: "*exp.Category", - Path: "*exp.Category", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Category", utils.INFIELD_SEP)}, - { - Tag: "*exp.Account", - Path: "*exp.Account", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Account", utils.INFIELD_SEP)}, - { - Tag: "*exp.Destination", - Path: "*exp.Destination", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Destination", utils.INFIELD_SEP)}, - { - Tag: "*exp.AnswerTime", - Path: "*exp.AnswerTime", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"AnswerTime", utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - { - Tag: "*exp.UsageVoice", - Path: "*exp.UsageVoice", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*voice"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_seconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageData", - Path: "*exp.UsageData", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*data"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageSMS", - Path: "*exp.UsageSMS", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*sms"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.Cost", - Path: "*exp.Cost", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Cost", utils.INFIELD_SEP), - RoundingDecimals: utils.IntPointer(5)}, - } - cdrVoice := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 1.01, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrData := &CDR{ - CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.DATA, OriginID: "abcdef", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Nanosecond, - RunID: utils.MetaDefault, Cost: 0.012, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrSMS := &CDR{ - CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.SMS, OriginID: "sdfwer", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(1), - RunID: utils.MetaDefault, Cost: 0.15, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdre, err := NewCDRExporter([]*CDR{cdrVoice, cdrData, cdrSMS}, cdreCfg, - utils.MetaFileCSV, "", "", "firstexport", - true, 1, '|', true, nil, &FilterS{cfg: cfg}) - if err != nil { - t.Error("Unexpected error received: ", err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - csvWriter := csv.NewWriter(writer) - if err := cdre.writeCsv(csvWriter); err != nil { - t.Error("Unexpected error: ", err) - } - expected := `*sms|sdfwer|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|1|0.15000 -*voice|dsafdsaf|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|10|1.01000 -*data|abcdef|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|10|0.01200` - result := strings.TrimSpace(writer.String()) - if len(result) != len(expected) { // export is async, cannot check order - t.Errorf("expected: \n%s received: \n%s.", expected, result) - } - if cdre.TotalCost() != 1.172 { - t.Error("unexpected TotalCost: ", cdre.TotalCost()) - } -} - -func TestExportWithFilter(t *testing.T) { - writer := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - cdreCfg := cfg.CdreProfiles[utils.MetaDefault] - cdreCfg.Filters = []string{"*string:~*req.Tenant:cgrates.org"} - cdreCfg.Fields = []*config.FCTemplate{ - { - Tag: "*exp.ToR", - Path: "*exp.ToR", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"ToR", utils.INFIELD_SEP)}, - { - Tag: "*exp.OriginID", - Path: "*exp.OriginID", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"OriginID", utils.INFIELD_SEP)}, - { - Tag: "*exp.RequestType", - Path: "*exp.RequestType", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"RequestType", utils.INFIELD_SEP)}, - { - Tag: "*exp.Tenant", - Path: "*exp.Tenant", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Tenant", utils.INFIELD_SEP)}, - { - Tag: "*exp.Category", - Path: "*exp.Category", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Category", utils.INFIELD_SEP)}, - { - Tag: "*exp.Account", - Path: "*exp.Account", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Account", utils.INFIELD_SEP)}, - { - Tag: "*exp.Destination", - Path: "*exp.Destination", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Destination", utils.INFIELD_SEP)}, - { - Tag: "*exp.AnswerTime", - Path: "*exp.AnswerTime", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"AnswerTime", utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - { - Tag: "*exp.UsageVoice", - Path: "*exp.UsageVoice", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*voice"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_seconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageData", - Path: "*exp.UsageData", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*data"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageSMS", - Path: "*exp.UsageSMS", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*sms"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.Cost", - Path: "*exp.Cost", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Cost", utils.INFIELD_SEP), - RoundingDecimals: utils.IntPointer(5)}, - } - cdrVoice := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 1.01, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrData := &CDR{ - CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.DATA, OriginID: "abcdef", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "AnotherTenant", Category: "call", //for data CDR use different Tenant - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Nanosecond, - RunID: utils.MetaDefault, Cost: 0.012, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrSMS := &CDR{ - CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.SMS, OriginID: "sdfwer", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(1), - RunID: utils.MetaDefault, Cost: 0.15, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdre, err := NewCDRExporter([]*CDR{cdrVoice, cdrData, cdrSMS}, cdreCfg, - utils.MetaFileCSV, "", "", "firstexport", - true, 1, '|', true, nil, &FilterS{cfg: cfg}) - if err != nil { - t.Error("Unexpected error received: ", err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - csvWriter := csv.NewWriter(writer) - if err := cdre.writeCsv(csvWriter); err != nil { - t.Error("Unexpected error: ", err) - } - expected := `*sms|sdfwer|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|1|0.15000 -*voice|dsafdsaf|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|10|1.01000` - result := strings.TrimSpace(writer.String()) - if len(result) != len(expected) { // export is async, cannot check order - t.Errorf("expected: \n%s received: \n%s.", expected, result) - } - if cdre.TotalCost() != 1.16 { - t.Error("unexpected TotalCost: ", cdre.TotalCost()) - } -} - -func TestExportWithFilter2(t *testing.T) { - writer := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - cdreCfg := cfg.CdreProfiles[utils.MetaDefault] - cdreCfg.Filters = []string{"*string:~*req.Tenant:cgrates.org", "*lte:~*req.Cost:0.5"} - cdreCfg.Fields = []*config.FCTemplate{ - { - Tag: "*exp.ToR", - Path: "*exp.ToR", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"ToR", utils.INFIELD_SEP)}, - { - Tag: "*exp.OriginID", - Path: "*exp.OriginID", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"OriginID", utils.INFIELD_SEP)}, - { - Tag: "*exp.RequestType", - Path: "*exp.RequestType", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"RequestType", utils.INFIELD_SEP)}, - { - Tag: "*exp.Tenant", - Path: "*exp.Tenant", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Tenant", utils.INFIELD_SEP)}, - { - Tag: "*exp.Category", - Path: "*exp.Category", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Category", utils.INFIELD_SEP)}, - { - Tag: "*exp.Account", - Path: "*exp.Account", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Account", utils.INFIELD_SEP)}, - { - Tag: "*exp.Destination", - Path: "*exp.Destination", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Destination", utils.INFIELD_SEP)}, - { - Tag: "*exp.AnswerTime", - Path: "*exp.AnswerTime", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"AnswerTime", utils.INFIELD_SEP), - Layout: "2006-01-02T15:04:05Z07:00"}, - { - Tag: "*exp.UsageVoice", - Path: "*exp.UsageVoice", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*voice"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_seconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageData", - Path: "*exp.UsageData", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*data"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.UsageSMS", - Path: "*exp.UsageSMS", - Type: "*composed", - Filters: []string{"*string:~*req.ToR:*sms"}, - Value: config.NewRSRParsersMustCompile("~*req.Usage{*duration_nanoseconds}", utils.INFIELD_SEP)}, - { - Tag: "*exp.Cost", - Path: "*exp.Cost", - Type: "*composed", - Value: config.NewRSRParsersMustCompile(utils.DynamicDataPrefix+utils.MetaReq+utils.NestingSep+"Cost", utils.INFIELD_SEP), - RoundingDecimals: utils.IntPointer(5)}, - } - cdrVoice := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.VOICE, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 1.01, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrData := &CDR{ - CGRID: utils.Sha1("abcdef", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.DATA, OriginID: "abcdef", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "AnotherTenant", Category: "call", //for data CDR use different Tenant - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(10) * time.Nanosecond, - RunID: utils.MetaDefault, Cost: 0.012, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdrSMS := &CDR{ - CGRID: utils.Sha1("sdfwer", time.Unix(1383813745, 0).UTC().String()), - ToR: utils.SMS, OriginID: "sdfwer", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Unix(1383813745, 0).UTC(), - AnswerTime: time.Unix(1383813746, 0).UTC(), - Usage: time.Duration(1), - RunID: utils.MetaDefault, Cost: 0.15, - ExtraFields: map[string]string{"extra1": "val_extra1", - "extra2": "val_extra2", "extra3": "val_extra3"}, - } - cdre, err := NewCDRExporter([]*CDR{cdrVoice, cdrData, cdrSMS}, cdreCfg, - utils.MetaFileCSV, "", "", "firstexport", - true, 1, '|', true, nil, &FilterS{cfg: cfg}) - if err != nil { - t.Error("Unexpected error received: ", err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - csvWriter := csv.NewWriter(writer) - if err := cdre.writeCsv(csvWriter); err != nil { - t.Error("Unexpected error: ", err) - } - expected := `*sms|sdfwer|*rated|cgrates.org|call|1001|1002|2013-11-07T08:42:26Z|1|0.15000` - result := strings.TrimSpace(writer.String()) - if len(result) != len(expected) { // export is async, cannot check order - t.Errorf("expected: \n%s received: \n%s.", expected, result) - } - if cdre.TotalCost() != 0.15 { - t.Error("unexpected TotalCost: ", cdre.TotalCost()) - } -} diff --git a/engine/cdrefwv_test.go b/engine/cdrefwv_test.go deleted file mode 100644 index 50e96ac5b..000000000 --- a/engine/cdrefwv_test.go +++ /dev/null @@ -1,444 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package engine - -import ( - "bytes" - "math" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -var contentJsnCfgFlds = []*config.FcTemplateJsonCfg{ - { - Tag: utils.StringPointer("TypeOfRecord"), - Path: utils.StringPointer("*hdr.TypeOfRecord"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("10"), - Width: utils.IntPointer(2)}, - { - Tag: utils.StringPointer("Filler1"), - Path: utils.StringPointer("*hdr.Filler1"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(3)}, - { - Tag: utils.StringPointer("DistributorCode"), - Path: utils.StringPointer("*hdr.DistributorCode"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("VOI"), - Width: utils.IntPointer(3)}, - { - Tag: utils.StringPointer("FileSeqNr"), - Path: utils.StringPointer("*hdr.FileSeqNr"), - Type: utils.StringPointer(utils.META_HANDLER), - Value: utils.StringPointer(metaExportID), - Width: utils.IntPointer(5), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaZeroLeft)}, - { - Tag: utils.StringPointer("LastCdr"), - Path: utils.StringPointer("*hdr.LastCdr"), - Type: utils.StringPointer(utils.META_HANDLER), - Width: utils.IntPointer(12), - Value: utils.StringPointer(metaLastCDRAtime), - Layout: utils.StringPointer("020106150400")}, - { - Tag: utils.StringPointer("FileCreationfTime"), - Path: utils.StringPointer("*hdr.FileCreationfTime"), - Type: utils.StringPointer(utils.META_HANDLER), - Value: utils.StringPointer(metaTimeNow), - Width: utils.IntPointer(12), - Layout: utils.StringPointer("020106150400")}, - { - Tag: utils.StringPointer("FileVersion"), - Path: utils.StringPointer("*hdr.FileVersion"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("01"), - Width: utils.IntPointer(2)}, - { - Tag: utils.StringPointer("Filler2"), - Path: utils.StringPointer("*hdr.Filler2"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(105)}, - { - Tag: utils.StringPointer("TypeOfRecord"), - Path: utils.StringPointer("*exp.TypeOfRecord"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("20"), - Width: utils.IntPointer(2)}, - { - Tag: utils.StringPointer("Account"), - Path: utils.StringPointer("*exp.Account"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.Account), - Width: utils.IntPointer(12), - Strip: utils.StringPointer(utils.MetaLeft), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("Subject"), - Path: utils.StringPointer("*exp.Subject"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.Subject), - Width: utils.IntPointer(5), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("CLI"), - Path: utils.StringPointer("*exp.CLI"), - Type: utils.StringPointer(utils.META_COMPOSED), - Width: utils.IntPointer(15), - Value: utils.StringPointer("cli"), - Strip: utils.StringPointer(utils.MetaXRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("Destination"), - Path: utils.StringPointer("*exp.Destination"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.Destination), - Width: utils.IntPointer(24), - Strip: utils.StringPointer(utils.MetaXRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("ToR"), - Path: utils.StringPointer("*exp.ToR"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("02"), - Width: utils.IntPointer(2)}, - { - Tag: utils.StringPointer("SubtypeTOR"), - Path: utils.StringPointer("*exp.SubtypeTOR"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("11"), - Padding: utils.StringPointer(utils.MetaRight), - Width: utils.IntPointer(4)}, - { - Tag: utils.StringPointer("SetupTime"), - Path: utils.StringPointer("*exp.SetupTime"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.SetupTime), - Width: utils.IntPointer(12), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight), - Layout: utils.StringPointer("020106150400")}, - { - Tag: utils.StringPointer("Duration"), - Path: utils.StringPointer("*exp.Duration"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.Usage), - Width: utils.IntPointer(6), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight), - Layout: utils.StringPointer(utils.SECONDS)}, - { - Tag: utils.StringPointer("DataVolume"), - Path: utils.StringPointer("*exp.DataVolume"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(6)}, - { - Tag: utils.StringPointer("TaxCode"), - Path: utils.StringPointer("*exp.TaxCode"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("1"), - Width: utils.IntPointer(1)}, - { - Tag: utils.StringPointer("OperatorCode"), - Path: utils.StringPointer("*exp.OperatorCode"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("opercode"), - Width: utils.IntPointer(2), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("ProductId"), - Path: utils.StringPointer("*exp.ProductId"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~productid"), - Width: utils.IntPointer(5), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("NetworkId"), - Path: utils.StringPointer("*exp.NetworkId"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("3"), - Width: utils.IntPointer(1)}, - { - Tag: utils.StringPointer("CallId"), - Path: utils.StringPointer("*exp.CallId"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~" + utils.OriginID), - Width: utils.IntPointer(16), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("Filler"), - Path: utils.StringPointer("*exp.Filler"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(8)}, - { - Tag: utils.StringPointer("Filler"), - Path: utils.StringPointer("*exp.Filler"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(8)}, - { - Tag: utils.StringPointer("TerminationCode"), - Path: utils.StringPointer("*exp.TerminationCode"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("~operator;~product"), - Width: utils.IntPointer(5), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaRight)}, - { - Tag: utils.StringPointer("Cost"), - Path: utils.StringPointer("*exp.Cost"), - Type: utils.StringPointer(utils.META_COMPOSED), - Width: utils.IntPointer(9), - Value: utils.StringPointer("~" + utils.COST), - Padding: utils.StringPointer(utils.MetaZeroLeft), - Rounding_decimals: utils.IntPointer(5)}, - { - Tag: utils.StringPointer("DestinationPrivacy"), - Path: utils.StringPointer("*exp.DestinationPrivacy"), - Type: utils.StringPointer(utils.MetaMaskedDestination), - Width: utils.IntPointer(1)}, - { - Tag: utils.StringPointer("TypeOfRecord"), - Path: utils.StringPointer("*trl.TypeOfRecord"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("90"), - Width: utils.IntPointer(2)}, - { - Tag: utils.StringPointer("Filler1"), - Path: utils.StringPointer("*trl.Filler1"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(3)}, - { - Tag: utils.StringPointer("DistributorCode"), - Path: utils.StringPointer("*trl.DistributorCode"), - Type: utils.StringPointer(utils.META_CONSTANT), - Value: utils.StringPointer("VOI"), - Width: utils.IntPointer(3)}, - { - Tag: utils.StringPointer("FileSeqNr"), - Path: utils.StringPointer("*trl.FileSeqNr"), - Type: utils.StringPointer(utils.META_HANDLER), - Value: utils.StringPointer(metaExportID), - Width: utils.IntPointer(5), - Strip: utils.StringPointer(utils.MetaRight), - Padding: utils.StringPointer(utils.MetaZeroLeft)}, - { - Tag: utils.StringPointer("NumberOfRecords"), - Path: utils.StringPointer("*trl.NumberOfRecords"), - Type: utils.StringPointer(utils.META_HANDLER), - Value: utils.StringPointer(metaNrCDRs), - Width: utils.IntPointer(6), - Padding: utils.StringPointer(utils.MetaZeroLeft)}, - { - Tag: utils.StringPointer("CdrsDuration"), - Path: utils.StringPointer("*trl.CdrsDuration"), - Type: utils.StringPointer(utils.META_HANDLER), - Value: utils.StringPointer(metaDurCDRs), - Width: utils.IntPointer(8), - Padding: utils.StringPointer(utils.MetaZeroLeft), - Layout: utils.StringPointer(utils.SECONDS)}, - { - Tag: utils.StringPointer("FirstCdrTime"), - Path: utils.StringPointer("*trl.FirstCdrTime"), - Type: utils.StringPointer(utils.META_HANDLER), - Width: utils.IntPointer(12), - Value: utils.StringPointer(metaFirstCDRAtime), - Layout: utils.StringPointer("020106150400")}, - { - Tag: utils.StringPointer("LastCdrTime"), - Path: utils.StringPointer("*trl.LastCdrTime"), - Type: utils.StringPointer(utils.META_HANDLER), - Width: utils.IntPointer(12), - Value: utils.StringPointer(metaLastCDRAtime), - Layout: utils.StringPointer("020106150400")}, - { - Tag: utils.StringPointer("Filler2"), - Path: utils.StringPointer("*trl.Filler2"), - Type: utils.StringPointer(utils.META_FILLER), - Width: utils.IntPointer(93)}, -} - -var contentCfgFlds []*config.FCTemplate - -// Write one CDR and test it's results only for content buffer -func TestWriteCdr(t *testing.T) { - var err error - wrBuf := &bytes.Buffer{} - cfg, _ := config.NewDefaultCGRConfig() - if contentCfgFlds, err = config.FCTemplatesFromFCTemplatesJsonCfg(contentJsnCfgFlds, utils.INFIELD_SEP); err != nil { - t.Error(err) - } - cdreCfg := &config.CdreCfg{ - ExportFormat: utils.MetaFileFWV, - Fields: contentCfgFlds, - } - cdr := &CDR{ - CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), - ToR: utils.VOICE, OrderID: 1, OriginID: "dsafdsaf", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", Category: "call", - Account: "1001", Subject: "1001", Destination: "1002", - SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, - RunID: utils.MetaDefault, Cost: 2.34567, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, - } - - cdre, err := NewCDRExporter([]*CDR{cdr}, cdreCfg, utils.MetaFileFWV, "", "", "fwv_1", - true, 1, '|', cfg.GeneralCfg().HttpSkipTlsVerify, nil, nil) - if err != nil { - t.Error(err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Fatal(err) - } - eHeader := "10 VOIfwv_107111308420018011511340001 \n" - eContentOut := "201001 1001 1002 0211 07111308420010 1 3dsafdsaf 0002.34570\n" - eTrailer := "90 VOIfwv_100000100000010071113084200071113084200 \n" - if err := cdre.writeOut(wrBuf); err != nil { - t.Error(err) - } - allOut := wrBuf.String() - eAllOut := eHeader + eContentOut + eTrailer - if math.Mod(float64(len(allOut)), 145) != 0 { - t.Errorf("Unexpected export content length %d, have output \n%q, \n expecting: \n%q", - len(allOut), allOut, eAllOut) - } else if len(allOut) != len(eAllOut) { - t.Errorf("Output does not match expected length. Have output \n%q, \n expecting: \n%q", - allOut, eAllOut) - } - // Test stats - if !cdre.firstCdrATime.Equal(cdr.AnswerTime) { - t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime) - } else if !cdre.lastCdrATime.Equal(cdr.AnswerTime) { - t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime) - } else if cdre.numberOfRecords != 1 { - t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords) - } else if cdre.totalDuration != cdr.Usage { - t.Error("Unexpected total duration in the stats: ", cdre.totalDuration) - } else if cdre.totalCost != utils.Round(cdr.Cost, 5, utils.ROUNDING_MIDDLE) { - t.Error("Unexpected total cost in the stats: ", cdre.totalCost) - } - - if cdre.FirstOrderID() != 1 { - t.Error("Unexpected FirstOrderId", cdre.FirstOrderID()) - } - if cdre.LastOrderID() != 1 { - t.Error("Unexpected LastOrderId", cdre.LastOrderID()) - } - if cdre.TotalCost() != utils.Round(cdr.Cost, 5, utils.ROUNDING_MIDDLE) { - t.Error("Unexpected TotalCost: ", cdre.TotalCost()) - } -} - -func TestWriteCdrs(t *testing.T) { - wrBuf := &bytes.Buffer{} - cdreCfg := &config.CdreCfg{ - ExportFormat: utils.MetaFileFWV, - Fields: contentCfgFlds, - } - cdr1 := &CDR{CGRID: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), - ToR: utils.VOICE, OrderID: 2, OriginID: "aaa1", OriginHost: "192.168.1.1", - RequestType: utils.META_RATED, Tenant: "cgrates.org", - Category: "call", Account: "1001", Subject: "1001", Destination: "1010", - SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, RunID: utils.MetaDefault, Cost: 2.25, - ExtraFields: map[string]string{"productnumber": "12341", "fieldextr2": "valextr2"}, - } - cdr2 := &CDR{CGRID: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()), - ToR: utils.VOICE, OrderID: 4, OriginID: "aaa2", OriginHost: "192.168.1.2", - RequestType: utils.META_PREPAID, Tenant: "cgrates.org", - Category: "call", Account: "1002", Subject: "1002", Destination: "1011", - SetupTime: time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 7, 42, 26, 0, time.UTC), - Usage: time.Duration(5) * time.Minute, - RunID: utils.MetaDefault, Cost: 1.40001, - ExtraFields: map[string]string{"productnumber": "12342", "fieldextr2": "valextr2"}, - } - cdr3 := &CDR{} - cdr4 := &CDR{CGRID: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()), - ToR: utils.VOICE, OrderID: 3, OriginID: "aaa4", OriginHost: "192.168.1.4", - RequestType: utils.META_POSTPAID, Tenant: "cgrates.org", - Category: "call", Account: "1004", Subject: "1004", Destination: "1013", - SetupTime: time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 9, 42, 26, 0, time.UTC), - Usage: time.Duration(20) * time.Second, - RunID: utils.MetaDefault, Cost: 2.34567, - ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"}, - } - cfg, _ := config.NewDefaultCGRConfig() - cdre, err := NewCDRExporter([]*CDR{cdr1, cdr2, cdr3, cdr4}, cdreCfg, - utils.MetaFileFWV, "", "", "fwv_1", true, 1, utils.CSV_SEP, - cfg.GeneralCfg().HttpSkipTlsVerify, nil, nil) - if err != nil { - t.Error(err) - } - if err = cdre.processCDRs(); err != nil { - t.Error(err) - } - if err = cdre.composeHeader(); err != nil { - t.Error(err) - } - if err = cdre.composeTrailer(); err != nil { - t.Error(err) - } - if err := cdre.writeOut(wrBuf); err != nil { - t.Error(err) - } - if len(wrBuf.String()) != 725 { - t.Error("Output buffer does not contain expected info. Expecting len: 725, got: ", len(wrBuf.String())) - } - // Test stats - if !cdre.firstCdrATime.Equal(cdr2.AnswerTime) { - t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime) - } - if !cdre.lastCdrATime.Equal(cdr4.AnswerTime) { - t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime) - } - if cdre.numberOfRecords != 3 { - t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords) - } - if cdre.totalDuration != time.Duration(330)*time.Second { - t.Error("Unexpected total duration in the stats: ", cdre.totalDuration) - } - if cdre.totalCost != 5.99568 { - t.Error("Unexpected total cost in the stats: ", cdre.totalCost) - } - if cdre.FirstOrderID() != 2 { - t.Error("Unexpected FirstOrderId", cdre.FirstOrderID()) - } - if cdre.LastOrderID() != 4 { - t.Error("Unexpected LastOrderId", cdre.LastOrderID()) - } - if cdre.TotalCost() != 5.99568 { - t.Error("Unexpected TotalCost: ", cdre.TotalCost()) - } -} diff --git a/engine/cdrs.go b/engine/cdrs.go index 704aeaa68..3e6b017b1 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -399,29 +399,8 @@ func (cdrS *CDRServer) statSProcessEvent(cgrEv *utils.CGREventWithArgDispatcher) return } -// exportCDRs will export the CDRs received -func (cdrS *CDRServer) exportCDRs(cdrs []*CDR) (err error) { - for _, exportID := range cdrS.cgrCfg.CdrsCfg().OnlineCDRExports { - expTpl := cdrS.cgrCfg.CdreProfiles[exportID] // not checking for existence of profile since this should be done in a higher layer - var cdre *CDRExporter - if cdre, err = NewCDRExporter(cdrs, expTpl, expTpl.ExportFormat, - expTpl.ExportPath, cdrS.cgrCfg.GeneralCfg().FailedPostsDir, - "CDRSReplication", expTpl.Synchronous, expTpl.Attempts, - expTpl.FieldSeparator, cdrS.cgrCfg.GeneralCfg().HttpSkipTlsVerify, - cdrS.cgrCfg.CdrsCfg().AttributeSConns, cdrS.filterS); err != nil { - utils.Logger.Err(fmt.Sprintf(" Building CDRExporter for online exports got error: <%s>", err.Error())) - continue - } - if err = cdre.ExportCDRs(); err != nil { - utils.Logger.Err(fmt.Sprintf(" Replicating CDR: %+v, got error: <%s>", cdrs, err.Error())) - continue - } - } - return -} - // eeSProcessEvent will process the event with the EEs component -func (cdrS *CDRServer) eeSProcessEvent(cgrEv *utils.CGREventWithOpts) (err error) { +func (cdrS *CDRServer) eeSProcessEvent(cgrEv *utils.CGREventWithIDs) (err error) { var reply string if err = cdrS.connMgr.Call(cdrS.cgrCfg.CdrsCfg().EEsConns, nil, utils.EventExporterSv1ProcessEvent, @@ -583,19 +562,17 @@ func (cdrS *CDRServer) processEvent(ev *utils.CGREventWithOpts, } var partiallyExecuted bool // from here actions are optional and a general error is returned if export { - if len(cdrS.cgrCfg.CdrsCfg().OnlineCDRExports) != 0 { - if err = cdrS.exportCDRs(cdrs); err != nil { - utils.Logger.Warning( - fmt.Sprintf("<%s> error: <%s> exporting CDRs %+v", - utils.CDRs, err.Error(), cdrs)) - partiallyExecuted = true - } - } if len(cdrS.cgrCfg.CdrsCfg().EEsConns) != 0 { for _, cgrEv := range cgrEvs { - evWithOpts := &utils.CGREventWithOpts{ - CGREvent: cgrEv.CGREvent, - ArgDispatcher: cgrEv.ArgDispatcher} + evWithOpts := &utils.CGREventWithIDs{ + CGREventWithOpts: &utils.CGREventWithOpts{ + CGREvent: cgrEv.CGREvent, + ArgDispatcher: cgrEv.ArgDispatcher, + }, + } + if len(cdrS.cgrCfg.CdrsCfg().OnlineCDRExports) != 0 { + evWithOpts.IDs = cdrS.cgrCfg.CdrsCfg().OnlineCDRExports + } if err = cdrS.eeSProcessEvent(evWithOpts); err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> error: <%s> exporting cdr %+v", @@ -717,7 +694,7 @@ func (cdrS *CDRServer) V1ProcessCDR(cdr *CDRWithOpts, reply *string) (err error) !cdr.PreRated, // rate the CDR if is not PreRated cdrS.cgrCfg.CdrsCfg().StoreCdrs, false, // no rerate - (len(cdrS.cgrCfg.CdrsCfg().OnlineCDRExports) != 0 || len(cdrS.cgrCfg.CdrsCfg().EEsConns) != 0), + len(cdrS.cgrCfg.CdrsCfg().OnlineCDRExports) != 0 || len(cdrS.cgrCfg.CdrsCfg().EEsConns) != 0, len(cdrS.cgrCfg.CdrsCfg().ThresholdSConns) != 0, len(cdrS.cgrCfg.CdrsCfg().StatSConns) != 0); err != nil { return diff --git a/general_tests/cdre_it_test.go b/general_tests/cdre_it_test.go deleted file mode 100644 index 8bec91db7..000000000 --- a/general_tests/cdre_it_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package general_tests - -import ( - "net/rpc" - "path" - "testing" - "time" - - v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - cdreCfg *config.CGRConfig - cdreRPC *rpc.Client - cdreCfgPath string - cdreConfigDIR string - - sTestsCDRE = []func(t *testing.T){ - testCDREInitCfg, - testCDREInitDataDb, - testCDREResetStorDb, - testCDREStartEngine, - testCDRERpcConn, - testCDREGetCdrs, - testCDREExportNotFound, - testCDREProcessCdr, - testCDREExport, - testCDREStopEngine, - } -) - -func TestCDREIT(t *testing.T) { - switch *dbType { - case utils.MetaInternal: - cdreConfigDIR = "tutinternal_new" - case utils.MetaMySQL: - cdreConfigDIR = "tutmysql" - case utils.MetaMongo: - cdreConfigDIR = "tutmongonew" - case utils.MetaPostgres: - t.SkipNow() - default: - t.Fatal("Unknown Database type") - } - - for _, stest := range sTestsCDRE { - t.Run(cdreConfigDIR, stest) - } -} - -func testCDREInitCfg(t *testing.T) { - var err error - cdreCfgPath = path.Join(*dataDir, "conf", "samples", cdreConfigDIR) - cdreCfg, err = config.NewCGRConfigFromPath(cdreCfgPath) - if err != nil { - t.Error(err) - } - cdreCfg.DataFolderPath = *dataDir -} - -func testCDREInitDataDb(t *testing.T) { - if err := engine.InitDataDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -func testCDREResetStorDb(t *testing.T) { - if err := engine.InitStorDb(cdreCfg); err != nil { - t.Fatal(err) - } -} - -func testCDREStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(cdreCfgPath, *waitRater); err != nil { - t.Fatal(err) - } -} - -func testCDRERpcConn(t *testing.T) { - var err error - cdreRPC, err = newRPCClient(cdreCfg.ListenCfg()) - if err != nil { - t.Fatal(err) - } -} - -func testCDREGetCdrs(t *testing.T) { - var reply []*engine.ExternalCDR - req := utils.RPCCDRsFilter{} - if err := cdreRPC.Call(utils.APIerSv1GetCDRs, &req, &reply); err.Error() != utils.ErrNotFound.Error() { - t.Error("Unexpected error: ", err.Error()) - } -} - -func testCDREExportNotFound(t *testing.T) { - var replyExport v1.RplExportedCDRs - exportArgs := v1.ArgExportCDRs{ - ExportArgs: map[string]interface{}{ - utils.ExportPath: "/tmp", - utils.ExportFileName: "TestTutITExportCDR.csv", - utils.ExportTemplate: "TestTutITExportCDR", - }, - RPCCDRsFilter: utils.RPCCDRsFilter{}, - } - if err := cdreRPC.Call(utils.APIerSv1ExportCDRs, &exportArgs, - &replyExport); err.Error() != utils.ErrNotFound.Error() { - t.Error(err) - } -} - -func testCDREProcessCdr(t *testing.T) { - cdr := &engine.CDR{ - ToR: utils.VOICE, - OriginID: "testCDREProcessCdr", - OriginHost: "192.168.1.1", - Source: "TestTutITExportCDR", - RequestType: utils.META_RATED, - Tenant: "cgrates.org", - Category: "call", - Account: "1001", - Subject: "1001", - Destination: "1003", - SetupTime: time.Date(2016, 11, 30, 17, 5, 24, 0, time.UTC), - AnswerTime: time.Date(2016, 11, 30, 17, 6, 4, 0, time.UTC), - Usage: time.Duration(98) * time.Second, - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, - } - cdr.ComputeCGRID() - var reply string - if err := cdreRPC.Call(utils.CDRsV1ProcessEvent, - &engine.ArgV1ProcessEvent{ - Flags: []string{utils.ConcatenatedKey(utils.MetaChargers, "false")}, - CGREvent: *cdr.AsCGREvent(), - }, &reply); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if reply != utils.OK { - t.Error("Unexpected reply received: ", reply) - } -} - -func testCDREExport(t *testing.T) { - // time.Sleep(100 * time.Millisecond) - var replyExport v1.RplExportedCDRs - exportArgs := v1.ArgExportCDRs{ - ExportArgs: map[string]interface{}{ - utils.ExportPath: "/tmp", - utils.ExportFileName: "TestTutITExportCDR.csv", - utils.ExportTemplate: "TestTutITExportCDR", - }, - RPCCDRsFilter: utils.RPCCDRsFilter{}, - } - if err := cdreRPC.Call(utils.APIerSv1ExportCDRs, &exportArgs, &replyExport); err != nil { - t.Error(err) - } else if replyExport.TotalRecords != 1 { - t.Errorf("Unexpected total records: %+v", replyExport.TotalRecords) - } - // expFilePath := path.Join(*exportArgs.ExportPath, *exportArgs.ExportFileName) - // if err := os.Remove(expFilePath); err != nil { - // t.Error(err) - // } -} - -func testCDREStopEngine(t *testing.T) { - if err := engine.KillEngine(*waitRater); err != nil { - t.Error(err) - } -} diff --git a/utils/cgrevent.go b/utils/cgrevent.go index d79a87fe0..c0a38989b 100644 --- a/utils/cgrevent.go +++ b/utils/cgrevent.go @@ -285,3 +285,9 @@ func (ev *CGREventWithOpts) Clone() (clned *CGREventWithOpts) { } return } + +// CGREventWithIDs is the CGREventWithOpts with ExporterIDs +type CGREventWithIDs struct { + IDs []string + *CGREventWithOpts +} diff --git a/utils/consts.go b/utils/consts.go index 2ff5f72d2..1b06e7ec8 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -19,9 +19,6 @@ along with this program. If not, see package utils var ( - CDRExportFormats = NewStringSet([]string{DRYRUN, MetaFileCSV, MetaFileFWV, MetaHTTPjsonCDR, MetaHTTPjsonMap, - MetaHTTPjson, MetaHTTPPost, MetaAMQPjsonCDR, MetaAMQPjsonMap, MetaAMQPV1jsonMap, MetaSQSjsonMap, - MetaKafkajsonMap, MetaS3jsonMap}) MainCDRFields = NewStringSet([]string{CGRID, Source, OriginHost, OriginID, ToR, RequestType, Tenant, Category, Account, Subject, Destination, SetupTime, AnswerTime, Usage, COST, RATED, Partial, RunID, PreRated, CostSource, CostDetails, ExtraInfo, OrderID}) @@ -236,7 +233,6 @@ const ( ZERO_RATING_SUBJECT_PREFIX = "*zero" OK = "OK" MetaFileXML = "*file_xml" - CDRE = "cdre" MASK_CHAR = "*" CONCATENATED_KEY_SEP = ":" UNIT_TEST = "UNIT_TEST" @@ -249,10 +245,8 @@ const ( VOICE = "*voice" MAX_COST_FREE = "*free" MAX_COST_DISCONNECT = "*disconnect" - SECONDS = "seconds" META_OUT = "*out" META_ANY = "*any" - ASR = "ASR" ACD = "ACD" TASKS_KEY = "tasks" ACTION_PLAN_PREFIX = "apl_" @@ -1254,8 +1248,6 @@ const ( APIerSv1RemoveActionPlan = "APIerSv1.RemoveActionPlan" APIerSv1RemoveActions = "APIerSv1.RemoveActions" APIerSv1RemoveBalances = "APIerSv1.RemoveBalances" - APIerSv1ReloadCdrcConfig = "APIerSv1.ReloadCdrcConfig" - APIerSv1ReloadCdreConfig = "APIerSv1.ReloadCdreConfig" APIerSv1GetLoadHistory = "APIerSv1.GetLoadHistory" APIerSv1GetLoadIDs = "APIerSv1.GetLoadIDs" APIerSv1GetLoadTimes = "APIerSv1.GetLoadTimes" @@ -2149,7 +2141,6 @@ const ( // CGRConfig const ( - CdreProfiles = "cdre" // from JSON LoaderCfg = "loaders" // from JSON HttpAgentCfg = "http_agent" // from JSON RpcConns = "rpc_conns" // from JSON