From bdf354afeb07908fff2f537957eefe4111554a69 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 20 Oct 2014 12:46:47 +0200 Subject: [PATCH] ApierV2.ExportTPToFolder --- apier/v2/tp.go | 28 ++++++++++++++++++++++- config/test_data.txt | 16 +++++++------- data/conf/cgrates.cfg | 2 +- engine/tpexporter.go | 48 +++++++++++++++++++++++++++++++++++----- utils/apitpdata.go | 3 ++- utils/cgrrecordwriter.go | 6 +++++ 6 files changed, 86 insertions(+), 17 deletions(-) diff --git a/apier/v2/tp.go b/apier/v2/tp.go index 06cc41003..32ffd071b 100644 --- a/apier/v2/tp.go +++ b/apier/v2/tp.go @@ -21,6 +21,7 @@ package v2 import ( "fmt" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -33,6 +34,31 @@ func (self *ApierV2) RemTP(tpid string, reply *string) error { return nil } -func (self *ApierV2) ExportTPToFolder(attrs utils.AttrDirExportTP, exported *utils.DirExportedTP) error { +func (self *ApierV2) ExportTPToFolder(attrs utils.AttrDirExportTP, exported *utils.ExportedTPStats) error { + if len(*attrs.TPid) == 0 { + return fmt.Errorf("%s:TPid", utils.ERR_MANDATORY_IE_MISSING) + } + dir := "" + if attrs.ExportDir != nil { + dir = *attrs.ExportDir + } + fileFormat := "" + if attrs.FileFormat != nil { + fileFormat = *attrs.FileFormat + } + sep := "" + if attrs.FieldSeparator != nil { + sep = *attrs.FieldSeparator + } + tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, nil) + if err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } + if err := tpExporter.Run(); err != nil { + return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) + } else { + *exported = *tpExporter.ExportStats() + } + return nil } diff --git a/config/test_data.txt b/config/test_data.txt index 4ff5f7237..68061d40e 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -14,25 +14,25 @@ accountdb_port = test # Accounting subsystem port to reach the database. accountdb_name = test # Accounting subsystem database name to connect to. accountdb_user = test # Accounting subsystem username to use when connecting to database. accountdb_passwd = test # Accounting subsystem password to use when connecting to database. -stordb_type = test # Log/scategoryed database type to use: -stordb_host = test # The host to connect to. Values that start with / are for UNIX domain sockets. -stordb_port = test # The port to reach the logdb. -stordb_name = test # The name of the log database to connect to. +stordb_type = test # Log/scategoryed database type to use: +stordb_host = test # The host to connect to. Values that start with / are for UNIX domain sockets. +stordb_port = test # The port to reach the logdb. +stordb_name = test # The name of the log database to connect to. stordb_user = test # Username to use when connecting to logdb. stordb_passwd = test # Password to use when connecting to logdb. -stordb_max_open_conns = 99 # Maximum database connections opened -stordb_max_idle_conns = 99 # Maximum database connections idle +stordb_max_open_conns = 99 # Maximum database connections opened +stordb_max_idle_conns = 99 # Maximum database connections idle dbdata_encoding = test # The encoding used to scategorye object data in strings: rpc_json_listen = test # RPC JSON listening address rpc_gob_listen = test # RPC GOB listening address -http_listen = test # HTTP listening address +http_listen = TEST # HTTP listening address default_reqtype = test # Default request type to consider when missing from requests: <""|prepaid|postpaid|pseudoprepaid|rated>. default_category = test # Default Type of Record to consider when missing from requests. default_tenant = test # Default Tenant to consider when missing from requests. default_subject = test # Default rating Subject to consider when missing from requests. rounding_decimals = 99 # Number of decimals to round floats/costs at http_skip_tls_veify = true # If enabled Http Client will accept any TLS certificate -tpexport_dir = test # Path towards export folder for offline Tariff Plans +tpexport_dir = test # Path towards export folder for offline Tariff Plans [balancer] enabled = true # Start Balancer service: . diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 97b9aaaa1..e112c28ef 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -36,7 +36,7 @@ # rounding_decimals = 10 # System level precision for floats # http_skip_tls_veify = false # If enabled Http Client will accept any TLS certificate # tpexport_dir = /var/log/cgrates/tpe # Path towards export folder for offline Tariff Plans -# xmlcfg_path = # Path towards additional config defined in xml file +# xmlcfg_path = # Path towards additional config defined in xml file [balancer] # enabled = false # Start Balancer service: . diff --git a/engine/tpexporter.go b/engine/tpexporter.go index 1b69fb4de..c37bd30ea 100644 --- a/engine/tpexporter.go +++ b/engine/tpexporter.go @@ -27,6 +27,7 @@ import ( "path" "unicode/utf8" + "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" ) @@ -37,6 +38,18 @@ var ( ) func NewTPExporter(storDb LoadStorage, tpID, dir, fileFormat, sep string, dryRunBuff *bytes.Buffer) (*TPExporter, error) { + if len(tpID) == 0 { + return nil, errors.New("Missing TPid") + } + if len(dir) == 0 { + dir = config.CgrConfig().TpExportDir + } + if len(fileFormat) == 0 { + fileFormat = utils.CSV + } + if len(sep) == 0 { + sep = "," + } if !utils.IsSliceMember(TPExportFormats, fileFormat) { return nil, errors.New("Unsupported file format") } @@ -58,15 +71,17 @@ func NewTPExporter(storDb LoadStorage, tpID, dir, fileFormat, sep string, dryRun // Export TariffPlan to a folder type TPExporter struct { - storDb LoadStorage // StorDb connection handle - tpID string // Load data on this tpid - dirPath string // Directory path to export to - fileFormat string // The file format - sep rune // Separator in the csv file - dryRunBuffer *bytes.Buffer // Will be written in case of dryRun so we can read it from tests + storDb LoadStorage // StorDb connection handle + tpID string // Load data on this tpid + dirPath string // Directory path to export to + fileFormat string // The file format + sep rune // Separator in the csv file + dryRunBuffer *bytes.Buffer // Will be written in case of dryRun so we can read it from tests + exportedFiles []string } func (self *TPExporter) Run() error { + self.removeFiles() // Make sure we clean the folder before starting with new one for _, fHandler := range []func() error{ self.exportTimings, self.exportDestinations, @@ -100,6 +115,9 @@ func (self *TPExporter) removeFiles() error { // General method to write the content out to a file func (self *TPExporter) writeOut(fileName string, tpData []utils.ExportedData) error { + if len(tpData) == 0 { + return nil + } var writerOut utils.CgrRecordWriter if self.fileFormat == utils.DRYRUN { writerOut = utils.NewCgrIORecordWriter(self.dryRunBuffer) @@ -120,6 +138,7 @@ func (self *TPExporter) writeOut(fileName string, tpData []utils.ExportedData) e } } } + writerOut.Flush() return nil } @@ -138,6 +157,7 @@ func (self *TPExporter) exportTimings() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -156,6 +176,7 @@ func (self *TPExporter) exportDestinations() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -174,6 +195,7 @@ func (self *TPExporter) exportRates() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -192,6 +214,7 @@ func (self *TPExporter) exportDestinationRates() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -210,6 +233,7 @@ func (self *TPExporter) exportRatingPlans() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -228,6 +252,7 @@ func (self *TPExporter) exportRatingProfiles() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -246,6 +271,7 @@ func (self *TPExporter) exportSharedGroups() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -264,6 +290,7 @@ func (self *TPExporter) exportActions() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -282,6 +309,7 @@ func (self *TPExporter) exportActionPlans() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -300,6 +328,7 @@ func (self *TPExporter) exportActionTriggers() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -318,6 +347,7 @@ func (self *TPExporter) exportAccountActions() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -336,6 +366,7 @@ func (self *TPExporter) exportDerivedChargers() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } @@ -354,5 +385,10 @@ func (self *TPExporter) exportCdrStats() error { if err := self.writeOut(fileName, exportedData); err != nil { return err } + self.exportedFiles = append(self.exportedFiles, fileName) return nil } + +func (self *TPExporter) ExportStats() *utils.ExportedTPStats { + return &utils.ExportedTPStats{ExportDir: self.dirPath, ExportedFiles: self.exportedFiles} +} diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 43a6ec72a..f4901b8c0 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -782,9 +782,10 @@ type AttrDirExportTP struct { FileFormat *string // Format of the exported file FieldSeparator *string // Separator used between fields ExportDir *string // If provided it overwrites the configured export directory + Compress *bool // If true the folder will be compressed after export performed } -type DirExportedTP struct { +type ExportedTPStats struct { ExportDir string // Full path to the newly generated export file ExportedFiles []string // List of exported files } diff --git a/utils/cgrrecordwriter.go b/utils/cgrrecordwriter.go index 4684fe377..86f4fb5aa 100644 --- a/utils/cgrrecordwriter.go +++ b/utils/cgrrecordwriter.go @@ -25,6 +25,7 @@ import ( // Writer for one line, compatible with csv.Writer interface on Write type CgrRecordWriter interface { Write([]string) error + Flush() } func NewCgrIORecordWriter(w io.Writer) *CgrIORecordWriter { @@ -43,3 +44,8 @@ func (self *CgrIORecordWriter) Write(record []string) error { } return nil } + +// ToDo: make sure we properly handle this method +func (self *CgrIORecordWriter) Flush() { + return +}