From 4bb3e745f1acc8aabf489237038eab0a64e98c42 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 22 Mar 2014 22:16:59 +0100 Subject: [PATCH] Adding config xml template for exporter with fixed width CDRs together with tests --- apier/tutfsjson_local_test.go | 6 +- config/config.go | 94 +++++++++++++++++-------- config/config_local_test.go | 47 +++++++++++++ config/config_test.go | 6 ++ config/test_data.txt | 2 +- config/xmlconfig.go | 45 ++++++++---- config/xmlconfig_test.go | 13 +++- data/conf/cgrates.cfg | 5 +- data/conf/samples/cgr_addconfig.xml | 20 ++++++ data/conf/samples/config_local_test.cfg | 13 ++++ local_test.sh | 5 +- utils/consts.go | 4 +- 12 files changed, 206 insertions(+), 54 deletions(-) create mode 100644 config/config_local_test.go create mode 100644 data/conf/samples/cgr_addconfig.xml create mode 100644 data/conf/samples/config_local_test.cfg diff --git a/apier/tutfsjson_local_test.go b/apier/tutfsjson_local_test.go index 264ad5937..b2e45a3be 100644 --- a/apier/tutfsjson_local_test.go +++ b/apier/tutfsjson_local_test.go @@ -413,7 +413,7 @@ func TestMaxDebit1001(t *testing.T) { for _, blnc := range blncLst { if blnc.SharedGroup == "SHARED_A" && blnc.Value != 5 { t.Errorf("Unexpected value for shared balance: %f", blnc.Value) - } else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.4 { + } else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.7 { t.Errorf("Unexpected value for general balance: %f", blnc.Value) } } @@ -448,9 +448,9 @@ func TestMaxDebit1007(t *testing.T) { } blncLst := acnt.BalanceMap["*monetary*out"] for _, blnc := range blncLst { - if blnc.SharedGroup == "SHARED_A" && blnc.Value != 4 { + if blnc.SharedGroup == "SHARED_A" && blnc.Value != 4.7 { t.Errorf("Unexpected value for shared balance: %f", blnc.Value) - } else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.4 { + } else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.7 { t.Errorf("Unexpected value for general balance: %f", blnc.Value) } } diff --git a/config/config.go b/config/config.go index 932d70d66..82b69a1bd 100644 --- a/config/config.go +++ b/config/config.go @@ -21,6 +21,8 @@ package config import ( "errors" "fmt" + "os" + "strings" "time" "code.google.com/p/goconf/conf" @@ -59,37 +61,39 @@ type CGRConfig struct { RatingDBUser string // The user to sign in as. RatingDBPass string // The user's password. AccountDBType string - AccountDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - AccountDBPort string // The port to bind to. - AccountDBName string // The name of the database to connect to. - AccountDBUser string // The user to sign in as. - AccountDBPass string // The user's password. - StorDBType string // Should reflect the database type used to store logs - StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. - StorDBPort string // Th e port to bind to. - StorDBName string // The name of the database to connect to. - StorDBUser string // The user to sign in as. - StorDBPass string // The user's password. - DBDataEncoding string // The encoding used to store object data in strings: - RPCJSONListen string // RPC JSON listening address - RPCGOBListen string // RPC GOB listening address - HTTPListen string // HTTP listening address - DefaultReqType string // Use this request type if not defined on top - DefaultTOR string // set default type of record - DefaultTenant string // set default tenant - DefaultSubject string // set default rating subject, useful in case of fallback - RoundingMethod string // Rounding method for the end price: <*up|*middle|*down> - RoundingDecimals int // Number of decimals to round end prices at - RaterEnabled bool // start standalone server (no balancer) - RaterBalancer string // balancer address host:port + AccountDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + AccountDBPort string // The port to bind to. + AccountDBName string // The name of the database to connect to. + AccountDBUser string // The user to sign in as. + AccountDBPass string // The user's password. + StorDBType string // Should reflect the database type used to store logs + StorDBHost string // The host to connect to. Values that start with / are for UNIX domain sockets. + StorDBPort string // Th e port to bind to. + StorDBName string // The name of the database to connect to. + StorDBUser string // The user to sign in as. + StorDBPass string // The user's password. + DBDataEncoding string // The encoding used to store object data in strings: + RPCJSONListen string // RPC JSON listening address + RPCGOBListen string // RPC GOB listening address + HTTPListen string // HTTP listening address + DefaultReqType string // Use this request type if not defined on top + DefaultTOR string // set default type of record + DefaultTenant string // set default tenant + DefaultSubject string // set default rating subject, useful in case of fallback + RoundingMethod string // Rounding method for the end price: <*up|*middle|*down> + RoundingDecimals int // Number of decimals to round end prices at + XmlCfgDocument *CgrXmlCfgDocument // Load additional configuration inside xml document + RaterEnabled bool // start standalone server (no balancer) + RaterBalancer string // balancer address host:port BalancerEnabled bool SchedulerEnabled bool CDRSEnabled bool // Enable CDR Server service CDRSExtraFields []*utils.RSRField // Extra fields to store in CDRs CDRSMediator string // Address where to reach the Mediator. Empty for disabling mediation. <""|internal> CdreCdrFormat string // Format of the exported CDRs. - CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs CdreDir string // Path towards exported cdrs directory + CdreExportedFields []*utils.RSRField // List of fields in the exported CDRs + CdreFWXmlTemplate *CgrXmlCdreFwCfg // Use this configuration as export template in case of fixed fields length CdrcEnabled bool // Enable CDR client functionality CdrcCdrs string // Address where to reach CDR server CdrcCdrsMethod string // Mechanism to use when posting CDRs on server @@ -184,6 +188,7 @@ func (self *CGRConfig) setDefaults() error { self.DefaultSubject = "cgrates" self.RoundingMethod = utils.ROUNDING_MIDDLE self.RoundingDecimals = 4 + self.XmlCfgDocument = nil self.RaterEnabled = false self.RaterBalancer = "" self.BalancerEnabled = false @@ -277,6 +282,14 @@ func (self *CGRConfig) setDefaults() error { } func (self *CGRConfig) checkConfigSanity() error { + // Cdre sanity check for fixed_width + if self.CdreCdrFormat == utils.FIXED_WIDTH { + if self.XmlCfgDocument == nil { + return errors.New("Need XmlConfigurationDocument for fixed_width cdr export") + } else if self.CdreFWXmlTemplate == nil { + return errors.New("Need XmlTemplate for fixed_width cdr export") + } + } // SessionManager should have same fields config length for session emulation if len(self.SMReqTypeFields) != len(self.SMRunIds) || len(self.SMDirectionFields) != len(self.SMRunIds) || @@ -435,6 +448,19 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("global", "rounding_decimals"); hasOpt { cfg.RoundingDecimals, _ = c.GetInt("global", "rounding_decimals") } + // XML config path defined, try loading the document + if hasOpt = c.HasOption("global", "xmlcfg_path"); hasOpt { + xmlCfgPath, _ := c.GetString("global", "xmlcfg_path") + xmlFile, err := os.Open(xmlCfgPath) + if err != nil { + return nil, err + } + if cgrXmlCfgDoc, err := ParseCgrXmlConfig(xmlFile); err != nil { + return nil, err + } else { + cfg.XmlCfgDocument = cgrXmlCfgDoc + } + } if hasOpt = c.HasOption("rater", "enabled"); hasOpt { cfg.RaterEnabled, _ = c.GetBool("rater", "enabled") } @@ -464,12 +490,20 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("cdre", "cdr_format"); hasOpt { cfg.CdreCdrFormat, _ = c.GetString("cdre", "cdr_format") } - if hasOpt = c.HasOption("cdre", "exported_fields"); hasOpt { - extraFieldsStr, _ := c.GetString("cdre", "exported_fields") - if extraFields, err := ParseRSRFields(extraFieldsStr); err != nil { - return nil, errParse - } else { - cfg.CdreExportedFields = extraFields + if hasOpt = c.HasOption("cdre", "export_template"); hasOpt { // Load configs for csv normally from template, fixed_width from xml file + exportTemplate, _ := c.GetString("cdre", "export_template") + if cfg.CdreCdrFormat != utils.FIXED_WIDTH { // Csv most likely + if extraFields, err := ParseRSRFields(exportTemplate); err != nil { + return nil, errParse + } else { + cfg.CdreExportedFields = extraFields + } + } else if strings.HasPrefix(exportTemplate, utils.XML_PROFILE_PREFIX) { + if xmlTemplate, err := cfg.XmlCfgDocument.GetCdreFWCfg(exportTemplate[len(utils.XML_PROFILE_PREFIX):]); err != nil { + return nil, err + } else { + cfg.CdreFWXmlTemplate = xmlTemplate + } } } if hasOpt = c.HasOption("cdre", "export_dir"); hasOpt { diff --git a/config/config_local_test.go b/config/config_local_test.go new file mode 100644 index 000000000..eac978f11 --- /dev/null +++ b/config/config_local_test.go @@ -0,0 +1,47 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2013 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package config + +import ( + "flag" + "path" + "testing" +) + +var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args +var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") + +func TestLoadXmlCfg(t *testing.T) { + if !*testLocal { + return + } + cfgPath := path.Join(*dataDir, "conf", "samples", "config_local_test.cfg") + cfg, err := NewCGRConfig(&cfgPath) + if err != nil { + t.Error(err) + } + if cfg.XmlCfgDocument == nil { + t.Error("Did not load the XML Config Document") + } + if cdreFWCfg, err := cfg.XmlCfgDocument.GetCdreFWCfg("CDREFW-A"); err != nil { + t.Error(err) + } else if cdreFWCfg == nil { + t.Error("Could not retrieve CDRExporter FixedWidth config instance") + } +} diff --git a/config/config_test.go b/config/config_test.go index b1ace7da3..a528c3f8d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -73,6 +73,7 @@ func TestDefaults(t *testing.T) { eCfg.DefaultSubject = "cgrates" eCfg.RoundingMethod = utils.ROUNDING_MIDDLE eCfg.RoundingDecimals = 4 + eCfg.XmlCfgDocument = nil eCfg.RaterEnabled = false eCfg.RaterBalancer = "" eCfg.BalancerEnabled = false @@ -182,6 +183,11 @@ func TestSanityCheck(t *testing.T) { if err := cfg.checkConfigSanity(); err == nil { t.Error("Failed to detect config insanity") } + cfg = &CGRConfig{} + cfg.CdreCdrFormat = utils.FIXED_WIDTH + if err := cfg.checkConfigSanity(); err == nil { + t.Error("Failed to detect fixed_width dependency on xml configuration") + } } // Load config from file and make sure we have all set diff --git a/config/test_data.txt b/config/test_data.txt index 6079dc4c6..a68fa2c38 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -50,7 +50,7 @@ mediator = test # Address where to reach the Mediator. Empty for disabling me [cdre] cdr_format = test # Exported CDRs format export_dir = test # Path where the exported CDRs will be placed -exported_fields = test # List of fields in the exported CDRs +export_template = test # List of fields in the exported CDRs [cdrc] enabled = true # Enable CDR client functionality diff --git a/config/xmlconfig.go b/config/xmlconfig.go index 752e5403e..a46d7b2e8 100644 --- a/config/xmlconfig.go +++ b/config/xmlconfig.go @@ -20,7 +20,7 @@ package config import ( "encoding/xml" - "errors" + "fmt" "github.com/cgrates/cgrates/utils" "io" ) @@ -37,9 +37,10 @@ func ParseCgrXmlConfig(reader io.Reader) (*CgrXmlCfgDocument, error) { // Define a format for configuration file, one doc contains more configuration instances, identified by section, type and id type CgrXmlCfgDocument struct { - XMLName xml.Name `xml:"document"` - Type string `xml:"type,attr"` - Configurations []*CgrXmlConfiguration `xml:"configuration"` + XMLName xml.Name `xml:"document"` + Type string `xml:"type,attr"` + Configurations []*CgrXmlConfiguration `xml:"configuration"` + cdrefws map[string]*CgrXmlCdreFwCfg // Cache for processed fixed width config instances, key will be the id of the instance } // Storage for raw configuration @@ -48,7 +49,7 @@ type CgrXmlConfiguration struct { Section string `xml:"section,attr"` Type string `xml:"type,attr"` Id string `xml:"id,attr"` - RawConfig []byte `xml:",innerxml"` + RawConfig []byte `xml:",innerxml"` // Used to store the configuration struct, as raw so we can store different types } // The CdrExporter Fixed Width configuration instance @@ -84,20 +85,34 @@ type CgrXmlCfgCdrField struct { Width string `xml:"width,attr"` } -func (xmlCfg *CgrXmlCfgDocument) GetCdreFWCfg(instName string) (*CgrXmlCdreFwCfg, error) { - cdrefwCfg := new(CgrXmlCdreFwCfg) +// Avoid building from raw config string always, so build cache here +func (xmlCfg *CgrXmlCfgDocument) cacheCdreFWCfgs() error { + xmlCfg.cdrefws = make(map[string]*CgrXmlCdreFwCfg) for _, cfgInst := range xmlCfg.Configurations { - if cfgInst.Section != "cdre" || cfgInst.Type != utils.CDR_FIXED_WIDTH || cfgInst.Id != instName { - continue + if cfgInst.Section == utils.CDRE || cfgInst.Type == utils.FIXED_WIDTH { + cdrefwCfg := new(CgrXmlCdreFwCfg) + rawConfig := append([]byte(""), cfgInst.RawConfig...) // Encapsulate the rawConfig in one element so we can Unmarshall + rawConfig = append(rawConfig, []byte("")...) + if err := xml.Unmarshal(rawConfig, cdrefwCfg); err != nil { + return err + } else if cdrefwCfg == nil { + return fmt.Errorf("Could not unmarshal CgrXmlCdreFwCfg: %s", cfgInst.Id) + } else { // All good, cache the config instance + xmlCfg.cdrefws[cfgInst.Id] = cdrefwCfg + } } - rawConfig := append([]byte(""), cfgInst.RawConfig...) - rawConfig = append(rawConfig, []byte("")...) - if err := xml.Unmarshal(rawConfig, cdrefwCfg); err != nil { // Encapsulate the rawConfig in one element so we can Unmarshall + } + return nil +} + +func (xmlCfg *CgrXmlCfgDocument) GetCdreFWCfg(instName string) (*CgrXmlCdreFwCfg, error) { + if len(xmlCfg.cdrefws) == 0 { // First time, cache also + if err := xmlCfg.cacheCdreFWCfgs(); err != nil { return nil, err - } else if cdrefwCfg == nil { - return nil, errors.New("Could not unmarshal CgrXmlCdreFwCfg") } - return cdrefwCfg, nil + } + if cfg, hasIt := xmlCfg.cdrefws[instName]; hasIt { + return cfg, nil } return nil, nil } diff --git a/config/xmlconfig_test.go b/config/xmlconfig_test.go index 4834e9330..59ba58c37 100644 --- a/config/xmlconfig_test.go +++ b/config/xmlconfig_test.go @@ -28,7 +28,7 @@ var cfgDoc *CgrXmlCfgDocument // Will be populated by first test func TestParseXmlConfig(t *testing.T) { cfgXmlStr := ` - +
@@ -89,6 +89,17 @@ func TestParseXmlConfig(t *testing.T) { } } +func TestCacheCdreFWCfgs(t *testing.T) { + if len(cfgDoc.cdrefws) != 0 { + t.Error("Cache should be empty before caching") + } + if err := cfgDoc.cacheCdreFWCfgs(); err != nil { + t.Error(err) + } else if len(cfgDoc.cdrefws) != 1 { + t.Error("Did not cache") + } +} + func TestGetCdreFWCfg(t *testing.T) { cdreFWCfg, err := cfgDoc.GetCdreFWCfg("CDRE-FW1") if err != nil { diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index cf66d26e9..c1670ac75 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -33,6 +33,7 @@ # default_subject = cgrates # Default rating Subject to consider when missing from requests. # rounding_method = *middle # Rounding method for floats/costs: <*up|*middle|*down> # rounding_decimals = 4 # Number of decimals to round float/costs at +# xmlcfg_path = # Path towards additional config defined in xml file [balancer] # enabled = false # Start Balancer service: . @@ -52,8 +53,8 @@ [cdre] # cdr_format = csv # Exported CDRs format # export_dir = /var/log/cgrates/cdr/cdrexport/csv # Path where the exported CDRs will be placed -# exported_fields = cgrid,mediation_runid,accid,cdrhost,reqtype,direction,tenant,tor,account,subject,destination,setup_time,answer_time,duration,cost - # List of fields in the exported CDRs +# export_template = cgrid,mediation_runid,accid,cdrhost,reqtype,direction,tenant,tor,account,subject,destination,setup_time,answer_time,duration,cost + # Exported fields template <""|fld1,fld2|*xml:instance_name> [cdrc] # enabled = false # Enable CDR client functionality diff --git a/data/conf/samples/cgr_addconfig.xml b/data/conf/samples/cgr_addconfig.xml new file mode 100644 index 000000000..6bc5260e0 --- /dev/null +++ b/data/conf/samples/cgr_addconfig.xml @@ -0,0 +1,20 @@ + + + +
+ + + +
+ + + + + + + + + + + + diff --git a/data/conf/samples/config_local_test.cfg b/data/conf/samples/config_local_test.cfg new file mode 100644 index 000000000..c4c4770a8 --- /dev/null +++ b/data/conf/samples/config_local_test.cfg @@ -0,0 +1,13 @@ +# CGRateS Configuration file +# +# This file contains the default configuration hardcoded into CGRateS. +# This is what you get when you load CGRateS with an empty configuration file. +# [global] must exist in all files, rest of the configuration is inter-changeable. + +[global] +xmlcfg_path = /usr/share/cgrates/conf/samples/cgr_addconfig.xml # Path towards additional config defined in xml file + +[cdre] +cdr_format = fixed_width # Exported CDRs format +export_template = *xml:CDREFW-A # Exported fields template <""|fld1,fld2|*xml:instance_name> + diff --git a/local_test.sh b/local_test.sh index 2de420cd0..5fb181031 100755 --- a/local_test.sh +++ b/local_test.sh @@ -10,9 +10,12 @@ go test github.com/cgrates/cgrates/cdrc -local cdrc=$? go test github.com/cgrates/cgrates/mediator -local med=$? +go test github.com/cgrates/cgrates/config -local +cfg=$? -exit $gen && $ap && $en && $cdrc && $med + +exit $gen && $ap && $en && $cdrc && $med && $cfg diff --git a/utils/consts.go b/utils/consts.go index 3405475d1..2f572e37d 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -91,7 +91,9 @@ const ( INTERNAL = "internal" ZERO_RATING_SUBJECT_PREFIX = "*zero" OK = "OK" - CDR_FIXED_WIDTH = "cdr_fixed_width" + FIXED_WIDTH = "fixed_width" + XML_PROFILE_PREFIX = "*xml:" + CDRE = "cdre" ) var (