From 758f399acbe6094ea5ef2bf349d6502aef77bccc Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 13 Jan 2014 13:15:16 +0100 Subject: [PATCH] Adding static fields inside cdrc config --- cdrc/cdrc.go | 65 ++++++++++++++++++++++++++--------------- cdrc/cdrc_test.go | 28 +++++++++++++----- config/config.go | 4 +-- config/config_test.go | 2 +- data/conf/cgrates.cfg | 2 +- engine/storage_mysql.go | 3 ++ 6 files changed, 69 insertions(+), 35 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index 537f29691..d6e539f97 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -51,10 +51,8 @@ func NewCdrc(config *config.CGRConfig) (*Cdrc, error) { return nil, fmt.Errorf("Folder %s does not exist", dir) } } - if cdrc.cgrCfg.CdrcCdrType == CSV { - if err := cdrc.parseFieldIndexesFromConfig(); err != nil { - return nil, err - } + if err := cdrc.parseFieldsConfig(); err != nil { + return nil, err } cdrc.httpClient = new(http.Client) return cdrc, nil @@ -62,7 +60,7 @@ func NewCdrc(config *config.CGRConfig) (*Cdrc, error) { type Cdrc struct { cgrCfg *config.CGRConfig - fieldIndxes map[string]int // Key is the name of the field, int is the position in the csv file + cfgCdrFields map[string]string // Key is the name of the field httpClient *http.Client } @@ -79,19 +77,23 @@ func (self *Cdrc) Run() error { return nil } -// Parses fieldIndex strings into fieldIndex integers needed -func (self *Cdrc) parseFieldIndexesFromConfig() error { +// Loads all fields (primary and extra) into cfgCdrFields, do some pre-checks (eg: in case of csv make sure that values are integers) +func (self *Cdrc) parseFieldsConfig() error { var err error - self.fieldIndxes = make(map[string]int) - fieldKeys := []string{utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, utils.TOR, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.ANSWER_TIME, utils.DURATION} - fieldIdxStrs := []string{self.cgrCfg.CdrcAccIdField, self.cgrCfg.CdrcReqTypeField, self.cgrCfg.CdrcDirectionField, self.cgrCfg.CdrcTenantField, self.cgrCfg.CdrcTorField, - self.cgrCfg.CdrcAccountField, self.cgrCfg.CdrcSubjectField, self.cgrCfg.CdrcDestinationField, self.cgrCfg.CdrcAnswerTimeField, self.cgrCfg.CdrcDurationField} - for i, strVal := range fieldIdxStrs { - if self.fieldIndxes[fieldKeys[i]], err = strconv.Atoi(strVal); err != nil { - return fmt.Errorf("Cannot parse configuration field %s into integer", fieldKeys[i]) - } - } - // Add extra fields here, extra fields in the form of []string{"indxInCsv1:fieldName1","indexInCsv2:fieldName2"} + self.cfgCdrFields = map[string]string{ + utils.ACCID: self.cgrCfg.CdrcAccIdField, + utils.REQTYPE: self.cgrCfg.CdrcReqTypeField, + utils.DIRECTION: self.cgrCfg.CdrcDirectionField, + utils.TENANT: self.cgrCfg.CdrcTenantField, + utils.TOR: self.cgrCfg.CdrcTorField, + utils.ACCOUNT: self.cgrCfg.CdrcAccountField, + utils.SUBJECT: self.cgrCfg.CdrcSubjectField, + utils.DESTINATION: self.cgrCfg.CdrcDestinationField, + utils.ANSWER_TIME: self.cgrCfg.CdrcAnswerTimeField, + utils.DURATION: self.cgrCfg.CdrcDurationField, + } + + // Add extra fields here, config extra fields in the form of []string{"fieldName1:indxInCsv1","fieldName2: indexInCsv2"} for _, fieldWithIdx := range self.cgrCfg.CdrcExtraFields { splt := strings.Split(fieldWithIdx, ":") if len(splt) != 2 { @@ -100,8 +102,14 @@ func (self *Cdrc) parseFieldIndexesFromConfig() error { if utils.IsSliceMember(utils.PrimaryCdrFields, splt[0]) { return errors.New("Extra cdrc.extra_fields overwriting primary fields") } - if self.fieldIndxes[splt[1]], err = strconv.Atoi(splt[0]); err != nil { - return fmt.Errorf("Cannot parse configuration cdrc extra field %s into integer", splt[1]) + self.cfgCdrFields[splt[0]] = splt[1] + } + // Fields populated, do some sanity checks here + for cdrField, cfgVal := range self.cfgCdrFields { + if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cgrCfg.CdrcCdrType) && !strings.HasPrefix(cfgVal, utils.STATIC_VALUE_PREFIX) { + if _, err = strconv.Atoi(cfgVal); err != nil { + return fmt.Errorf("Cannot parse configuration field %s into integer", cdrField) + } } } return nil @@ -112,11 +120,22 @@ func (self *Cdrc) cdrAsHttpForm(record []string) (url.Values, error) { // engine.Logger.Info(fmt.Sprintf("Processing record %v", record)) v := url.Values{} v.Set(utils.CDRSOURCE, self.cgrCfg.CdrcSourceId) - for fldName, idx := range self.fieldIndxes { - if len(record) <= idx { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, fldName) + for cfgFieldName, cfgFieldVal := range self.cfgCdrFields { + var fieldVal string + if strings.HasPrefix(cfgFieldVal, utils.STATIC_VALUE_PREFIX) { + fieldVal = cfgFieldVal[1:] + } else if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cgrCfg.CdrcCdrType) { + if cfgFieldIdx, err := strconv.Atoi(cfgFieldVal); err != nil { // Should in theory never happen since we have already parsed config + return nil, err + } else if len(record) <= cfgFieldIdx { + return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) + } else { + fieldVal = record[cfgFieldIdx] + } + } else { // Modify here when we add more supported cdr formats + fieldVal = "UNKNOWN" } - v.Set(fldName, record[idx]) + v.Set(cfgFieldName, fieldVal) } return v, nil } diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index 3eee88c06..87f72cf4e 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -24,28 +24,40 @@ import ( "testing" ) -func TestParseFieldIndexesFromConfig(t *testing.T) { +func TestParseFieldsConfig(t *testing.T) { // Test default config cgrConfig, _ := config.NewDefaultCGRConfig() // Test primary field index definition cgrConfig.CdrcAccIdField = "detect_me" cdrc := &Cdrc{cgrCfg: cgrConfig} - if err := cdrc.parseFieldIndexesFromConfig(); err == nil { + if err := cdrc.parseFieldsConfig(); err == nil { t.Error("Failed detecting error in accounting id definition", err) } + cgrConfig.CdrcAccIdField = "^static_val" + cgrConfig.CdrcSubjectField = "1" + cdrc = &Cdrc{cgrCfg: cgrConfig} + if err := cdrc.parseFieldsConfig(); err != nil { + t.Error("Failed to corectly parse primary fields %v", cdrc.cfgCdrFields) + } + cgrConfig.CdrcExtraFields = []string{"^static_val:orig_ip"} // Test extra field index definition cgrConfig.CdrcAccIdField = "0" // Put back as int - cgrConfig.CdrcExtraFields = []string{"supplier1", "11:orig_ip"} + cgrConfig.CdrcExtraFields = []string{"supplier1", "orig_ip:11"} cdrc = &Cdrc{cgrCfg: cgrConfig} - if err := cdrc.parseFieldIndexesFromConfig(); err == nil { + if err := cdrc.parseFieldsConfig(); err == nil { t.Error("Failed detecting error in extra fields definition", err) } + cgrConfig.CdrcExtraFields = []string{"supplier1:^top_supplier", "orig_ip:11"} + cdrc = &Cdrc{cgrCfg: cgrConfig} + if err := cdrc.parseFieldsConfig(); err != nil { + t.Errorf("Failed to corectly parse extra fields %v",cdrc.cfgCdrFields) + } } func TestCdrAsHttpForm(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() cdrc := &Cdrc{cgrCfg: cgrConfig} - if err := cdrc.parseFieldIndexesFromConfig(); err != nil { + if err := cdrc.parseFieldsConfig(); err != nil { t.Error("Failed parsing default fieldIndexesFromConfig", err) } cdrRow := []string{"firstField", "secondField"} @@ -64,7 +76,7 @@ func TestCdrAsHttpForm(t *testing.T) { if cdrAsForm.Get(utils.REQTYPE) != "prepaid" { t.Error("Unexpected CDR value received", cdrAsForm.Get(utils.REQTYPE)) } - if cdrAsForm.Get("supplier") != "supplier1" { - t.Error("Unexpected CDR value received", cdrAsForm.Get(utils.REQTYPE)) - } + //if cdrAsForm.Get("supplier") != "supplier1" { + // t.Error("Unexpected CDR value received", cdrAsForm.Get("supplier")) + //} } diff --git a/config/config.go b/config/config.go index 9920290e4..2db5edf22 100644 --- a/config/config.go +++ b/config/config.go @@ -110,7 +110,7 @@ type CGRConfig struct { CdrcDestinationField string // Destination field identifier. Use index numbers in case of .csv cdrs. CdrcAnswerTimeField string // Answer time field identifier. Use index numbers in case of .csv cdrs. CdrcDurationField string // Duration field identifier. Use index numbers in case of .csv cdrs. - CdrcExtraFields []string // Field identifiers of the fields to add in extra fields section, special format in case of .csv "index1:field1,index2:field2" + CdrcExtraFields []string // Field identifiers of the fields to add in extra fields section, special format in case of .csv "field1:index1,field2:index2" SMEnabled bool SMSwitchType string SMRater string // address where to access rater. Can be internal, direct rater address or the address of a balancer @@ -200,7 +200,7 @@ func (self *CGRConfig) setDefaults() error { self.CdrcDestinationField = "7" self.CdrcAnswerTimeField = "8" self.CdrcDurationField = "9" - self.CdrcExtraFields = []string{"10:supplier","11:orig_ip"} + self.CdrcExtraFields = []string{} self.MediatorEnabled = false self.MediatorListen = "127.0.0.1:2032" self.MediatorRater = "127.0.0.1:2012" diff --git a/config/config_test.go b/config/config_test.go index 0a70f3934..0e43924d8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -103,7 +103,7 @@ func TestDefaults(t *testing.T) { eCfg.CdrcDestinationField = "7" eCfg.CdrcAnswerTimeField = "8" eCfg.CdrcDurationField = "9" - eCfg.CdrcExtraFields = []string{"10:supplier","11:orig_ip"} + eCfg.CdrcExtraFields = []string{} eCfg.MediatorEnabled = false eCfg.MediatorListen = "127.0.0.1:2032" eCfg.MediatorRater = "127.0.0.1:2012" diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 1af1f445f..fa3900a67 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -74,7 +74,7 @@ # destination_field = 7 # Destination field identifier. Use index numbers in case of .csv cdrs. # answer_time_field = 8 # Answer time field identifier. Use index numbers in case of .csv cdrs. # duration_field = 9 # Duration field identifier. Use index numbers in case of .csv cdrs. -# extra_fields = 10:supplier,11:orig_ip # Extra fields identifiers. For .csv, format: :[,:] +# extra_fields = # Extra fields identifiers. For .csv, format: :[...,:] [mediator] # enabled = false # Starts Mediator service: . diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 1df69b393..543243883 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -33,5 +33,8 @@ func NewMySQLStorage(host, port, name, user, password string) (Storage, error) { if err != nil { return nil, err } + if err := db.Ping(); err != nil { + return nil, err + } return &MySQLStorage{&SQLStorage{db}}, nil }