From 4ff80bb2f2ee95ab57421bbbbd548fbd07f612cc Mon Sep 17 00:00:00 2001 From: Anevo Date: Wed, 28 Mar 2018 10:13:55 -0400 Subject: [PATCH] Config sanity to accept loaderTypes, MetaComposed and MetaString --- config/config.go | 51 +++++++++++++++---------- config/config_json.go | 8 ++-- config/config_json_test.go | 16 ++++---- config/config_test.go | 16 ++++---- config/libconfig_json.go | 6 +-- config/loadersconfig.go | 36 ++++++++--------- data/conf/samples/tutmysql/cgrates.json | 33 ++++++++++++++++ 7 files changed, 105 insertions(+), 61 deletions(-) diff --git a/config/config.go b/config/config.go index 445602c81..05da1ba6c 100755 --- a/config/config.go +++ b/config/config.go @@ -42,14 +42,14 @@ const ( ) var ( - DBDefaults DbDefaults - cgrCfg *CGRConfig // will be shared - dfltFsConnConfig *FsConnConfig // Default FreeSWITCH Connection configuration, built out of json default configuration - dfltKamConnConfig *KamConnConfig // Default Kamailio Connection configuration - dfltHaPoolConfig *HaPoolConfig - dfltAstConnCfg *AsteriskConnCfg - dfltLoadersConfig *LoaderSConfig - dfltLoaderSDataTypeConfig *LoaderSDataType + DBDefaults DbDefaults + cgrCfg *CGRConfig // will be shared + dfltFsConnConfig *FsConnConfig // Default FreeSWITCH Connection configuration, built out of json default configuration + dfltKamConnConfig *KamConnConfig // Default Kamailio Connection configuration + dfltHaPoolConfig *HaPoolConfig + dfltAstConnCfg *AsteriskConnCfg + dfltLoaderConfig *LoaderConfig + dfltLoaderDataTypeConfig *LoaderDataType ) func NewDbDefaults() DbDefaults { @@ -345,7 +345,7 @@ type CGRConfig struct { statsCfg *StatSCfg // Configuration for StatS thresholdSCfg *ThresholdSCfg // configuration for ThresholdS supplierSCfg *SupplierSCfg // configuration for SupplierS - loaderSCfg []*LoaderSConfig // configuration for Loader + loaderCfg []*LoaderConfig // configuration for Loader MailerServer string // The server to use when sending emails out MailerAuthUser string // Authenticate to email server using this user MailerAuthPass string // Authenticate to email server with this password @@ -475,13 +475,24 @@ func (self *CGRConfig) checkConfigSanity() error { } } // Loaders sanity checks - for _, ldrSCfg := range self.loaderSCfg { + for _, ldrSCfg := range self.loaderCfg { if !ldrSCfg.Enabled { continue } for _, dir := range []string{ldrSCfg.TpInDir, ldrSCfg.TpOutDir} { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { - fmt.Errorf("Nonexistent folder: %s", dir) + return fmt.Errorf("<%s> Nonexistent folder: %s", utils.LoaderS, dir) + } + } + for _, data := range ldrSCfg.Data { + if data.Type != utils.MetaAttributes { + return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type) + } + + for _, field := range data.Fields { + if field.Type != utils.META_COMPOSED && field.Type != utils.MetaString { + return fmt.Errorf("<%s> invalid field type %s for %s at %s", utils.LoaderS, field.Type, data.Type, field.Tag) + } } } } @@ -790,7 +801,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { return err } - jsnLoaderSCfg, err := jsnCfg.LoaderSJsonCfg() + jsnLoaderCfg, err := jsnCfg.LoaderJsonCfg() if err != nil { return err } @@ -1158,13 +1169,13 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { } } - if jsnLoaderSCfg != nil { - if self.loaderSCfg == nil { - self.loaderSCfg = make([]*LoaderSConfig, len(jsnLoaderSCfg)) + if jsnLoaderCfg != nil { + if self.loaderCfg == nil { + self.loaderCfg = make([]*LoaderConfig, len(jsnLoaderCfg)) } - for idx, profile := range jsnLoaderSCfg { - self.loaderSCfg[idx] = NewDfltLoadersConfig() - self.loaderSCfg[idx].loadFromJsonCfg(profile) + for idx, profile := range jsnLoaderCfg { + self.loaderCfg[idx] = NewDfltLoaderConfig() + self.loaderCfg[idx].loadFromJsonCfg(profile) } } @@ -1405,6 +1416,6 @@ func (cfg *CGRConfig) CacheCfg() CacheConfig { return cfg.cacheConfig } -func (cfg *CGRConfig) LoaderSCfg() []*LoaderSConfig { - return cfg.loaderSCfg +func (cfg *CGRConfig) LoaderCfg() []*LoaderConfig { + return cfg.loaderCfg } diff --git a/config/config_json.go b/config/config_json.go index 17779319b..64628eaeb 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -61,7 +61,7 @@ const ( THRESHOLDS_JSON = "thresholds" SupplierSJson = "suppliers" FILTERS_JSON = "filters" - LoaderSJson = "loaders" + LoaderJson = "loaders" MAILER_JSN = "mailer" SURETAX_JSON = "suretax" ) @@ -401,12 +401,12 @@ func (self CgrJsonCfg) SupplierSJsonCfg() (*SupplierSJsonCfg, error) { return cfg, nil } -func (self CgrJsonCfg) LoaderSJsonCfg() ([]*LoaderSJsonCfg, error) { - rawCfg, hasKey := self[LoaderSJson] +func (self CgrJsonCfg) LoaderJsonCfg() ([]*LoaderJsonCfg, error) { + rawCfg, hasKey := self[LoaderJson] if !hasKey { return nil, nil } - cfg := make([]*LoaderSJsonCfg, 0) + cfg := make([]*LoaderJsonCfg, 0) if err := json.Unmarshal(*rawCfg, &cfg); err != nil { return nil, err } diff --git a/config/config_json_test.go b/config/config_json_test.go index 9ece2468e..b2ca20a08 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -760,8 +760,8 @@ func TestDfSupplierSJsonCfg(t *testing.T) { } } -func TestDfLoaderSJsonCfg(t *testing.T) { - dataType := &LoaderSJsonDataType{ +func TestDfLoaderJsonCfg(t *testing.T) { + dataType := &LoaderJsonDataType{ Type: utils.StringPointer(utils.MetaAttributes), File_name: utils.StringPointer("Attributes.csv"), Fields: &[]*CdrFieldJsonCfg{ @@ -809,8 +809,8 @@ func TestDfLoaderSJsonCfg(t *testing.T) { Value: utils.StringPointer("9")}, }, } - eCfg := []*LoaderSJsonCfg{ - &LoaderSJsonCfg{ + eCfg := []*LoaderJsonCfg{ + &LoaderJsonCfg{ ID: utils.StringPointer(utils.META_DEFAULT), Enabled: utils.BoolPointer(false), Dry_run: utils.BoolPointer(false), @@ -820,12 +820,12 @@ func TestDfLoaderSJsonCfg(t *testing.T) { Address: utils.StringPointer(utils.MetaInternal), }}, Field_separator: utils.StringPointer(","), - Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in"), - Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out"), - Data: &[]*LoaderSJsonDataType{dataType}, + Tp_in_dir: utils.StringPointer("/var/spool/cgrates/tploader/in"), + Tp_out_dir: utils.StringPointer("/var/spool/cgrates/tploader/out"), + Data: &[]*LoaderJsonDataType{dataType}, }, } - if cfg, err := dfCgrJsonCfg.LoaderSJsonCfg(); err != nil { + if cfg, err := dfCgrJsonCfg.LoaderJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { t.Errorf("Expecting: \n%s\n, received: \n%s\n: ", diff --git a/config/config_test.go b/config/config_test.go index f872a8ad6..31823fd31 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -917,8 +917,8 @@ func TestDbDefaults(t *testing.T) { } func TestLoaderDefaults(t *testing.T) { - eCfg := []*LoaderSConfig{ - &LoaderSConfig{ + eCfg := []*LoaderConfig{ + &LoaderConfig{ Id: utils.META_DEFAULT, Enabled: false, DryRun: false, @@ -930,10 +930,10 @@ func TestLoaderDefaults(t *testing.T) { }, }, FieldSeparator: ",", - TpInDir: "/var/spool/cgrates/loader/in", - TpOutDir: "/var/spool/cgrates/loader/out", - Data: []*LoaderSDataType{ - &LoaderSDataType{ + TpInDir: "/var/spool/cgrates/tploader/in", + TpOutDir: "/var/spool/cgrates/tploader/out", + Data: []*LoaderDataType{ + &LoaderDataType{ Type: utils.MetaAttributes, Filename: utils.AttributesCsv, Fields: []*CfgCdrField{ @@ -984,7 +984,7 @@ func TestLoaderDefaults(t *testing.T) { }, }, } - if !reflect.DeepEqual(eCfg, cgrCfg.loaderSCfg) { - t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderSCfg) + if !reflect.DeepEqual(eCfg, cgrCfg.loaderCfg) { + t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderCfg) } } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 747e42860..7ed002943 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -429,13 +429,13 @@ type SupplierSJsonCfg struct { Stats_conns *[]*HaPoolJsonCfg } -type LoaderSJsonDataType struct { +type LoaderJsonDataType struct { Type *string File_name *string Fields *[]*CdrFieldJsonCfg } -type LoaderSJsonCfg struct { +type LoaderJsonCfg struct { ID *string Enabled *bool Dry_run *bool @@ -445,7 +445,7 @@ type LoaderSJsonCfg struct { Field_separator *string Tp_in_dir *string Tp_out_dir *string - Data *[]*LoaderSJsonDataType + Data *[]*LoaderJsonDataType } // Mailer config section diff --git a/config/loadersconfig.go b/config/loadersconfig.go index a264a179f..ba3b41f7c 100644 --- a/config/loadersconfig.go +++ b/config/loadersconfig.go @@ -22,15 +22,15 @@ import ( "time" ) -func NewDfltLoadersConfig() *LoaderSConfig { - if dfltLoadersConfig == nil { - return new(LoaderSConfig) +func NewDfltLoaderConfig() *LoaderConfig { + if dfltLoaderConfig == nil { + return new(LoaderConfig) } - dfltVal := *dfltLoadersConfig + dfltVal := *dfltLoaderConfig return &dfltVal } -type LoaderSConfig struct { +type LoaderConfig struct { // rename to LoaderConfig Id string Enabled bool DryRun bool @@ -40,24 +40,24 @@ type LoaderSConfig struct { FieldSeparator string TpInDir string TpOutDir string - Data []*LoaderSDataType + Data []*LoaderDataType } -func NewDfltLoaderSDataTypeConfig() *LoaderSDataType { - if dfltLoaderSDataTypeConfig == nil { - return new(LoaderSDataType) // No defaults, most probably we are building the defaults now +func NewDfltLoaderDataTypeConfig() *LoaderDataType { + if dfltLoaderDataTypeConfig == nil { + return new(LoaderDataType) // No defaults, most probably we are building the defaults now } - dfltVal := *dfltLoaderSDataTypeConfig // Copy the value instead of it's pointer + dfltVal := *dfltLoaderDataTypeConfig // Copy the value instead of it's pointer return &dfltVal } -type LoaderSDataType struct { +type LoaderDataType struct { //rename to LoaderDataType Type string Filename string Fields []*CfgCdrField } -func (self *LoaderSDataType) loadFromJsonCfg(jsnCfg *LoaderSJsonDataType) error { +func (self *LoaderDataType) loadFromJsonCfg(jsnCfg *LoaderJsonDataType) error { if jsnCfg == nil { return nil } @@ -76,7 +76,7 @@ func (self *LoaderSDataType) loadFromJsonCfg(jsnCfg *LoaderSJsonDataType) error return nil } -func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error { +func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error { if jsnCfg == nil { return nil } @@ -112,9 +112,9 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error { self.TpOutDir = *jsnCfg.Tp_out_dir } if jsnCfg.Data != nil { - self.Data = make([]*LoaderSDataType, len(*jsnCfg.Data)) + self.Data = make([]*LoaderDataType, len(*jsnCfg.Data)) for idx, jsnLoCfg := range *jsnCfg.Data { - self.Data[idx] = NewDfltLoaderSDataTypeConfig() + self.Data[idx] = NewDfltLoaderDataTypeConfig() self.Data[idx].loadFromJsonCfg(jsnLoCfg) } } @@ -122,8 +122,8 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error { } // Clone itself into a new LoadersConfig -func (self *LoaderSConfig) Clone() *LoaderSConfig { - clnLoader := new(LoaderSConfig) +func (self *LoaderConfig) Clone() *LoaderConfig { + clnLoader := new(LoaderConfig) clnLoader.Id = self.Id clnLoader.Enabled = self.Enabled clnLoader.DryRun = self.DryRun @@ -137,7 +137,7 @@ func (self *LoaderSConfig) Clone() *LoaderSConfig { clnLoader.FieldSeparator = self.FieldSeparator clnLoader.TpInDir = self.TpInDir clnLoader.TpOutDir = self.TpOutDir - clnLoader.Data = make([]*LoaderSDataType, len(self.Data)) + clnLoader.Data = make([]*LoaderDataType, len(self.Data)) for idx, fld := range self.Data { clonedVal := *fld clnLoader.Data[idx] = &clonedVal diff --git a/data/conf/samples/tutmysql/cgrates.json b/data/conf/samples/tutmysql/cgrates.json index b0c7b76a9..73464211a 100644 --- a/data/conf/samples/tutmysql/cgrates.json +++ b/data/conf/samples/tutmysql/cgrates.json @@ -109,6 +109,39 @@ }, }, +"loaders": [ + { + "id": "*default", // identifier of the Loader + "enabled": false, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/var/spool/cgrates/tploader/in", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp", // absolute path towards the directory where processed CDRs will be moved + "data":[ // data profiles to load + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "2"}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "3"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "4"}, + {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "5"}, + {"tag": "Initial", "field_id": "Initial", "type": "*composed", "value": "6"}, + {"tag": "Substitute", "field_id": "Substitute", "type": "*composed", "value": "7"}, + {"tag": "Append", "field_id": "Append", "type": "*composed", "value": "8"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"}, + ], + }, + ], + }, +], "cdrstats": { "enabled": true,