uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-12 02:26:26 +05:00
Code Issues Packages Projects Releases Wiki Activity

Config sanity to accept loaderTypes, MetaComposed and MetaString

Browse Source
This commit is contained in:
Anevo
2018-03-28 10:13:55 -04:00
committed by Dan Christian Bogos
parent cbfeff169d
commit 4ff80bb2f2
7 changed files with 105 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
config/config.go
View File

@@ -42,14 +42,14 @@ const (
)
var (
DBDefaults DbDefaults
cgrCfg *CGRConfig // will be shared
dfltFsConnConfig *FsConnConfig // Default FreeSWITCH Connection configuration, built out of json default configuration
dfltKamConnConfig *KamConnConfig // Default Kamailio Connection configuration
dfltHaPoolConfig *HaPoolConfig
dfltAstConnCfg *AsteriskConnCfg
dfltLoadersConfig *LoaderSConfig
dfltLoaderSDataTypeConfig *LoaderSDataType
DBDefaults DbDefaults
cgrCfg *CGRConfig // will be shared
dfltFsConnConfig *FsConnConfig // Default FreeSWITCH Connection configuration, built out of json default configuration
dfltKamConnConfig *KamConnConfig // Default Kamailio Connection configuration
dfltHaPoolConfig *HaPoolConfig
dfltAstConnCfg *AsteriskConnCfg
dfltLoaderConfig *LoaderConfig
dfltLoaderDataTypeConfig *LoaderDataType
)
func NewDbDefaults() DbDefaults {
@@ -345,7 +345,7 @@ type CGRConfig struct {
statsCfg *StatSCfg // Configuration for StatS
thresholdSCfg *ThresholdSCfg // configuration for ThresholdS
supplierSCfg *SupplierSCfg // configuration for SupplierS
loaderSCfg []*LoaderSConfig // configuration for Loader
loaderCfg []*LoaderConfig // configuration for Loader
MailerServer string // The server to use when sending emails out
MailerAuthUser string // Authenticate to email server using this user
MailerAuthPass string // Authenticate to email server with this password
@@ -475,13 +475,24 @@ func (self *CGRConfig) checkConfigSanity() error {
}
}
// Loaders sanity checks
for _, ldrSCfg := range self.loaderSCfg {
for _, ldrSCfg := range self.loaderCfg {
if !ldrSCfg.Enabled {
continue
}
for _, dir := range []string{ldrSCfg.TpInDir, ldrSCfg.TpOutDir} {
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
fmt.Errorf("Nonexistent folder: %s", dir)
return fmt.Errorf("<%s> Nonexistent folder: %s", utils.LoaderS, dir)
}
}
for _, data := range ldrSCfg.Data {
if data.Type != utils.MetaAttributes {
return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type)
}
for _, field := range data.Fields {
if field.Type != utils.META_COMPOSED && field.Type != utils.MetaString {
return fmt.Errorf("<%s> invalid field type %s for %s at %s", utils.LoaderS, field.Type, data.Type, field.Tag)
}
}
}
}
@@ -790,7 +801,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) {
return err
}
jsnLoaderSCfg, err := jsnCfg.LoaderSJsonCfg()
jsnLoaderCfg, err := jsnCfg.LoaderJsonCfg()
if err != nil {
return err
}
@@ -1158,13 +1169,13 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) {
}
}
if jsnLoaderSCfg != nil {
if self.loaderSCfg == nil {
self.loaderSCfg = make([]*LoaderSConfig, len(jsnLoaderSCfg))
if jsnLoaderCfg != nil {
if self.loaderCfg == nil {
self.loaderCfg = make([]*LoaderConfig, len(jsnLoaderCfg))
}
for idx, profile := range jsnLoaderSCfg {
self.loaderSCfg[idx] = NewDfltLoadersConfig()
self.loaderSCfg[idx].loadFromJsonCfg(profile)
for idx, profile := range jsnLoaderCfg {
self.loaderCfg[idx] = NewDfltLoaderConfig()
self.loaderCfg[idx].loadFromJsonCfg(profile)
}
}
@@ -1405,6 +1416,6 @@ func (cfg *CGRConfig) CacheCfg() CacheConfig {
return cfg.cacheConfig
}
func (cfg *CGRConfig) LoaderSCfg() []*LoaderSConfig {
return cfg.loaderSCfg
func (cfg *CGRConfig) LoaderCfg() []*LoaderConfig {
return cfg.loaderCfg
}

8
config/config_json.go
View File

@@ -61,7 +61,7 @@ const (
THRESHOLDS_JSON = "thresholds"
SupplierSJson = "suppliers"
FILTERS_JSON = "filters"
LoaderSJson = "loaders"
LoaderJson = "loaders"
MAILER_JSN = "mailer"
SURETAX_JSON = "suretax"
)
@@ -401,12 +401,12 @@ func (self CgrJsonCfg) SupplierSJsonCfg() (*SupplierSJsonCfg, error) {
return cfg, nil
}
func (self CgrJsonCfg) LoaderSJsonCfg() ([]*LoaderSJsonCfg, error) {
rawCfg, hasKey := self[LoaderSJson]
func (self CgrJsonCfg) LoaderJsonCfg() ([]*LoaderJsonCfg, error) {
rawCfg, hasKey := self[LoaderJson]
if !hasKey {
return nil, nil
}
cfg := make([]*LoaderSJsonCfg, 0)
cfg := make([]*LoaderJsonCfg, 0)
if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
return nil, err
}

16
config/config_json_test.go
View File

@@ -760,8 +760,8 @@ func TestDfSupplierSJsonCfg(t *testing.T) {
}
}
func TestDfLoaderSJsonCfg(t *testing.T) {
dataType := &LoaderSJsonDataType{
func TestDfLoaderJsonCfg(t *testing.T) {
dataType := &LoaderJsonDataType{
Type: utils.StringPointer(utils.MetaAttributes),
File_name: utils.StringPointer("Attributes.csv"),
Fields: &[]*CdrFieldJsonCfg{
@@ -809,8 +809,8 @@ func TestDfLoaderSJsonCfg(t *testing.T) {
Value: utils.StringPointer("9")},
},
}
eCfg := []*LoaderSJsonCfg{
&LoaderSJsonCfg{
eCfg := []*LoaderJsonCfg{
&LoaderJsonCfg{
ID: utils.StringPointer(utils.META_DEFAULT),
Enabled: utils.BoolPointer(false),
Dry_run: utils.BoolPointer(false),
@@ -820,12 +820,12 @@ func TestDfLoaderSJsonCfg(t *testing.T) {
Address: utils.StringPointer(utils.MetaInternal),
}},
Field_separator: utils.StringPointer(","),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out"),
Data: &[]*LoaderSJsonDataType{dataType},
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/tploader/in"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/tploader/out"),
Data: &[]*LoaderJsonDataType{dataType},
},
}
if cfg, err := dfCgrJsonCfg.LoaderSJsonCfg(); err != nil {
if cfg, err := dfCgrJsonCfg.LoaderJsonCfg(); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eCfg, cfg) {
t.Errorf("Expecting: \n%s\n, received: \n%s\n: ",

16
config/config_test.go
View File

@@ -917,8 +917,8 @@ func TestDbDefaults(t *testing.T) {
}
func TestLoaderDefaults(t *testing.T) {
eCfg := []*LoaderSConfig{
&LoaderSConfig{
eCfg := []*LoaderConfig{
&LoaderConfig{
Id: utils.META_DEFAULT,
Enabled: false,
DryRun: false,
@@ -930,10 +930,10 @@ func TestLoaderDefaults(t *testing.T) {
},
},
FieldSeparator: ",",
TpInDir: "/var/spool/cgrates/loader/in",
TpOutDir: "/var/spool/cgrates/loader/out",
Data: []*LoaderSDataType{
&LoaderSDataType{
TpInDir: "/var/spool/cgrates/tploader/in",
TpOutDir: "/var/spool/cgrates/tploader/out",
Data: []*LoaderDataType{
&LoaderDataType{
Type: utils.MetaAttributes,
Filename: utils.AttributesCsv,
Fields: []*CfgCdrField{
@@ -984,7 +984,7 @@ func TestLoaderDefaults(t *testing.T) {
},
},
}
if !reflect.DeepEqual(eCfg, cgrCfg.loaderSCfg) {
t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderSCfg)
if !reflect.DeepEqual(eCfg, cgrCfg.loaderCfg) {
t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderCfg)
}
}

6
config/libconfig_json.go
View File

@@ -429,13 +429,13 @@ type SupplierSJsonCfg struct {
Stats_conns *[]*HaPoolJsonCfg
}
type LoaderSJsonDataType struct {
type LoaderJsonDataType struct {
Type *string
File_name *string
Fields *[]*CdrFieldJsonCfg
}
type LoaderSJsonCfg struct {
type LoaderJsonCfg struct {
ID *string
Enabled *bool
Dry_run *bool
@@ -445,7 +445,7 @@ type LoaderSJsonCfg struct {
Field_separator *string
Tp_in_dir *string
Tp_out_dir *string
Data *[]*LoaderSJsonDataType
Data *[]*LoaderJsonDataType
}
// Mailer config section

36
config/loadersconfig.go
View File

@@ -22,15 +22,15 @@ import (
"time"
)
func NewDfltLoadersConfig() *LoaderSConfig {
if dfltLoadersConfig == nil {
return new(LoaderSConfig)
func NewDfltLoaderConfig() *LoaderConfig {
if dfltLoaderConfig == nil {
return new(LoaderConfig)
}
dfltVal := *dfltLoadersConfig
dfltVal := *dfltLoaderConfig
return &dfltVal
}
type LoaderSConfig struct {
type LoaderConfig struct { // rename to LoaderConfig
Id string
Enabled bool
DryRun bool
@@ -40,24 +40,24 @@ type LoaderSConfig struct {
FieldSeparator string
TpInDir string
TpOutDir string
Data []*LoaderSDataType
Data []*LoaderDataType
}
func NewDfltLoaderSDataTypeConfig() *LoaderSDataType {
if dfltLoaderSDataTypeConfig == nil {
return new(LoaderSDataType) // No defaults, most probably we are building the defaults now
func NewDfltLoaderDataTypeConfig() *LoaderDataType {
if dfltLoaderDataTypeConfig == nil {
return new(LoaderDataType) // No defaults, most probably we are building the defaults now
}
dfltVal := *dfltLoaderSDataTypeConfig // Copy the value instead of it's pointer
dfltVal := *dfltLoaderDataTypeConfig // Copy the value instead of it's pointer
return &dfltVal
}
type LoaderSDataType struct {
type LoaderDataType struct { //rename to LoaderDataType
Type string
Filename string
Fields []*CfgCdrField
}
func (self *LoaderSDataType) loadFromJsonCfg(jsnCfg *LoaderSJsonDataType) error {
func (self *LoaderDataType) loadFromJsonCfg(jsnCfg *LoaderJsonDataType) error {
if jsnCfg == nil {
return nil
}
@@ -76,7 +76,7 @@ func (self *LoaderSDataType) loadFromJsonCfg(jsnCfg *LoaderSJsonDataType) error
return nil
}
func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error {
func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error {
if jsnCfg == nil {
return nil
}
@@ -112,9 +112,9 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error {
self.TpOutDir = *jsnCfg.Tp_out_dir
}
if jsnCfg.Data != nil {
self.Data = make([]*LoaderSDataType, len(*jsnCfg.Data))
self.Data = make([]*LoaderDataType, len(*jsnCfg.Data))
for idx, jsnLoCfg := range *jsnCfg.Data {
self.Data[idx] = NewDfltLoaderSDataTypeConfig()
self.Data[idx] = NewDfltLoaderDataTypeConfig()
self.Data[idx].loadFromJsonCfg(jsnLoCfg)
}
}
@@ -122,8 +122,8 @@ func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderSJsonCfg) error {
}
// Clone itself into a new LoadersConfig
func (self *LoaderSConfig) Clone() *LoaderSConfig {
clnLoader := new(LoaderSConfig)
func (self *LoaderConfig) Clone() *LoaderConfig {
clnLoader := new(LoaderConfig)
clnLoader.Id = self.Id
clnLoader.Enabled = self.Enabled
clnLoader.DryRun = self.DryRun
@@ -137,7 +137,7 @@ func (self *LoaderSConfig) Clone() *LoaderSConfig {
clnLoader.FieldSeparator = self.FieldSeparator
clnLoader.TpInDir = self.TpInDir
clnLoader.TpOutDir = self.TpOutDir
clnLoader.Data = make([]*LoaderSDataType, len(self.Data))
clnLoader.Data = make([]*LoaderDataType, len(self.Data))
for idx, fld := range self.Data {
clonedVal := *fld
clnLoader.Data[idx] = &clonedVal

33
data/conf/samples/tutmysql/cgrates.json
View File

@@ -109,6 +109,39 @@
},
},
"loaders": [
{
"id": "*default", // identifier of the Loader
"enabled": false, // starts as service: <true|false>.
"dry_run": false, // do not send the CDRs to CDRS, just parse them
"run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify
"lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing
"caches_conns": [
{"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234>
],
"field_separator": ",", // separator used in case of csv files
"tp_in_dir": "/var/spool/cgrates/tploader/in", // absolute path towards the directory where the CDRs are stored
"tp_out_dir": "/tmp", // absolute path towards the directory where processed CDRs will be moved
"data":[ // data profiles to load
{
"type": "*attributes", // data source type
"file_name": "Attributes.csv", // file name in the tp_in_dir
"fields": [
{"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true},
{"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true},
{"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "2"},
{"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "3"},
{"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "4"},
{"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "5"},
{"tag": "Initial", "field_id": "Initial", "type": "*composed", "value": "6"},
{"tag": "Substitute", "field_id": "Substitute", "type": "*composed", "value": "7"},
{"tag": "Append", "field_id": "Append", "type": "*composed", "value": "8"},
{"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"},
],
},
],
},
],
"cdrstats": {
"enabled": true,