From 90339dbc0f8755198106cc15b23d77214dc4b7f8 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 13 Mar 2018 19:25:40 +0100 Subject: [PATCH] Config JSON with sample loaders --- config/config_defaults.go | 35 ++++++++++++++++++ config/config_json.go | 13 +++++++ config/config_json_test.go | 72 ++++++++++++++++++++++++++++++++++++++ config/libconfig_json.go | 18 ++++++++++ utils/consts.go | 6 ++++ 5 files changed, 144 insertions(+) diff --git a/config/config_defaults.go b/config/config_defaults.go index 4f8176f63..fbb79a980 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -450,6 +450,41 @@ const CGRATES_CFG_JSON = ` }, +"loaders": [ + { + "id": "*default", // identifier of the Loader + "enabled": false, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "max_open_files": 1024, // maximum simultaneous files to process, 0 for unlimited + "tp_in_dir": "/var/spool/cgrates/tploader/in", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/var/spool/cgrates/tploader/out", // absolute path towards the directory where processed CDRs will be moved + + "data":[ // data profiles to load + { + "type": "*attributes", // data source type + "file_name": "Attributes.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "2"}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "3"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "4"}, + {"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "5"}, + {"tag": "Initial", "field_id": "Initial", "type": "*composed", "value": "6"}, + {"tag": "Substitute", "field_id": "Substitute", "type": "*composed", "value": "7"}, + {"tag": "Append", "field_id": "Append", "type": "*composed", "value": "8"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"}, + ], + }, + ], + }, +], + + "mailer": { "server": "localhost", // the server to use when sending emails out "auth_user": "cgrates", // authenticate to email server using this user diff --git a/config/config_json.go b/config/config_json.go index f6d83ef76..17779319b 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -61,6 +61,7 @@ const ( THRESHOLDS_JSON = "thresholds" SupplierSJson = "suppliers" FILTERS_JSON = "filters" + LoaderSJson = "loaders" MAILER_JSN = "mailer" SURETAX_JSON = "suretax" ) @@ -400,6 +401,18 @@ func (self CgrJsonCfg) SupplierSJsonCfg() (*SupplierSJsonCfg, error) { return cfg, nil } +func (self CgrJsonCfg) LoaderSJsonCfg() ([]*LoaderSJsonCfg, error) { + rawCfg, hasKey := self[LoaderSJson] + if !hasKey { + return nil, nil + } + cfg := make([]*LoaderSJsonCfg, 0) + if err := json.Unmarshal(*rawCfg, &cfg); err != nil { + return nil, err + } + return cfg, nil +} + func (self CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) { rawCfg, hasKey := self[MAILER_JSN] if !hasKey { diff --git a/config/config_json_test.go b/config/config_json_test.go index c690127d2..e1131cafc 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -760,6 +760,78 @@ func TestDfSupplierSJsonCfg(t *testing.T) { } } +func TestDfLoaderSJsonCfg(t *testing.T) { + dataType := &LoaderSJsonDataType{ + Type: utils.StringPointer(utils.MetaAttributes), + File_name: utils.StringPointer("Attributes.csv"), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer("TenantID"), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ProfileID"), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Contexts"), + Field_id: utils.StringPointer(utils.Contexts), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer(utils.FilterIDs), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FieldName"), + Field_id: utils.StringPointer(utils.FieldName), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Initial"), + Field_id: utils.StringPointer(utils.Initial), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Substitute"), + Field_id: utils.StringPointer(utils.Substitute), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Append"), + Field_id: utils.StringPointer(utils.Append), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer(utils.Weight), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + }, + } + eCfg := []*LoaderSJsonCfg{ + &LoaderSJsonCfg{ + ID: utils.StringPointer(utils.META_DEFAULT), + Enabled: utils.BoolPointer(false), + Dry_run: utils.BoolPointer(false), + Caches_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{ + Address: utils.StringPointer(utils.MetaInternal), + }}, + Field_separator: utils.StringPointer(","), + Max_open_files: utils.IntPointer(1024), + Tp_in_dir: utils.StringPointer("/var/spool/cgrates/tploader/in"), + Tp_out_dir: utils.StringPointer("/var/spool/cgrates/tploader/out"), + Data: &[]*LoaderSJsonDataType{dataType}, + }, + } + if cfg, err := dfCgrJsonCfg.LoaderSJsonCfg(); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(eCfg, cfg) { + t.Errorf("Expecting: \n%s\n, received: \n%s\n: ", + utils.ToIJSON(eCfg), utils.ToIJSON(cfg)) + } +} + func TestDfMailerJsonCfg(t *testing.T) { eCfg := &MailerJsonCfg{ Server: utils.StringPointer("localhost"), diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 12ab1a941..3e7fcb4cb 100755 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -427,6 +427,24 @@ type SupplierSJsonCfg struct { Stats_conns *[]*HaPoolJsonCfg } +type LoaderSJsonDataType struct { + Type *string + File_name *string + Fields *[]*CdrFieldJsonCfg +} + +type LoaderSJsonCfg struct { + ID *string + Enabled *bool + Dry_run *bool + Caches_conns *[]*HaPoolJsonCfg + Field_separator *string + Max_open_files *int + Tp_in_dir *string + Tp_out_dir *string + Data *[]*LoaderSJsonDataType +} + // Mailer config section type MailerJsonCfg struct { Server *string diff --git a/utils/consts.go b/utils/consts.go index 7943977bc..41333a3e0 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -156,6 +156,7 @@ const ( Tenant = "Tenant" Category = "Category" Context = "Context" + Contexts = "Contexts" Account = "Account" Subject = "Subject" Destination = "Destination" @@ -523,6 +524,11 @@ const ( MetaCGRReply = "*cgrReply" CacheS = "CacheS" CGR_ACD = "cgr_acd" + FilterIDs = "FilterIDs" + FieldName = "FieldName" + Initial = "Initial" + Substitute = "Substitute" + Append = "Append" ) //MetaMetrics