Config JSON with sample loaders

2026-02-24 16:48:45 +05:00 · 2018-03-13 19:25:40 +01:00
parent 7eb7586c48
commit 90339dbc0f
5 changed files with 144 additions and 0 deletions
--- a/config/config_defaults.go
+++ b/config/config_defaults.go
@@ -450,6 +450,41 @@ const CGRATES_CFG_JSON = `
 },


+"loaders": [
+	{
+		"id": "*default",									// identifier of the Loader
+		"enabled": false,									// starts as service: <true|false>.
+		"dry_run": false,									// do not send the CDRs to CDRS, just parse them
+		"caches_conns": [
+			{"address": "*internal"},                       // address where to reach the CacheS for data reload, empty for no reloads  <""|*internal|x.y.z.y:1234>
+		],
+		"field_separator": ",",								// separator used in case of csv files
+		"max_open_files": 1024,								// maximum simultaneous files to process, 0 for unlimited
+		"tp_in_dir": "/var/spool/cgrates/tploader/in",		// absolute path towards the directory where the CDRs are stored
+		"tp_out_dir": "/var/spool/cgrates/tploader/out",	// absolute path towards the directory where processed CDRs will be moved
+	
+		"data":[											// data profiles to load
+			{
+				"type": "*attributes",						// data source type
+				"file_name": "Attributes.csv", 				// file name in the tp_in_dir
+				"fields": [
+					{"tag": "TenantID", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true},
+					{"tag": "ProfileID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true},
+					{"tag": "Contexts", "field_id": "Contexts", "type": "*composed", "value": "2"},
+					{"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "3"},
+					{"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "4"},
+					{"tag": "FieldName", "field_id": "FieldName", "type": "*composed", "value": "5"},
+					{"tag": "Initial", "field_id": "Initial", "type": "*composed", "value": "6"},
+					{"tag": "Substitute", "field_id": "Substitute", "type": "*composed", "value": "7"},
+					{"tag": "Append", "field_id": "Append", "type": "*composed", "value": "8"},
+					{"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"},
+				],
+			},
+		],
+	},
+],
+
+
 "mailer": {
 	"server": "localhost",								// the server to use when sending emails out
 	"auth_user": "cgrates",								// authenticate to email server using this user
--- a/config/config_json.go
+++ b/config/config_json.go
@@ -61,6 +61,7 @@ const (
 	THRESHOLDS_JSON    = "thresholds"
 	SupplierSJson      = "suppliers"
 	FILTERS_JSON       = "filters"
+	LoaderSJson        = "loaders"
 	MAILER_JSN         = "mailer"
 	SURETAX_JSON       = "suretax"
 )
@@ -400,6 +401,18 @@ func (self CgrJsonCfg) SupplierSJsonCfg() (*SupplierSJsonCfg, error) {
 	return cfg, nil
 }

+func (self CgrJsonCfg) LoaderSJsonCfg() ([]*LoaderSJsonCfg, error) {
+	rawCfg, hasKey := self[LoaderSJson]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make([]*LoaderSJsonCfg, 0)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
 func (self CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) {
 	rawCfg, hasKey := self[MAILER_JSN]
 	if !hasKey {
--- a/config/config_json_test.go
+++ b/config/config_json_test.go
@@ -760,6 +760,78 @@ func TestDfSupplierSJsonCfg(t *testing.T) {
 	}
 }

+func TestDfLoaderSJsonCfg(t *testing.T) {
+	dataType := &LoaderSJsonDataType{
+		Type:      utils.StringPointer(utils.MetaAttributes),
+		File_name: utils.StringPointer("Attributes.csv"),
+		Fields: &[]*CdrFieldJsonCfg{
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("TenantID"),
+				Field_id:  utils.StringPointer(utils.Tenant),
+				Type:      utils.StringPointer(utils.META_COMPOSED),
+				Value:     utils.StringPointer("0"),
+				Mandatory: utils.BoolPointer(true)},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("ProfileID"),
+				Field_id:  utils.StringPointer(utils.ID),
+				Type:      utils.StringPointer(utils.META_COMPOSED),
+				Value:     utils.StringPointer("1"),
+				Mandatory: utils.BoolPointer(true)},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("Contexts"),
+				Field_id: utils.StringPointer(utils.Contexts),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("2")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"),
+				Field_id: utils.StringPointer(utils.FilterIDs),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("3")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"),
+				Field_id: utils.StringPointer("ActivationInterval"),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("4")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("FieldName"),
+				Field_id: utils.StringPointer(utils.FieldName),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("5")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("Initial"),
+				Field_id: utils.StringPointer(utils.Initial),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("6")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("Substitute"),
+				Field_id: utils.StringPointer(utils.Substitute),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("7")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("Append"),
+				Field_id: utils.StringPointer(utils.Append),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("8")},
+			&CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"),
+				Field_id: utils.StringPointer(utils.Weight),
+				Type:     utils.StringPointer(utils.META_COMPOSED),
+				Value:    utils.StringPointer("9")},
+		},
+	}
+	eCfg := []*LoaderSJsonCfg{
+		&LoaderSJsonCfg{
+			ID:      utils.StringPointer(utils.META_DEFAULT),
+			Enabled: utils.BoolPointer(false),
+			Dry_run: utils.BoolPointer(false),
+			Caches_conns: &[]*HaPoolJsonCfg{&HaPoolJsonCfg{
+				Address: utils.StringPointer(utils.MetaInternal),
+			}},
+			Field_separator: utils.StringPointer(","),
+			Max_open_files:  utils.IntPointer(1024),
+			Tp_in_dir:       utils.StringPointer("/var/spool/cgrates/tploader/in"),
+			Tp_out_dir:      utils.StringPointer("/var/spool/cgrates/tploader/out"),
+			Data:            &[]*LoaderSJsonDataType{dataType},
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.LoaderSJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Errorf("Expecting: \n%s\n, received: \n%s\n: ",
+			utils.ToIJSON(eCfg), utils.ToIJSON(cfg))
+	}
+}
+
 func TestDfMailerJsonCfg(t *testing.T) {
 	eCfg := &MailerJsonCfg{
 		Server:        utils.StringPointer("localhost"),
--- a/config/libconfig_json.go
+++ b/config/libconfig_json.go
@@ -427,6 +427,24 @@ type SupplierSJsonCfg struct {
 	Stats_conns           *[]*HaPoolJsonCfg
 }

+type LoaderSJsonDataType struct {
+	Type      *string
+	File_name *string
+	Fields    *[]*CdrFieldJsonCfg
+}
+
+type LoaderSJsonCfg struct {
+	ID              *string
+	Enabled         *bool
+	Dry_run         *bool
+	Caches_conns    *[]*HaPoolJsonCfg
+	Field_separator *string
+	Max_open_files  *int
+	Tp_in_dir       *string
+	Tp_out_dir      *string
+	Data            *[]*LoaderSJsonDataType
+}
+
 // Mailer config section
 type MailerJsonCfg struct {
 	Server        *string
--- a/utils/consts.go
+++ b/utils/consts.go
@@ -156,6 +156,7 @@ const (
 	Tenant                        = "Tenant"
 	Category                      = "Category"
 	Context                       = "Context"
+	Contexts                      = "Contexts"
 	Account                       = "Account"
 	Subject                       = "Subject"
 	Destination                   = "Destination"
@@ -523,6 +524,11 @@ const (
 	MetaCGRReply                 = "*cgrReply"
 	CacheS                       = "CacheS"
 	CGR_ACD                      = "cgr_acd"
+	FilterIDs                    = "FilterIDs"
+	FieldName                    = "FieldName"
+	Initial                      = "Initial"
+	Substitute                   = "Substitute"
+	Append                       = "Append"
 )

 //MetaMetrics