diff --git a/config/config.go b/config/config.go index 375a590bf..c6de12f08 100755 --- a/config/config.go +++ b/config/config.go @@ -171,6 +171,7 @@ func NewDefaultCGRConfig() (*CGRConfig, error) { dfltFsConnConfig = cfg.fsAgentCfg.EventSocketConns[0] // We leave it crashing here on purpose if no Connection defaults defined dfltKamConnConfig = cfg.kamAgentCfg.EvapiConns[0] dfltAstConnCfg = cfg.asteriskAgentCfg.AsteriskConns[0] + dfltLoaderConfig = cfg.loaderCfg[0].Clone() if err := cfg.checkConfigSanity(); err != nil { return nil, err } @@ -490,7 +491,8 @@ func (self *CGRConfig) checkConfigSanity() error { } for _, data := range ldrSCfg.Data { if !utils.IsSliceMember([]string{utils.MetaAttributes, - utils.MetaResources, utils.MetaFilters}, data.Type) { + utils.MetaResources, utils.MetaFilters, utils.MetaStats, + utils.MetaSuppliers, utils.MetaThresholds}, data.Type) { return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type) } @@ -670,7 +672,6 @@ func (self *CGRConfig) checkConfigSanity() error { // Loads from json configuration object, will be used for defaults, config from file and reload, might need lock func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { - // Load sections out of JSON config, stop on error jsnGeneralCfg, err := jsnCfg.GeneralJsonCfg() if err != nil { @@ -1182,9 +1183,7 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) { } if jsnLoaderCfg != nil { - if self.loaderCfg == nil { - self.loaderCfg = make([]*LoaderConfig, len(jsnLoaderCfg)) - } + self.loaderCfg = make([]*LoaderConfig, len(jsnLoaderCfg)) for idx, profile := range jsnLoaderCfg { self.loaderCfg[idx] = NewDfltLoaderConfig() self.loaderCfg[idx].loadFromJsonCfg(profile) diff --git a/config/config_defaults.go b/config/config_defaults.go index bd084fb1a..f80d3b645 100755 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -482,6 +482,93 @@ const CGRATES_CFG_JSON = ` {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"}, ], }, + { + "type": "*filters", // data source type + "file_name": "Filters.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterType", "field_id": "FilterType", "type": "*composed", "value": "2"}, + {"tag": "FilterFieldName", "field_id": "FilterFieldName", "type": "*composed", "value": "3"}, + {"tag": "FilterFieldValues", "field_id": "FilterFieldValues", "type": "*composed", "value": "4"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "5"}, + ], + }, + { + "type": "*resources", // data source type + "file_name": "Resources.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "TTL", "field_id": "UsageTTL", "type": "*composed", "value": "4"}, + {"tag": "Limit", "field_id": "Limit", "type": "*composed", "value": "5"}, + {"tag": "AllocationMessage", "field_id": "AllocationMessage", "type": "*composed", "value": "6"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "7"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "8"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "9"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "10"}, + ], + }, + { + "type": "*stats", // data source type + "file_name": "Stats.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "QueueLength", "field_id": "QueueLength", "type": "*composed", "value": "4"}, + {"tag": "TTL", "field_id": "TTL", "type": "*composed", "value": "5"}, + {"tag": "Metrics", "field_id": "Metrics", "type": "*composed", "value": "6"}, + {"tag": "MetricParams", "field_id": "Parameters", "type": "*composed", "value": "7"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "8"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "10"}, + {"tag": "MinItems", "field_id": "MinItems", "type": "*composed", "value": "11"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "12"}, + ], + }, + { + "type": "*thresholds", // data source type + "file_name": "Thresholds.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "Recurrent", "field_id": "Recurrent", "type": "*composed", "value": "4"}, + {"tag": "MinHits", "field_id": "MinHits", "type": "*composed", "value": "5"}, + {"tag": "MinSleep", "field_id": "MinSleep", "type": "*composed", "value": "6"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "7"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "8"}, + {"tag": "ActionIDs", "field_id": "ActionIDs", "type": "*composed", "value": "9"}, + {"tag": "Async", "field_id": "Async", "type": "*composed", "value": "10"}, + ], + }, + { + "type": "*suppliers", // data source type + "file_name": "Suppliers.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "Sorting", "field_id": "Sorting", "type": "*composed", "value": "4"}, + {"tag": "SortingParamameters", "field_id": "SortingParamameters", "type": "*composed", "value": "5"}, + {"tag": "SupplierID", "field_id": "SupplierID", "type": "*composed", "value": "6"}, + {"tag": "SupplierFilterIDs", "field_id": "SupplierFilterIDs", "type": "*composed", "value": "7"}, + {"tag": "SupplierAccountIDs", "field_id": "SupplierAccountIDs", "type": "*composed", "value": "8"}, + {"tag": "SupplierRatingPlanIDs", "field_id": "SupplierRatingPlanIDs", "type": "*composed", "value": "9"}, + {"tag": "SupplierResourceIDs", "field_id": "SupplierResourceIDs", "type": "*composed", "value": "10"}, + {"tag": "SupplierStatIDs", "field_id": "SupplierStatIDs", "type": "*composed", "value": "11"}, + {"tag": "SupplierWeight", "field_id": "SupplierWeight", "type": "*composed", "value": "12"}, + {"tag": "SupplierBlocker", "field_id": "SupplierBlocker", "type": "*composed", "value": "13"}, + {"tag": "SupplierParameters", "field_id": "SupplierParameters", "type": "*composed", "value": "14"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "15"}, + ], + }, ], }, ], diff --git a/config/config_json_test.go b/config/config_json_test.go index 5bc1d985a..676ab7b79 100755 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -761,54 +761,6 @@ func TestDfSupplierSJsonCfg(t *testing.T) { } func TestDfLoaderJsonCfg(t *testing.T) { - dataType := &LoaderJsonDataType{ - Type: utils.StringPointer(utils.MetaAttributes), - File_name: utils.StringPointer("Attributes.csv"), - Fields: &[]*CdrFieldJsonCfg{ - &CdrFieldJsonCfg{Tag: utils.StringPointer("TenantID"), - Field_id: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("0"), - Mandatory: utils.BoolPointer(true)}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("ProfileID"), - Field_id: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("1"), - Mandatory: utils.BoolPointer(true)}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("Contexts"), - Field_id: utils.StringPointer(utils.Contexts), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("2")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), - Field_id: utils.StringPointer(utils.FilterIDs), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("3")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), - Field_id: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("4")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("FieldName"), - Field_id: utils.StringPointer(utils.FieldName), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("5")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("Initial"), - Field_id: utils.StringPointer(utils.Initial), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("6")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("Substitute"), - Field_id: utils.StringPointer(utils.Substitute), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("7")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("Append"), - Field_id: utils.StringPointer(utils.Append), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("8")}, - &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), - Field_id: utils.StringPointer(utils.Weight), - Type: utils.StringPointer(utils.META_COMPOSED), - Value: utils.StringPointer("9")}, - }, - } eCfg := []*LoaderJsonCfg{ &LoaderJsonCfg{ ID: utils.StringPointer(utils.META_DEFAULT), @@ -822,13 +774,330 @@ func TestDfLoaderJsonCfg(t *testing.T) { Field_separator: utils.StringPointer(","), Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in"), Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out"), - Data: &[]*LoaderJsonDataType{dataType}, + Data: &[]*LoaderJsonDataType{ + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaAttributes), + File_name: utils.StringPointer(utils.AttributesCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer("TenantID"), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ProfileID"), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Contexts"), + Field_id: utils.StringPointer(utils.Contexts), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer(utils.FilterIDs), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FieldName"), + Field_id: utils.StringPointer(utils.FieldName), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Initial"), + Field_id: utils.StringPointer(utils.Initial), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Substitute"), + Field_id: utils.StringPointer(utils.Substitute), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Append"), + Field_id: utils.StringPointer(utils.Append), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer(utils.Weight), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + }, + }, + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaFilters), + File_name: utils.StringPointer(utils.FiltersCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.Tenant), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.ID), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterType"), + Field_id: utils.StringPointer("FilterType"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterFieldName"), + Field_id: utils.StringPointer("FilterFieldName"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterFieldValues"), + Field_id: utils.StringPointer("FilterFieldValues"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + }, + }, + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaResources), + File_name: utils.StringPointer(utils.ResourcesCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.Tenant), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.ID), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer("FilterIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("TTL"), + Field_id: utils.StringPointer("UsageTTL"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Limit"), + Field_id: utils.StringPointer("Limit"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("AllocationMessage"), + Field_id: utils.StringPointer("AllocationMessage"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Blocker"), + Field_id: utils.StringPointer("Blocker"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Stored"), + Field_id: utils.StringPointer("Stored"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer("Weight"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ThresholdIDs"), + Field_id: utils.StringPointer("ThresholdIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("10")}, + }, + }, + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaStats), + File_name: utils.StringPointer(utils.StatsCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.Tenant), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.ID), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer("FilterIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("QueueLength"), + Field_id: utils.StringPointer("QueueLength"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("TTL"), + Field_id: utils.StringPointer("TTL"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Metrics"), + Field_id: utils.StringPointer("Metrics"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("MetricParams"), + Field_id: utils.StringPointer("Parameters"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Blocker"), + Field_id: utils.StringPointer("Blocker"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Stored"), + Field_id: utils.StringPointer("Stored"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer("Weight"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("10")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("MinItems"), + Field_id: utils.StringPointer("MinItems"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("11")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ThresholdIDs"), + Field_id: utils.StringPointer("ThresholdIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("12")}, + }, + }, + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaThresholds), + File_name: utils.StringPointer(utils.ThresholdsCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.Tenant), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.ID), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer("FilterIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Recurrent"), + Field_id: utils.StringPointer("Recurrent"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("MinHits"), + Field_id: utils.StringPointer("MinHits"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("MinSleep"), + Field_id: utils.StringPointer("MinSleep"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Blocker"), + Field_id: utils.StringPointer("Blocker"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer("Weight"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActionIDs"), + Field_id: utils.StringPointer("ActionIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Async"), + Field_id: utils.StringPointer("Async"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("10")}, + }, + }, + &LoaderJsonDataType{ + Type: utils.StringPointer(utils.MetaSuppliers), + File_name: utils.StringPointer(utils.SuppliersCsv), + Fields: &[]*CdrFieldJsonCfg{ + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.Tenant), + Field_id: utils.StringPointer(utils.Tenant), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("0"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer(utils.ID), + Field_id: utils.StringPointer(utils.ID), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("1"), + Mandatory: utils.BoolPointer(true)}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("FilterIDs"), + Field_id: utils.StringPointer("FilterIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("2")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("ActivationInterval"), + Field_id: utils.StringPointer("ActivationInterval"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("3")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Sorting"), + Field_id: utils.StringPointer("Sorting"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("4")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SortingParamameters"), + Field_id: utils.StringPointer("SortingParamameters"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("5")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierID"), + Field_id: utils.StringPointer("SupplierID"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("6")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierFilterIDs"), + Field_id: utils.StringPointer("SupplierFilterIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("7")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierAccountIDs"), + Field_id: utils.StringPointer("SupplierAccountIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("8")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierRatingPlanIDs"), + Field_id: utils.StringPointer("SupplierRatingPlanIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("9")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierResourceIDs"), + Field_id: utils.StringPointer("SupplierResourceIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("10")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierStatIDs"), + Field_id: utils.StringPointer("SupplierStatIDs"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("11")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierWeight"), + Field_id: utils.StringPointer("SupplierWeight"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("12")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierBlocker"), + Field_id: utils.StringPointer("SupplierBlocker"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("13")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("SupplierParameters"), + Field_id: utils.StringPointer("SupplierParameters"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("14")}, + &CdrFieldJsonCfg{Tag: utils.StringPointer("Weight"), + Field_id: utils.StringPointer("Weight"), + Type: utils.StringPointer(utils.META_COMPOSED), + Value: utils.StringPointer("15")}, + }, + }, + }, }, } if cfg, err := dfCgrJsonCfg.LoaderJsonCfg(); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCfg, cfg) { - t.Errorf("Expecting: \n%s\n, received: \n%s\n: ", + t.Errorf("Expecting: %s, received: %s ", utils.ToIJSON(eCfg), utils.ToIJSON(cfg)) } } diff --git a/config/config_test.go b/config/config_test.go index d96e6e515..d1ec8198c 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -914,7 +914,7 @@ func TestDbDefaults(t *testing.T) { } } -func TestLoaderDefaults(t *testing.T) { +func TestCgrCfgLoaderDefaults(t *testing.T) { eCfg := []*LoaderConfig{ &LoaderConfig{ Id: utils.META_DEFAULT, @@ -979,11 +979,280 @@ func TestLoaderDefaults(t *testing.T) { Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP)}, }, }, + &LoaderDataType{ + Type: utils.MetaFilters, + Filename: utils.FiltersCsv, + Fields: []*CfgCdrField{ + &CfgCdrField{Tag: "Tenant", + FieldId: "Tenant", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("0", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "ID", + FieldId: "ID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "FilterType", + FieldId: "FilterType", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "FilterFieldName", + FieldId: "FilterFieldName", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "FilterFieldValues", + FieldId: "FilterFieldValues", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActivationInterval", + FieldId: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP)}, + }, + }, + &LoaderDataType{ + Type: utils.MetaResources, + Filename: utils.ResourcesCsv, + Fields: []*CfgCdrField{ + &CfgCdrField{Tag: "Tenant", + FieldId: "Tenant", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("0", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "ID", + FieldId: "ID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "FilterIDs", + FieldId: "FilterIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActivationInterval", + FieldId: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TTL", + FieldId: "UsageTTL", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Limit", + FieldId: "Limit", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "AllocationMessage", + FieldId: "AllocationMessage", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Blocker", + FieldId: "Blocker", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Stored", + FieldId: "Stored", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Weight", + FieldId: "Weight", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ThresholdIDs", + FieldId: "ThresholdIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP)}, + }, + }, + &LoaderDataType{ + Type: utils.MetaStats, + Filename: utils.StatsCsv, + Fields: []*CfgCdrField{ + &CfgCdrField{Tag: "Tenant", + FieldId: "Tenant", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("0", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "ID", + FieldId: "ID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "FilterIDs", + FieldId: "FilterIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActivationInterval", + FieldId: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "QueueLength", + FieldId: "QueueLength", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "TTL", + FieldId: "TTL", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Metrics", + FieldId: "Metrics", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "MetricParams", + FieldId: "Parameters", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Blocker", + FieldId: "Blocker", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Stored", + FieldId: "Stored", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Weight", + FieldId: "Weight", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "MinItems", + FieldId: "MinItems", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ThresholdIDs", + FieldId: "ThresholdIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP)}, + }, + }, + &LoaderDataType{ + Type: utils.MetaThresholds, + Filename: utils.ThresholdsCsv, + Fields: []*CfgCdrField{ + &CfgCdrField{Tag: "Tenant", + FieldId: "Tenant", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("0", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "ID", + FieldId: "ID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "FilterIDs", + FieldId: "FilterIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActivationInterval", + FieldId: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Recurrent", + FieldId: "Recurrent", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "MinHits", + FieldId: "MinHits", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "MinSleep", + FieldId: "MinSleep", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Blocker", + FieldId: "Blocker", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Weight", + FieldId: "Weight", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActionIDs", + FieldId: "ActionIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Async", + FieldId: "Async", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP)}, + }, + }, + &LoaderDataType{ + Type: utils.MetaSuppliers, + Filename: utils.SuppliersCsv, + Fields: []*CfgCdrField{ + &CfgCdrField{Tag: "Tenant", + FieldId: "Tenant", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("0", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "ID", + FieldId: "ID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("1", utils.INFIELD_SEP), + Mandatory: true}, + &CfgCdrField{Tag: "FilterIDs", + FieldId: "FilterIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "ActivationInterval", + FieldId: "ActivationInterval", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Sorting", + FieldId: "Sorting", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SortingParamameters", + FieldId: "SortingParamameters", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierID", + FieldId: "SupplierID", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierFilterIDs", + FieldId: "SupplierFilterIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierAccountIDs", + FieldId: "SupplierAccountIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierRatingPlanIDs", + FieldId: "SupplierRatingPlanIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierResourceIDs", + FieldId: "SupplierResourceIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierStatIDs", + FieldId: "SupplierStatIDs", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierWeight", + FieldId: "SupplierWeight", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierBlocker", + FieldId: "SupplierBlocker", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "SupplierParameters", + FieldId: "SupplierParameters", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("14", utils.INFIELD_SEP)}, + &CfgCdrField{Tag: "Weight", + FieldId: "Weight", + Type: utils.META_COMPOSED, + Value: utils.ParseRSRFieldsMustCompile("15", utils.INFIELD_SEP)}, + }, + }, }, }, } if !reflect.DeepEqual(eCfg, cgrCfg.loaderCfg) { - t.Errorf("received: %+v, expecting: %+v", eCfg, cgrCfg.loaderCfg) + t.Errorf("received: %+v, expecting: %+v", + utils.ToJSON(eCfg), utils.ToJSON(cgrCfg.loaderCfg)) } } diff --git a/config/loadersconfig.go b/config/loadersconfig.go index ba3b41f7c..7463b1a61 100644 --- a/config/loadersconfig.go +++ b/config/loadersconfig.go @@ -58,10 +58,10 @@ type LoaderDataType struct { //rename to LoaderDataType } func (self *LoaderDataType) loadFromJsonCfg(jsnCfg *LoaderJsonDataType) error { + if jsnCfg == nil { return nil } - var err error if jsnCfg.Type != nil { self.Type = *jsnCfg.Type } @@ -69,8 +69,10 @@ func (self *LoaderDataType) loadFromJsonCfg(jsnCfg *LoaderJsonDataType) error { self.Filename = *jsnCfg.File_name } if jsnCfg.Fields != nil { - if self.Fields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Fields); err != nil { + if fields, err := CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Fields); err != nil { return err + } else { + self.Fields = fields } } return nil @@ -96,11 +98,12 @@ func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error { self.LockFileName = *jsnCfg.Lock_filename } if jsnCfg.Caches_conns != nil { - self.CacheSConns = make([]*HaPoolConfig, len(*jsnCfg.Caches_conns)) + cacheConns := make([]*HaPoolConfig, len(*jsnCfg.Caches_conns)) for idx, jsnHaCfg := range *jsnCfg.Caches_conns { - self.CacheSConns[idx] = NewDfltHaPoolConfig() - self.CacheSConns[idx].loadFromJsonCfg(jsnHaCfg) + cacheConns[idx] = NewDfltHaPoolConfig() + cacheConns[idx].loadFromJsonCfg(jsnHaCfg) } + self.CacheSConns = cacheConns } if jsnCfg.Field_separator != nil { self.FieldSeparator = *jsnCfg.Field_separator @@ -112,12 +115,14 @@ func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error { self.TpOutDir = *jsnCfg.Tp_out_dir } if jsnCfg.Data != nil { - self.Data = make([]*LoaderDataType, len(*jsnCfg.Data)) + data := make([]*LoaderDataType, len(*jsnCfg.Data)) for idx, jsnLoCfg := range *jsnCfg.Data { - self.Data[idx] = NewDfltLoaderDataTypeConfig() - self.Data[idx].loadFromJsonCfg(jsnLoCfg) + data[idx] = NewDfltLoaderDataTypeConfig() + data[idx].loadFromJsonCfg(jsnLoCfg) } + self.Data = data } + return nil } diff --git a/data/conf/samples/tutmysql/cgrates.json b/data/conf/samples/tutmysql/cgrates.json index 75ee1fd7c..18d3e97b7 100644 --- a/data/conf/samples/tutmysql/cgrates.json +++ b/data/conf/samples/tutmysql/cgrates.json @@ -117,8 +117,8 @@ "loaders": [ { - "id": "FilterResourceLoader", // identifier of the Loader - "enabled": false, // starts as service: . + "id": "TeoLoader", // identifier of the Loader + "enabled": true, // starts as service: . "dry_run": false, // do not send the CDRs to CDRS, just parse them "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing @@ -129,18 +129,6 @@ "tp_in_dir": "/tmp/In", // absolute path towards the directory where the CDRs are stored "tp_out_dir": "/tmp/Out", // absolute path towards the directory where processed CDRs will be moved "data":[ - { - "type": "*filters", // data source type - "file_name": "Filters.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, - {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, - {"tag": "FilterType", "field_id": "FilterType", "type": "*composed", "value": "2"}, - {"tag": "FilterFieldName", "field_id": "FilterFieldName", "type": "*composed", "value": "3"}, - {"tag": "FilterFieldValues", "field_id": "FilterFieldValues", "type": "*composed", "value": "4"}, - {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "5"}, - ], - }, { "type": "*resources", // data source type "file_name": "Resources.csv", // file name in the tp_in_dir @@ -158,10 +146,96 @@ {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "10"}, ], }, + { + "type": "*stats", // data source type + "file_name": "Stats.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "QueueLength", "field_id": "QueueLength", "type": "*composed", "value": "4"}, + {"tag": "TTL", "field_id": "TTL", "type": "*composed", "value": "5"}, + {"tag": "Metrics", "field_id": "Metrics", "type": "*composed", "value": "6"}, + {"tag": "MetricParams", "field_id": "Parameters", "type": "*composed", "value": "7"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "8"}, + {"tag": "Stored", "field_id": "Stored", "type": "*composed", "value": "9"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "10"}, + {"tag": "MinItems", "field_id": "MinItems", "type": "*composed", "value": "11"}, + {"tag": "ThresholdIDs", "field_id": "ThresholdIDs", "type": "*composed", "value": "12"}, + ], + }, + { + "type": "*thresholds", // data source type + "file_name": "Thresholds.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "Recurrent", "field_id": "Recurrent", "type": "*composed", "value": "4"}, + {"tag": "MinHits", "field_id": "MinHits", "type": "*composed", "value": "5"}, + {"tag": "MinSleep", "field_id": "MinSleep", "type": "*composed", "value": "6"}, + {"tag": "Blocker", "field_id": "Blocker", "type": "*composed", "value": "7"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "8"}, + {"tag": "ActionIDs", "field_id": "ActionIDs", "type": "*composed", "value": "9"}, + {"tag": "Async", "field_id": "Async", "type": "*composed", "value": "10"}, + ], + }, + { + "type": "*suppliers", // data source type + "file_name": "Suppliers.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterIDs", "field_id": "FilterIDs", "type": "*composed", "value": "2"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "3"}, + {"tag": "Sorting", "field_id": "Sorting", "type": "*composed", "value": "4"}, + {"tag": "SortingParamameters", "field_id": "SortingParamameters", "type": "*composed", "value": "5"}, + {"tag": "SupplierID", "field_id": "SupplierID", "type": "*composed", "value": "6"}, + {"tag": "SupplierFilterIDs", "field_id": "SupplierFilterIDs", "type": "*composed", "value": "7"}, + {"tag": "SupplierAccountIDs", "field_id": "SupplierAccountIDs", "type": "*composed", "value": "8"}, + {"tag": "SupplierRatingPlanIDs", "field_id": "SupplierRatingPlanIDs", "type": "*composed", "value": "9"}, + {"tag": "SupplierResourceIDs", "field_id": "SupplierResourceIDs", "type": "*composed", "value": "10"}, + {"tag": "SupplierStatIDs", "field_id": "SupplierStatIDs", "type": "*composed", "value": "11"}, + {"tag": "SupplierWeight", "field_id": "SupplierWeight", "type": "*composed", "value": "12"}, + {"tag": "SupplierBlocker", "field_id": "SupplierBlocker", "type": "*composed", "value": "13"}, + {"tag": "SupplierParameters", "field_id": "SupplierParameters", "type": "*composed", "value": "14"}, + {"tag": "Weight", "field_id": "Weight", "type": "*composed", "value": "15"}, + ], + }, + ], + }, + { + "id": "FilterLoader", // identifier of the Loader + "enabled": true, // starts as service: . + "dry_run": false, // do not send the CDRs to CDRS, just parse them + "run_delay": 0, // sleep interval in seconds between consecutive runs, 0 to use automation via inotify + "lock_filename": ".cgr.lock", // Filename containing concurrency lock in case of delayed processing + "caches_conns": [ + {"address": "*internal"}, // address where to reach the CacheS for data reload, empty for no reloads <""|*internal|x.y.z.y:1234> + ], + "field_separator": ",", // separator used in case of csv files + "tp_in_dir": "/tmp/FilterIn", // absolute path towards the directory where the CDRs are stored + "tp_out_dir": "/tmp/FilterOut", // absolute path towards the directory where processed CDRs will be moved + "data":[ + { + "type": "*filters", // data source type + "file_name": "Filters.csv", // file name in the tp_in_dir + "fields": [ + {"tag": "Tenant", "field_id": "Tenant", "type": "*composed", "value": "0", "mandatory": true}, + {"tag": "ID", "field_id": "ID", "type": "*composed", "value": "1", "mandatory": true}, + {"tag": "FilterType", "field_id": "FilterType", "type": "*composed", "value": "2"}, + {"tag": "FilterFieldName", "field_id": "FilterFieldName", "type": "*composed", "value": "3"}, + {"tag": "FilterFieldValues", "field_id": "FilterFieldValues", "type": "*composed", "value": "4"}, + {"tag": "ActivationInterval", "field_id": "ActivationInterval", "type": "*composed", "value": "5"}, + ], + }, ], }, ], + "cdrstats": { "enabled": true, }, diff --git a/loaders/loader.go b/loaders/loader.go index 7e4e53aa6..8a87d3c04 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -321,6 +321,83 @@ func (ldr *Loader) storeLoadedData(loaderType string, } } } + case utils.MetaStats: + for _, lDataSet := range lds { + stsModels := make(engine.TpStatsS, len(lDataSet)) + for i, ld := range lDataSet { + stsModels[i] = new(engine.TpStats) + if err = utils.UpdateStructWithIfaceMap(stsModels[i], ld); err != nil { + return + } + } + for _, tpSts := range stsModels.AsTPStats() { + stsPrf, err := engine.APItoStats(tpSts, ldr.timezone) + if err != nil { + return err + } + if ldr.dryRun { + utils.Logger.Info( + fmt.Sprintf("<%s-%s> DRY_RUN: StatsQueueProfile: %s", + utils.LoaderS, ldr.ldrID, utils.ToJSON(stsPrf))) + continue + } + if err := ldr.dm.SetStatQueueProfile(stsPrf, true); err != nil { + return err + } + } + } + case utils.MetaThresholds: + for _, lDataSet := range lds { + thModels := make(engine.TpThresholdS, len(lDataSet)) + for i, ld := range lDataSet { + thModels[i] = new(engine.TpThreshold) + if err = utils.UpdateStructWithIfaceMap(thModels[i], ld); err != nil { + return + } + } + + for _, tpTh := range thModels.AsTPThreshold() { + thPrf, err := engine.APItoThresholdProfile(tpTh, ldr.timezone) + if err != nil { + return err + } + if ldr.dryRun { + utils.Logger.Info( + fmt.Sprintf("<%s-%s> DRY_RUN: ThresholdProfile: %s", + utils.LoaderS, ldr.ldrID, utils.ToJSON(thPrf))) + continue + } + if err := ldr.dm.SetThresholdProfile(thPrf, true); err != nil { + return err + } + } + } + case utils.MetaSuppliers: + for _, lDataSet := range lds { + sppModels := make(engine.TpSuppliers, len(lDataSet)) + for i, ld := range lDataSet { + sppModels[i] = new(engine.TpSupplier) + if err = utils.UpdateStructWithIfaceMap(sppModels[i], ld); err != nil { + return + } + } + + for _, tpSpp := range sppModels.AsTPSuppliers() { + spPrf, err := engine.APItoSupplierProfile(tpSpp, ldr.timezone) + if err != nil { + return err + } + if ldr.dryRun { + utils.Logger.Info( + fmt.Sprintf("<%s-%s> DRY_RUN: SupplierProfile: %s", + utils.LoaderS, ldr.ldrID, utils.ToJSON(spPrf))) + continue + } + if err := ldr.dm.SetSupplierProfile(spPrf, true); err != nil { + return err + } + } + } } return }