diff --git a/config/config_defaults.go b/config/config_defaults.go index 3a616033e..2f216a2e2 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -134,7 +134,7 @@ const CGRATES_CFG_JSON = ` "max_idle_conns": 10, // maximum database connections idle, not applying for mongo "conn_max_lifetime": 0, // maximum amount of time in seconds a connection may be reused (0 for unlimited), not applying for mongo "query_timeout":"10s", - "sslmode":"disable", // sslmode in case of *postgres + "sslMode":"disable", // sslmode in case of *postgres "mysql_location": "Local", // the location the time from mysql is retrived }, "items":{ @@ -312,20 +312,35 @@ const CGRATES_CFG_JSON = ` { "id": "*default", // identifier of the EventReader profile "type": "*none", // reader type <*file_csv> - "row_length" : 0, // Number of fields from csv file - "field_separator": ",", // separator used in case of csv files - "header_define_character": ":", // the starting character for header definition used in case of CSV files "run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together "concurrent_requests": 1024, // maximum simultaneous requests/files to process, 0 for unlimited "source_path": "/var/spool/cgrates/ers/in", // read data from this path "processed_path": "/var/spool/cgrates/ers/out", // move processed data here "opts": { + // FileCSV and PartialCSV + "csvRowLength": 0, // Number of fields from csv file + "csvFieldSeparator": ",", // separator used when reading the fields + "csvHeaderDefineChar": ":", // the starting character for header definition used in case of CSV files + // "csvLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + + // PartialCSV + "csvCacheExpiryAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> + // "csvRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + + // FlatStore + "fstRowLength": 0, // Number of fields from csv file + "fstFieldSeparator": ",", // separator used when reading the fields + // "fstFailedCallsPrefix": "" // Used in case of flatstore CDRs to avoid searching for BYE records + // "fstRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + // "fstLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + + // FileXML + "xmlRootPath": "", // path towards one event in case of XML CDRs + + // "queueID": "cgrates_cdrs", // the queue id for AMQP, AMQPv1, SQS and S3 readers from were the events are read // "queueIDProcessed": "", // the queue id for AMQP, AMQPv1, SQS and S3 readers were the events are sent after they are processed - // FileCSV, FlatStore and PartialCSV - // "lazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field - // AMQP // "consumerTag": "cgrates", // the ID of the consumer // "exchange": "", @@ -347,11 +362,11 @@ const CGRATES_CFG_JSON = ` // SQL // "dbName": "cgrates", // the name of the database from were the events are read // "tableName": "cdrs", // the name of the table from were the events are read - // "sslmode": "disable", // the ssl mode for postgres db + // "sslMode": "disable", // the ssl mode for postgres db // "dbNameProcessed": "", // the name of the database were the events are sent after they are processed // "tableNameProcessed": "", // the name of the table were the events are sent after they are processed - // "sslmodeProcessed": "", // the ssl mode for postgres db + // "sslModeProcessed": "", // the ssl mode for postgres db // SQS and S3 // "awsRegion": "", @@ -363,10 +378,9 @@ const CGRATES_CFG_JSON = ` // "awsKeyProcessed": "", // "awsSecretProcessed": "", // "awsTokenProcessed": "", - // "folderPathProcessed": "", // only for S3 event posting + // "s3FolderPathProcessed": "", // only for S3 event posting }, - "xml_root_path": "", // path towards one event in case of XML CDRs "tenant": "", // tenant used by import "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> "filters": [], // limit parsing based on the filters diff --git a/config/config_it_test.go b/config/config_it_test.go index b5afde656..ed840aa12 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -411,37 +411,30 @@ func testCGRConfigReloadERs(t *testing.T) { SessionSConns: []string{utils.MetaLocalHost}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Flags: flagsDefault, - Fields: content, - CacheDumpFields: []*FCTemplate{}, - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Flags: flagsDefault, + Fields: content, + CacheDumpFields: []*FCTemplate{}, + Opts: make(map[string]interface{}), }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - Filters: []string{}, - Flags: flags, - Fields: content, - CacheDumpFields: []*FCTemplate{}, - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Filters: []string{}, + Flags: flags, + Fields: content, + CacheDumpFields: []*FCTemplate{}, + Opts: make(map[string]interface{}), }, }, } diff --git a/config/config_json_test.go b/config/config_json_test.go index aa1d6fdd6..74ed7f2cd 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -1881,23 +1881,27 @@ func TestDfEventReaderCfg(t *testing.T) { Sessions_conns: &[]string{utils.MetaInternal}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer(utils.MetaDefault), - Type: utils.StringPointer(utils.MetaNone), - Row_length: utils.IntPointer(0), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("0"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), - Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), - Xml_root_path: utils.StringPointer(utils.EmptyString), - Tenant: utils.StringPointer(utils.EmptyString), - Timezone: utils.StringPointer(utils.EmptyString), - Filters: &[]string{}, - Flags: &[]string{}, - Fields: &cdrFields, - Cache_dump_fields: &[]*FcTemplateJsonCfg{}, - Opts: make(map[string]interface{}), + Id: utils.StringPointer(utils.MetaDefault), + Type: utils.StringPointer(utils.MetaNone), + Run_delay: utils.StringPointer("0"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), + Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), + Tenant: utils.StringPointer(utils.EmptyString), + Timezone: utils.StringPointer(utils.EmptyString), + Filters: &[]string{}, + Flags: &[]string{}, + Fields: &cdrFields, + Cache_dump_fields: &[]*FcTemplateJsonCfg{}, + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } diff --git a/config/config_test.go b/config/config_test.go index 23e8c0226..656fbd2ef 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2227,23 +2227,27 @@ func TestERSConfig(t *testing.T) { SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: make([]*FCTemplate, 0), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -4534,26 +4538,27 @@ func TestV1GetConfigSectionERS(t *testing.T) { utils.SessionSConnsCfg: []string{utils.MetaInternal}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: utils.EmptyString, - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: utils.EmptyString, - utils.TimezoneCfg: utils.EmptyString, - utils.XMLRootPathCfg: utils.EmptyString, - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: utils.MetaNone, - utils.FailedCallsPrefixCfg: utils.EmptyString, - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", - utils.FieldsCfg: []string{}, - utils.OptsCfg: make(map[string]interface{}), + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: utils.EmptyString, + utils.TimezoneCfg: utils.EmptyString, + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: utils.MetaNone, + utils.FieldsCfg: []string{}, + utils.OptsCfg: map[string]interface{}{ + utils.PartialCSVCacheExpiryActionOpt: utils.MetaPostCDR, + "csvFieldSeparator": ",", + utils.HeaderDefineCharOpt: ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + utils.XMLRootPathOpt: "", + }, }, }, }, @@ -4865,7 +4870,7 @@ func TestV1GetConfigAsJSONDataDB(t *testing.T) { func TestV1GetConfigAsJSONStorDB(t *testing.T) { var reply string - expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_accounts":{"remote":false,"replicate":false},"*tp_action_profiles":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rate_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"conn_max_lifetime":0,"max_idle_conns":10,"max_open_conns":100,"mysql_location":"Local","query_timeout":"10s","sslmode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` + expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_accounts":{"remote":false,"replicate":false},"*tp_action_profiles":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rate_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"conn_max_lifetime":0,"max_idle_conns":10,"max_open_conns":100,"mysql_location":"Local","query_timeout":"10s","sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` cfgCgr := NewDefaultCGRConfig() if err := cfgCgr.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Sections: []string{StorDBJSON}}, &reply); err != nil { t.Error(err) @@ -5186,7 +5191,7 @@ func TestV1GetConfigAsJSONCfgEES(t *testing.T) { func TestV1GetConfigAsJSONCfgERS(t *testing.T) { var reply string - expected := `{"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"failed_calls_prefix":"","field_separator":",","fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"header_define_character":":","id":"*default","opts":{},"partial_cache_expiry_action":"","partial_record_cache":"0","processed_path":"/var/spool/cgrates/ers/out","row_length":0,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none","xml_root_path":""}],"sessions_conns":["*internal"]}}` + expected := `{"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]}}` cgrCfg := NewDefaultCGRConfig() if err := cgrCfg.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Sections: []string{ERsJSON}}, &reply); err != nil { t.Error(err) @@ -5354,7 +5359,7 @@ func TestV1GetConfigAsJSONAllConfig(t *testing.T) { } }` var reply string - expected := `{"accounts":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"max_iterations":1000,"max_usage":"259200000000000","nested_fields":false,"prefix_indexed_fields":[],"rates_conns":[],"suffix_indexed_fields":[],"thresholds_conns":[]},"actions":{"accounts_conns":[],"cdrs_conns":[],"ees_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"stats_conns":[],"suffix_indexed_fields":[],"tenants":[],"thresholds_conns":[]},"admins":{"actions_conns":[],"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false},"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"admins_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"actions_conns":[],"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"config_db":{"db_host":"","db_name":"","db_password":"","db_port":0,"db_type":"*internal","db_user":"","items":{},"opts":{"query_timeout":"10s","redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"remote_conn_id":"","remote_conns":null,"replication_cache":"","replication_conns":null,"replication_filtered":false},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*accounts":{"remote":false,"replicate":false},"*action_profiles":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rate_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"query_timeout":"10s","redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"failed_calls_prefix":"","field_separator":",","fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"header_define_character":":","id":"*default","opts":{},"partial_cache_expiry_action":"","partial_record_cache":"0","processed_path":"/var/spool/cgrates/ers/out","row_length":0,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none","xml_root_path":""}],"sessions_conns":["*internal"]},"filters":{"admins_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":[],"low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"actions_conns":["*localhost"],"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"MinCost","tag":"MinCost","type":"*variable","value":"~*req.5"},{"path":"MaxCost","tag":"MaxCost","type":"*variable","value":"~*req.6"},{"path":"MaxCostStrategy","tag":"MaxCostStrategy","type":"*variable","value":"~*req.7"},{"path":"RateID","tag":"RateID","type":"*variable","value":"~*req.8"},{"path":"RateFilterIDs","tag":"RateFilterIDs","type":"*variable","value":"~*req.9"},{"path":"RateActivationTimes","tag":"RateActivationTimes","type":"*variable","value":"~*req.10"},{"path":"RateWeight","tag":"RateWeight","type":"*variable","value":"~*req.11"},{"path":"RateBlocker","tag":"RateBlocker","type":"*variable","value":"~*req.12"},{"path":"RateIntervalStart","tag":"RateIntervalStart","type":"*variable","value":"~*req.13"},{"path":"RateFixedFee","tag":"RateFixedFee","type":"*variable","value":"~*req.14"},{"path":"RateRecurrentFee","tag":"RateRecurrentFee","type":"*variable","value":"~*req.15"},{"path":"RateUnit","tag":"RateUnit","type":"*variable","value":"~*req.16"},{"path":"RateIncrement","tag":"RateIncrement","type":"*variable","value":"~*req.17"}],"file_name":"RateProfiles.csv","flags":null,"type":"*rate_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"Schedule","tag":"Schedule","type":"*variable","value":"~*req.5"},{"path":"TargetType","tag":"TargetType","type":"*variable","value":"~*req.6"},{"path":"TargetIDs","tag":"TargetIDs","type":"*variable","value":"~*req.7"},{"path":"ActionID","tag":"ActionID","type":"*variable","value":"~*req.8"},{"path":"ActionFilterIDs","tag":"ActionFilterIDs","type":"*variable","value":"~*req.9"},{"path":"ActionBlocker","tag":"ActionBlocker","type":"*variable","value":"~*req.10"},{"path":"ActionTTL","tag":"ActionTTL","type":"*variable","value":"~*req.11"},{"path":"ActionType","tag":"ActionType","type":"*variable","value":"~*req.12"},{"path":"ActionOpts","tag":"ActionOpts","type":"*variable","value":"~*req.13"},{"path":"ActionPath","tag":"ActionPath","type":"*variable","value":"~*req.14"},{"path":"ActionValue","tag":"ActionValue","type":"*variable","value":"~*req.15"}],"file_name":"ActionProfiles.csv","flags":null,"type":"*action_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"BalanceID","tag":"BalanceID","type":"*variable","value":"~*req.5"},{"path":"BalanceFilterIDs","tag":"BalanceFilterIDs","type":"*variable","value":"~*req.6"},{"path":"BalanceWeight","tag":"BalanceWeight","type":"*variable","value":"~*req.7"},{"path":"BalanceBlocker","tag":"BalanceBlocker","type":"*variable","value":"~*req.8"},{"path":"BalanceType","tag":"BalanceType","type":"*variable","value":"~*req.9"},{"path":"BalanceOpts","tag":"BalanceOpts","type":"*variable","value":"~*req.10"},{"path":"BalanceCostIncrements","tag":"BalanceCostIncrements","type":"*variable","value":"~*req.11"},{"path":"BalanceAttributeIDs","tag":"BalanceAttributeIDs","type":"*variable","value":"~*req.12"},{"path":"BalanceRateProfileIDs","tag":"BalanceRateProfileIDs","type":"*variable","value":"~*req.13"},{"path":"BalanceUnitFactors","tag":"BalanceUnitFactors","type":"*variable","value":"~*req.14"},{"path":"BalanceUnits","tag":"BalanceUnits","type":"*variable","value":"~*req.15"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.16"}],"file_name":"Accounts.csv","flags":null,"type":"*accounts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rates":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rate_indexed_selects":true,"rate_nested_fields":false,"rate_prefix_indexed_fields":[],"rate_suffix_indexed_fields":[],"suffix_indexed_fields":[],"verbosity":1000},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"sessions":{"actions_conns":[],"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","replication_conns":[],"resources_conns":[],"routes_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":"1s","sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_accounts":{"remote":false,"replicate":false},"*tp_action_profiles":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rate_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"conn_max_lifetime":0,"max_idle_conns":10,"max_open_conns":100,"mysql_location":"Local","query_timeout":"10s","sslmode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` + expected := `{"accounts":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"max_iterations":1000,"max_usage":"259200000000000","nested_fields":false,"prefix_indexed_fields":[],"rates_conns":[],"suffix_indexed_fields":[],"thresholds_conns":[]},"actions":{"accounts_conns":[],"cdrs_conns":[],"ees_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"stats_conns":[],"suffix_indexed_fields":[],"tenants":[],"thresholds_conns":[]},"admins":{"actions_conns":[],"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false},"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"admins_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"actions_conns":[],"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"config_db":{"db_host":"","db_name":"","db_password":"","db_port":0,"db_type":"*internal","db_user":"","items":{},"opts":{"query_timeout":"10s","redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"remote_conn_id":"","remote_conns":null,"replication_cache":"","replication_conns":null,"replication_filtered":false},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*accounts":{"remote":false,"replicate":false},"*action_profiles":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rate_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"query_timeout":"10s","redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"admins_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":[],"low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"actions_conns":["*localhost"],"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"MinCost","tag":"MinCost","type":"*variable","value":"~*req.5"},{"path":"MaxCost","tag":"MaxCost","type":"*variable","value":"~*req.6"},{"path":"MaxCostStrategy","tag":"MaxCostStrategy","type":"*variable","value":"~*req.7"},{"path":"RateID","tag":"RateID","type":"*variable","value":"~*req.8"},{"path":"RateFilterIDs","tag":"RateFilterIDs","type":"*variable","value":"~*req.9"},{"path":"RateActivationTimes","tag":"RateActivationTimes","type":"*variable","value":"~*req.10"},{"path":"RateWeight","tag":"RateWeight","type":"*variable","value":"~*req.11"},{"path":"RateBlocker","tag":"RateBlocker","type":"*variable","value":"~*req.12"},{"path":"RateIntervalStart","tag":"RateIntervalStart","type":"*variable","value":"~*req.13"},{"path":"RateFixedFee","tag":"RateFixedFee","type":"*variable","value":"~*req.14"},{"path":"RateRecurrentFee","tag":"RateRecurrentFee","type":"*variable","value":"~*req.15"},{"path":"RateUnit","tag":"RateUnit","type":"*variable","value":"~*req.16"},{"path":"RateIncrement","tag":"RateIncrement","type":"*variable","value":"~*req.17"}],"file_name":"RateProfiles.csv","flags":null,"type":"*rate_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"Schedule","tag":"Schedule","type":"*variable","value":"~*req.5"},{"path":"TargetType","tag":"TargetType","type":"*variable","value":"~*req.6"},{"path":"TargetIDs","tag":"TargetIDs","type":"*variable","value":"~*req.7"},{"path":"ActionID","tag":"ActionID","type":"*variable","value":"~*req.8"},{"path":"ActionFilterIDs","tag":"ActionFilterIDs","type":"*variable","value":"~*req.9"},{"path":"ActionBlocker","tag":"ActionBlocker","type":"*variable","value":"~*req.10"},{"path":"ActionTTL","tag":"ActionTTL","type":"*variable","value":"~*req.11"},{"path":"ActionType","tag":"ActionType","type":"*variable","value":"~*req.12"},{"path":"ActionOpts","tag":"ActionOpts","type":"*variable","value":"~*req.13"},{"path":"ActionPath","tag":"ActionPath","type":"*variable","value":"~*req.14"},{"path":"ActionValue","tag":"ActionValue","type":"*variable","value":"~*req.15"}],"file_name":"ActionProfiles.csv","flags":null,"type":"*action_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.4"},{"path":"BalanceID","tag":"BalanceID","type":"*variable","value":"~*req.5"},{"path":"BalanceFilterIDs","tag":"BalanceFilterIDs","type":"*variable","value":"~*req.6"},{"path":"BalanceWeight","tag":"BalanceWeight","type":"*variable","value":"~*req.7"},{"path":"BalanceBlocker","tag":"BalanceBlocker","type":"*variable","value":"~*req.8"},{"path":"BalanceType","tag":"BalanceType","type":"*variable","value":"~*req.9"},{"path":"BalanceOpts","tag":"BalanceOpts","type":"*variable","value":"~*req.10"},{"path":"BalanceCostIncrements","tag":"BalanceCostIncrements","type":"*variable","value":"~*req.11"},{"path":"BalanceAttributeIDs","tag":"BalanceAttributeIDs","type":"*variable","value":"~*req.12"},{"path":"BalanceRateProfileIDs","tag":"BalanceRateProfileIDs","type":"*variable","value":"~*req.13"},{"path":"BalanceUnitFactors","tag":"BalanceUnitFactors","type":"*variable","value":"~*req.14"},{"path":"BalanceUnits","tag":"BalanceUnits","type":"*variable","value":"~*req.15"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.16"}],"file_name":"Accounts.csv","flags":null,"type":"*accounts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redis_ca_certificate":"","redis_client_certificate":"","redis_client_key":"","redis_cluster":false,"redis_cluster_ondown_delay":"0","redis_cluster_sync":"5s","redis_sentinel":"","redis_tls":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rates":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rate_indexed_selects":true,"rate_nested_fields":false,"rate_prefix_indexed_fields":[],"rate_suffix_indexed_fields":[],"suffix_indexed_fields":[],"verbosity":1000},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"sessions":{"actions_conns":[],"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","replication_conns":[],"resources_conns":[],"routes_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":"1s","sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_accounts":{"remote":false,"replicate":false},"*tp_action_profiles":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rate_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"conn_max_lifetime":0,"max_idle_conns":10,"max_open_conns":100,"mysql_location":"Local","query_timeout":"10s","sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSON) if err != nil { t.Fatal(err) @@ -5403,19 +5408,16 @@ func TestCgrCdfEventReader(t *testing.T) { SessionSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaSessionS)}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -5441,7 +5443,15 @@ func TestCgrCdfEventReader(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: []*FCTemplate{}, - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -5493,19 +5503,16 @@ func TestCgrCdfEventExporter(t *testing.T) { func TestCgrCfgEventReaderDefault(t *testing.T) { eCfg := &EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -5531,7 +5538,15 @@ func TestCgrCfgEventReaderDefault(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, } for _, v := range eCfg.Fields { v.ComputePath() diff --git a/config/configsanity.go b/config/configsanity.go index 62f6a9de1..afa1f20c4 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -475,14 +475,61 @@ func (cfg *CGRConfig) checkConfigSanity() error { } switch rdr.Type { - case utils.MetaFileCSV, utils.MetaPartialCSV, utils.MetaFlatstore: + case utils.MetaFileCSV, utils.MetaPartialCSV: for _, dir := range []string{rdr.ProcessedPath, rdr.SourcePath} { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) } } - if rdr.FieldSep == utils.EmptyString { - return fmt.Errorf("<%s> empty FieldSep for reader with ID: %s", utils.ERs, rdr.ID) + if fldSep, has := rdr.Opts[utils.CSV+utils.FieldSepOpt]; has && + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.CSV+utils.FieldSepOpt, rdr.ID) + } + if rowl, has := rdr.Opts[utils.CSV+utils.RowLengthOpt]; has { + if _, err := utils.IfaceAsTInt64(rowl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.RowLengthOpt, err.Error(), rdr.ID) + } + } + if lq, has := rdr.Opts[utils.CSV+utils.LazyQuotes]; has { + if _, err := utils.IfaceAsBool(lq); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.LazyQuotes, err.Error(), rdr.ID) + } + } + if rdr.Type == utils.MetaPartialCSV { + if act, has := rdr.Opts[utils.PartialCSVCacheExpiryActionOpt]; has && (utils.IfaceAsString(act) != utils.MetaDumpToFile || + utils.IfaceAsString(act) != utils.MetaPostCDR) { + return fmt.Errorf("<%s> wrong partial expiry action for reader with ID: %s", utils.ERs, rdr.ID) + } + if ttl, has := rdr.Opts[utils.PartialCSVRecordCacheOpt]; has { + if _, err := utils.IfaceAsDuration(ttl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.PartialCSVRecordCacheOpt, err.Error(), rdr.ID) + } + } + } + case utils.MetaFlatstore: + for _, dir := range []string{rdr.ProcessedPath, rdr.SourcePath} { + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) + } + } + if fldSep, has := rdr.Opts[utils.FlatstorePrfx+utils.FieldSepOpt]; has && + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.FieldSepOpt, rdr.ID) + } + if rowl, has := rdr.Opts[utils.FlatstorePrfx+utils.RowLengthOpt]; has { + if _, err := utils.IfaceAsTInt64(rowl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.RowLengthOpt, err.Error(), rdr.ID) + } + } + if lq, has := rdr.Opts[utils.FlatstorePrfx+utils.LazyQuotes]; has { + if _, err := utils.IfaceAsBool(lq); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.LazyQuotes, err.Error(), rdr.ID) + } + } + if ttl, has := rdr.Opts[utils.FstPartialRecordCacheOpt]; has { + if _, err := utils.IfaceAsDuration(ttl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FstPartialRecordCacheOpt, err.Error(), rdr.ID) + } } case utils.MetaKafkajsonMap: if rdr.RunDelay > 0 { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 24a06f13d..d8436b875 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -664,9 +664,9 @@ func TestConfigSanityEventReader(t *testing.T) { Type: utils.MetaFileCSV, ProcessedPath: "/", SourcePath: "/", - FieldSep: "", + Opts: map[string]interface{}{"csvFieldSeparator": ""}, }} - expected = " empty FieldSep for reader with ID: test3" + expected = " empty csvFieldSeparator for reader with ID: test3" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } @@ -674,7 +674,7 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test4", Type: utils.MetaKafkajsonMap, RunDelay: 1, - FieldSep: utils.InInFieldSep, + Opts: map[string]interface{}{"csvFieldSeparator": ","}, } expected = " the RunDelay field can not be bigger than zero for reader with ID: test4" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -684,7 +684,6 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test5", Type: utils.MetaFileXML, RunDelay: 0, - FieldSep: utils.InInFieldSep, ProcessedPath: "not/a/path", SourcePath: "not/a/path", } @@ -697,7 +696,6 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test5", Type: utils.MetaFileFWV, RunDelay: 0, - FieldSep: utils.InInFieldSep, ProcessedPath: "not/a/path", SourcePath: "not/a/path", } diff --git a/config/erscfg.go b/config/erscfg.go index c16e568bb..d7e5b80f6 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -111,26 +111,19 @@ func (erS *ERsCfg) AsMapInterface(separator string) (initialMP map[string]interf // EventReaderCfg the event for the Event Reader type EventReaderCfg struct { - ID string - Type string - RowLength int - FieldSep string - HeaderDefineChar string - RunDelay time.Duration - ConcurrentReqs int - SourcePath string - ProcessedPath string - Opts map[string]interface{} - XMLRootPath utils.HierarchyPath - Tenant RSRParsers - Timezone string - Filters []string - Flags utils.FlagsWithParams - FailedCallsPrefix string // Used in case of flatstore CDRs to avoid searching for BYE records - PartialRecordCache time.Duration // Duration to cache partial records when not pairing - PartialCacheExpiryAction string - Fields []*FCTemplate - CacheDumpFields []*FCTemplate + ID string + Type string + RunDelay time.Duration + ConcurrentReqs int + SourcePath string + ProcessedPath string + Opts map[string]interface{} + Tenant RSRParsers + Timezone string + Filters []string + Flags utils.FlagsWithParams + Fields []*FCTemplate + CacheDumpFields []*FCTemplate } func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplates map[string][]*FCTemplate, sep string) (err error) { @@ -143,15 +136,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Type != nil { er.Type = *jsnCfg.Type } - if jsnCfg.Row_length != nil { - er.RowLength = *jsnCfg.Row_length - } - if jsnCfg.Field_separator != nil { - er.FieldSep = *jsnCfg.Field_separator - } - if jsnCfg.Header_define_character != nil { - er.HeaderDefineChar = *jsnCfg.Header_define_character - } if jsnCfg.Run_delay != nil { if er.RunDelay, err = utils.ParseDurationWithNanosecs(*jsnCfg.Run_delay); err != nil { return @@ -166,9 +150,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Processed_path != nil { er.ProcessedPath = *jsnCfg.Processed_path } - if jsnCfg.Xml_root_path != nil { - er.XMLRootPath = utils.ParseHierarchyPath(*jsnCfg.Xml_root_path, utils.EmptyString) - } if jsnCfg.Tenant != nil { if er.Tenant, err = NewRSRParsers(*jsnCfg.Tenant, sep); err != nil { return err @@ -183,17 +164,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Flags != nil { er.Flags = utils.FlagsWithParamsFromSlice(*jsnCfg.Flags) } - if jsnCfg.Failed_calls_prefix != nil { - er.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix - } - if jsnCfg.Partial_record_cache != nil { - if er.PartialRecordCache, err = utils.ParseDurationWithNanosecs(*jsnCfg.Partial_record_cache); err != nil { - return err - } - } - if jsnCfg.Partial_cache_expiry_action != nil { - er.PartialCacheExpiryAction = *jsnCfg.Partial_cache_expiry_action - } if jsnCfg.Fields != nil { if er.Fields, err = FCTemplatesFromFCTemplatesJSONCfg(*jsnCfg.Fields, sep); err != nil { return err @@ -225,22 +195,16 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat // Clone returns a deep copy of EventReaderCfg func (er EventReaderCfg) Clone() (cln *EventReaderCfg) { cln = &EventReaderCfg{ - ID: er.ID, - Type: er.Type, - FieldSep: er.FieldSep, - HeaderDefineChar: er.HeaderDefineChar, - RunDelay: er.RunDelay, - ConcurrentReqs: er.ConcurrentReqs, - SourcePath: er.SourcePath, - ProcessedPath: er.ProcessedPath, - XMLRootPath: er.XMLRootPath.Clone(), - Tenant: er.Tenant.Clone(), - Timezone: er.Timezone, - Flags: er.Flags.Clone(), - FailedCallsPrefix: er.FailedCallsPrefix, - PartialCacheExpiryAction: er.PartialCacheExpiryAction, - PartialRecordCache: er.PartialRecordCache, - Opts: make(map[string]interface{}), + ID: er.ID, + Type: er.Type, + RunDelay: er.RunDelay, + ConcurrentReqs: er.ConcurrentReqs, + SourcePath: er.SourcePath, + ProcessedPath: er.ProcessedPath, + Tenant: er.Tenant.Clone(), + Timezone: er.Timezone, + Flags: er.Flags.Clone(), + Opts: make(map[string]interface{}), } if er.Filters != nil { cln.Filters = utils.CloneStringSlice(er.Filters) @@ -266,23 +230,16 @@ func (er EventReaderCfg) Clone() (cln *EventReaderCfg) { // AsMapInterface returns the config as a map[string]interface{} func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string]interface{}) { initialMP = map[string]interface{}{ - utils.IDCfg: er.ID, - utils.TypeCfg: er.Type, - utils.RowLengthCfg: er.RowLength, - utils.FieldSepCfg: er.FieldSep, - utils.HeaderDefCharCfg: er.HeaderDefineChar, - utils.ConcurrentRequestsCfg: er.ConcurrentReqs, - utils.SourcePathCfg: er.SourcePath, - utils.ProcessedPathCfg: er.ProcessedPath, - utils.TenantCfg: er.Tenant.GetRule(separator), - utils.XMLRootPathCfg: er.XMLRootPath.AsString("/", len(er.XMLRootPath) != 0 && len(er.XMLRootPath[0]) != 0), - utils.TimezoneCfg: er.Timezone, - utils.FiltersCfg: er.Filters, - utils.FlagsCfg: []string{}, - utils.FailedCallsPrefixCfg: er.FailedCallsPrefix, - utils.PartialCacheExpiryActionCfg: er.PartialCacheExpiryAction, - utils.PartialRecordCacheCfg: "0", - utils.RunDelayCfg: "0", + utils.IDCfg: er.ID, + utils.TypeCfg: er.Type, + utils.ConcurrentRequestsCfg: er.ConcurrentReqs, + utils.SourcePathCfg: er.SourcePath, + utils.ProcessedPathCfg: er.ProcessedPath, + utils.TenantCfg: er.Tenant.GetRule(separator), + utils.TimezoneCfg: er.Timezone, + utils.FiltersCfg: er.Filters, + utils.FlagsCfg: []string{}, + utils.RunDelayCfg: "0", } opts := make(map[string]interface{}) @@ -315,35 +272,24 @@ func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string } else if er.RunDelay < 0 { initialMP[utils.RunDelayCfg] = "-1" } - - if er.PartialRecordCache != 0 { - initialMP[utils.PartialRecordCacheCfg] = er.PartialRecordCache.String() - } return } // EventReaderSJsonCfg is the configuration of a single EventReader type EventReaderJsonCfg struct { - Id *string - Type *string - Row_length *int - Field_separator *string - Header_define_character *string - Run_delay *string - Concurrent_requests *int - Source_path *string - Processed_path *string - Opts map[string]interface{} - Xml_root_path *string - Tenant *string - Timezone *string - Filters *[]string - Flags *[]string - Failed_calls_prefix *string - Partial_record_cache *string - Partial_cache_expiry_action *string - Fields *[]*FcTemplateJsonCfg - Cache_dump_fields *[]*FcTemplateJsonCfg + Id *string + Type *string + Run_delay *string + Concurrent_requests *int + Source_path *string + Processed_path *string + Opts map[string]interface{} + Tenant *string + Timezone *string + Filters *[]string + Flags *[]string + Fields *[]*FcTemplateJsonCfg + Cache_dump_fields *[]*FcTemplateJsonCfg } func diffEventReaderJsonCfg(d *EventReaderJsonCfg, v1, v2 *EventReaderCfg, separator string) *EventReaderJsonCfg { @@ -356,15 +302,6 @@ func diffEventReaderJsonCfg(d *EventReaderJsonCfg, v1, v2 *EventReaderCfg, separ if v1.Type != v2.Type { d.Type = utils.StringPointer(v2.Type) } - if v1.RowLength != v2.RowLength { - d.Row_length = utils.IntPointer(v2.RowLength) - } - if v1.FieldSep != v2.FieldSep { - d.Field_separator = utils.StringPointer(v2.FieldSep) - } - if v1.HeaderDefineChar != v2.HeaderDefineChar { - d.Header_define_character = utils.StringPointer(v2.HeaderDefineChar) - } if v1.RunDelay != v2.RunDelay { d.Run_delay = utils.StringPointer(v2.RunDelay.String()) } @@ -378,11 +315,6 @@ func diffEventReaderJsonCfg(d *EventReaderJsonCfg, v1, v2 *EventReaderCfg, separ d.Processed_path = utils.StringPointer(v2.ProcessedPath) } d.Opts = diffMap(d.Opts, v1.Opts, v2.Opts) - xml1 := v1.XMLRootPath.AsString("/", len(v1.XMLRootPath) != 0 && len(v1.XMLRootPath[0]) != 0) - xml2 := v2.XMLRootPath.AsString("/", len(v2.XMLRootPath) != 0 && len(v2.XMLRootPath[0]) != 0) - if xml1 != xml2 { - d.Xml_root_path = utils.StringPointer(xml2) - } tnt1 := v1.Tenant.GetRule(separator) tnt2 := v2.Tenant.GetRule(separator) if tnt1 != tnt2 { @@ -399,15 +331,6 @@ func diffEventReaderJsonCfg(d *EventReaderJsonCfg, v1, v2 *EventReaderCfg, separ if !utils.SliceStringEqual(flgs1, flgs2) { d.Flags = &flgs2 } - if v1.FailedCallsPrefix != v2.FailedCallsPrefix { - d.Failed_calls_prefix = utils.StringPointer(v2.FailedCallsPrefix) - } - if v1.PartialRecordCache != v2.PartialRecordCache { - d.Partial_record_cache = utils.StringPointer(v2.PartialRecordCache.String()) - } - if v1.PartialCacheExpiryAction != v2.PartialCacheExpiryAction { - d.Partial_cache_expiry_action = utils.StringPointer(v2.PartialCacheExpiryAction) - } var flds []*FcTemplateJsonCfg if d.Fields != nil { flds = *d.Fields diff --git a/config/erscfg_test.go b/config/erscfg_test.go index 0081dd96a..d65b19eec 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -65,19 +65,16 @@ func TestERSClone(t *testing.T) { SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -103,25 +100,27 @@ func TestERSClone(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: "*file_csv", - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), - Timezone: utils.EmptyString, - Filters: []string{"randomFiletrs"}, - Flags: utils.FlagsWithParams{}, - FailedCallsPrefix: "randomPrefix", - PartialRecordCache: time.Second, - PartialCacheExpiryAction: "randomAction", + ID: "file_reader1", + Type: "*file_csv", + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), + Timezone: utils.EmptyString, + Filters: []string{"randomFiletrs"}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -133,7 +132,14 @@ func TestERSClone(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, Opts: map[string]interface{}{ - utils.MetaDefault: "randomVal", + utils.MetaDefault: "randomVal", + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", }, }, }, @@ -180,21 +186,6 @@ func TestEventReaderloadFromJsonCase1(t *testing.T) { } } -func TestEventReaderloadFromJsonCase2(t *testing.T) { - cfgJSON := &ERsJsonCfg{ - Readers: &[]*EventReaderJsonCfg{ - { - Partial_record_cache: utils.StringPointer("1ss"), - }, - }, - } - expected := "time: unknown unit \"ss\" in duration \"1ss\"" - jsoncfg := NewDefaultCGRConfig() - if err = jsoncfg.ersCfg.loadFromJSONCfg(cfgJSON, jsoncfg.templates, jsoncfg.generalCfg.RSRSep, jsoncfg.dfltEvRdr); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - func TestEventReaderloadFromJsonCase3(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ @@ -216,19 +207,16 @@ func TestERSLoadFromjsonCfg(t *testing.T) { SessionSConns: []string{"conn1", "conn3"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -254,22 +242,27 @@ func TestERSLoadFromjsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -295,7 +288,15 @@ func TestERSLoadFromjsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -403,23 +404,16 @@ func TestERSloadFromJsonCase3(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.AnswerTime), @@ -437,19 +431,16 @@ func TestERSloadFromJsonCase3(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -475,25 +466,27 @@ func TestERSloadFromJsonCase3(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, @@ -503,7 +496,15 @@ func TestERSloadFromJsonCase3(t *testing.T) { }, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -534,24 +535,17 @@ func TestERSloadFromJsonCase4(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), - Fields: &[]*FcTemplateJsonCfg{}, + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, + Fields: &[]*FcTemplateJsonCfg{}, Cache_dump_fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer("OriginID"), @@ -568,19 +562,16 @@ func TestERSloadFromJsonCase4(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -606,26 +597,28 @@ func TestERSloadFromJsonCase4(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, - Fields: []*FCTemplate{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{}, CacheDumpFields: []*FCTemplate{ { Tag: "OrderID", @@ -634,7 +627,15 @@ func TestERSloadFromJsonCase4(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.OrderID", utils.InfieldSep), }, }, - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -687,19 +688,16 @@ func TestEventReaderSameID(t *testing.T) { SessionSConns: []string{"conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -725,29 +723,41 @@ func TestEventReaderSameID(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -819,24 +829,17 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: "", - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -850,15 +853,20 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.OptsCfg: make(map[string]interface{}), + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { utils.CacheDumpFieldsCfg: []map[string]interface{}{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -872,19 +880,23 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "file_reader1", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/tmp/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "-1", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/tmp/ers/in", - utils.TenantCfg: "~*req.Destination1", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: "", - utils.OptsCfg: make(map[string]interface{}), + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "file_reader1", + utils.ProcessedPathCfg: "/tmp/ers/out", + utils.RunDelayCfg: "-1", + utils.SourcePathCfg: "/tmp/ers/in", + utils.TenantCfg: "~*req.Destination1", + utils.TimezoneCfg: "", + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -926,24 +938,17 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: "", - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -957,7 +962,15 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.OptsCfg: make(map[string]interface{}), + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { utils.CacheDumpFieldsCfg: []map[string]interface{}{ @@ -965,9 +978,6 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { }, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -981,20 +991,23 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.FiltersCfg: []string{"randomFilter"}, - utils.FlagsCfg: []string{"randomFlag"}, - utils.IDCfg: "file_reader1", - utils.ProcessedPathCfg: "/tmp/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "10s", - utils.PartialRecordCacheCfg: "1s", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/tmp/ers/in", - utils.TenantCfg: "~*req.Destination1", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: "", + utils.FiltersCfg: []string{"randomFilter"}, + utils.FlagsCfg: []string{"randomFlag"}, + utils.IDCfg: "file_reader1", + utils.ProcessedPathCfg: "/tmp/ers/out", + utils.RunDelayCfg: "10s", + utils.SourcePathCfg: "/tmp/ers/in", + utils.TenantCfg: "~*req.Destination1", + utils.TimezoneCfg: "", utils.OptsCfg: map[string]interface{}{ - utils.KafkaGroupID: "test", + utils.KafkaGroupID: "test", + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", }, }, }, @@ -1012,23 +1025,16 @@ func TestERsloadFromJsonCfg(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.CGRID), @@ -1047,19 +1053,16 @@ func TestERsloadFromJsonCfg(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -1085,25 +1088,27 @@ func TestERsloadFromJsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, @@ -1115,7 +1120,15 @@ func TestERsloadFromJsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } diff --git a/config/stordbcfg_test.go b/config/stordbcfg_test.go index b9db299a6..d6d3242ec 100644 --- a/config/stordbcfg_test.go +++ b/config/stordbcfg_test.go @@ -198,7 +198,7 @@ func TestStorDbCfgAsMapInterface(t *testing.T) { "max_idle_conns": 10, "conn_max_lifetime": 0, "query_timeout":"10s", - "sslmode":"disable", + "sslMode":"disable", "mysql_location": "UTC", }, "items":{ diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 57279944d..89d1480cd 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -116,7 +116,7 @@ // "max_idle_conns": 10, // maximum database connections idle, not applying for mongo // "conn_max_lifetime": 0, // maximum amount of time in seconds a connection may be reused (0 for unlimited), not applying for mongo // "query_timeout":"10s", -// "sslmode":"disable", // sslmode in case of *postgres +// "sslMode":"disable", // sslmode in case of *postgres // "mysql_location": "Local", // the location the time from mysql is retrived // }, // "items":{ @@ -325,20 +325,35 @@ // { // "id": "*default", // identifier of the EventReader profile // "type": "*none", // reader type <*file_csv> -// "row_length" : 0, // Number of fields from csv file -// "field_separator": ",", // separator used in case of csv files -// "header_define_character": ":", // the starting character for header definition used in case of CSV files // "run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together // "concurrent_requests": 1024, // maximum simultaneous requests/files to process, 0 for unlimited // "source_path": "/var/spool/cgrates/ers/in", // read data from this path // "processed_path": "/var/spool/cgrates/ers/out", // move processed data here // "opts": { +// // FileCSV and PartialCSV +// "csvRowLength": 0, // Number of fields from csv file +// "csvFieldSeparator": ",", // separator used when reading the fields +// "csvHeaderDefineChar": ":", // the starting character for header definition used in case of CSV files +// // "csvLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + +// // PartialCSV +// "csvCacheExpiryAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> +// // "csvRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + +// // FlatStore +// "fstRowLength": 0, // Number of fields from csv file +// "fstFieldSeparator": ",", // separator used when reading the fields +// // "fstFailedCallsPrefix": "" // Used in case of flatstore CDRs to avoid searching for BYE records +// // "fstRecordCacheTTL": "1s" // Duration to cache partial records when not pairing +// // "fstLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + +// // FileXML +// "xmlRootPath": "", // path towards one event in case of XML CDRs + + // // "queueID": "cgrates_cdrs", // the queue id for AMQP, AMQPv1, SQS and S3 readers from were the events are read // // "queueIDProcessed": "", // the queue id for AMQP, AMQPv1, SQS and S3 readers were the events are sent after they are processed -// // FileCSV, FlatStore and PartialCSV -// // "lazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field - // // AMQP // // "consumerTag": "cgrates", // the ID of the consumer // // "exchange": "", @@ -360,11 +375,11 @@ // // SQL // // "dbName": "cgrates", // the name of the database from were the events are read // // "tableName": "cdrs", // the name of the table from were the events are read -// // "sslmode": "disable", // the ssl mode for postgres db +// // "sslMode": "disable", // the ssl mode for postgres db // // "dbNameProcessed": "", // the name of the database were the events are sent after they are processed // // "tableNameProcessed": "", // the name of the table were the events are sent after they are processed -// // "sslmodeProcessed": "", // the ssl mode for postgres db +// // "sslModeProcessed": "", // the ssl mode for postgres db // // SQS and S3 // // "awsRegion": "", @@ -376,10 +391,9 @@ // // "awsKeyProcessed": "", // // "awsSecretProcessed": "", // // "awsTokenProcessed": "", -// // "folderPathProcessed": "", // only for S3 event posting +// // "s3FolderPathProcessed": "", // only for S3 event posting // }, -// "xml_root_path": "", // path towards one event in case of XML CDRs // "tenant": "", // tenant used by import // "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> // "filters": [], // limit parsing based on the filters diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index e9171a125..1355e7c98 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -396,7 +396,7 @@ "opts": { "dbName": "exportedDatabase", // if dbName is not present "cgrates" will be used as default "tableName": "expTable", // tableName is mandatory in opts for sql exporter - "sslmode": "disable", + "sslMode": "disable", "maxIdleConns": "10", "maxOpenConns": "100", "maxConnLifetime": "0", @@ -417,7 +417,7 @@ "opts": { "dbName": "exportedDatabase", "tableName": "expTable", - "sslmode": "disable", + "sslMode": "disable", "maxIdleConns": "10", "maxOpenConns": "100", "maxConnLifetime": "0", diff --git a/data/conf/samples/ers_internal/cgrates.json b/data/conf/samples/ers_internal/cgrates.json index 24f80338d..7c2df0d80 100644 --- a/data/conf/samples/ers_internal/cgrates.json +++ b/data/conf/samples/ers_internal/cgrates.json @@ -90,7 +90,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -115,7 +117,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -137,7 +138,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -152,7 +152,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -177,7 +176,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -207,7 +208,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -261,8 +264,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -302,8 +307,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -329,12 +336,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -375,7 +384,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_mongo/cgrates.json b/data/conf/samples/ers_mongo/cgrates.json index 842473c24..547e8cde3 100644 --- a/data/conf/samples/ers_mongo/cgrates.json +++ b/data/conf/samples/ers_mongo/cgrates.json @@ -91,7 +91,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -116,7 +118,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -138,7 +139,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -153,7 +153,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -178,7 +177,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -208,7 +209,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -262,8 +265,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -303,8 +308,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -330,12 +337,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -376,7 +385,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_mysql/cgrates.json b/data/conf/samples/ers_mysql/cgrates.json index ec187541b..9850ce9ef 100644 --- a/data/conf/samples/ers_mysql/cgrates.json +++ b/data/conf/samples/ers_mysql/cgrates.json @@ -88,7 +88,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -113,7 +115,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -135,7 +136,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -150,7 +150,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -175,7 +174,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -205,7 +206,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -259,8 +262,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -300,8 +305,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -327,12 +334,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -373,7 +382,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_postgres/cgrates.json b/data/conf/samples/ers_postgres/cgrates.json index 5440672fa..191ca9237 100644 --- a/data/conf/samples/ers_postgres/cgrates.json +++ b/data/conf/samples/ers_postgres/cgrates.json @@ -85,7 +85,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -110,7 +112,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -132,7 +133,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -147,7 +147,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -172,7 +171,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -202,7 +203,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -256,8 +259,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -297,8 +302,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -324,12 +331,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -370,7 +379,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_reload/first_reload/cgrates.json b/data/conf/samples/ers_reload/first_reload/cgrates.json index 32b036672..1f78d17b5 100644 --- a/data/conf/samples/ers_reload/first_reload/cgrates.json +++ b/data/conf/samples/ers_reload/first_reload/cgrates.json @@ -79,11 +79,13 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/ers2/out", + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, diff --git a/data/conf/samples/ers_reload/second_reload/cgrates.json b/data/conf/samples/ers_reload/second_reload/cgrates.json index 9cac8b7d9..e202b1272 100644 --- a/data/conf/samples/ers_reload/second_reload/cgrates.json +++ b/data/conf/samples/ers_reload/second_reload/cgrates.json @@ -79,11 +79,13 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], "processed_path": "/tmp/ers2/out", + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, @@ -104,7 +106,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -126,7 +127,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -141,7 +141,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -166,12 +165,13 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "flags": ["*dryrun"], - "field_separator": ";", "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], "filters":["*string:~*req.3:1002"], + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, diff --git a/ees/sql.go b/ees/sql.go index aecb58551..8ada1c034 100644 --- a/ees/sql.go +++ b/ees/sql.go @@ -75,7 +75,7 @@ func (sqlEe *SQLEe) NewSQLEeURL(cgrCfg *config.CGRConfig) (dialect gorm.Dialecto dbname = utils.IfaceAsString(vals) } ssl := utils.SQLDefaultSSLMode - if vals, has := cgrCfg.EEsCfg().Exporters[sqlEe.cfgIdx].Opts[utils.SQLSSLMode]; has { + if vals, has := cgrCfg.EEsCfg().Exporters[sqlEe.cfgIdx].Opts[utils.SSLModeCfg]; has { ssl = utils.IfaceAsString(vals) } // tableName is mandatory in opts diff --git a/ees/sql_test.go b/ees/sql_test.go index 3b1957780..fd44acb39 100644 --- a/ees/sql_test.go +++ b/ees/sql_test.go @@ -62,7 +62,7 @@ func TestNewSQLeUrl(t *testing.T) { cgrCfg := config.NewDefaultCGRConfig() cgrCfg.EEsCfg().Exporters[0].Opts[utils.SQLTableName] = "expTable" cgrCfg.EEsCfg().Exporters[0].Opts[utils.SQLDBName] = "postgres" - cgrCfg.EEsCfg().Exporters[0].Opts[utils.SQLSSLMode] = "test" + cgrCfg.EEsCfg().Exporters[0].Opts[utils.SSLModeCfg] = "test" newIDb := engine.NewInternalDB(nil, nil, true) newDM := engine.NewDataManager(newIDb, cgrCfg.CacheCfg(), nil) filterS := engine.NewFilterS(cgrCfg, nil, newDM) diff --git a/ers/amqpv1_it_test.go b/ers/amqpv1_it_test.go index cd9763064..d8fe45288 100644 --- a/ers/amqpv1_it_test.go +++ b/ers/amqpv1_it_test.go @@ -140,17 +140,14 @@ func TestAmqpv1NewAMQPv1ER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } @@ -174,17 +171,14 @@ func TestAmqpv1NewAMQPv1ER2(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } diff --git a/ers/filecsv.go b/ers/filecsv.go index a3e3365fd..e6b2bb35c 100644 --- a/ers/filecsv.go +++ b/ers/filecsv.go @@ -131,7 +131,7 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.CSV); err != nil { utils.Logger.Err( fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) @@ -142,6 +142,7 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + hdrDefChar := utils.IfaceAsString(rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts[utils.HeaderDefineCharOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -151,8 +152,8 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { return } if rowNr == 0 && len(record) > 0 && - strings.HasPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) { - record[0] = strings.TrimPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) + strings.HasPrefix(record[0], hdrDefChar) { + record[0] = strings.TrimPrefix(record[0], hdrDefChar) // map the templates indxAls = make(map[string]int) for i, hdr := range record { diff --git a/ers/filefwv_it_test.go b/ers/filefwv_it_test.go index 96b6890a7..c90e7401d 100644 --- a/ers/filefwv_it_test.go +++ b/ers/filefwv_it_test.go @@ -223,38 +223,30 @@ func TestFWVFileConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: make(map[string]interface{}), }, { - ID: "file_reader2", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader2", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: make(map[string]interface{}), }, } expected := cfg.ERsCfg().Readers[0] diff --git a/ers/filejson_test.go b/ers/filejson_test.go index 96912b0d9..d2a136065 100644 --- a/ers/filejson_test.go +++ b/ers/filejson_test.go @@ -57,40 +57,34 @@ func TestFileJSONConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfgIdx := 0 cfg.ERsCfg().Readers[cfgIdx] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*config.FCTemplate{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*config.FCTemplate{}, } rdr := &JSONFileER{ cgrCfg: cfg, cfgIdx: cfgIdx, } expected := &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*config.FCTemplate{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*config.FCTemplate{}, } result := rdr.Config() if !reflect.DeepEqual(result, expected) { diff --git a/ers/filexml.go b/ers/filexml.go index 0e2802d26..997e30b06 100644 --- a/ers/filexml.go +++ b/ers/filexml.go @@ -135,7 +135,8 @@ func (rdr *XMLFileER) processFile(fPath, fName string) (err error) { if err != nil { return err } - xmlElmts := xmlquery.Find(doc, rdr.Config().XMLRootPath.AsString("/", true)) + xmlRootPath := utils.ParseHierarchyPath(utils.IfaceAsString(rdr.Config().Opts[utils.XMLRootPathOpt]), utils.EmptyString) + xmlElmts := xmlquery.Find(doc, xmlRootPath.AsString("/", true)) rowNr := 0 // This counts the rows in the file, not really number of CDRs evsPosted := 0 timeStart := time.Now() @@ -143,7 +144,7 @@ func (rdr *XMLFileER) processFile(fPath, fName string) (err error) { for _, xmlElmt := range xmlElmts { rowNr++ // increment the rowNr after checking if it's not the end of file agReq := agents.NewAgentRequest( - config.NewXMLProvider(xmlElmt, rdr.Config().XMLRootPath), reqVars, + config.NewXMLProvider(xmlElmt, xmlRootPath), reqVars, nil, nil, nil, rdr.Config().Tenant, rdr.cgrCfg.GeneralCfg().DefaultTenant, utils.FirstNonEmpty(rdr.Config().Timezone, diff --git a/ers/filexml_test.go b/ers/filexml_test.go index 965db5840..a3a7bb92b 100644 --- a/ers/filexml_test.go +++ b/ers/filexml_test.go @@ -51,17 +51,14 @@ func TestERSNewXMLFileER(t *testing.T) { func TestERSXMLFileERConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 0, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 0, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), } result1, err := NewXMLFileER(cfg, 0, nil, nil, nil, nil) if err != nil { @@ -76,17 +73,14 @@ func TestERSXMLFileERConfig(t *testing.T) { func TestERSXMLFileERServeNil(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 0, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 0, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), } result1, err := NewXMLFileER(cfg, 0, nil, nil, nil, nil) if err != nil { diff --git a/ers/flatstore.go b/ers/flatstore.go index 5ab0c1abe..8ffcffe72 100644 --- a/ers/flatstore.go +++ b/ers/flatstore.go @@ -60,8 +60,14 @@ func NewFlatstoreER(cfg *config.CGRConfig, cfgIdx int, for i := 0; i < cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs; i++ { flatER.conReqs <- processFile // Empty initiate so we do not need to wait later when we pop } - flatER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[cfgIdx].PartialRecordCache, false, flatER.dumpToFile) - return flatER, err + var ttl time.Duration + if ttlOpt, has := flatER.Config().Opts[utils.FstPartialRecordCacheOpt]; has { + if ttl, err = utils.IfaceAsDuration(ttlOpt); err != nil { + return + } + } + flatER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, ttl, false, flatER.dumpToFile) + return flatER, nil } // FlatstoreER implements EventReader interface for Flatstore CDR @@ -138,9 +144,9 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.FlatstorePrfx); err != nil { utils.Logger.Err( - fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", + fmt.Sprintf("<%s> failed creating flatStore reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) return } @@ -148,6 +154,7 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + failCallPrfx := utils.IfaceAsString(rdr.Config().Opts[utils.FstFailedCallsPrefixOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -156,7 +163,7 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { } return } - if strings.HasPrefix(fName, rdr.Config().FailedCallsPrefix) { // Use the first index since they should be the same in all configs + if strings.HasPrefix(fName, failCallPrfx) { // Use the first index since they should be the same in all configs record = append(record, "0") // Append duration 0 for failed calls flatstore CDR } else { pr, err := NewUnpairedRecord(record, utils.FirstNonEmpty(rdr.Config().Timezone, @@ -308,7 +315,7 @@ func (rdr *FlatstoreER) dumpToFile(itmID string, value interface{}) { return } csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = rune(rdr.Config().FieldSep[0]) + csvWriter.Comma = rune(utils.IfaceAsString(rdr.Config().Opts[utils.FlatstorePrfx+utils.FieldSepOpt])[0]) if err = csvWriter.Write(unpRcd.Values); err != nil { utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", utils.ERs, unpRcd.Values, dumpFilePath, err.Error())) diff --git a/ers/flatstore_test.go b/ers/flatstore_test.go index 9c2535691..1c96f4517 100644 --- a/ers/flatstore_test.go +++ b/ers/flatstore_test.go @@ -49,38 +49,30 @@ func TestFlatstoreConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: map[string]interface{}{utils.FlatstorePrfx + utils.RowLengthOpt: 5}, }, { - ID: "file_reader2", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader2", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: map[string]interface{}{utils.FlatstorePrfx + utils.RowLengthOpt: 5}, }, } expected := cfg.ERsCfg().Readers[0] diff --git a/ers/libers.go b/ers/libers.go index a321f612b..0e96d7a15 100644 --- a/ers/libers.go +++ b/ers/libers.go @@ -36,15 +36,19 @@ func getProcessOptions(opts map[string]interface{}) (proc map[string]interface{} return } -func newCSVReader(file io.Reader, rowLenght int, fieldSep string, opts map[string]interface{}) (csvReader *csv.Reader, err error) { +func newCSVReader(file io.Reader, opts map[string]interface{}, prfx string) (csvReader *csv.Reader, err error) { csvReader = csv.NewReader(file) - csvReader.FieldsPerRecord = rowLenght + var rowLength int64 + if rowLength, err = utils.IfaceAsTInt64(opts[prfx+utils.RowLengthOpt]); err != nil { + return + } + csvReader.FieldsPerRecord = int(rowLength) csvReader.Comment = utils.CommentChar csvReader.Comma = utils.CSVSep - if len(fieldSep) > 0 { - csvReader.Comma = rune(fieldSep[0]) + if fieldSep, has := opts[prfx+utils.FieldSepOpt]; has { + csvReader.Comma = rune(utils.IfaceAsString(fieldSep)[0]) } - if val, has := opts[utils.LazyQuotes]; has { + if val, has := opts[prfx+utils.LazyQuotes]; has { csvReader.LazyQuotes, err = utils.IfaceAsBool(val) } return diff --git a/ers/partial_csv.go b/ers/partial_csv.go index 9fb9095da..80220f48d 100644 --- a/ers/partial_csv.go +++ b/ers/partial_csv.go @@ -57,17 +57,21 @@ func NewPartialCSVFileER(cfg *config.CGRConfig, cfgIdx int, rdrExit: rdrExit, conReqs: make(chan struct{}, cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs)} - var function func(itmID string, value interface{}) - if cfg.ERsCfg().Readers[cfgIdx].PartialCacheExpiryAction == utils.MetaDumpToFile { + function := pCSVFileER.postCDR + if utils.IfaceAsString(pCSVFileER.Config().Opts[utils.PartialCSVCacheExpiryActionOpt]) == utils.MetaDumpToFile { function = pCSVFileER.dumpToFile - } else { - function = pCSVFileER.postCDR } var processFile struct{} for i := 0; i < cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs; i++ { pCSVFileER.conReqs <- processFile // Empty initiate so we do not need to wait later when we pop } - pCSVFileER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[cfgIdx].PartialRecordCache, false, function) + var ttl time.Duration + if ttlOpt, has := pCSVFileER.Config().Opts[utils.PartialCSVRecordCacheOpt]; has { + if ttl, err = utils.IfaceAsDuration(ttlOpt); err != nil { + return + } + } + pCSVFileER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, ttl, false, function) return pCSVFileER, nil } @@ -145,7 +149,7 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.CSV); err != nil { utils.Logger.Err( fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) @@ -156,6 +160,7 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + hdrDefChar := utils.IfaceAsString(rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts[utils.HeaderDefineCharOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -165,8 +170,8 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { return } if rowNr == 0 && len(record) > 0 && - strings.HasPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) { - record[0] = strings.TrimPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) + strings.HasPrefix(record[0], hdrDefChar) { + record[0] = strings.TrimPrefix(record[0], hdrDefChar) // map the templates indxAls = make(map[string]int) for i, hdr := range record { @@ -321,7 +326,7 @@ func (rdr *PartialCSVFileER) dumpToFile(itmID string, value interface{}) { return } csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = rune(rdr.Config().FieldSep[0]) + csvWriter.Comma = rune(utils.IfaceAsString(rdr.Config().Opts[utils.CSV+utils.FieldSepOpt])[0]) if err = csvWriter.Write(record); err != nil { utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", utils.ERs, record, dumpFilePath, err.Error())) diff --git a/ers/partial_csv_it_test.go b/ers/partial_csv_it_test.go index 109b6c1cf..90429456f 100644 --- a/ers/partial_csv_it_test.go +++ b/ers/partial_csv_it_test.go @@ -274,7 +274,7 @@ func TestNewPartialCSVFileERCase2(t *testing.T) { func TestNewPartialCSVFileERCase3(t *testing.T) { cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].PartialCacheExpiryAction = utils.MetaDumpToFile + cfg.ERsCfg().Readers[0].Opts[utils.PartialCSVCacheExpiryActionOpt] = utils.MetaDumpToFile fltr := &engine.FilterS{} result, err := NewPartialCSVFileER(cfg, 0, nil, nil, fltr, nil) if err != nil { @@ -301,23 +301,19 @@ func TestPartialCSVConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -333,23 +329,19 @@ func TestPartialCSVConfig(t *testing.T) { conReqs: nil, } expected := &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), } result := testStruct.Config() if !reflect.DeepEqual(result, expected) { @@ -361,23 +353,19 @@ func TestPartialCSVServe1(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -402,23 +390,19 @@ func TestPartialCSVServe3(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -444,23 +428,19 @@ func TestPartialCSVServe4(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -527,23 +507,19 @@ func TestPartialCSVServe2(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} diff --git a/ers/s3_it_test.go b/ers/s3_it_test.go index bd9b6d3d6..142f45528 100644 --- a/ers/s3_it_test.go +++ b/ers/s3_it_test.go @@ -151,30 +151,24 @@ func TestNewS3ER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 1, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } @@ -208,17 +202,14 @@ func TestNewS3ERCase2(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } diff --git a/ers/sql.go b/ers/sql.go index 2f6bc4020..f84ff66cc 100644 --- a/ers/sql.go +++ b/ers/sql.go @@ -272,7 +272,7 @@ func (rdr *SQLEventReader) setURL(inURL, outURL string, opts map[string]interfac dbname = utils.IfaceAsString(vals) } ssl := utils.SQLDefaultSSLMode - if vals, has := opts[utils.SQLSSLMode]; has { + if vals, has := opts[utils.SSLModeCfg]; has { ssl = utils.IfaceAsString(vals) } @@ -321,7 +321,7 @@ func (rdr *SQLEventReader) setURL(inURL, outURL string, opts map[string]interfac outDBname = utils.IfaceAsString(vals) } outSSL = utils.SQLDefaultSSLMode - if vals, has := processedOpt[utils.SQLSSLMode]; has { + if vals, has := processedOpt[utils.SSLModeCfg]; has { outSSL = utils.IfaceAsString(vals) } rdr.expTableName = utils.CDRsTBL diff --git a/ers/sql_test.go b/ers/sql_test.go index 3f2b8f600..f5c7004c7 100644 --- a/ers/sql_test.go +++ b/ers/sql_test.go @@ -43,11 +43,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "dbName": "cgrates2", "tableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "dbNameProcessed": "cgrates3", "tableNameProcessed": "cdrs3", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -78,11 +78,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "dbName": "cgrates2", "tableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "dbNameProcessed": "cgrates3", "tableNameProcessed": "cdrs3", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -113,11 +113,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "dbName": "cgrates2", "tableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "dbNameProcessed": "cgrates2", "tableNameProcessed": "cdrs2", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { diff --git a/ers/sqs_test.go b/ers/sqs_test.go index bd1400793..15922d74b 100644 --- a/ers/sqs_test.go +++ b/ers/sqs_test.go @@ -37,17 +37,14 @@ func TestNewSQSER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, @@ -68,17 +65,14 @@ func TestSQSERServeRunDelay0(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, @@ -97,17 +91,14 @@ func TestSQSERServe(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, diff --git a/utils/consts.go b/utils/consts.go index ab7668b3d..7b2315916 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -428,6 +428,7 @@ const ( MetaDateTime = "*datetime" MetaMaskedDestination = "*masked_destination" MetaUnixTimestamp = "*unix_timestamp" + MetaPostCDR = "*post_cdr" MetaDumpToFile = "*dump_to_file" NonTransactional = "" DataDB = "data_db" @@ -1767,7 +1768,7 @@ const ( PrefixIndexedFieldsCfg = "prefix_indexed_fields" SuffixIndexedFieldsCfg = "suffix_indexed_fields" QueryTimeoutCfg = "query_timeout" - SSLModeCfg = "sslmode" + SSLModeCfg = "sslMode" ItemsCfg = "items" OptsCfg = "opts" Tenants = "tenants" @@ -2128,22 +2129,16 @@ const ( // EventReaderCfg const ( - IDCfg = "id" - CacheCfg = "cache" - RowLengthCfg = "row_length" - FieldSepCfg = "field_separator" - HeaderDefCharCfg = "header_define_character" - RunDelayCfg = "run_delay" - SourcePathCfg = "source_path" - ProcessedPathCfg = "processed_path" - XMLRootPathCfg = "xml_root_path" - TenantCfg = "tenant" - FlagsCfg = "flags" - FailedCallsPrefixCfg = "failed_calls_prefix" - PartialRecordCacheCfg = "partial_record_cache" - PartialCacheExpiryActionCfg = "partial_cache_expiry_action" - FieldsCfg = "fields" - CacheDumpFieldsCfg = "cache_dump_fields" + IDCfg = "id" + CacheCfg = "cache" + FieldSepCfg = "field_separator" + RunDelayCfg = "run_delay" + SourcePathCfg = "source_path" + ProcessedPathCfg = "processed_path" + TenantCfg = "tenant" + FlagsCfg = "flags" + FieldsCfg = "fields" + CacheDumpFieldsCfg = "cache_dump_fields" ) // RegistrarCCfg @@ -2323,7 +2318,7 @@ const ( AWSKey = "awsKey" AWSSecret = "awsSecret" AWSToken = "awsToken" - AWSFolderPath = "folderPath" + AWSFolderPath = "s3FolderPath" KafkaTopic = "topic" KafkaGroupID = "groupID" KafkaMaxWait = "maxWait" @@ -2340,19 +2335,34 @@ const ( AMQPDefaultConsumerTag = "cgrates" AMQPConsumerTag = "consumerTag" - LazyQuotes = "lazyQuotes" - KafkaDefaultTopic = "cgrates" KafkaDefaultGroupID = "cgrates" KafkaDefaultMaxWait = time.Millisecond SQLDBName = "dbName" SQLTableName = "tableName" - SQLSSLMode = "sslmode" SQLDefaultSSLMode = "disable" SQLDefaultDBName = "cgrates" ProcessedOpt = "Processed" + + // fileCSV + RowLengthOpt = "RowLength" + FieldSepOpt = "FieldSeparator" + HeaderDefineCharOpt = "csvHeaderDefineChar" + LazyQuotes = "LazyQuotes" + + // partialCSV + PartialCSVCacheExpiryActionOpt = "csvCacheExpiryAction" + PartialCSVRecordCacheOpt = "csvRecordCacheTTL" + + // flatStore + FlatstorePrfx = "fst" + FstFailedCallsPrefixOpt = "fstFailedCallsPrefix" + FstPartialRecordCacheOpt = "fstRecordCacheTTL" + + // fileXML + XMLRootPathOpt = "xmlRootPath" ) // Analyzers constants