From 5105d3aa85b342b4a407a5d52ab4e804b5cbf1c7 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 27 Apr 2021 17:19:57 +0300 Subject: [PATCH] Updated ers config --- config/config_defaults.go | 44 +- config/config_it_test.go | 51 +- config/config_json_test.go | 38 +- config/config_test.go | 151 ++-- config/configsanity.go | 53 +- config/configsanity_test.go | 8 +- config/erscfg.go | 113 +-- config/erscfg_test.go | 651 +++++++++--------- config/libconfig_json.go | 33 +- config/stordbcfg_test.go | 2 +- data/conf/cgrates/cgrates.json | 32 +- data/conf/samples/ees/cgrates.json | 4 +- data/conf/samples/ers_internal/cgrates.json | 36 +- data/conf/samples/ers_mongo/cgrates.json | 36 +- data/conf/samples/ers_mysql/cgrates.json | 36 +- data/conf/samples/ers_postgres/cgrates.json | 36 +- .../ers_reload/first_reload/cgrates.json | 4 +- .../ers_reload/second_reload/cgrates.json | 12 +- ers/amqpv1_it_test.go | 38 +- ers/filecsv.go | 7 +- ers/filecsv_it_test.go | 1 - ers/filefwv_it_test.go | 52 +- ers/filejson_test.go | 50 +- ers/filexml.go | 5 +- ers/filexml_test.go | 38 +- ers/flatstore.go | 19 +- ers/flatstore_it_test.go | 16 +- ers/flatstore_test.go | 52 +- ers/libers.go | 14 +- ers/partial_csv.go | 23 +- ers/partial_csv_it_test.go | 184 +++-- ers/s3_it_test.go | 57 +- ers/sql_test.go | 12 +- ers/sqs_test.go | 57 +- utils/consts.go | 48 +- 35 files changed, 1019 insertions(+), 994 deletions(-) diff --git a/config/config_defaults.go b/config/config_defaults.go index 571122576..012fd468e 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -138,7 +138,7 @@ const CGRATES_CFG_JSON = ` "sqlMaxIdleConns": 10, // maximum database connections idle, not applying for mongo "sqlConnMaxLifetime": 0, // maximum amount of time in seconds a connection may be reused (0 for unlimited), not applying for mongo "mongoQueryTimeout":"10s", // timeout for query when mongo is used - "sslmode":"disable", // postgresSSLMode in case of *postgres + "sslMode":"disable", // postgresSSLMode in case of *postgres "mysqlLocation": "Local", // the location the time from mysql is retrived }, "items":{ @@ -361,20 +361,35 @@ const CGRATES_CFG_JSON = ` { "id": "*default", // identifier of the EventReader profile "type": "*none", // reader type <*file_csv> - "row_length" : 0, // Number of fields from csv file - "field_separator": ",", // separator used in case of csv files - "header_define_character": ":", // the starting character for header definition used in case of CSV files "run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together "concurrent_requests": 1024, // maximum simultaneous requests/files to process, 0 for unlimited "source_path": "/var/spool/cgrates/ers/in", // read data from this path "processed_path": "/var/spool/cgrates/ers/out", // move processed data here "opts": { + // FileCSV and PartialCSV + "csvRowLength": 0, // Number of fields from csv file + "csvFieldSeparator": ",", // separator used when reading the fields + "csvHeaderDefineChar": ":", // the starting character for header definition used in case of CSV files + // "csvLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + + // PartialCSV + "csvCacheExpiryAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> + // "csvRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + + // FlatStore + "fstRowLength": 0, // Number of fields from csv file + "fstFieldSeparator": ",", // separator used when reading the fields + // "fstFailedCallsPrefix": "" // Used in case of flatstore CDRs to avoid searching for BYE records + // "fstRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + // "fstLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + + // FileXML + "xmlRootPath": "", // path towards one event in case of XML CDRs + + // "queueID": "cgrates_cdrs", // the queue id for AMQP, AMQPv1, SQS and S3 readers from were the events are read // "queueIDProcessed": "", // the queue id for AMQP, AMQPv1, SQS and S3 readers were the events are sent after they are processed - // FileCSV, FlatStore and PartialCSV - // "lazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field - // AMQP // "consumerTag": "cgrates", // the ID of the consumer // "amqpExchange": "", @@ -394,13 +409,13 @@ const CGRATES_CFG_JSON = ` // SQL - // "sqlDBName": "cgrates", // the name of the database from were the events are read - // "sqlTableName": "cdrs", // the name of the table from were the events are read - // "sslmode": "disable", // the postgresSSLMode for postgres db + // "sqldbName": "cgrates", // the name of the database from were the events are read + // "sqltableName": "cdrs", // the name of the table from were the events are read + // "sslMode": "disable", // the postgresSSLMode for postgres db - // "sqlDBNameProcessed": "", // the name of the database were the events are sent after they are processed: "", // the name of the database were the events are sent after they are processed - // "sqlTableNameProcessed": "", // the name of the table were the events are sent after they are processed - // "sslmodeProcessed": "", // the postgresSSLMode for postgres db + // "sqldbNameProcessed": "", // the name of the database were the events are sent after they are processed + // "sqltableNameProcessed": "", // the name of the table were the events are sent after they are processed + // "sslModeProcessed": "", // the postgresSSLMode for postgres db // SQS and S3 // "awsRegion": "", @@ -412,10 +427,9 @@ const CGRATES_CFG_JSON = ` // "awsKeyProcessed": "", // "awsSecretProcessed": "", // "awsTokenProcessed": "", - // "folderPathProcessed": "", // only for S3 event posting + // "s3FolderPathProcessed": "", // only for S3 event posting }, - "xml_root_path": "", // path towards one event in case of XML CDRs "tenant": "", // tenant used by import "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> "filters": [], // limit parsing based on the filters diff --git a/config/config_it_test.go b/config/config_it_test.go index 01398526c..7ede18f4e 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -561,37 +561,30 @@ func testCGRConfigReloadERs(t *testing.T) { SessionSConns: []string{utils.MetaLocalHost}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Flags: flagsDefault, - Fields: content, - CacheDumpFields: []*FCTemplate{}, - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Flags: flagsDefault, + Fields: content, + CacheDumpFields: []*FCTemplate{}, + Opts: make(map[string]interface{}), }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - Filters: []string{}, - Flags: flags, - Fields: content, - CacheDumpFields: []*FCTemplate{}, - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Filters: []string{}, + Flags: flags, + Fields: content, + CacheDumpFields: []*FCTemplate{}, + Opts: make(map[string]interface{}), }, }, } diff --git a/config/config_json_test.go b/config/config_json_test.go index e0a4935c5..8259b84da 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -1787,23 +1787,27 @@ func TestDfEventReaderCfg(t *testing.T) { Sessions_conns: &[]string{utils.MetaInternal}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer(utils.MetaDefault), - Type: utils.StringPointer(utils.MetaNone), - Row_length: utils.IntPointer(0), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("0"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), - Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), - Xml_root_path: utils.StringPointer(utils.EmptyString), - Tenant: utils.StringPointer(utils.EmptyString), - Timezone: utils.StringPointer(utils.EmptyString), - Filters: &[]string{}, - Flags: &[]string{}, - Fields: &cdrFields, - Cache_dump_fields: &[]*FcTemplateJsonCfg{}, - Opts: make(map[string]interface{}), + Id: utils.StringPointer(utils.MetaDefault), + Type: utils.StringPointer(utils.MetaNone), + Run_delay: utils.StringPointer("0"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), + Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), + Tenant: utils.StringPointer(utils.EmptyString), + Timezone: utils.StringPointer(utils.EmptyString), + Filters: &[]string{}, + Flags: &[]string{}, + Fields: &cdrFields, + Cache_dump_fields: &[]*FcTemplateJsonCfg{}, + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } diff --git a/config/config_test.go b/config/config_test.go index e7d3b0542..b217b13da 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2366,23 +2366,27 @@ func TestERSConfig(t *testing.T) { SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: make([]*FCTemplate, 0), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -4366,26 +4370,27 @@ func TestV1GetConfigSectionERS(t *testing.T) { utils.SessionSConnsCfg: []string{utils.MetaInternal}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: utils.EmptyString, - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: utils.EmptyString, - utils.TimezoneCfg: utils.EmptyString, - utils.XMLRootPathCfg: []string{utils.EmptyString}, - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: utils.MetaNone, - utils.FailedCallsPrefixCfg: utils.EmptyString, - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", - utils.FieldsCfg: []string{}, - utils.OptsCfg: make(map[string]interface{}), + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: utils.EmptyString, + utils.TimezoneCfg: utils.EmptyString, + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: utils.MetaNone, + utils.FieldsCfg: []string{}, + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, }, @@ -4713,7 +4718,7 @@ func TestV1GetConfigAsJSONDataDB(t *testing.T) { func TestV1GetConfigAsJSONStorDB(t *testing.T) { var reply string - expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslmode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` + expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` cfgCgr := NewDefaultCGRConfig() if err := cfgCgr.V1GetConfigAsJSON(&SectionWithAPIOpts{Section: STORDB_JSN}, &reply); err != nil { t.Error(err) @@ -5045,7 +5050,7 @@ func TestV1GetConfigAsJSONCfgEES(t *testing.T) { func TestV1GetConfigAsJSONCfgERS(t *testing.T) { var reply string - expected := `{"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"failed_calls_prefix":"","field_separator":",","fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"header_define_character":":","id":"*default","opts":{},"partial_cache_expiry_action":"","partial_record_cache":"0","processed_path":"/var/spool/cgrates/ers/out","row_length":0,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none","xml_root_path":[""]}],"sessions_conns":["*internal"]}}` + expected := `{"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]}}` cgrCfg := NewDefaultCGRConfig() if err := cgrCfg.V1GetConfigAsJSON(&SectionWithAPIOpts{Section: ERsJson}, &reply); err != nil { t.Error(err) @@ -5213,7 +5218,7 @@ func TestV1GetConfigAsJSONAllConfig(t *testing.T) { } }` var reply string - expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"apiers_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_account_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destination_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"remote":false,"replicate":false},"*accounts":{"remote":false,"replicate":false},"*action_plans":{"remote":false,"replicate":false},"*action_triggers":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rating_plans":{"remote":false,"replicate":false},"*rating_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*shared_groups":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"failed_calls_prefix":"","field_separator":",","fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"header_define_character":":","id":"*default","opts":{},"partial_cache_expiry_action":"","partial_record_cache":"0","processed_path":"/var/spool/cgrates/ers/out","row_length":0,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none","xml_root_path":[""]}],"sessions_conns":["*internal"]},"filters":{"apiers_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"caches_conns":["*internal"],"dynaprepaid_actionplans":[],"enabled":false,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"stats_conns":[],"thresholds_conns":[]},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sessions":{"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslmode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` + expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"apiers_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_account_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destination_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"remote":false,"replicate":false},"*accounts":{"remote":false,"replicate":false},"*action_plans":{"remote":false,"replicate":false},"*action_triggers":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rating_plans":{"remote":false,"replicate":false},"*rating_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*shared_groups":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"apiers_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"caches_conns":["*internal"],"dynaprepaid_actionplans":[],"enabled":false,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"stats_conns":[],"thresholds_conns":[]},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sessions":{"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100,"sslMode":"disable"},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSON) if err != nil { t.Fatal(err) @@ -5262,19 +5267,16 @@ func TestCgrCdfEventReader(t *testing.T) { SessionSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaSessionS)}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -5300,7 +5302,15 @@ func TestCgrCdfEventReader(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: []*FCTemplate{}, - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -5352,19 +5362,16 @@ func TestCgrCdfEventExporter(t *testing.T) { func TestCgrCfgEventReaderDefault(t *testing.T) { eCfg := &EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -5390,7 +5397,15 @@ func TestCgrCfgEventReaderDefault(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, } for _, v := range eCfg.Fields { v.ComputePath() diff --git a/config/configsanity.go b/config/configsanity.go index e93f8a6f4..03acf7ae3 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -695,14 +695,61 @@ func (cfg *CGRConfig) checkConfigSanity() error { } switch rdr.Type { - case utils.MetaFileCSV, utils.MetaPartialCSV, utils.MetaFlatstore: + case utils.MetaFileCSV, utils.MetaPartialCSV: for _, dir := range []string{rdr.ProcessedPath, rdr.SourcePath} { if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) } } - if rdr.FieldSep == utils.EmptyString { - return fmt.Errorf("<%s> empty FieldSep for reader with ID: %s", utils.ERs, rdr.ID) + if fldSep, has := rdr.Opts[utils.CSV+utils.FieldSepOpt]; has && + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.CSV+utils.FieldSepOpt, rdr.ID) + } + if rowl, has := rdr.Opts[utils.CSV+utils.RowLengthOpt]; has { + if _, err := utils.IfaceAsTInt64(rowl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.RowLengthOpt, err.Error(), rdr.ID) + } + } + if lq, has := rdr.Opts[utils.CSV+utils.LazyQuotes]; has { + if _, err := utils.IfaceAsBool(lq); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.LazyQuotes, err.Error(), rdr.ID) + } + } + if rdr.Type == utils.MetaPartialCSV { + if act, has := rdr.Opts[utils.PartialCSVCacheExpiryActionOpt]; has && (utils.IfaceAsString(act) != utils.MetaDumpToFile || + utils.IfaceAsString(act) != utils.MetaPostCDR) { + return fmt.Errorf("<%s> wrong partial expiry action for reader with ID: %s", utils.ERs, rdr.ID) + } + if ttl, has := rdr.Opts[utils.PartialCSVRecordCacheOpt]; has { + if _, err := utils.IfaceAsDuration(ttl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.PartialCSVRecordCacheOpt, err.Error(), rdr.ID) + } + } + } + case utils.MetaFlatstore: + for _, dir := range []string{rdr.ProcessedPath, rdr.SourcePath} { + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) + } + } + if fldSep, has := rdr.Opts[utils.FlatstorePrfx+utils.FieldSepOpt]; has && + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.FieldSepOpt, rdr.ID) + } + if rowl, has := rdr.Opts[utils.FlatstorePrfx+utils.RowLengthOpt]; has { + if _, err := utils.IfaceAsTInt64(rowl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.RowLengthOpt, err.Error(), rdr.ID) + } + } + if lq, has := rdr.Opts[utils.FlatstorePrfx+utils.LazyQuotes]; has { + if _, err := utils.IfaceAsBool(lq); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FlatstorePrfx+utils.LazyQuotes, err.Error(), rdr.ID) + } + } + if ttl, has := rdr.Opts[utils.FstPartialRecordCacheOpt]; has { + if _, err := utils.IfaceAsDuration(ttl); err != nil { + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.FstPartialRecordCacheOpt, err.Error(), rdr.ID) + } } case utils.MetaKafkajsonMap: if rdr.RunDelay > 0 { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 7e1b7521e..2f6f6ce5c 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1243,9 +1243,9 @@ func TestConfigSanityEventReader(t *testing.T) { Type: utils.MetaFileCSV, ProcessedPath: "/", SourcePath: "/", - FieldSep: "", + Opts: map[string]interface{}{"csvFieldSeparator": ""}, }} - expected = " empty FieldSep for reader with ID: test3" + expected = " empty csvFieldSeparator for reader with ID: test3" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } @@ -1253,7 +1253,7 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test4", Type: utils.MetaKafkajsonMap, RunDelay: 1, - FieldSep: utils.InInFieldSep, + Opts: map[string]interface{}{"csvFieldSeparator": ","}, } expected = " the RunDelay field can not be bigger than zero for reader with ID: test4" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1263,7 +1263,6 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test5", Type: utils.MetaFileXML, RunDelay: 0, - FieldSep: utils.InInFieldSep, ProcessedPath: "not/a/path", SourcePath: "not/a/path", } @@ -1276,7 +1275,6 @@ func TestConfigSanityEventReader(t *testing.T) { ID: "test5", Type: utils.MetaFileFWV, RunDelay: 0, - FieldSep: utils.InInFieldSep, ProcessedPath: "not/a/path", SourcePath: "not/a/path", } diff --git a/config/erscfg.go b/config/erscfg.go index 1459adb93..2d9845dfd 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -126,26 +126,19 @@ func (erS *ERsCfg) AsMapInterface(separator string) (initialMP map[string]interf // EventReaderCfg the event for the Event Reader type EventReaderCfg struct { - ID string - Type string - RowLength int - FieldSep string - HeaderDefineChar string - RunDelay time.Duration - ConcurrentReqs int - SourcePath string - ProcessedPath string - Opts map[string]interface{} - XMLRootPath utils.HierarchyPath - Tenant RSRParsers - Timezone string - Filters []string - Flags utils.FlagsWithParams - FailedCallsPrefix string // Used in case of flatstore CDRs to avoid searching for BYE records - PartialRecordCache time.Duration // Duration to cache partial records when not pairing - PartialCacheExpiryAction string - Fields []*FCTemplate - CacheDumpFields []*FCTemplate + ID string + Type string + RunDelay time.Duration + ConcurrentReqs int + SourcePath string + ProcessedPath string + Opts map[string]interface{} + Tenant RSRParsers + Timezone string + Filters []string + Flags utils.FlagsWithParams + Fields []*FCTemplate + CacheDumpFields []*FCTemplate } func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplates map[string][]*FCTemplate, sep string) (err error) { @@ -158,15 +151,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Type != nil { er.Type = *jsnCfg.Type } - if jsnCfg.Row_length != nil { - er.RowLength = *jsnCfg.Row_length - } - if jsnCfg.Field_separator != nil { - er.FieldSep = *jsnCfg.Field_separator - } - if jsnCfg.Header_define_character != nil { - er.HeaderDefineChar = *jsnCfg.Header_define_character - } if jsnCfg.Run_delay != nil { if er.RunDelay, err = utils.ParseDurationWithNanosecs(*jsnCfg.Run_delay); err != nil { return @@ -181,9 +165,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Processed_path != nil { er.ProcessedPath = *jsnCfg.Processed_path } - if jsnCfg.Xml_root_path != nil { - er.XMLRootPath = utils.ParseHierarchyPath(*jsnCfg.Xml_root_path, utils.EmptyString) - } if jsnCfg.Tenant != nil { if er.Tenant, err = NewRSRParsers(*jsnCfg.Tenant, sep); err != nil { return err @@ -201,17 +182,6 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat if jsnCfg.Flags != nil { er.Flags = utils.FlagsWithParamsFromSlice(*jsnCfg.Flags) } - if jsnCfg.Failed_calls_prefix != nil { - er.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix - } - if jsnCfg.Partial_record_cache != nil { - if er.PartialRecordCache, err = utils.ParseDurationWithNanosecs(*jsnCfg.Partial_record_cache); err != nil { - return err - } - } - if jsnCfg.Partial_cache_expiry_action != nil { - er.PartialCacheExpiryAction = *jsnCfg.Partial_cache_expiry_action - } if jsnCfg.Fields != nil { if er.Fields, err = FCTemplatesFromFCTemplatesJSONCfg(*jsnCfg.Fields, sep); err != nil { return err @@ -243,22 +213,16 @@ func (er *EventReaderCfg) loadFromJSONCfg(jsnCfg *EventReaderJsonCfg, msgTemplat // Clone returns a deep copy of EventReaderCfg func (er EventReaderCfg) Clone() (cln *EventReaderCfg) { cln = &EventReaderCfg{ - ID: er.ID, - Type: er.Type, - FieldSep: er.FieldSep, - HeaderDefineChar: er.HeaderDefineChar, - RunDelay: er.RunDelay, - ConcurrentReqs: er.ConcurrentReqs, - SourcePath: er.SourcePath, - ProcessedPath: er.ProcessedPath, - XMLRootPath: er.XMLRootPath.Clone(), - Tenant: er.Tenant.Clone(), - Timezone: er.Timezone, - Flags: er.Flags.Clone(), - FailedCallsPrefix: er.FailedCallsPrefix, - PartialCacheExpiryAction: er.PartialCacheExpiryAction, - PartialRecordCache: er.PartialRecordCache, - Opts: make(map[string]interface{}), + ID: er.ID, + Type: er.Type, + RunDelay: er.RunDelay, + ConcurrentReqs: er.ConcurrentReqs, + SourcePath: er.SourcePath, + ProcessedPath: er.ProcessedPath, + Tenant: er.Tenant.Clone(), + Timezone: er.Timezone, + Flags: er.Flags.Clone(), + Opts: make(map[string]interface{}), } if er.Filters != nil { cln.Filters = make([]string, len(er.Filters)) @@ -287,23 +251,16 @@ func (er EventReaderCfg) Clone() (cln *EventReaderCfg) { // AsMapInterface returns the config as a map[string]interface{} func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string]interface{}) { initialMP = map[string]interface{}{ - utils.IDCfg: er.ID, - utils.TypeCfg: er.Type, - utils.RowLengthCfg: er.RowLength, - utils.FieldSepCfg: er.FieldSep, - utils.HeaderDefCharCfg: er.HeaderDefineChar, - utils.ConcurrentRequestsCfg: er.ConcurrentReqs, - utils.SourcePathCfg: er.SourcePath, - utils.ProcessedPathCfg: er.ProcessedPath, - utils.TenantCfg: er.Tenant.GetRule(separator), - utils.XMLRootPathCfg: []string(er.XMLRootPath), - utils.TimezoneCfg: er.Timezone, - utils.FiltersCfg: er.Filters, - utils.FlagsCfg: []string{}, - utils.FailedCallsPrefixCfg: er.FailedCallsPrefix, - utils.PartialCacheExpiryActionCfg: er.PartialCacheExpiryAction, - utils.PartialRecordCacheCfg: "0", - utils.RunDelayCfg: "0", + utils.IDCfg: er.ID, + utils.TypeCfg: er.Type, + utils.ConcurrentRequestsCfg: er.ConcurrentReqs, + utils.SourcePathCfg: er.SourcePath, + utils.ProcessedPathCfg: er.ProcessedPath, + utils.TenantCfg: er.Tenant.GetRule(separator), + utils.TimezoneCfg: er.Timezone, + utils.FiltersCfg: er.Filters, + utils.FlagsCfg: []string{}, + utils.RunDelayCfg: "0", } opts := make(map[string]interface{}) @@ -336,9 +293,5 @@ func (er *EventReaderCfg) AsMapInterface(separator string) (initialMP map[string } else if er.RunDelay < 0 { initialMP[utils.RunDelayCfg] = "-1" } - - if er.PartialRecordCache != 0 { - initialMP[utils.PartialRecordCacheCfg] = er.PartialRecordCache.String() - } return } diff --git a/config/erscfg_test.go b/config/erscfg_test.go index 5dd4391b6..ffdc18bdb 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -65,19 +65,16 @@ func TestERSClone(t *testing.T) { SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -103,25 +100,27 @@ func TestERSClone(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: "*file_csv", - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), - Timezone: utils.EmptyString, - Filters: []string{"randomFiletrs"}, - Flags: utils.FlagsWithParams{}, - FailedCallsPrefix: "randomPrefix", - PartialRecordCache: time.Second, - PartialCacheExpiryAction: "randomAction", + ID: "file_reader1", + Type: "*file_csv", + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), + Timezone: utils.EmptyString, + Filters: []string{"randomFiletrs"}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -133,7 +132,14 @@ func TestERSClone(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, Opts: map[string]interface{}{ - utils.MetaDefault: "randomVal", + utils.MetaDefault: "randomVal", + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", }, }, }, @@ -180,21 +186,6 @@ func TestEventReaderloadFromJsonCase1(t *testing.T) { } } -func TestEventReaderloadFromJsonCase2(t *testing.T) { - cfgJSON := &ERsJsonCfg{ - Readers: &[]*EventReaderJsonCfg{ - { - Partial_record_cache: utils.StringPointer("1ss"), - }, - }, - } - expected := "time: unknown unit \"ss\" in duration \"1ss\"" - jsoncfg := NewDefaultCGRConfig() - if err = jsoncfg.ersCfg.loadFromJSONCfg(cfgJSON, jsoncfg.templates, jsoncfg.generalCfg.RSRSep, jsoncfg.dfltEvRdr, jsoncfg.generalCfg.RSRSep); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - func TestEventReaderloadFromJsonCase3(t *testing.T) { cfgJSON := &ERsJsonCfg{ Readers: &[]*EventReaderJsonCfg{ @@ -216,19 +207,16 @@ func TestERSLoadFromjsonCfg(t *testing.T) { SessionSConns: []string{"conn1", "conn3"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -254,22 +242,27 @@ func TestERSLoadFromjsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -295,7 +288,15 @@ func TestERSLoadFromjsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -403,23 +404,16 @@ func TestERSloadFromJsonCase3(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.AnswerTime), @@ -437,19 +431,16 @@ func TestERSloadFromJsonCase3(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -475,25 +466,27 @@ func TestERSloadFromJsonCase3(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, @@ -503,7 +496,15 @@ func TestERSloadFromJsonCase3(t *testing.T) { }, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -534,24 +535,17 @@ func TestERSloadFromJsonCase4(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), - Fields: &[]*FcTemplateJsonCfg{}, + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, + Fields: &[]*FcTemplateJsonCfg{}, Cache_dump_fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer("OriginID"), @@ -568,19 +562,16 @@ func TestERSloadFromJsonCase4(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -606,26 +597,28 @@ func TestERSloadFromJsonCase4(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, - Fields: []*FCTemplate{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*FCTemplate{}, CacheDumpFields: []*FCTemplate{ { Tag: "OrderID", @@ -634,7 +627,15 @@ func TestERSloadFromJsonCase4(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.OrderID", utils.InfieldSep), }, }, - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -687,19 +688,16 @@ func TestEventReaderSameID(t *testing.T) { SessionSConns: []string{"conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -725,29 +723,41 @@ func TestEventReaderSameID(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -819,24 +829,17 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: []string{""}, - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -850,15 +853,20 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.OptsCfg: make(map[string]interface{}), + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { utils.CacheDumpFieldsCfg: []map[string]interface{}{}, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -872,19 +880,23 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "file_reader1", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/tmp/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "-1", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/tmp/ers/in", - utils.TenantCfg: "~*req.Destination1", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: []string{""}, - utils.OptsCfg: make(map[string]interface{}), + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "file_reader1", + utils.ProcessedPathCfg: "/tmp/ers/out", + utils.RunDelayCfg: "-1", + utils.SourcePathCfg: "/tmp/ers/in", + utils.TenantCfg: "~*req.Destination1", + utils.TimezoneCfg: "", + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } @@ -926,24 +938,17 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.PartialRecordCacheCfg: "0", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "0", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: []string{""}, - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -957,7 +962,15 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.OptsCfg: make(map[string]interface{}), + utils.OptsCfg: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { utils.CacheDumpFieldsCfg: []map[string]interface{}{ @@ -965,9 +978,6 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { }, utils.ConcurrentRequestsCfg: 1024, utils.TypeCfg: "*file_csv", - utils.FailedCallsPrefixCfg: "", - utils.FieldSepCfg: ",", - utils.HeaderDefCharCfg: ":", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -981,20 +991,23 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.AnswerTime", utils.TagCfg: "AnswerTime", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.12"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, - utils.FiltersCfg: []string{"randomFilter"}, - utils.FlagsCfg: []string{"randomFlag"}, - utils.IDCfg: "file_reader1", - utils.ProcessedPathCfg: "/tmp/ers/out", - utils.RowLengthCfg: 0, - utils.RunDelayCfg: "10s", - utils.PartialRecordCacheCfg: "1s", - utils.PartialCacheExpiryActionCfg: "", - utils.SourcePathCfg: "/tmp/ers/in", - utils.TenantCfg: "~*req.Destination1", - utils.TimezoneCfg: "", - utils.XMLRootPathCfg: []string{""}, + utils.FiltersCfg: []string{"randomFilter"}, + utils.FlagsCfg: []string{"randomFlag"}, + utils.IDCfg: "file_reader1", + utils.ProcessedPathCfg: "/tmp/ers/out", + utils.RunDelayCfg: "10s", + utils.SourcePathCfg: "/tmp/ers/in", + utils.TenantCfg: "~*req.Destination1", + utils.TimezoneCfg: "", utils.OptsCfg: map[string]interface{}{ - utils.KafkaGroupID: "test", + utils.KafkaGroupID: "test", + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", }, }, }, @@ -1012,23 +1025,16 @@ func TestERsloadFromJsonCfg(t *testing.T) { Sessions_conns: &[]string{"*conn1"}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer("file_reader1"), - Type: utils.StringPointer(utils.MetaFileCSV), - Row_length: utils.IntPointer(5), - Field_separator: utils.StringPointer(","), - Header_define_character: utils.StringPointer(":"), - Run_delay: utils.StringPointer("-1"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/tmp/ers/in"), - Processed_path: utils.StringPointer("/tmp/ers/out"), - Xml_root_path: utils.StringPointer(""), - Tenant: nil, - Timezone: utils.StringPointer(""), - Filters: nil, - Flags: &[]string{}, - Failed_calls_prefix: nil, - Partial_cache_expiry_action: utils.StringPointer(""), - Partial_record_cache: utils.StringPointer("0"), + Id: utils.StringPointer("file_reader1"), + Type: utils.StringPointer(utils.MetaFileCSV), + Run_delay: utils.StringPointer("-1"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/tmp/ers/in"), + Processed_path: utils.StringPointer("/tmp/ers/out"), + Tenant: nil, + Timezone: utils.StringPointer(""), + Filters: nil, + Flags: &[]string{}, Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.CGRID), @@ -1047,19 +1053,16 @@ func TestERsloadFromJsonCfg(t *testing.T) { SessionSConns: []string{"*conn1"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, @@ -1085,25 +1088,27 @@ func TestERsloadFromJsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - PartialCacheExpiryAction: "", - PartialRecordCache: 0, + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, Fields: []*FCTemplate{ { Tag: utils.CGRID, @@ -1115,7 +1120,15 @@ func TestERsloadFromJsonCfg(t *testing.T) { Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, CacheDumpFields: make([]*FCTemplate, 0), - Opts: make(map[string]interface{}), + Opts: map[string]interface{}{ + "csvCacheExpiryAction": "*post_cdr", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "fstFieldSeparator": ",", + "fstRowLength": 0., + "xmlRootPath": "", + }, }, }, } diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 1d0483b74..04f346d97 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -163,26 +163,19 @@ type ERsJsonCfg struct { // EventReaderSJsonCfg is the configuration of a single EventReader type EventReaderJsonCfg struct { - Id *string - Type *string - Row_length *int - Field_separator *string - Header_define_character *string - Run_delay *string - Concurrent_requests *int - Source_path *string - Processed_path *string - Opts map[string]interface{} - Xml_root_path *string - Tenant *string - Timezone *string - Filters *[]string - Flags *[]string - Failed_calls_prefix *string - Partial_record_cache *string - Partial_cache_expiry_action *string - Fields *[]*FcTemplateJsonCfg - Cache_dump_fields *[]*FcTemplateJsonCfg + Id *string + Type *string + Run_delay *string + Concurrent_requests *int + Source_path *string + Processed_path *string + Opts map[string]interface{} + Tenant *string + Timezone *string + Filters *[]string + Flags *[]string + Fields *[]*FcTemplateJsonCfg + Cache_dump_fields *[]*FcTemplateJsonCfg } // EEsJsonCfg contains the configuration of EventExporterService diff --git a/config/stordbcfg_test.go b/config/stordbcfg_test.go index 9b600a8f4..13947a68c 100644 --- a/config/stordbcfg_test.go +++ b/config/stordbcfg_test.go @@ -209,7 +209,7 @@ func TestStorDbCfgAsMapInterface(t *testing.T) { "sqlMaxIdleConns": 10, "sqlConnMaxLifetime": 0, "mongoQueryTimeout":"10s", - "sslmode":"disable", + "sslMode":"disable", "mysqlLocation": "UTC", }, "items":{ diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 9ed164621..f3162a636 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -117,7 +117,7 @@ // "sqlMaxIdleConns": 10, // maximum database connections idle, not applying for mongo // "sqlConnMaxLifetime": 0, // maximum amount of time in seconds a connection may be reused (0 for unlimited), not applying for mongo // "mongoQueryTimeout":"10s", -// "sslmode":"disable", // sslmode in case of *postgres +// "sslMode":"disable", // sslmode in case of *postgres // "mysqlLocation": "Local", // the location the time from mysql is retrived // }, // "items":{ @@ -340,20 +340,35 @@ // { // "id": "*default", // identifier of the EventReader profile // "type": "*none", // reader type <*file_csv> -// "row_length" : 0, // Number of fields from csv file -// "field_separator": ",", // separator used in case of csv files -// "header_define_character": ":", // the starting character for header definition used in case of CSV files // "run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together // "concurrent_requests": 1024, // maximum simultaneous requests/files to process, 0 for unlimited // "source_path": "/var/spool/cgrates/ers/in", // read data from this path // "processed_path": "/var/spool/cgrates/ers/out", // move processed data here // "opts": { +// // FileCSV and PartialCSV +// "csvRowLength": 0, // Number of fields from csv file +// "csvFieldSeparator": ",", // separator used when reading the fields +// "csvHeaderDefineChar": ":", // the starting character for header definition used in case of CSV files +// // "csvLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + +// // PartialCSV +// "csvCacheExpiryAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> +// // "csvRecordCacheTTL": "1s" // Duration to cache partial records when not pairing + +// // FlatStore +// "fstRowLength": 0, // Number of fields from csv file +// "fstFieldSeparator": ",", // separator used when reading the fields +// // "fstFailedCallsPrefix": "" // Used in case of flatstore CDRs to avoid searching for BYE records +// // "fstRecordCacheTTL": "1s" // Duration to cache partial records when not pairing +// // "fstLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field + +// // FileXML +// "xmlRootPath": "", // path towards one event in case of XML CDRs + + // // "queueID": "cgrates_cdrs", // the queue id for AMQP, AMQPv1, SQS and S3 readers from were the events are read // // "queueIDProcessed": "", // the queue id for AMQP, AMQPv1, SQS and S3 readers were the events are sent after they are processed -// // FileCSV, FlatStore and PartialCSV -// // "lazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field - // // AMQP // // "consumerTag": "cgrates", // the ID of the consumer // // "amqpExchange": "", @@ -391,10 +406,9 @@ // // "awsKeyProcessed": "", // // "awsSecretProcessed": "", // // "awsTokenProcessed": "", -// // "folderPathProcessed": "", // only for S3 event posting +// // "s3FolderPathProcessed": "", // only for S3 event posting // }, -// "xml_root_path": "", // path towards one event in case of XML CDRs // "tenant": "", // tenant used by import // "timezone": "", // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> // "filters": [], // limit parsing based on the filters diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index 3cf065465..29f4b464a 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -396,7 +396,7 @@ "opts": { "sqlDBName": "exportedDatabase", // if dbName is not present "cgrates" will be used as default "sqlTableName": "expTable", // tableName is mandatory in opts for sql exporter - "sslmode": "disable", + "sslMode": "disable", "sqlMaxIdleConns": "10", "sqlMaxOpenConns": "100", "sqlMaxConnLifetime": "0", @@ -417,7 +417,7 @@ "opts": { "sqlDBName": "exportedDatabase", "sqlTableName": "expTable", - "sslmode": "disable", + "sslMode": "disable", "sqlMaxIdleConns": "10", "sqlMaxOpenConns": "100", "sqlMaxConnLifetime": "0", diff --git a/data/conf/samples/ers_internal/cgrates.json b/data/conf/samples/ers_internal/cgrates.json index b3b6c5ee9..36715d226 100644 --- a/data/conf/samples/ers_internal/cgrates.json +++ b/data/conf/samples/ers_internal/cgrates.json @@ -90,7 +90,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -115,7 +117,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -137,7 +138,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -152,7 +152,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -177,7 +176,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -207,7 +208,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -261,8 +264,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -302,8 +307,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -329,12 +336,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -375,7 +384,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_mongo/cgrates.json b/data/conf/samples/ers_mongo/cgrates.json index be17b9612..7332e0b9c 100644 --- a/data/conf/samples/ers_mongo/cgrates.json +++ b/data/conf/samples/ers_mongo/cgrates.json @@ -91,7 +91,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -116,7 +118,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -138,7 +139,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -153,7 +153,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -178,7 +177,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -208,7 +209,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -262,8 +265,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -303,8 +308,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -330,12 +337,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -376,7 +385,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_mysql/cgrates.json b/data/conf/samples/ers_mysql/cgrates.json index 6436b14cf..9374ee050 100644 --- a/data/conf/samples/ers_mysql/cgrates.json +++ b/data/conf/samples/ers_mysql/cgrates.json @@ -88,7 +88,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -113,7 +115,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -135,7 +136,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -150,7 +150,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -175,7 +174,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -205,7 +206,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -259,8 +262,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -300,8 +305,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -327,12 +334,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -373,7 +382,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_postgres/cgrates.json b/data/conf/samples/ers_postgres/cgrates.json index a6cf46267..bb479a4dc 100644 --- a/data/conf/samples/ers_postgres/cgrates.json +++ b/data/conf/samples/ers_postgres/cgrates.json @@ -85,7 +85,9 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], @@ -110,7 +112,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -132,7 +133,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -147,7 +147,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -172,7 +171,9 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "field_separator": ";", + "opts": { + "csvFieldSeparator":";" + }, "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], @@ -202,7 +203,9 @@ "source_path": "/tmp/xmlErs/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/xmlErs/out", - "xml_root_path": "broadWorksCDR.cdrData", + "opts": { + "xmlRootPath": "broadWorksCDR.cdrData", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable", "value": "~*req.broadWorksCDR.cdrData.basicModule.localCallId", "mandatory": true}, @@ -256,8 +259,10 @@ "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], "processed_path": "/tmp/partErs1/out", - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*dump_to_file", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*dump_to_file", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -297,8 +302,10 @@ "source_path": "/tmp/partErs2/in", "processed_path": "/tmp/partErs2/out", "flags": ["*cdrs"], - "partial_record_cache": "500ms", - "partial_cache_expiry_action": "*post_cdr", + "opts": { + "csvRecordCacheTTL": "500ms", + "csvCacheExpiryAction": "*post_cdr", + }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "AccId1", "path": "*cgreq.OriginID", "type": "*composed", "value": "~*req.0"}, @@ -324,12 +331,14 @@ "enabled": true, "run_delay": "-1", "type": "*flatstore", - "field_separator": "|", + "opts": { + "fstFieldSeparator":"|", + "fstFailedCallsPrefix": "missed_calls", + "fstRecordCacheTTL": "500ms", + }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", - "failed_calls_prefix": "missed_calls", "flags": ["*cdrs"], - "partial_record_cache": "500ms", "fields":[ {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, @@ -370,7 +379,6 @@ { "id": "readerWithTemplate", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/readerWithTemplate/in", "flags": ["*cdrs","*log"], diff --git a/data/conf/samples/ers_reload/first_reload/cgrates.json b/data/conf/samples/ers_reload/first_reload/cgrates.json index a18b62153..079c012d5 100644 --- a/data/conf/samples/ers_reload/first_reload/cgrates.json +++ b/data/conf/samples/ers_reload/first_reload/cgrates.json @@ -79,11 +79,13 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*cdrs","*log"], "processed_path": "/tmp/ers2/out", + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, diff --git a/data/conf/samples/ers_reload/second_reload/cgrates.json b/data/conf/samples/ers_reload/second_reload/cgrates.json index aa2178b2d..b66f960a7 100644 --- a/data/conf/samples/ers_reload/second_reload/cgrates.json +++ b/data/conf/samples/ers_reload/second_reload/cgrates.json @@ -79,11 +79,13 @@ { "id": "file_reader2", "run_delay": "-1", - "field_separator": ";", "type": "*file_csv", "source_path": "/tmp/ers2/in", "flags": ["*dryrun"], "processed_path": "/tmp/ers2/out", + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, @@ -104,7 +106,6 @@ { "id": "init_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/init_session/in", "flags": ["*initiate","*accounts","*resources","*attributes","*log"], @@ -126,7 +127,6 @@ { "id": "terminate_session", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/terminate_session/in", "flags": ["*terminate","*accounts","*resources","*log"], @@ -141,7 +141,6 @@ { "id": "create_cdr", "run_delay": "-1", - "field_separator": ",", "type": "*file_csv", "source_path": "/tmp/cdrs/in", "flags": ["*cdrs","*log"], @@ -166,12 +165,13 @@ "id": "file_reader_with_filters", "run_delay": "-1", "type": "*file_csv", - "flags": ["*dryrun"], - "field_separator": ";", "source_path": "/tmp/ers_with_filters/in", "processed_path": "/tmp/ers_with_filters/out", "flags": ["*cdrs","*log"], "filters":["*string:~*req.3:1002"], + "opts": { + "csvFieldSeparator":";" + }, "content_fields":[ // import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value {"tag": "ToR", "path": "ToR", "type": "*variable", "value": "*voice", "mandatory": true}, {"tag": "OriginID", "path": "OriginID", "type": "*variable", "value": "~*req.0", "mandatory": true}, diff --git a/ers/amqpv1_it_test.go b/ers/amqpv1_it_test.go index b359e39e2..50826d361 100644 --- a/ers/amqpv1_it_test.go +++ b/ers/amqpv1_it_test.go @@ -140,17 +140,14 @@ func TestAmqpv1NewAMQPv1ER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } @@ -174,17 +171,14 @@ func TestAmqpv1NewAMQPv1ER2(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } diff --git a/ers/filecsv.go b/ers/filecsv.go index 05847d7a1..13b90c9c2 100644 --- a/ers/filecsv.go +++ b/ers/filecsv.go @@ -132,7 +132,7 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.CSV); err != nil { utils.Logger.Err( fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) @@ -143,6 +143,7 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + hdrDefChar := utils.IfaceAsString(rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts[utils.HeaderDefineCharOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -153,8 +154,8 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { return } if rowNr == 0 && len(record) > 0 && - strings.HasPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) { - record[0] = strings.TrimPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) + strings.HasPrefix(record[0], hdrDefChar) { + record[0] = strings.TrimPrefix(record[0], hdrDefChar) // map the templates indxAls = make(map[string]int) for i, hdr := range record { diff --git a/ers/filecsv_it_test.go b/ers/filecsv_it_test.go index b06bc5cfa..2d6d94fe0 100644 --- a/ers/filecsv_it_test.go +++ b/ers/filecsv_it_test.go @@ -376,7 +376,6 @@ func testCsvITKillEngine(t *testing.T) { func TestFileCSVProcessEvent(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0].ProcessedPath = "" - cfg.ERsCfg().Readers[0].HeaderDefineChar = ":" fltrs := &engine.FilterS{} filePath := "/tmp/TestFileCSVProcessEvent/" if err := os.MkdirAll(filePath, 0777); err != nil { diff --git a/ers/filefwv_it_test.go b/ers/filefwv_it_test.go index 7d911b33e..86993d87b 100644 --- a/ers/filefwv_it_test.go +++ b/ers/filefwv_it_test.go @@ -225,38 +225,30 @@ func TestFWVFileConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: make(map[string]interface{}), }, { - ID: "file_reader2", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader2", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: make(map[string]interface{}), }, } expected := cfg.ERsCfg().Readers[0] diff --git a/ers/filejson_test.go b/ers/filejson_test.go index 96912b0d9..d2a136065 100644 --- a/ers/filejson_test.go +++ b/ers/filejson_test.go @@ -57,40 +57,34 @@ func TestFileJSONConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfgIdx := 0 cfg.ERsCfg().Readers[cfgIdx] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*config.FCTemplate{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*config.FCTemplate{}, } rdr := &JSONFileER{ cgrCfg: cfg, cfgIdx: cfgIdx, } expected := &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: []*config.FCTemplate{}, + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: []*config.FCTemplate{}, } result := rdr.Config() if !reflect.DeepEqual(result, expected) { diff --git a/ers/filexml.go b/ers/filexml.go index 70666d085..dea7ae517 100644 --- a/ers/filexml.go +++ b/ers/filexml.go @@ -134,7 +134,8 @@ func (rdr *XMLFileER) processFile(fPath, fName string) (err error) { if err != nil { return err } - xmlElmts := xmlquery.Find(doc, rdr.Config().XMLRootPath.AsString("/", true)) + xmlRootPath := utils.ParseHierarchyPath(utils.IfaceAsString(rdr.Config().Opts[utils.XMLRootPathOpt]), utils.EmptyString) + xmlElmts := xmlquery.Find(doc, xmlRootPath.AsString("/", true)) rowNr := 0 // This counts the rows in the file, not really number of CDRs evsPosted := 0 timeStart := time.Now() @@ -142,7 +143,7 @@ func (rdr *XMLFileER) processFile(fPath, fName string) (err error) { for _, xmlElmt := range xmlElmts { rowNr++ // increment the rowNr after checking if it's not the end of file agReq := agents.NewAgentRequest( - config.NewXMLProvider(xmlElmt, rdr.Config().XMLRootPath), reqVars, + config.NewXMLProvider(xmlElmt, xmlRootPath), reqVars, nil, nil, nil, rdr.Config().Tenant, rdr.cgrCfg.GeneralCfg().DefaultTenant, utils.FirstNonEmpty(rdr.Config().Timezone, diff --git a/ers/filexml_test.go b/ers/filexml_test.go index 965db5840..a3a7bb92b 100644 --- a/ers/filexml_test.go +++ b/ers/filexml_test.go @@ -51,17 +51,14 @@ func TestERSNewXMLFileER(t *testing.T) { func TestERSXMLFileERConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 0, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 0, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), } result1, err := NewXMLFileER(cfg, 0, nil, nil, nil, nil) if err != nil { @@ -76,17 +73,14 @@ func TestERSXMLFileERConfig(t *testing.T) { func TestERSXMLFileERServeNil(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0] = &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 0, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 0, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), } result1, err := NewXMLFileER(cfg, 0, nil, nil, nil, nil) if err != nil { diff --git a/ers/flatstore.go b/ers/flatstore.go index 44216cc7d..240c64a36 100644 --- a/ers/flatstore.go +++ b/ers/flatstore.go @@ -59,8 +59,14 @@ func NewFlatstoreER(cfg *config.CGRConfig, cfgIdx int, for i := 0; i < cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs; i++ { flatER.conReqs <- processFile // Empty initiate so we do not need to wait later when we pop } - flatER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[cfgIdx].PartialRecordCache, false, flatER.dumpToFile) - return flatER, err + var ttl time.Duration + if ttlOpt, has := flatER.Config().Opts[utils.FstPartialRecordCacheOpt]; has { + if ttl, err = utils.IfaceAsDuration(ttlOpt); err != nil { + return + } + } + flatER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, ttl, false, flatER.dumpToFile) + return flatER, nil } // FlatstoreER implements EventReader interface for Flatstore CDR @@ -139,9 +145,9 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.FlatstorePrfx); err != nil { utils.Logger.Err( - fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", + fmt.Sprintf("<%s> failed creating flatStore reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) return } @@ -149,6 +155,7 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + failCallPrfx := utils.IfaceAsString(rdr.Config().Opts[utils.FstFailedCallsPrefixOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -158,7 +165,7 @@ func (rdr *FlatstoreER) processFile(fPath, fName string) (err error) { } return } - if strings.HasPrefix(fName, rdr.Config().FailedCallsPrefix) { // Use the first index since they should be the same in all configs + if strings.HasPrefix(fName, failCallPrfx) { // Use the first index since they should be the same in all configs record = append(record, "0") // Append duration 0 for failed calls flatstore CDR } else { pr, err := NewUnpairedRecord(record, utils.FirstNonEmpty(rdr.Config().Timezone, @@ -310,7 +317,7 @@ func (rdr *FlatstoreER) dumpToFile(itmID string, value interface{}) { return } csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = rune(rdr.Config().FieldSep[0]) + csvWriter.Comma = rune(utils.IfaceAsString(rdr.Config().Opts[utils.FlatstorePrfx+utils.FieldSepOpt])[0]) if err = csvWriter.Write(unpRcd.Values); err != nil { utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", utils.ERs, unpRcd.Values, dumpFilePath, err.Error())) diff --git a/ers/flatstore_it_test.go b/ers/flatstore_it_test.go index 5909b2e53..ba201ffe5 100644 --- a/ers/flatstore_it_test.go +++ b/ers/flatstore_it_test.go @@ -309,7 +309,7 @@ func TestFlatstoreProcessEvent2(t *testing.T) { if err != nil { t.Error(err) } - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[0].PartialRecordCache, false, eR.dumpToFile) + eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) eR.cache.Set("baToR", pr, nil) expEvent := &utils.CGREvent{ Tenant: "cgrates.org", @@ -329,7 +329,7 @@ func TestFlatstoreProcessEvent2(t *testing.T) { APIOpts: map[string]interface{}{}, } eR.conReqs <- struct{}{} - eR.Config().FailedCallsPrefix = "x" + eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" if err := eR.processFile(filePath, fname); err != nil { t.Error(err) } @@ -374,10 +374,10 @@ func TestFlatstoreProcessEvent2CacheNotSet(t *testing.T) { conReqs: make(chan struct{}, 1), } - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[0].PartialRecordCache, false, eR.dumpToFile) + eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) eR.conReqs <- struct{}{} - eR.Config().FailedCallsPrefix = "x" + eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" if err := eR.processFile(filePath, fname); err != nil { t.Error(err) } @@ -428,11 +428,11 @@ func TestFlatstoreProcessEvent2Error1(t *testing.T) { if err == nil || err.Error() != errExpect { t.Errorf("Expected %v but received %v", errExpect, err) } - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[0].PartialRecordCache, false, eR.dumpToFile) + eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) eR.cache.Set("baToR", pr, nil) eR.conReqs <- struct{}{} - eR.Config().FailedCallsPrefix = "x" + eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" if err := eR.processFile(filePath, fname); err != nil { t.Error(err) } @@ -487,10 +487,10 @@ func TestFlatstoreProcessEvent2Error2(t *testing.T) { if err != nil { t.Error(err) } - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[0].PartialRecordCache, false, eR.dumpToFile) + eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) eR.cache.Set("baToR", pr, nil) eR.conReqs <- struct{}{} - eR.Config().FailedCallsPrefix = "x" + eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" if err := eR.processFile(filePath, fname); err != nil { t.Error(err) } diff --git a/ers/flatstore_test.go b/ers/flatstore_test.go index 0eeba6484..ba116a8d8 100644 --- a/ers/flatstore_test.go +++ b/ers/flatstore_test.go @@ -50,38 +50,30 @@ func TestFlatstoreConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: "file_reader1", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader1", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: map[string]interface{}{utils.FlatstorePrfx + utils.RowLengthOpt: 5}, }, { - ID: "file_reader2", - Type: utils.MetaFileCSV, - RowLength: 5, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/tmp/ers/in", - ProcessedPath: "/tmp/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Opts: make(map[string]interface{}), + ID: "file_reader2", + Type: utils.MetaFileCSV, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/tmp/ers/in", + ProcessedPath: "/tmp/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Opts: map[string]interface{}{utils.FlatstorePrfx + utils.RowLengthOpt: 5}, }, } expected := cfg.ERsCfg().Readers[0] diff --git a/ers/libers.go b/ers/libers.go index a321f612b..0e96d7a15 100644 --- a/ers/libers.go +++ b/ers/libers.go @@ -36,15 +36,19 @@ func getProcessOptions(opts map[string]interface{}) (proc map[string]interface{} return } -func newCSVReader(file io.Reader, rowLenght int, fieldSep string, opts map[string]interface{}) (csvReader *csv.Reader, err error) { +func newCSVReader(file io.Reader, opts map[string]interface{}, prfx string) (csvReader *csv.Reader, err error) { csvReader = csv.NewReader(file) - csvReader.FieldsPerRecord = rowLenght + var rowLength int64 + if rowLength, err = utils.IfaceAsTInt64(opts[prfx+utils.RowLengthOpt]); err != nil { + return + } + csvReader.FieldsPerRecord = int(rowLength) csvReader.Comment = utils.CommentChar csvReader.Comma = utils.CSVSep - if len(fieldSep) > 0 { - csvReader.Comma = rune(fieldSep[0]) + if fieldSep, has := opts[prfx+utils.FieldSepOpt]; has { + csvReader.Comma = rune(utils.IfaceAsString(fieldSep)[0]) } - if val, has := opts[utils.LazyQuotes]; has { + if val, has := opts[prfx+utils.LazyQuotes]; has { csvReader.LazyQuotes, err = utils.IfaceAsBool(val) } return diff --git a/ers/partial_csv.go b/ers/partial_csv.go index 90070254a..736146834 100644 --- a/ers/partial_csv.go +++ b/ers/partial_csv.go @@ -56,17 +56,21 @@ func NewPartialCSVFileER(cfg *config.CGRConfig, cfgIdx int, rdrExit: rdrExit, conReqs: make(chan struct{}, cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs)} - var function func(itmID string, value interface{}) - if cfg.ERsCfg().Readers[cfgIdx].PartialCacheExpiryAction == utils.MetaDumpToFile { + function := pCSVFileER.postCDR + if utils.IfaceAsString(pCSVFileER.Config().Opts[utils.PartialCSVCacheExpiryActionOpt]) == utils.MetaDumpToFile { function = pCSVFileER.dumpToFile - } else { - function = pCSVFileER.postCDR } var processFile struct{} for i := 0; i < cfg.ERsCfg().Readers[cfgIdx].ConcurrentReqs; i++ { pCSVFileER.conReqs <- processFile // Empty initiate so we do not need to wait later when we pop } - pCSVFileER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, cfg.ERsCfg().Readers[cfgIdx].PartialRecordCache, false, function) + var ttl time.Duration + if ttlOpt, has := pCSVFileER.Config().Opts[utils.PartialCSVRecordCacheOpt]; has { + if ttl, err = utils.IfaceAsDuration(ttlOpt); err != nil { + return + } + } + pCSVFileER.cache = ltcache.NewCache(ltcache.UnlimitedCaching, ttl, false, function) return pCSVFileER, nil } @@ -144,7 +148,7 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { } defer file.Close() var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].RowLength, rdr.Config().FieldSep, rdr.Config().Opts); err != nil { + if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.CSV); err != nil { utils.Logger.Err( fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) @@ -155,6 +159,7 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { evsPosted := 0 timeStart := time.Now() reqVars := &utils.DataNode{Type: utils.NMMapType, Map: map[string]*utils.DataNode{utils.FileName: utils.NewLeafNode(fName)}} + hdrDefChar := utils.IfaceAsString(rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].Opts[utils.HeaderDefineCharOpt]) for { var record []string if record, err = csvReader.Read(); err != nil { @@ -165,8 +170,8 @@ func (rdr *PartialCSVFileER) processFile(fPath, fName string) (err error) { return } if rowNr == 0 && len(record) > 0 && - strings.HasPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) { - record[0] = strings.TrimPrefix(record[0], rdr.cgrCfg.ERsCfg().Readers[rdr.cfgIdx].HeaderDefineChar) + strings.HasPrefix(record[0], hdrDefChar) { + record[0] = strings.TrimPrefix(record[0], hdrDefChar) // map the templates indxAls = make(map[string]int) for i, hdr := range record { @@ -321,7 +326,7 @@ func (rdr *PartialCSVFileER) dumpToFile(itmID string, value interface{}) { return } csvWriter := csv.NewWriter(fileOut) - csvWriter.Comma = rune(rdr.Config().FieldSep[0]) + csvWriter.Comma = rune(utils.IfaceAsString(rdr.Config().Opts[utils.CSV+utils.FieldSepOpt])[0]) if err = csvWriter.Write(record); err != nil { utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", utils.ERs, record, dumpFilePath, err.Error())) diff --git a/ers/partial_csv_it_test.go b/ers/partial_csv_it_test.go index 3356ffbe3..f020db9f3 100644 --- a/ers/partial_csv_it_test.go +++ b/ers/partial_csv_it_test.go @@ -279,7 +279,7 @@ func TestNewPartialCSVFileERCase2(t *testing.T) { func TestNewPartialCSVFileERCase3(t *testing.T) { cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].PartialCacheExpiryAction = utils.MetaDumpToFile + cfg.ERsCfg().Readers[0].Opts[utils.PartialCSVCacheExpiryActionOpt] = utils.MetaDumpToFile fltr := &engine.FilterS{} result, err := NewPartialCSVFileER(cfg, 0, nil, nil, fltr, nil) if err != nil { @@ -306,23 +306,19 @@ func TestPartialCSVConfig(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -338,23 +334,19 @@ func TestPartialCSVConfig(t *testing.T) { conReqs: nil, } expected := &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), } result := testStruct.Config() if !reflect.DeepEqual(result, expected) { @@ -366,23 +358,19 @@ func TestPartialCSVServe1(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -407,23 +395,19 @@ func TestPartialCSVServe3(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -449,23 +433,19 @@ func TestPartialCSVServe4(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -532,23 +512,19 @@ func TestPartialCSVServe2(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - XMLRootPath: utils.HierarchyPath{utils.EmptyString}, - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: -1, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: nil, + Opts: make(map[string]interface{}), }, } fltr := &engine.FilterS{} @@ -658,7 +634,7 @@ func TestPartialCSVProcessEvent(t *testing.T) { func TestPartialCSVProcessEventPrefix(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers[0].ProcessedPath = "" - cfg.ERsCfg().Readers[0].HeaderDefineChar = ":" + cfg.ERsCfg().Readers[0].Opts[utils.HeaderDefineCharOpt] = ":" fltrs := &engine.FilterS{} filePath := "/tmp/TestPartialCSVProcessEvent/" if err := os.MkdirAll(filePath, 0777); err != nil { diff --git a/ers/s3_it_test.go b/ers/s3_it_test.go index aff9d1a4d..5613499f6 100644 --- a/ers/s3_it_test.go +++ b/ers/s3_it_test.go @@ -151,30 +151,24 @@ func TestNewS3ER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 1, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: -1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: -1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } @@ -209,17 +203,14 @@ func TestNewS3ERCase2(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } diff --git a/ers/sql_test.go b/ers/sql_test.go index b0ce78725..540006832 100644 --- a/ers/sql_test.go +++ b/ers/sql_test.go @@ -43,11 +43,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "sqlDBName": "cgrates2", "sqlTableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "sqlDBNameProcessed": "cgrates3", "sqlTableNameProcessed": "cdrs3", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -78,11 +78,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "sqlDBName": "cgrates2", "sqlTableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "sqlDBNameProcessed": "cgrates3", "sqlTableNameProcessed": "cdrs3", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { @@ -113,11 +113,11 @@ func TestSQLSetURL(t *testing.T) { if err := sql.setURL(inURL, outURL, map[string]interface{}{ "sqlDBName": "cgrates2", "sqlTableName": "cdrs2", - "sslmode": "enabled", + "sslMode": "enabled", "sqlDBNameProcessed": "cgrates2", "sqlTableNameProcessed": "cdrs2", - "sslmodeProcessed": "enabled", + "sslModeProcessed": "enabled", }); err != nil { t.Fatal(err) } else if expsql.connString != sql.connString { diff --git a/ers/sqs_test.go b/ers/sqs_test.go index 3515be490..21c963560 100644 --- a/ers/sqs_test.go +++ b/ers/sqs_test.go @@ -41,17 +41,14 @@ func TestNewSQSER(t *testing.T) { } cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, @@ -72,17 +69,14 @@ func TestSQSERServeRunDelay0(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, @@ -101,17 +95,14 @@ func TestSQSERServe(t *testing.T) { cfg := config.NewDefaultCGRConfig() cfg.ERsCfg().Readers = []*config.EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RowLength: 0, - FieldSep: ",", - HeaderDefineChar: ":", - RunDelay: 0, - ConcurrentReqs: 1, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Filters: []string{}, - Opts: make(map[string]interface{}), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Filters: []string{}, + Opts: make(map[string]interface{}), }, } rdr, err := NewSQSER(cfg, 0, nil, diff --git a/utils/consts.go b/utils/consts.go index 5638402c7..e8c01be4b 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -1963,7 +1963,7 @@ const ( PrefixIndexedFieldsCfg = "prefix_indexed_fields" SuffixIndexedFieldsCfg = "suffix_indexed_fields" MongoQueryTimeoutCfg = "mongoQueryTimeout" - SSLModeCfg = "sslmode" + SSLModeCfg = "sslMode" ItemsCfg = "items" OptsCfg = "opts" Tenants = "tenants" @@ -2340,22 +2340,16 @@ const ( // EventReaderCfg const ( - IDCfg = "id" - CacheCfg = "cache" - RowLengthCfg = "row_length" - FieldSepCfg = "field_separator" - HeaderDefCharCfg = "header_define_character" - RunDelayCfg = "run_delay" - SourcePathCfg = "source_path" - ProcessedPathCfg = "processed_path" - XMLRootPathCfg = "xml_root_path" - TenantCfg = "tenant" - FlagsCfg = "flags" - FailedCallsPrefixCfg = "failed_calls_prefix" - PartialRecordCacheCfg = "partial_record_cache" - PartialCacheExpiryActionCfg = "partial_cache_expiry_action" - FieldsCfg = "fields" - CacheDumpFieldsCfg = "cache_dump_fields" + IDCfg = "id" + CacheCfg = "cache" + FieldSepCfg = "field_separator" + RunDelayCfg = "run_delay" + SourcePathCfg = "source_path" + ProcessedPathCfg = "processed_path" + TenantCfg = "tenant" + FlagsCfg = "flags" + FieldsCfg = "fields" + CacheDumpFieldsCfg = "cache_dump_fields" ) // RegistrarCCfg @@ -2552,8 +2546,6 @@ const ( AMQPDefaultConsumerTag = "cgrates" AMQPConsumerTag = "consumerTag" - LazyQuotes = "lazyQuotes" - KafkaDefaultTopic = "cgrates" KafkaDefaultGroupID = "cgrates" KafkaDefaultMaxWait = time.Millisecond @@ -2567,9 +2559,23 @@ const ( S3Bucket = "s3BucketID" SQSQueueID = "SQSQueueID" + // fileCSV + RowLengthOpt = "RowLength" + FieldSepOpt = "FieldSeparator" + HeaderDefineCharOpt = "csvHeaderDefineChar" + LazyQuotes = "LazyQuotes" - //CSV - FieldSepOpt = "FieldSeparator" + // partialCSV + PartialCSVCacheExpiryActionOpt = "csvCacheExpiryAction" + PartialCSVRecordCacheOpt = "csvRecordCacheTTL" + + // flatStore + FlatstorePrfx = "fst" + FstFailedCallsPrefixOpt = "fstFailedCallsPrefix" + FstPartialRecordCacheOpt = "fstRecordCacheTTL" + + // fileXML + XMLRootPathOpt = "xmlRootPath" ) // Analyzers constants