From 73de0175915cb684a9a63aeb55db20e2e275a101 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 14 May 2021 12:45:22 +0300 Subject: [PATCH] Updated flatstore tests --- apier/v1/dispatchersv1_it_test.go | 4 +- config/config_defaults.go | 13 +- config/config_json_test.go | 43 +- config/config_test.go | 133 ++-- config/configsanity.go | 53 +- config/configsanity_test.go | 9 +- config/eescfg.go | 6 - config/eescfg_test.go | 55 +- config/erscfg.go | 17 +- config/erscfg_test.go | 355 +++++----- config/libconfig_json.go | 1 - data/conf/cgrates/cgrates.json | 44 +- data/conf/samples/ers_internal/cgrates.json | 29 +- data/conf/samples/ers_mongo/cgrates.json | 79 ++- data/conf/samples/ers_mysql/cgrates.json | 79 ++- data/conf/samples/ers_postgres/cgrates.json | 79 ++- ees/filecsv.go | 5 +- engine/dynamicdp.go | 3 +- ers/amqp.go | 2 +- ers/amqpv1.go | 2 +- ers/ers.go | 174 ++--- ers/filecsv.go | 16 +- ers/filefwv.go | 6 +- ers/filejson.go | 2 +- ers/filexml.go | 2 +- ers/flatstore_it_test.go | 445 +----------- ers/kafka.go | 2 +- ers/libers.go | 88 ++- ers/partial_csv_it_test.go | 713 +------------------- ers/s3.go | 2 +- ers/sql.go | 2 +- ers/sqs.go | 2 +- packages/debian/changelog | 2 + utils/consts.go | 47 +- 34 files changed, 700 insertions(+), 1814 deletions(-) diff --git a/apier/v1/dispatchersv1_it_test.go b/apier/v1/dispatchersv1_it_test.go index 10d90ddcb..8ef004267 100644 --- a/apier/v1/dispatchersv1_it_test.go +++ b/apier/v1/dispatchersv1_it_test.go @@ -204,8 +204,8 @@ func testDspDspv1GetProfileForEventWithMethod(t *testing.T) { ID: "testDspv2", Event: map[string]interface{}{}, APIOpts: map[string]interface{}{ - utils.Subsys: utils.MetaAny, - utils.OptsMethod: utils.DispatcherSv1GetProfileForEvent, + utils.Subsys: utils.MetaAny, + "*method": utils.DispatcherSv1GetProfileForEvent, }, } var reply engine.DispatcherProfile diff --git a/config/config_defaults.go b/config/config_defaults.go index 976982cf6..cf0cf364b 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -358,7 +358,7 @@ const CGRATES_CFG_JSON = ` "enabled": false, // starts the EventReader service: "sessions_conns":["*internal"], // RPC Connections IDs "partial_cache_ttl": "1s", // the duration to cache partial records when not pairing - "partial_cache_action": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> + "partial_cache_action": "*none", // the action that will be executed for the partial CSVs that are not matched<*post_cdr|*dump_to_file> // "partial_path": "/var/spool/cgrates/ers/partial", // the path were the partial events will be sent "readers": [ { @@ -371,7 +371,7 @@ const CGRATES_CFG_JSON = ` "opts": { // Partial // "partialPath": "/", // the path were the partial events will be sent - // "partialCacheAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> + // "partialCacheAction": "*none", // the action that will be executed for the partial CSVs that are not matched<*none|*post_cdr|*dump_to_file> "partialOrderField": "~*req.AnswerTime", // the field after what the events are order when merged // "partialcsvFieldSeparator": "," // separator used when dumping the fields @@ -514,12 +514,16 @@ const CGRATES_CFG_JSON = ` // SQS and S3 - // "sqsQueueID": "cgrates_cdrs", // the queue id for SQS exporters from were the events are exported - // "s3BucketID": "cgrates_cdrs", // the bucket id for S3 readers from where the events that are exported // "awsRegion": "", // AWSRegion // "awsKey": "", // AWSKey // "awsSecret": "", // AWSSecret // "awsToken": "", // AWSToken + + //SQS + // "sqsQueueID": "cgrates_cdrs", // the queue id for SQS exporters from were the events are exported + + // S3 + // "s3BucketID": "cgrates_cdrs", // the bucket id for S3 readers from where the events that are exported // "s3FolderPath": "", // S3FolderPath }, // extra options for exporter @@ -531,7 +535,6 @@ const CGRATES_CFG_JSON = ` "attribute_context": "", // context used to discover matching Attribute profiles "synchronous": false, // block processing until export has a result "attempts": 1, // export attempts - "field_separator": ",", // separator used in case of csv files "fields":[], // import fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value }, ], diff --git a/config/config_json_test.go b/config/config_json_test.go index 3d0204515..8700a199b 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -1787,32 +1787,30 @@ func TestDfEventReaderCfg(t *testing.T) { Sessions_conns: &[]string{utils.MetaInternal}, Readers: &[]*EventReaderJsonCfg{ { - Id: utils.StringPointer(utils.MetaDefault), - Type: utils.StringPointer(utils.MetaNone), - Run_delay: utils.StringPointer("0"), - Concurrent_requests: utils.IntPointer(1024), - Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), - Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), - Tenant: utils.StringPointer(utils.EmptyString), - Timezone: utils.StringPointer(utils.EmptyString), - Filters: &[]string{}, - Flags: &[]string{}, - Fields: &cdrFields, - Cache_dump_fields: &[]*FcTemplateJsonCfg{}, + Id: utils.StringPointer(utils.MetaDefault), + Type: utils.StringPointer(utils.MetaNone), + Run_delay: utils.StringPointer("0"), + Concurrent_requests: utils.IntPointer(1024), + Source_path: utils.StringPointer("/var/spool/cgrates/ers/in"), + Processed_path: utils.StringPointer("/var/spool/cgrates/ers/out"), + Tenant: utils.StringPointer(utils.EmptyString), + Timezone: utils.StringPointer(utils.EmptyString), + Filters: &[]string{}, + Flags: &[]string{}, + Fields: &cdrFields, + Cache_dump_fields: &[]*FcTemplateJsonCfg{}, + Partial_commit_fields: &[]*FcTemplateJsonCfg{}, Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + Partial_cache_ttl: utils.StringPointer("1s"), + Partial_cache_action: utils.StringPointer(utils.MetaNone), } dfCgrJSONCfg, err := NewCgrJsonCfgFromBytes([]byte(CGRATES_CFG_JSON)) if err != nil { @@ -1840,7 +1838,6 @@ func TestDfEventExporterCfg(t *testing.T) { { Id: utils.StringPointer(utils.MetaDefault), Type: utils.StringPointer(utils.MetaNone), - Field_separator: utils.StringPointer(","), Export_path: utils.StringPointer("/var/spool/cgrates/ees"), Attribute_context: utils.StringPointer(utils.EmptyString), Tenant: utils.StringPointer(utils.EmptyString), diff --git a/config/config_test.go b/config/config_test.go index 1ba6bd100..e2bf6deb2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2341,32 +2341,30 @@ func TestERSConfig(t *testing.T) { SessionSConns: []string{"*internal:*sessions"}, Readers: []*EventReaderCfg{ { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: make([]*FCTemplate, 0), + ID: utils.MetaDefault, + Type: utils.MetaNone, + RunDelay: 0, + ConcurrentReqs: 1024, + SourcePath: "/var/spool/cgrates/ers/in", + ProcessedPath: "/var/spool/cgrates/ers/out", + Tenant: nil, + Timezone: utils.EmptyString, + Filters: []string{}, + Flags: utils.FlagsWithParams{}, + Fields: nil, + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } cgrConfig := NewDefaultCGRConfig() if err != nil { @@ -2394,7 +2392,6 @@ func TestEEsNoLksConfig(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, @@ -4326,7 +4323,6 @@ func TestV1GetConfigSectionEES(t *testing.T) { utils.AttributeContextCfg: utils.EmptyString, utils.SynchronousCfg: false, utils.AttemptsCfg: 1, - utils.FieldSepCfg: ",", utils.FieldsCfg: []map[string]interface{}{}, }, }, @@ -4348,32 +4344,31 @@ func TestV1GetConfigSectionERS(t *testing.T) { utils.SessionSConnsCfg: []string{utils.MetaInternal}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RunDelayCfg: "0", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: utils.EmptyString, - utils.TimezoneCfg: utils.EmptyString, - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: utils.MetaNone, - utils.FieldsCfg: []string{}, + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: utils.EmptyString, + utils.TimezoneCfg: utils.EmptyString, + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.PartialCommitFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: utils.MetaNone, + utils.FieldsCfg: []string{}, utils.OptsCfg: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + utils.PartialCacheTTLCfg: "1s", + utils.PartialCacheActionCfg: utils.MetaNone, + utils.PartialPathCfg: utils.EmptyString, }, } cfgCgr := NewDefaultCGRConfig() @@ -5020,7 +5015,7 @@ func TestV1GetConfigAsJSONApierS(t *testing.T) { func TestV1GetConfigAsJSONCfgEES(t *testing.T) { var reply string - expected := `{"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]}}` + expected := `{"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]}}` cgrCfg := NewDefaultCGRConfig() if err := cgrCfg.V1GetConfigAsJSON(&SectionWithAPIOpts{Section: EEsJson}, &reply); err != nil { t.Error(err) @@ -5031,7 +5026,7 @@ func TestV1GetConfigAsJSONCfgEES(t *testing.T) { func TestV1GetConfigAsJSONCfgERS(t *testing.T) { var reply string - expected := `{"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstMadatoryACK":false,"fstMethod":"~*req.0","fstOriginID":"~*req.3;~*req.1;~*req.2","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]}}` + expected := `{"ers":{"enabled":false,"partial_cache_action":"*none","partial_cache_ttl":"1s","partial_path":"","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"partialOrderField":"~*req.AnswerTime","xmlRootPath":""},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]}}` cgrCfg := NewDefaultCGRConfig() if err := cgrCfg.V1GetConfigAsJSON(&SectionWithAPIOpts{Section: ERsJson}, &reply); err != nil { t.Error(err) @@ -5199,7 +5194,7 @@ func TestV1GetConfigAsJSONAllConfig(t *testing.T) { } }` var reply string - expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"apiers_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_account_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destination_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"remote":false,"replicate":false},"*accounts":{"remote":false,"replicate":false},"*action_plans":{"remote":false,"replicate":false},"*action_triggers":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rating_plans":{"remote":false,"replicate":false},"*rating_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*shared_groups":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","field_separator":",","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvCacheExpiryAction":"*post_cdr","csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"fstFieldSeparator":",","fstMadatoryACK":false,"fstMethod":"~*req.0","fstOriginID":"~*req.3;~*req.1;~*req.2","fstRowLength":0,"xmlRootPath":""},"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"apiers_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"caches_conns":["*internal"],"dynaprepaid_actionplans":[],"enabled":false,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"stats_conns":[],"thresholds_conns":[]},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sessions":{"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","postgresSSLMode":"disable","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` + expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"apiers_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"process_runs":1,"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*cdrs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*event_charges":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*session_costs":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_account_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_action_triggers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_actions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destination_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_destinations":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rates":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_plans":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_rating_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_shared_groups":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*tp_timings":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*versions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":""}},"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"remote":false,"replicate":false},"*accounts":{"remote":false,"replicate":false},"*action_plans":{"remote":false,"replicate":false},"*action_triggers":{"remote":false,"replicate":false},"*actions":{"remote":false,"replicate":false},"*attribute_profiles":{"remote":false,"replicate":false},"*charger_profiles":{"remote":false,"replicate":false},"*destinations":{"remote":false,"replicate":false},"*dispatcher_hosts":{"remote":false,"replicate":false},"*dispatcher_profiles":{"remote":false,"replicate":false},"*filters":{"remote":false,"replicate":false},"*indexes":{"remote":false,"replicate":false},"*load_ids":{"remote":false,"replicate":false},"*rating_plans":{"remote":false,"replicate":false},"*rating_profiles":{"remote":false,"replicate":false},"*resource_profiles":{"remote":false,"replicate":false},"*resources":{"remote":false,"replicate":false},"*reverse_destinations":{"remote":false,"replicate":false},"*route_profiles":{"remote":false,"replicate":false},"*shared_groups":{"remote":false,"replicate":false},"*statqueue_profiles":{"remote":false,"replicate":false},"*statqueues":{"remote":false,"replicate":false},"*threshold_profiles":{"remote":false,"replicate":false},"*thresholds":{"remote":false,"replicate":false},"*timings":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*file_csv":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"export_path":"/var/spool/cgrates/ees","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"tenant":"","timezone":"","type":"*none"}]},"ers":{"enabled":false,"partial_cache_action":"*none","partial_cache_ttl":"1s","partial_path":"","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"partialOrderField":"~*req.AnswerTime","xmlRootPath":""},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"apiers_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","password":"ClueCon","reconnects":5}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0","forceAttemptHttp2":true,"idleConnTimeout":"90s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.4"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lock_filename":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0","redisClusterSync":"5s","redisSentinel":"","redisTLS":false},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"mysql","out_stordb_user":"cgrates","users_filters":[]},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"caches_conns":["*internal"],"dynaprepaid_actionplans":[],"enabled":false,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"stats_conns":[],"thresholds_conns":[]},"registrarc":{"dispatcher":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"enabled":false,"hosts":{},"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sessions":{"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"remote":false,"replicate":false},"*session_costs":{"remote":false,"replicate":false},"*tp_account_actions":{"remote":false,"replicate":false},"*tp_action_plans":{"remote":false,"replicate":false},"*tp_action_triggers":{"remote":false,"replicate":false},"*tp_actions":{"remote":false,"replicate":false},"*tp_attributes":{"remote":false,"replicate":false},"*tp_chargers":{"remote":false,"replicate":false},"*tp_destination_rates":{"remote":false,"replicate":false},"*tp_destinations":{"remote":false,"replicate":false},"*tp_dispatcher_hosts":{"remote":false,"replicate":false},"*tp_dispatcher_profiles":{"remote":false,"replicate":false},"*tp_filters":{"remote":false,"replicate":false},"*tp_rates":{"remote":false,"replicate":false},"*tp_rating_plans":{"remote":false,"replicate":false},"*tp_rating_profiles":{"remote":false,"replicate":false},"*tp_resources":{"remote":false,"replicate":false},"*tp_routes":{"remote":false,"replicate":false},"*tp_shared_groups":{"remote":false,"replicate":false},"*tp_stats":{"remote":false,"replicate":false},"*tp_thresholds":{"remote":false,"replicate":false},"*tp_timings":{"remote":false,"replicate":false},"*versions":{"remote":false,"replicate":false}},"opts":{"mongoQueryTimeout":"10s","mysqlLocation":"Local","postgresSSLMode":"disable","sqlConnMaxLifetime":0,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"enabled":false,"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4}}` cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSON) if err != nil { t.Fatal(err) @@ -5282,21 +5277,19 @@ func TestCgrCdfEventReader(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: []*FCTemplate{}, + CacheDumpFields: []*FCTemplate{}, + PartialCommitFields: []*FCTemplate{}, Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } for _, profile := range eCfg.Readers { for _, v := range profile.Fields { @@ -5323,7 +5316,6 @@ func TestCgrCdfEventExporter(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, @@ -5380,18 +5372,14 @@ func TestCgrCfgEventReaderDefault(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, } for _, v := range eCfg.Fields { @@ -5407,7 +5395,6 @@ func TestCgrCfgEventExporterDefault(t *testing.T) { eCfg := &EventExporterCfg{ ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, diff --git a/config/configsanity.go b/config/configsanity.go index 4c5811734..d2a232ad8 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -681,6 +681,17 @@ func (cfg *CGRConfig) checkConfigSanity() error { } // EventReader sanity checks if cfg.ersCfg.Enabled { + // check the global partial options + if cfg.ersCfg.PartialCacheAction != utils.MetaNone && + cfg.ersCfg.PartialCacheAction != utils.MetaDumpToFile && + cfg.ersCfg.PartialCacheAction != utils.MetaPostCDR { + return fmt.Errorf("<%s> wrong partial expiry action", utils.ERs) + } + if cfg.ersCfg.PartialPath != utils.EmptyString { + if _, err := os.Stat(cfg.ersCfg.PartialPath); err != nil && os.IsNotExist(err) { + return fmt.Errorf("<%s> nonexistent partial folder: %s", utils.ERs, cfg.ersCfg.PartialPath) + } + } for _, connID := range cfg.ersCfg.SessionSConns { if strings.HasPrefix(connID, utils.MetaInternal) && !cfg.sessionSCfg.Enabled { return fmt.Errorf("<%s> not enabled but requested by <%s> component", utils.SessionS, utils.ERs) @@ -693,7 +704,30 @@ func (cfg *CGRConfig) checkConfigSanity() error { if !possibleReaderTypes.Has(rdr.Type) { return fmt.Errorf("<%s> unsupported data type: %s for reader with ID: %s", utils.ERs, rdr.Type, rdr.ID) } - + pAct := cfg.ersCfg.PartialCacheAction + if act, has := rdr.Opts[utils.PartialCacheActionOpt]; has { // check the action from opts + if pAct = utils.IfaceAsString(act); pAct != utils.MetaDumpToFile && + pAct != utils.MetaNone && + pAct != utils.MetaPostCDR { + return fmt.Errorf("<%s> wrong partial expiry action for reader with ID: %s", utils.ERs, rdr.ID) + } + } + if pAct != utils.MetaNone { // if is *none we do not process the evicted events + if fldSep, has := rdr.Opts[utils.PartialOrderFieldOpt]; has && // the field we order after must not be empty + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.PartialOrderFieldOpt, rdr.ID) + } + } else if pAct == utils.MetaDumpToFile { // only if the action is *dump_to_file + if path, has := rdr.Opts[utils.PartialPathOpt]; has { // the path from options needs to exists if overwriten by reader + if _, err := os.Stat(utils.IfaceAsString(path)); err != nil && os.IsNotExist(err) { + return fmt.Errorf("<%s> nonexistent partial folder: %s for reader with ID: %s", utils.ERs, path, rdr.ID) + } + } + if fldSep, has := rdr.Opts[utils.PartialCSVFieldSepartorOpt]; has && // the separtor must not be empty + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.PartialCSVFieldSepartorOpt, rdr.ID) + } + } switch rdr.Type { case utils.MetaFileCSV: paths := []string{rdr.ProcessedPath, rdr.SourcePath} @@ -705,18 +739,18 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> nonexistent folder: %s for reader with ID: %s", utils.ERs, dir, rdr.ID) } } - if fldSep, has := rdr.Opts[utils.CSV+utils.FieldSepOpt]; has && + if fldSep, has := rdr.Opts[utils.CSVFieldSepOpt]; has && utils.IfaceAsString(fldSep) == utils.EmptyString { - return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.CSV+utils.FieldSepOpt, rdr.ID) + return fmt.Errorf("<%s> empty %s for reader with ID: %s", utils.ERs, utils.CSVFieldSepOpt, rdr.ID) } - if rowl, has := rdr.Opts[utils.CSV+utils.RowLengthOpt]; has { + if rowl, has := rdr.Opts[utils.CSVRowLengthOpt]; has { if _, err := utils.IfaceAsTInt64(rowl); err != nil { - return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.RowLengthOpt, err.Error(), rdr.ID) + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSVRowLengthOpt, err.Error(), rdr.ID) } } - if lq, has := rdr.Opts[utils.CSV+utils.LazyQuotes]; has { + if lq, has := rdr.Opts[utils.CSVLazyQuotes]; has { if _, err := utils.IfaceAsBool(lq); err != nil { - return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSV+utils.LazyQuotes, err.Error(), rdr.ID) + return fmt.Errorf("<%s> error when converting %s: <%s> for reader with ID: %s", utils.ERs, utils.CSVLazyQuotes, err.Error(), rdr.ID) } } case utils.MetaKafkajsonMap: @@ -813,8 +847,9 @@ func (cfg *CGRConfig) checkConfigSanity() error { return fmt.Errorf("<%s> nonexistent folder: %s for exporter with ID: %s", utils.EEs, dir, exp.ID) } } - if exp.FieldSep == utils.EmptyString { - return fmt.Errorf("<%s> empty FieldSep for exporter with ID: %s", utils.EEs, exp.ID) + if fldSep, has := exp.Opts[utils.CSVFieldSepOpt]; has && + utils.IfaceAsString(fldSep) == utils.EmptyString { + return fmt.Errorf("<%s> empty %s for exporter with ID: %s", utils.EEs, utils.CSVFieldSepOpt, exp.ID) } case utils.MetaFileFWV: for _, dir := range []string{exp.ExportPath} { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index 2f6f6ce5c..874ad2517 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -1206,8 +1206,9 @@ func TestConfigSanityScheduler(t *testing.T) { func TestConfigSanityEventReader(t *testing.T) { cfg = NewDefaultCGRConfig() cfg.ersCfg = &ERsCfg{ - Enabled: true, - SessionSConns: []string{"unexistedConn"}, + Enabled: true, + SessionSConns: []string{"unexistedConn"}, + PartialCacheAction: utils.MetaNone, } expected := " connection with id: not defined" if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { @@ -1298,6 +1299,7 @@ func TestConfigSanityEventReader(t *testing.T) { }, }, }, + PartialCacheAction: utils.MetaNone, } //CacheDumpFields @@ -1425,7 +1427,8 @@ func TestConfigSanityEventExporter(t *testing.T) { cfg.eesCfg.Exporters[0].Type = utils.MetaFileCSV cfg.eesCfg.Exporters[0].ExportPath = "/" - expected = " empty FieldSep for exporter with ID: " + cfg.eesCfg.Exporters[0].Opts = map[string]interface{}{utils.CSVFieldSepOpt: ""} + expected = " empty csvFieldSeparator for exporter with ID: " if err := cfg.CheckConfigSanity(); err == nil || err.Error() != expected { t.Errorf("Expecting: %+q received: %+q", expected, err) } diff --git a/config/eescfg.go b/config/eescfg.go index b4beee454..255ce325a 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -165,7 +165,6 @@ type EventExporterCfg struct { AttributeSCtx string // context to use when querying AttributeS Synchronous bool Attempts int - FieldSep string Fields []*FCTemplate headerFields []*FCTemplate contentFields []*FCTemplate @@ -217,9 +216,6 @@ func (eeC *EventExporterCfg) loadFromJSONCfg(jsnEec *EventExporterJsonCfg, msgTe if jsnEec.Attempts != nil { eeC.Attempts = *jsnEec.Attempts } - if jsnEec.Field_separator != nil { - eeC.FieldSep = *jsnEec.Field_separator - } if jsnEec.Fields != nil { eeC.Fields, err = FCTemplatesFromFCTemplatesJSONCfg(*jsnEec.Fields, separator) if err != nil { @@ -285,7 +281,6 @@ func (eeC EventExporterCfg) Clone() (cln *EventExporterCfg) { AttributeSCtx: eeC.AttributeSCtx, Synchronous: eeC.Synchronous, Attempts: eeC.Attempts, - FieldSep: eeC.FieldSep, Fields: make([]*FCTemplate, len(eeC.Fields)), headerFields: make([]*FCTemplate, len(eeC.headerFields)), contentFields: make([]*FCTemplate, len(eeC.contentFields)), @@ -335,7 +330,6 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str utils.TypeCfg: eeC.Type, utils.ExportPathCfg: eeC.ExportPath, utils.TenantCfg: eeC.Tenant.GetRule(separator), - utils.FieldSepCfg: eeC.FieldSep, utils.TimezoneCfg: eeC.Timezone, utils.FiltersCfg: eeC.Filters, utils.FlagsCfg: flgs, diff --git a/config/eescfg_test.go b/config/eescfg_test.go index 9bea9cc7a..d10d7660d 100644 --- a/config/eescfg_test.go +++ b/config/eescfg_test.go @@ -77,7 +77,6 @@ func TestEESClone(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Synchronous: false, Tenant: NewRSRParsersMustCompile("", utils.InfieldSep), ExportPath: "/var/spool/cgrates/ees", @@ -96,7 +95,6 @@ func TestEESClone(t *testing.T) { { ID: utils.CGRateSLwr, Type: utils.MetaNone, - FieldSep: ",", Synchronous: false, Tenant: NewRSRParsersMustCompile("~*req.Destination1", utils.InfieldSep), ExportPath: "/var/spool/cgrates/ees", @@ -298,7 +296,6 @@ func TestEventExporterSameID(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, @@ -315,7 +312,6 @@ func TestEventExporterSameID(t *testing.T) { { ID: "file_exporter1", Type: utils.MetaFileCSV, - FieldSep: ",", Tenant: nil, Timezone: utils.EmptyString, Filters: []string{}, @@ -387,17 +383,16 @@ func TestEEsCfgloadFromJsonCfgCase1(t *testing.T) { }, Exporters: &[]*EventExporterJsonCfg{ { - Id: utils.StringPointer("CSVExporter"), - Type: utils.StringPointer("*file_csv"), - Filters: &[]string{}, - Attribute_ids: &[]string{}, - Flags: &[]string{"*dryrun"}, - Export_path: utils.StringPointer("/tmp/testCSV"), - Tenant: nil, - Timezone: utils.StringPointer("UTC"), - Synchronous: utils.BoolPointer(true), - Attempts: utils.IntPointer(1), - Field_separator: utils.StringPointer(","), + Id: utils.StringPointer("CSVExporter"), + Type: utils.StringPointer("*file_csv"), + Filters: &[]string{}, + Attribute_ids: &[]string{}, + Flags: &[]string{"*dryrun"}, + Export_path: utils.StringPointer("/tmp/testCSV"), + Tenant: nil, + Timezone: utils.StringPointer("UTC"), + Synchronous: utils.BoolPointer(true), + Attempts: utils.IntPointer(1), Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.CGRID), @@ -423,7 +418,6 @@ func TestEEsCfgloadFromJsonCfgCase1(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, @@ -448,7 +442,6 @@ func TestEEsCfgloadFromJsonCfgCase1(t *testing.T) { Timezone: "UTC", Synchronous: true, Attempts: 1, - FieldSep: ",", headerFields: []*FCTemplate{}, trailerFields: []*FCTemplate{}, contentFields: []*FCTemplate{ @@ -498,17 +491,16 @@ func TestEEsCfgloadFromJsonCfgCase2(t *testing.T) { }, Exporters: &[]*EventExporterJsonCfg{ { - Id: utils.StringPointer("CSVExporter"), - Type: utils.StringPointer("*file_csv"), - Filters: &[]string{}, - Attribute_ids: &[]string{}, - Flags: &[]string{"*dryrun"}, - Export_path: utils.StringPointer("/tmp/testCSV"), - Tenant: nil, - Timezone: utils.StringPointer("UTC"), - Synchronous: utils.BoolPointer(true), - Attempts: utils.IntPointer(1), - Field_separator: utils.StringPointer(","), + Id: utils.StringPointer("CSVExporter"), + Type: utils.StringPointer("*file_csv"), + Filters: &[]string{}, + Attribute_ids: &[]string{}, + Flags: &[]string{"*dryrun"}, + Export_path: utils.StringPointer("/tmp/testCSV"), + Tenant: nil, + Timezone: utils.StringPointer("UTC"), + Synchronous: utils.BoolPointer(true), + Attempts: utils.IntPointer(1), Fields: &[]*FcTemplateJsonCfg{ { Tag: utils.StringPointer(utils.AnswerTime), @@ -540,7 +532,6 @@ func TestEEsCfgloadFromJsonCfgCase2(t *testing.T) { { ID: utils.MetaDefault, Type: utils.MetaNone, - FieldSep: ",", Tenant: nil, ExportPath: "/var/spool/cgrates/ees", Attempts: 1, @@ -565,7 +556,6 @@ func TestEEsCfgloadFromJsonCfgCase2(t *testing.T) { Timezone: "UTC", Synchronous: true, Attempts: 1, - FieldSep: ",", headerFields: []*FCTemplate{}, trailerFields: []*FCTemplate{}, contentFields: []*FCTemplate{ @@ -686,7 +676,6 @@ func TestEEsCfgAsMapInterface(t *testing.T) { utils.AttributeContextCfg: utils.EmptyString, utils.SynchronousCfg: false, utils.AttemptsCfg: 1, - utils.FieldSepCfg: ",", utils.FieldsCfg: []map[string]interface{}{ { utils.TagCfg: utils.CGRID, @@ -713,8 +702,8 @@ func TestEEsCfgAsMapInterface(t *testing.T) { eMap[utils.ExportersCfg].([]map[string]interface{})[0][utils.FieldsCfg] = nil if !reflect.DeepEqual(rcv[utils.ExportersCfg].([]map[string]interface{})[1], eMap[utils.ExportersCfg].([]map[string]interface{})[0]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(rcv[utils.ExportersCfg].([]map[string]interface{})[1]), - utils.ToJSON(eMap[utils.ExportersCfg].([]map[string]interface{})[0])) + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[utils.ExportersCfg].([]map[string]interface{})[1]), + utils.ToJSON(rcv[utils.ExportersCfg].([]map[string]interface{})[0])) } rcv[utils.ExportersCfg] = nil eMap[utils.ExportersCfg] = nil diff --git a/config/erscfg.go b/config/erscfg.go index 7df044283..96ac1024a 100644 --- a/config/erscfg.go +++ b/config/erscfg.go @@ -100,9 +100,12 @@ func (erS *ERsCfg) appendERsReaders(jsnReaders *[]*EventReaderJsonCfg, msgTempla // Clone returns a deep copy of ERsCfg func (erS *ERsCfg) Clone() (cln *ERsCfg) { cln = &ERsCfg{ - Enabled: erS.Enabled, - SessionSConns: make([]string, len(erS.SessionSConns)), - Readers: make([]*EventReaderCfg, len(erS.Readers)), + Enabled: erS.Enabled, + SessionSConns: make([]string, len(erS.SessionSConns)), + Readers: make([]*EventReaderCfg, len(erS.Readers)), + PartialCacheTTL: erS.PartialCacheTTL, + PartialCacheAction: erS.PartialCacheAction, + PartialPath: erS.PartialPath, } for idx, sConn := range erS.SessionSConns { cln.SessionSConns[idx] = sConn @@ -116,7 +119,13 @@ func (erS *ERsCfg) Clone() (cln *ERsCfg) { // AsMapInterface returns the config as a map[string]interface{} func (erS *ERsCfg) AsMapInterface(separator string) (initialMP map[string]interface{}) { initialMP = map[string]interface{}{ - utils.EnabledCfg: erS.Enabled, + utils.EnabledCfg: erS.Enabled, + utils.PartialCacheTTLCfg: "0", + utils.PartialCacheActionCfg: erS.PartialCacheAction, + utils.PartialPathCfg: erS.PartialPath, + } + if erS.PartialCacheTTL != 0 { + initialMP[utils.PartialCacheTTLCfg] = erS.PartialCacheTTL.String() } if erS.SessionSConns != nil { sessionSConns := make([]string, len(erS.SessionSConns)) diff --git a/config/erscfg_test.go b/config/erscfg_test.go index d87fd9fa2..98037c145 100644 --- a/config/erscfg_test.go +++ b/config/erscfg_test.go @@ -99,18 +99,14 @@ func TestERSClone(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -134,21 +130,19 @@ func TestERSClone(t *testing.T) { {Tag: utils.ToR, Path: utils.MetaCgreq + utils.NestingSep + utils.ToR, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - utils.MetaDefault: "randomVal", - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + utils.MetaDefault: "randomVal", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { @@ -247,18 +241,14 @@ func TestERSLoadFromjsonCfg(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -296,21 +286,19 @@ func TestERSLoadFromjsonCfg(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { @@ -477,18 +465,14 @@ func TestERSloadFromJsonCase3(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -510,21 +494,20 @@ func TestERSloadFromJsonCase3(t *testing.T) { Layout: time.RFC3339, }, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } msgTemplates := map[string][]*FCTemplate{ "randomTemplate": { @@ -614,18 +597,14 @@ func TestERSloadFromJsonCase4(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -648,20 +627,19 @@ func TestERSloadFromJsonCase4(t *testing.T) { Value: NewRSRParsersMustCompile("~*req.OrderID", utils.InfieldSep), }, }, + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } msgTemplates := map[string][]*FCTemplate{ "randomTemplate": { @@ -746,18 +724,14 @@ func TestEventReaderSameID(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -775,21 +749,20 @@ func TestEventReaderSameID(t *testing.T) { {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { @@ -859,17 +832,18 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RunDelayCfg: "0", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.PartialCommitFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -884,22 +858,18 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.OptsCfg: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*file_csv", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.PartialCommitFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*file_csv", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -922,19 +892,17 @@ func TestERsCfgAsMapInterfaceCase1(t *testing.T) { utils.TenantCfg: "~*req.Destination1", utils.TimezoneCfg: "", utils.OptsCfg: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + utils.PartialCacheTTLCfg: "1s", + utils.PartialCacheActionCfg: utils.MetaNone, + utils.PartialPathCfg: "", } if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) @@ -974,17 +942,18 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { utils.SessionSConnsCfg: []string{"conn1", "conn3"}, utils.ReadersCfg: []map[string]interface{}{ { - utils.FiltersCfg: []string{}, - utils.FlagsCfg: []string{}, - utils.IDCfg: "*default", - utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", - utils.RunDelayCfg: "0", - utils.SourcePathCfg: "/var/spool/cgrates/ers/in", - utils.TenantCfg: "", - utils.TimezoneCfg: "", - utils.CacheDumpFieldsCfg: []map[string]interface{}{}, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*none", + utils.FiltersCfg: []string{}, + utils.FlagsCfg: []string{}, + utils.IDCfg: "*default", + utils.ProcessedPathCfg: "/var/spool/cgrates/ers/out", + utils.RunDelayCfg: "0", + utils.SourcePathCfg: "/var/spool/cgrates/ers/in", + utils.TenantCfg: "", + utils.TimezoneCfg: "", + utils.CacheDumpFieldsCfg: []map[string]interface{}{}, + utils.PartialCommitFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*none", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -999,24 +968,20 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.Usage", utils.TagCfg: "Usage", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.13"}, }, utils.OptsCfg: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { utils.CacheDumpFieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, }, - utils.ConcurrentRequestsCfg: 1024, - utils.TypeCfg: "*file_csv", + utils.PartialCommitFieldsCfg: []map[string]interface{}{}, + utils.ConcurrentRequestsCfg: 1024, + utils.TypeCfg: "*file_csv", utils.FieldsCfg: []map[string]interface{}{ {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.ToR", utils.TagCfg: "ToR", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.2"}, {utils.MandatoryCfg: true, utils.PathCfg: "*cgreq.OriginID", utils.TagCfg: "OriginID", utils.TypeCfg: "*variable", utils.ValueCfg: "~*req.3"}, @@ -1039,20 +1004,18 @@ func TestERSCfgAsMapInterfaceCase2(t *testing.T) { utils.TenantCfg: "~*req.Destination1", utils.TimezoneCfg: "", utils.OptsCfg: map[string]interface{}{ - utils.KafkaGroupID: "test", - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + utils.KafkaGroupID: "test", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + utils.PartialCacheTTLCfg: "1s", + utils.PartialCacheActionCfg: utils.MetaNone, + utils.PartialPathCfg: "", } if cfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) @@ -1129,18 +1092,14 @@ func TestERsloadFromJsonCfg(t *testing.T) { {Tag: utils.Usage, Path: utils.MetaCgreq + utils.NestingSep + utils.Usage, Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, { @@ -1164,21 +1123,19 @@ func TestERsloadFromJsonCfg(t *testing.T) { {Tag: "CustomTag2", Path: "CustomPath2", Type: utils.MetaVariable, Value: NewRSRParsersMustCompile("CustomValue2", utils.InfieldSep), Mandatory: true, Layout: time.RFC3339}, }, - CacheDumpFields: make([]*FCTemplate, 0), + CacheDumpFields: make([]*FCTemplate, 0), + PartialCommitFields: make([]*FCTemplate, 0), Opts: map[string]interface{}{ - "csvCacheExpiryAction": "*post_cdr", - "csvFieldSeparator": ",", - "csvHeaderDefineChar": ":", - "csvRowLength": 0., - "fstFieldSeparator": ",", - "fstRowLength": 0., - "xmlRootPath": "", - "fstMadatoryACK": false, - "fstMethod": "~*req.0", - "fstOriginID": "~*req.3;~*req.1;~*req.2", + "csvFieldSeparator": ",", + "csvHeaderDefineChar": ":", + "csvRowLength": 0., + "xmlRootPath": "", + "partialOrderField": "~*req.AnswerTime", }, }, }, + PartialCacheTTL: time.Second, + PartialCacheAction: utils.MetaNone, } for _, profile := range expectedERsCfg.Readers { for _, v := range profile.Fields { diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 1e4e8c1bc..7f7f58128 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -204,7 +204,6 @@ type EventExporterJsonCfg struct { Attribute_context *string Synchronous *bool Attempts *int - Field_separator *string Fields *[]*FcTemplateJsonCfg } diff --git a/data/conf/cgrates/cgrates.json b/data/conf/cgrates/cgrates.json index 2e44a7bf6..eed147286 100755 --- a/data/conf/cgrates/cgrates.json +++ b/data/conf/cgrates/cgrates.json @@ -333,9 +333,12 @@ // }, -// "ers": { // EventReaderService -// "enabled": false, // starts the EventReader service: -// "sessions_conns":["*internal"], // RPC Connections IDs +// "ers": { // EventReaderService +// "enabled": false, // starts the EventReader service: +// "sessions_conns":["*internal"], // RPC Connections IDs +// "partial_cache_ttl": "1s", // the duration to cache partial records when not pairing +// "partial_cache_action": "*none", // the action that will be executed for the partial CSVs that are not matched<*post_cdr|*dump_to_file> +// // "partial_path": "/var/spool/cgrates/ers/partial", // the path were the partial events will be sent // "readers": [ // { // "id": "*default", // identifier of the EventReader profile @@ -345,23 +348,18 @@ // "source_path": "/var/spool/cgrates/ers/in", // read data from this path // "processed_path": "/var/spool/cgrates/ers/out", // move processed data here // "opts": { -// // FileCSV and PartialCSV +// // Partial +// // "partialPath": "/", // the path were the partial events will be sent +// // "partialCacheAction": "*none", // the action that will be executed for the partial CSVs that are not matched<*none|*post_cdr|*dump_to_file> +// "partialOrderField": "~*req.AnswerTime", // the field after what the events are order when merged +// // "partialcsvFieldSeparator": "," // separator used when dumping the fields + +// // FileCSV // "csvRowLength": 0, // Number of fields from csv file // "csvFieldSeparator": ",", // separator used when reading the fields // "csvHeaderDefineChar": ":", // the starting character for header definition used in case of CSV files // // "csvLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field -// // PartialCSV -// "csvCacheExpiryAction": "*post_cdr", // the action that will be exeuted for the partial CSVs that are not matched<*post_cdr|*dump_to_file> -// // "csvRecordCacheTTL": "1s" // Duration to cache partial records when not pairing - -// // FlatStore -// "fstRowLength": 0, // Number of fields from csv file -// "fstFieldSeparator": ",", // separator used when reading the fields -// // "fstFailedCallsPrefix": "" // Used in case of flatstore CDRs to avoid searching for BYE records -// // "fstRecordCacheTTL": "1s" // Duration to cache partial records when not pairing -// // "fstLazyQuotes": false, // if a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field - // // FileXML // "xmlRootPath": "", // path towards one event in case of XML CDRs @@ -435,6 +433,7 @@ // {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.12", "mandatory": true}, // {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.13", "mandatory": true}, // ], +// "partial_commit_fields": [], // "cache_dump_fields": [], // }, // ], @@ -489,18 +488,22 @@ // // AMQP // // "amqpQueueID": "cgrates_cdrs", // the queue id for AMQP exporters from were the events are exported // // "amqpRoutingKey": "", // RoutingKey -// // "sqlExchange": "", // Exchange -// // "sqlExchangeType": "", // ExchangeType +// // "amqpExchange": "", // Exchange +// // "amqpExchangeType": "", // ExchangeType // // SQS and S3 -// // "sqsQueueID": "cgrates_cdrs", // the queue id for SQS exporters from were the events are exported -// // "s3BucketID": "cgrates_cdrs", // the bucket id for S3 readers from where the events that are exported // // "awsRegion": "", // AWSRegion // // "awsKey": "", // AWSKey // // "awsSecret": "", // AWSSecret // // "awsToken": "", // AWSToken -// // "s3FolderPath": "", // AWSFolderPath + +// //SQS +// // "sqsQueueID": "cgrates_cdrs", // the queue id for SQS exporters from were the events are exported + +// // S3 +// // "s3BucketID": "cgrates_cdrs", // the bucket id for S3 readers from where the events that are exported +// // "s3FolderPath": "", // S3FolderPath // }, // extra options for exporter // "tenant": "", // tenant used in filterS.Pass @@ -511,7 +514,6 @@ // "attribute_context": "", // context used to discover matching Attribute profiles // "synchronous": false, // block processing until export has a result // "attempts": 1, // export attempts -// "field_separator": ",", // separator used in case of csv files // "fields":[], // import fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value // }, // ], diff --git a/data/conf/samples/ers_internal/cgrates.json b/data/conf/samples/ers_internal/cgrates.json index c218f8515..46bf73eec 100644 --- a/data/conf/samples/ers_internal/cgrates.json +++ b/data/conf/samples/ers_internal/cgrates.json @@ -266,7 +266,7 @@ "flags": ["*cdrs"], "processed_path": "", "opts": { - "csvCacheExpiryAction": "*dump_to_file", + "partialCacheAction": "*dump_to_file", "partialOrderField": "~*req.AnswerTime", "partialPath": "/tmp/partErs1/out", }, @@ -300,7 +300,6 @@ {"tag": "SetupTime", "path":"*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "AnswerTime", "path":"*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "Usage", "path":"*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"tag": "Cost", "path":"*exp.Cost", "type": "*variable", "value": "~*req.Cost","rounding_decimals":5}, ], }, { @@ -312,7 +311,7 @@ "processed_path": "", "flags": ["*cdrs"], "opts": { - "csvCacheExpiryAction": "*post_cdr", + "partialCacheAction": "*post_cdr", "partialOrderField": "~*req.AnswerTime", "partialPath": "/tmp/partErs2/out", }, @@ -373,24 +372,22 @@ {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "1", "filters":["*string:~*req.0:BYE"]}, ], "partial_commit_fields": [ - // {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, - // {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.OriginID", "mandatory": true}, - // {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value":"~*req.RequestType", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, - // {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*constant", "value": "cgrates.org", "mandatory": true}, - // {"tag": "Category", "path": "*cgreq.Category", "type": "*constant", "value": "call", "mandatory": true}, - // {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value":"~*req.Account", "mandatory": true}, - // {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value":"~*req.Subject", "mandatory": true}, - // {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value":"~*req.Destination" , "mandatory": true}, - // {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value":"~*req.SetupTime" , "mandatory": true}, - // {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value":"~*req.AnswerTime" , "mandatory": true}, - // {"tag": "EndTime", "path": "*cgreq.EndTime", "type": "*variable","value": "~*req.EndTime"}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*req.EndTime;~*req.AnswerTime", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls","*exists:~*opts.invite:","*exists:~*opts.bye:"]}, - // {"tag": "DisconnectCause", "path": "*cgreq.DisconnectCause", "type": "*variable", "value":"~*req.DisconnectCause", "mandatory": true}, - // {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value":"~*req.DialogId"}, {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*exists:~*opts.invite:","*exists:~*opts.bye:"]}, ], "cache_dump_fields": [ + {"tag": "OriginHost", "path": "*exp.OriginHost", "type": "*variable", "value":"~*req.OriginHost"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value":"~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value":"~*req.RequestType"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value":"~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value":"~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value":"~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value":"~*req.SetupTime"}, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value":"~*req.AnswerTime"}, + {"tag": "EndTime", "path": "*exp.EndTime", "type": "*variable", "value":"~*req.EndTime"}, + {"tag": "DisconnectCause", "path": "*exp.DisconnectCause", "type": "*variable", "value":"~*req.DisconnectCause"}, + {"tag": "DialogId", "path": "*exp.DialogId", "type": "*variable", "value":"~*req.DialogId"}, ], }, diff --git a/data/conf/samples/ers_mongo/cgrates.json b/data/conf/samples/ers_mongo/cgrates.json index e4972d42a..db2d47513 100644 --- a/data/conf/samples/ers_mongo/cgrates.json +++ b/data/conf/samples/ers_mongo/cgrates.json @@ -76,9 +76,11 @@ }, + "ers": { "enabled": true, "sessions_conns": ["*internal"], + "partial_cache_ttl": "500ms", "readers": [ { "id": "file_reader1", @@ -264,10 +266,11 @@ "type": "*file_csv", "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], - "processed_path": "/tmp/partErs1/out", + "processed_path": "", "opts": { - "csvCacheExpiryAction": "*dump_to_file", + "partialCacheAction": "*dump_to_file", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs1/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -287,8 +290,8 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, ], "cache_dump_fields": [ {"tag": "OriginID", "path":"*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, @@ -299,7 +302,6 @@ {"tag": "SetupTime", "path":"*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "AnswerTime", "path":"*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "Usage", "path":"*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"tag": "Cost", "path":"*exp.Cost", "type": "*variable", "value": "~*req.Cost","rounding_decimals":5}, ], }, { @@ -308,11 +310,12 @@ "run_delay": "-1", "type": "*file_csv", "source_path": "/tmp/partErs2/in", - "processed_path": "/tmp/partErs2/out", + "processed_path": "", "flags": ["*cdrs"], "opts": { - "csvCacheExpiryAction": "*post_cdr", + "partialCacheAction": "*post_cdr", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs2/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -332,39 +335,63 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} ], }, { "id": "FlatstoreOsips", "enabled": true, "run_delay": "-1", - "type": "*flatstore", + "type": "*file_csv", "opts": { - "fstFieldSeparator":"|", - "fstFailedCallsPrefix": "missed_calls", - "fstRecordCacheTTL": "500ms", + "csvFieldSeparator":"|", + "partialcsvFieldSeparator": "|", + "partialCacheAction": "*dump_to_file", + "partialOrderField": "~*opts.order", + "partialPath": "/tmp/flatstoreErs/out", }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", "flags": ["*cdrs"], "fields":[ - {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, + {"tag": "OriginHost", "path": "*cgreq.OriginHost", "type": "*constant","value":"flatStore", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*invite.7", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*constant", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*constant", "value": "call", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*invite.9", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*bye.6;~*invite.6", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls"]}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.7", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.9", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "EndTime", "path": "*cgreq.EndTime", "type": "*variable","value": "~*req.6", "mandatory": true, "filters": ["*string:~*req.0:BYE"]}, {"tag": "DisconnectCause", "path": "*cgreq.DisconnectCause", "type": "*variable", "value": "~*req.4; ;~*req.5", "mandatory": true}, - {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"} + {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Invite", "path": "*opts.invite", "type": "*constant", "value": "true", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Bye", "path": "*opts.bye", "type": "*constant", "value": "true", "filters":["*string:~*req.0:BYE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "0", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "1", "filters":["*string:~*req.0:BYE"]}, ], + "partial_commit_fields": [ + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*req.EndTime;~*req.AnswerTime", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls","*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + ], + "cache_dump_fields": [ + {"tag": "OriginHost", "path": "*exp.OriginHost", "type": "*variable", "value":"~*req.OriginHost"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value":"~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value":"~*req.RequestType"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value":"~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value":"~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value":"~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value":"~*req.SetupTime"}, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value":"~*req.AnswerTime"}, + {"tag": "EndTime", "path": "*exp.EndTime", "type": "*variable", "value":"~*req.EndTime"}, + {"tag": "DisconnectCause", "path": "*exp.DisconnectCause", "type": "*variable", "value":"~*req.DisconnectCause"}, + {"tag": "DialogId", "path": "*exp.DialogId", "type": "*variable", "value":"~*req.DialogId"}, + ], + }, { "id": "JSONReader", @@ -400,7 +427,7 @@ {"tag": "ExtraInfo2", "path": "*cgreq.ExtraInfo2", "type": "*constant", "value": "ExtraInfo2"}, ], } - ] + ], }, diff --git a/data/conf/samples/ers_mysql/cgrates.json b/data/conf/samples/ers_mysql/cgrates.json index 10f2e6bdc..0c7a037e2 100644 --- a/data/conf/samples/ers_mysql/cgrates.json +++ b/data/conf/samples/ers_mysql/cgrates.json @@ -73,9 +73,11 @@ }, + "ers": { "enabled": true, "sessions_conns": ["*internal"], + "partial_cache_ttl": "500ms", "readers": [ { "id": "file_reader1", @@ -261,10 +263,11 @@ "type": "*file_csv", "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], - "processed_path": "/tmp/partErs1/out", + "processed_path": "", "opts": { - "csvCacheExpiryAction": "*dump_to_file", + "partialCacheAction": "*dump_to_file", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs1/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -284,8 +287,8 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, ], "cache_dump_fields": [ {"tag": "OriginID", "path":"*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, @@ -296,7 +299,6 @@ {"tag": "SetupTime", "path":"*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "AnswerTime", "path":"*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "Usage", "path":"*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"tag": "Cost", "path":"*exp.Cost", "type": "*variable", "value": "~*req.Cost","rounding_decimals":5}, ], }, { @@ -305,11 +307,12 @@ "run_delay": "-1", "type": "*file_csv", "source_path": "/tmp/partErs2/in", - "processed_path": "/tmp/partErs2/out", + "processed_path": "", "flags": ["*cdrs"], "opts": { - "csvCacheExpiryAction": "*post_cdr", + "partialCacheAction": "*post_cdr", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs2/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -329,39 +332,63 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} ], }, { "id": "FlatstoreOsips", "enabled": true, "run_delay": "-1", - "type": "*flatstore", + "type": "*file_csv", "opts": { - "fstFieldSeparator":"|", - "fstFailedCallsPrefix": "missed_calls", - "fstRecordCacheTTL": "500ms", + "csvFieldSeparator":"|", + "partialcsvFieldSeparator": "|", + "partialCacheAction": "*dump_to_file", + "partialOrderField": "~*opts.order", + "partialPath": "/tmp/flatstoreErs/out", }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", "flags": ["*cdrs"], "fields":[ - {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, + {"tag": "OriginHost", "path": "*cgreq.OriginHost", "type": "*constant","value":"flatStore", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*invite.7", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*constant", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*constant", "value": "call", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*invite.9", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*bye.6;~*invite.6", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls"]}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.7", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.9", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "EndTime", "path": "*cgreq.EndTime", "type": "*variable","value": "~*req.6", "mandatory": true, "filters": ["*string:~*req.0:BYE"]}, {"tag": "DisconnectCause", "path": "*cgreq.DisconnectCause", "type": "*variable", "value": "~*req.4; ;~*req.5", "mandatory": true}, - {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"} + {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Invite", "path": "*opts.invite", "type": "*constant", "value": "true", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Bye", "path": "*opts.bye", "type": "*constant", "value": "true", "filters":["*string:~*req.0:BYE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "0", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "1", "filters":["*string:~*req.0:BYE"]}, ], + "partial_commit_fields": [ + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*req.EndTime;~*req.AnswerTime", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls","*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + ], + "cache_dump_fields": [ + {"tag": "OriginHost", "path": "*exp.OriginHost", "type": "*variable", "value":"~*req.OriginHost"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value":"~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value":"~*req.RequestType"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value":"~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value":"~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value":"~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value":"~*req.SetupTime"}, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value":"~*req.AnswerTime"}, + {"tag": "EndTime", "path": "*exp.EndTime", "type": "*variable", "value":"~*req.EndTime"}, + {"tag": "DisconnectCause", "path": "*exp.DisconnectCause", "type": "*variable", "value":"~*req.DisconnectCause"}, + {"tag": "DialogId", "path": "*exp.DialogId", "type": "*variable", "value":"~*req.DialogId"}, + ], + }, { "id": "JSONReader", @@ -397,7 +424,7 @@ {"tag": "ExtraInfo2", "path": "*cgreq.ExtraInfo2", "type": "*constant", "value": "ExtraInfo2"}, ], } - ] + ], }, diff --git a/data/conf/samples/ers_postgres/cgrates.json b/data/conf/samples/ers_postgres/cgrates.json index 29033304d..3f5fd29bf 100644 --- a/data/conf/samples/ers_postgres/cgrates.json +++ b/data/conf/samples/ers_postgres/cgrates.json @@ -70,9 +70,11 @@ }, + "ers": { "enabled": true, "sessions_conns": ["*internal"], + "partial_cache_ttl": "500ms", "readers": [ { "id": "file_reader1", @@ -258,10 +260,11 @@ "type": "*file_csv", "source_path": "/tmp/partErs1/in", "flags": ["*cdrs"], - "processed_path": "/tmp/partErs1/out", + "processed_path": "", "opts": { - "csvCacheExpiryAction": "*dump_to_file", + "partialCacheAction": "*dump_to_file", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs1/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -281,8 +284,8 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]}, ], "cache_dump_fields": [ {"tag": "OriginID", "path":"*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, @@ -293,7 +296,6 @@ {"tag": "SetupTime", "path":"*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "AnswerTime", "path":"*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime", "layout": "2006-01-02T15:04:05Z07:00"}, {"tag": "Usage", "path":"*exp.Usage", "type": "*variable", "value": "~*req.Usage"}, - {"tag": "Cost", "path":"*exp.Cost", "type": "*variable", "value": "~*req.Cost","rounding_decimals":5}, ], }, { @@ -302,11 +304,12 @@ "run_delay": "-1", "type": "*file_csv", "source_path": "/tmp/partErs2/in", - "processed_path": "/tmp/partErs2/out", + "processed_path": "", "flags": ["*cdrs"], "opts": { - "csvCacheExpiryAction": "*post_cdr", + "partialCacheAction": "*post_cdr", "partialOrderField": "~*req.AnswerTime", + "partialPath": "/tmp/partErs2/out", }, "fields":[ {"tag": "ToR", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, @@ -326,39 +329,63 @@ {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.4", "mandatory": true}, {"tag": "Usage", "path": "*cgreq.Usage", "type": "*variable", "value": "~*req.6:s/^(\\d+)$/${1}s/", "mandatory": true}, {"tag": "Partial", "path": "*cgreq.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]},// keep this here for partial cdr field - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, - {"tag": "Partial", "path": "*opts.Partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true", "filters":["*string:~*req.10:partial"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false", "filters":["*notstring:~*req.10:partial"]} ], }, { "id": "FlatstoreOsips", "enabled": true, "run_delay": "-1", - "type": "*flatstore", + "type": "*file_csv", "opts": { - "fstFieldSeparator":"|", - "fstFailedCallsPrefix": "missed_calls", - "fstRecordCacheTTL": "500ms", + "csvFieldSeparator":"|", + "partialcsvFieldSeparator": "|", + "partialCacheAction": "*dump_to_file", + "partialOrderField": "~*opts.order", + "partialPath": "/tmp/flatstoreErs/out", }, "source_path": "/tmp/flatstoreErs/in", "processed_path": "/tmp/flatstoreErs/out", "flags": ["*cdrs"], "fields":[ - {"tag": "Tor", "path": "*cgreq.ToR", "type": "*constant", "value": "*voice", "mandatory": true}, + {"tag": "OriginHost", "path": "*cgreq.OriginHost", "type": "*constant","value":"flatStore", "mandatory": true}, {"tag": "OriginID", "path": "*cgreq.OriginID", "type": "*variable","value":"~*req.3;~*req.1;~*req.2", "mandatory": true}, - {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*invite.7", "mandatory": true}, - {"tag": "Tenant", "path": "*cgreq.Tenant", "type": "*constant", "value": "cgrates.org", "mandatory": true}, - {"tag": "Category", "path": "*cgreq.Category", "type": "*constant", "value": "call", "mandatory": true}, - {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*invite.8", "mandatory": true}, - {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*invite.9", "mandatory": true}, - {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*invite.6", "mandatory": true}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, - {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*bye.6;~*invite.6", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls"]}, + {"tag": "RequestType", "path": "*cgreq.RequestType", "type": "*variable", "value": "~*req.7", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Account", "path": "*cgreq.Account", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Subject", "path": "*cgreq.Subject", "type": "*variable", "value": "~*req.8", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "Destination", "path": "*cgreq.Destination", "type": "*variable", "value": "~*req.9", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "SetupTime", "path": "*cgreq.SetupTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "AnswerTime", "path": "*cgreq.AnswerTime", "type": "*variable", "value": "~*req.6", "mandatory": true,"filters": ["*string:~*req.0:INVITE"]}, + {"tag": "EndTime", "path": "*cgreq.EndTime", "type": "*variable","value": "~*req.6", "mandatory": true, "filters": ["*string:~*req.0:BYE"]}, {"tag": "DisconnectCause", "path": "*cgreq.DisconnectCause", "type": "*variable", "value": "~*req.4; ;~*req.5", "mandatory": true}, - {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"} + {"tag": "DialogId", "path": "*cgreq.DialogId", "type": "*variable", "value": "~*req.11"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "true"}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Invite", "path": "*opts.invite", "type": "*constant", "value": "true", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Bye", "path": "*opts.bye", "type": "*constant", "value": "true", "filters":["*string:~*req.0:BYE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "0", "filters":["*string:~*req.0:INVITE"]}, + {"tag": "Order", "path": "*opts.order", "type": "*constant", "value": "1", "filters":["*string:~*req.0:BYE"]}, ], + "partial_commit_fields": [ + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*constant","value": "0", "mandatory": true, "filters": ["*prefix:~*vars.FileName:missed_calls"]}, + {"tag": "Usage", "path": "*cgreq.Usage", "type": "*usage_difference","value": "~*req.EndTime;~*req.AnswerTime", "mandatory": true, "filters": ["*notprefix:~*vars.FileName:missed_calls","*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + {"tag": "Partial", "path": "*opts.*partial", "type": "*constant", "value": "false","filters": ["*exists:~*opts.invite:","*exists:~*opts.bye:"]}, + ], + "cache_dump_fields": [ + {"tag": "OriginHost", "path": "*exp.OriginHost", "type": "*variable", "value":"~*req.OriginHost"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value":"~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value":"~*req.RequestType"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value":"~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value":"~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value":"~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value":"~*req.SetupTime"}, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value":"~*req.AnswerTime"}, + {"tag": "EndTime", "path": "*exp.EndTime", "type": "*variable", "value":"~*req.EndTime"}, + {"tag": "DisconnectCause", "path": "*exp.DisconnectCause", "type": "*variable", "value":"~*req.DisconnectCause"}, + {"tag": "DialogId", "path": "*exp.DialogId", "type": "*variable", "value":"~*req.DialogId"}, + ], + }, { "id": "JSONReader", @@ -394,7 +421,7 @@ {"tag": "ExtraInfo2", "path": "*cgreq.ExtraInfo2", "type": "*constant", "value": "ExtraInfo2"}, ], } - ] + ], }, diff --git a/ees/filecsv.go b/ees/filecsv.go index f25182314..1233c1138 100644 --- a/ees/filecsv.go +++ b/ees/filecsv.go @@ -65,10 +65,7 @@ func (fCsv *FileCSVee) init() (err error) { } fCsv.csvWriter = csv.NewWriter(fCsv.file) fCsv.csvWriter.Comma = utils.CSVSep - if len(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].FieldSep) > 0 { - fCsv.csvWriter.Comma = rune(fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].FieldSep[0]) - } - if fieldSep, has := fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Opts[utils.CSV+utils.FieldSepOpt]; has { + if fieldSep, has := fCsv.cgrCfg.EEsCfg().Exporters[fCsv.cfgIdx].Opts[utils.CSVFieldSepOpt]; has { fCsv.csvWriter.Comma = rune(utils.IfaceAsString(fieldSep)[0]) } return fCsv.composeHeader() diff --git a/engine/dynamicdp.go b/engine/dynamicdp.go index ba68f79dc..d605c5fb2 100644 --- a/engine/dynamicdp.go +++ b/engine/dynamicdp.go @@ -66,8 +66,7 @@ var initialDPPrefixes = utils.NewStringSet([]string{ utils.MetaCgrep, utils.MetaRep, utils.MetaAct, utils.MetaEC, utils.MetaUCH, utils.MetaOpts, utils.MetaHdr, utils.MetaTrl, utils.MetaCfg, - utils.MetaTenant, utils.MetaInvite, utils.MetaBye, - utils.MetaAck}) + utils.MetaTenant}) func (dDP *dynamicDP) FieldAsInterface(fldPath []string) (val interface{}, err error) { if len(fldPath) == 0 { diff --git a/ers/amqp.go b/ers/amqp.go index cb7d13744..9851f91e4 100644 --- a/ers/amqp.go +++ b/ers/amqp.go @@ -204,7 +204,7 @@ func (rdr *AMQPER) processMessage(msg []byte) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/amqpv1.go b/ers/amqpv1.go index f9f68ccc2..64a9e973d 100644 --- a/ers/amqpv1.go +++ b/ers/amqpv1.go @@ -177,7 +177,7 @@ func (rdr *AMQPv1ER) processMessage(msg []byte) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/ers.go b/ers/ers.go index 133d5b4d7..e41b42228 100644 --- a/ers/ers.go +++ b/ers/ers.go @@ -23,11 +23,9 @@ import ( "fmt" "os" "path" - "sort" "sync" "time" - "github.com/cgrates/cgrates/agents" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/sessions" @@ -320,25 +318,23 @@ func (erS *ERService) closeAllRdrs() { } } -const ( - partialOpt = "*partial" -) - type erEvents struct { events []*utils.CGREvent rdrCfg *config.EventReaderCfg } +// processPartialEvent process the event as a partial event func (erS *ERService) processPartialEvent(ev *utils.CGREvent, rdrCfg *config.EventReaderCfg) (err error) { + // to identify the event the originID and originHost is used to create the CGRID orgID, err := ev.FieldAsString(utils.OriginID) - if err == utils.ErrNotFound { + if err == utils.ErrNotFound { // the field is missing ignore the event utils.Logger.Warning( fmt.Sprintf("<%s> Missing field for partial event <%s>", utils.ERs, utils.ToJSON(ev))) return } orgHost, err := ev.FieldAsString(utils.OriginHost) - if err == utils.ErrNotFound { + if err == utils.ErrNotFound { // the field is missing ignore the event utils.Logger.Warning( fmt.Sprintf("<%s> Missing field for partial event <%s>", utils.ERs, utils.ToJSON(ev))) @@ -346,7 +342,7 @@ func (erS *ERService) processPartialEvent(ev *utils.CGREvent, rdrCfg *config.Eve } cgrID := utils.Sha1(orgID, orgHost) - evs, has := erS.partialCache.Get(cgrID) + evs, has := erS.partialCache.Get(cgrID) // get the existing events from cache var cgrEvs *erEvents if !has || evs == nil { cgrEvs = &erEvents{ @@ -360,107 +356,44 @@ func (erS *ERService) processPartialEvent(ev *utils.CGREvent, rdrCfg *config.Eve } var cgrEv *utils.CGREvent - if cgrEv, err = erS.preparePartialEvents(cgrEvs.events, cgrEvs.rdrCfg); err != nil { + if cgrEv, err = mergePartialEvents(cgrEvs.events, cgrEvs.rdrCfg, erS.filterS, // merge the events + erS.cfg.GeneralCfg().DefaultTenant, + erS.cfg.GeneralCfg().DefaultTimezone, + erS.cfg.GeneralCfg().RSRSep); err != nil { return } - if partial := cgrEv.APIOpts[partialOpt]; !utils.IsSliceMember([]string{"false", utils.EmptyString}, utils.IfaceAsString(partial)) { + if partial := cgrEv.APIOpts[utils.PartialOpt]; !utils.IsSliceMember([]string{utils.FalseStr, utils.EmptyString}, + utils.IfaceAsString(partial)) { // if is still partial set it back in cache erS.partialCache.Set(cgrID, cgrEvs, nil) return } - // complete CDR - if len(cgrEvs.events) != 1 { + // complete event + if len(cgrEvs.events) != 1 { // remove it from cache if there were events in cache erS.partialCache.Set(cgrID, nil, nil) // set it with nil in cache to ignore when we expire the item erS.partialCache.Remove(cgrID) } - go func() { erS.rdrEvents <- &erEvent{cgrEvent: cgrEv, rdrCfg: rdrCfg} }() - return -} - -func (erS *ERService) preparePartialEvents(cgrEvs []*utils.CGREvent, cfg *config.EventReaderCfg) (cgrEv *utils.CGREvent, err error) { - cgrEv = cgrEvs[0] - if len(cgrEvs) != 1 { - ordFld := utils.IfaceAsString(cfg.Opts[utils.PartialOrderFieldOpt]) - if ordFld == utils.EmptyString { - return nil, utils.NewErrMandatoryIeMissing(utils.PartialOrderFieldOpt) - } - fields := make([]interface{}, len(cgrEvs)) - - var ordPath config.RSRParsers - if ordPath, err = config.NewRSRParsers(ordFld, erS.cfg.GeneralCfg().RSRSep); err != nil { - return nil, err - } - - for i, ev := range cgrEvs { - if fields[i], err = ordPath.ParseDataProviderWithInterfaces(ev.AsDataProvider()); err != nil { - return - } - if fldStr, castStr := fields[i].(string); castStr { // attempt converting string since deserialization fails here (ie: time.Time fields) - fields[i] = utils.StringToInterface(fldStr) - } - } - //sort CGREvents based on partialOrderFieldOpt - sort.Slice(cgrEvs, func(i, j int) bool { - gt, serr := utils.GreaterThan(fields[i], fields[j], true) - if serr != nil { - err = serr - } - return gt - }) - if err != nil { - return - } - - // compose the CGREvent from slice - cgrEv = &utils.CGREvent{ - Tenant: cgrEvs[0].Tenant, - ID: utils.UUIDSha1Prefix(), - Time: utils.TimePointer(time.Now()), - Event: make(map[string]interface{}), - APIOpts: make(map[string]interface{}), - } - for _, ev := range cgrEvs { - for key, value := range ev.Event { - cgrEv.Event[key] = value - } - for key, val := range ev.APIOpts { - cgrEv.APIOpts[key] = val - } - } - } - if len(cfg.PartialCommitFields) != 0 { - agReq := agents.NewAgentRequest( - utils.MapStorage(cgrEv.Event), nil, - nil, nil, cgrEv.APIOpts, cfg.Tenant, - erS.cfg.GeneralCfg().DefaultTenant, - utils.FirstNonEmpty(cfg.Timezone, - erS.cfg.GeneralCfg().DefaultTimezone), - erS.filterS, nil) // create an AgentRequest - if err = agReq.SetFields(cfg.PartialCommitFields); err != nil { - utils.Logger.Warning( - fmt.Sprintf("<%s> processing partial event: <%s>, ignoring due to error: <%s>", - utils.ERs, utils.ToJSON(cgrEv), err.Error())) - return - } - cgrEv = utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) - } - + go func() { erS.rdrEvents <- &erEvent{cgrEvent: cgrEv, rdrCfg: rdrCfg} }() // put the event on the complete events chanel( in a goroutine to not block the select from ListenAndServe) return } +// onEvicted the function that is called when a element is removed from cache func (erS *ERService) onEvicted(id string, value interface{}) { - if value == nil { + if value == nil { // is already complete and sent to erS return } eEvs := value.(*erEvents) action := erS.cfg.ERsCfg().PartialCacheAction - if cAct, has := eEvs.rdrCfg.Opts[utils.PartialCacheAction]; has { + if cAct, has := eEvs.rdrCfg.Opts[utils.PartialCacheActionOpt]; has { // if the option is present overwrite the global cache action action = utils.IfaceAsString(cAct) } switch action { - case utils.MetaNone: - case utils.MetaPostCDR: - cgrEv, err := erS.preparePartialEvents(eEvs.events, eEvs.rdrCfg) + case utils.MetaNone: // do nothing with the events + case utils.MetaPostCDR: // merge the events and post the to erS + cgrEv, err := mergePartialEvents(eEvs.events, eEvs.rdrCfg, erS.filterS, + erS.cfg.GeneralCfg().DefaultTenant, + erS.cfg.GeneralCfg().DefaultTimezone, + erS.cfg.GeneralCfg().RSRSep) if err != nil { utils.Logger.Warning( fmt.Sprintf("<%s> failed posting expired parial events <%s> due error <%s>", @@ -468,15 +401,43 @@ func (erS *ERService) onEvicted(id string, value interface{}) { return } erS.rdrEvents <- &erEvent{cgrEvent: cgrEv, rdrCfg: eEvs.rdrCfg} - case utils.MetaDumpToFile: - tmz := utils.FirstNonEmpty(eEvs.rdrCfg.Timezone, erS.cfg.GeneralCfg().DefaultTimezone) + case utils.MetaDumpToFile: // apply the cacheDumpFields to the united events and write the record to file expPath := erS.cfg.ERsCfg().PartialPath if path, has := eEvs.rdrCfg.Opts[utils.PartialPathOpt]; has { expPath = utils.IfaceAsString(path) } - if expPath == utils.EmptyString { // do not send the partial events to any file + if expPath == utils.EmptyString { // do not write the partial event to file return } + cgrEv, err := mergePartialEvents(eEvs.events, eEvs.rdrCfg, erS.filterS, // merge the partial events + erS.cfg.GeneralCfg().DefaultTenant, + erS.cfg.GeneralCfg().DefaultTimezone, + erS.cfg.GeneralCfg().RSRSep) + if err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> failed posting expired parial events <%s> due error <%s>", + utils.ERs, utils.ToJSON(eEvs.events), err.Error())) + return + } + // convert the event to record + eeReq := engine.NewEventRequest(utils.MapStorage(cgrEv.Event), + utils.MapStorage{}, cgrEv.APIOpts, + eEvs.rdrCfg.Tenant, erS.cfg.GeneralCfg().DefaultTenant, + utils.FirstNonEmpty(eEvs.rdrCfg.Timezone, erS.cfg.GeneralCfg().DefaultTimezone), + erS.filterS, map[string]*utils.OrderedNavigableMap{ + utils.MetaExp: utils.NewOrderedNavigableMap(), + }) + + if err = eeReq.SetFields(eEvs.rdrCfg.CacheDumpFields); err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> Converting CDR with CGRID: <%s> to record , ignoring due to error: <%s>", + utils.ERs, id, err.Error())) + return + } + + record := eeReq.OrdNavMP[utils.MetaExp].OrderedFieldsAsStrings() + + // open the file and write the record dumpFilePath := path.Join(expPath, fmt.Sprintf("%s.%d%s", id, time.Now().Unix(), utils.TmpSuffix)) fileOut, err := os.Create(dumpFilePath) @@ -485,34 +446,17 @@ func (erS *ERService) onEvicted(id string, value interface{}) { utils.ERs, dumpFilePath, err.Error())) return } - defer fileOut.Close() csvWriter := csv.NewWriter(fileOut) - if fldSep, has := eEvs.rdrCfg.Opts[utils.PartialCSVFieldSepartor]; has { + if fldSep, has := eEvs.rdrCfg.Opts[utils.PartialCSVFieldSepartorOpt]; has { csvWriter.Comma = rune(utils.IfaceAsString(fldSep)[0]) } - for _, ev := range eEvs.events { - oNm := map[string]*utils.OrderedNavigableMap{ - utils.MetaExp: utils.NewOrderedNavigableMap(), - } - eeReq := engine.NewEventRequest(utils.MapStorage(ev.Event), utils.MapStorage{}, ev.APIOpts, - eEvs.rdrCfg.Tenant, erS.cfg.GeneralCfg().DefaultTenant, - tmz, erS.filterS, oNm) - if err = eeReq.SetFields(eEvs.rdrCfg.CacheDumpFields); err != nil { - utils.Logger.Warning( - fmt.Sprintf("<%s> Converting CDR with CGRID: <%s> to record , ignoring due to error: <%s>", - utils.ERs, id, err.Error())) - return - } - - record := eeReq.OrdNavMP[utils.MetaExp].OrderedFieldsAsStrings() - if err = csvWriter.Write(record); err != nil { - utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", - utils.ERs, record, dumpFilePath, err.Error())) - return - } + if err = csvWriter.Write(record); err != nil { + utils.Logger.Err(fmt.Sprintf("<%s> Failed writing partial record %v to file: %s, error: %s", + utils.ERs, record, dumpFilePath, err.Error())) } csvWriter.Flush() + fileOut.Close() } } diff --git a/ers/filecsv.go b/ers/filecsv.go index 9172dd99f..d86a0d45f 100644 --- a/ers/filecsv.go +++ b/ers/filecsv.go @@ -132,13 +132,23 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { return } defer file.Close() - var csvReader *csv.Reader - if csvReader, err = newCSVReader(file, rdr.Config().Opts, utils.CSV); err != nil { + csvReader := csv.NewReader(file) + var rowLength int64 + if rowLength, err = utils.IfaceAsTInt64(rdr.Config().Opts[utils.CSVRowLengthOpt]); err != nil { utils.Logger.Err( fmt.Sprintf("<%s> failed creating CSV reader for <%s>, due to option parsing error: <%s>", utils.ERs, rdr.Config().ID, err.Error())) return } + csvReader.FieldsPerRecord = int(rowLength) + csvReader.Comment = utils.CommentChar + csvReader.Comma = utils.CSVSep + if fieldSep, has := rdr.Config().Opts[utils.CSVFieldSepOpt]; has { + csvReader.Comma = rune(utils.IfaceAsString(fieldSep)[0]) + } + if val, has := rdr.Config().Opts[utils.CSVLazyQuotes]; has { + csvReader.LazyQuotes, err = utils.IfaceAsBool(val) + } var indxAls map[string]int rowNr := 0 // This counts the rows in the file, not really number of CDRs evsPosted := 0 @@ -190,7 +200,7 @@ func (rdr *CSVFileER) processFile(fPath, fName string) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/filefwv.go b/ers/filefwv.go index f90fc3be2..c04409c29 100644 --- a/ers/filefwv.go +++ b/ers/filefwv.go @@ -222,7 +222,7 @@ func (rdr *FWVFileER) processFile(fPath, fName string) (err error) { rdr.offset += rdr.lineLen // increase the offset cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ @@ -314,7 +314,7 @@ func (rdr *FWVFileER) processTrailer(file *os.File, rowNr, evsPosted int, absPat } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ @@ -360,7 +360,7 @@ func (rdr *FWVFileER) createHeaderMap(record string, rowNr, evsPosted int, absPa rdr.offset += rdr.headerOffset // increase the offset cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/filejson.go b/ers/filejson.go index 0581532f0..7fb64cf99 100644 --- a/ers/filejson.go +++ b/ers/filejson.go @@ -172,7 +172,7 @@ func (rdr *JSONFileER) processFile(fPath, fName string) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/filexml.go b/ers/filexml.go index e21577d68..c1455fa3a 100644 --- a/ers/filexml.go +++ b/ers/filexml.go @@ -168,7 +168,7 @@ func (rdr *XMLFileER) processFile(fPath, fName string) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/flatstore_it_test.go b/ers/flatstore_it_test.go index 871bdee15..6858dfc36 100644 --- a/ers/flatstore_it_test.go +++ b/ers/flatstore_it_test.go @@ -186,7 +186,7 @@ func testFlatstoreITHandleCdr1File(t *testing.T) { if len(filesOutDir) != 5 { t.Errorf("Unexpected number of files in output directory: %+v, %q", len(filesOutDir), ids) } - ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9p@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n" + ePartContent := "flatStore|dd0c4c617a9919d29a6175cdff223a9p@0:0:0:0:0:0:0:02daec40c548625ac|*prepaid|1001|1001|1002|1436454408|1436454408|200 OK|3401:2069362475\n" tmpl := path.Join("/tmp/flatstoreErs/out", "f7aed15c98b31fea0e3b02b52fc947879a3c5bbc.*.tmp") if match, err := filepath.Glob(tmpl); err != nil { t.Error(err) @@ -221,446 +221,3 @@ func testFlatstoreITKillEngine(t *testing.T) { t.Error(err) } } - -/* -func TestFlatstoreProcessEvent(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - cfg.ERsCfg().Readers[0].Opts[utils.FstFailedCallsPrefixOpt] = "file" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - file.Write([]byte("INVITE,a,ToR,b,c,d,e,f,g,h,i,j,k,l")) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.cache = ltcache.NewCache(-1, 0, false, eR.dumpToFile) - expEvent := &utils.CGREvent{ - Tenant: "cgrates.org", - Event: map[string]interface{}{ - utils.AccountField: "g", - utils.AnswerTime: "k", - utils.Category: "f", - utils.Destination: "i", - utils.OriginID: "b", - utils.RequestType: "c", - utils.SetupTime: "j", - utils.Subject: "h", - utils.Tenant: "e", - utils.ToR: "ToR", - utils.Usage: "l", - }, - APIOpts: map[string]interface{}{}, - } - eR.conReqs <- struct{}{} - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - select { - case data := <-eR.rdrEvents: - expEvent.ID = data.cgrEvent.ID - expEvent.Time = data.cgrEvent.Time - if !reflect.DeepEqual(data.cgrEvent, expEvent) { - t.Errorf("Expected %v but received %v", utils.ToJSON(expEvent), utils.ToJSON(data.cgrEvent)) - } - case <-time.After(50 * time.Millisecond): - t.Error("Time limit exceeded") - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -//Test the case in which the file name does not match a prefix -func TestFlatstoreProcessEvent2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - cfg.ERsCfg().Readers[0].Fields = append(cfg.ERsCfg().Readers[0].Fields, &config.FCTemplate{ - Tag: "Usage", - Path: "*cgreq.Usage", - Type: utils.MetaUsageDifference, - Value: config.NewRSRParsersMustCompile("~*bye.6;~*invite.6", utils.InfieldSep), - }) - for _, v := range cfg.ERsCfg().Readers[0].Fields { - v.ComputePath() - } - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - //baToR - file.Write([]byte("INVITE,a,ToR,b,c,d,2013-12-30T15:00:01Z,f,g,h,i,j,k,l")) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - record := []string{utils.FstBye, "a", "ToR", "b", "c", "d", "2013-12-30T16:00:01Z", "f", "g", "h", "i", "j", "k", "l"} - pr := &fstRecord{method: utils.FstBye, values: record, fileName: fname} - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, nil) - eR.cache.Set(utils.ConcatenatedKey("baToR", utils.FstBye), pr, []string{"baToR"}) - expEvent := &utils.CGREvent{ - Tenant: "cgrates.org", - Event: map[string]interface{}{ - utils.AccountField: "g", - utils.AnswerTime: "k", - utils.Category: "f", - utils.Destination: "i", - utils.OriginID: "b", - utils.RequestType: "c", - utils.SetupTime: "j", - utils.Subject: "h", - utils.Tenant: "2013-12-30T15:00:01Z", - utils.ToR: "ToR", - utils.Usage: "1h0m0s", - }, - APIOpts: map[string]interface{}{}, - } - eR.conReqs <- struct{}{} - eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - select { - case data := <-eR.rdrEvents: - expEvent.ID = data.cgrEvent.ID - expEvent.Time = data.cgrEvent.Time - if !reflect.DeepEqual(data.cgrEvent, expEvent) { - t.Errorf("Expected %v but received %v", utils.ToJSON(expEvent), utils.ToJSON(data.cgrEvent)) - } - case <-time.After(50 * time.Millisecond): - t.Error("Time limit exceeded") - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestFlatstoreProcessEvent2CacheNotSet(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - file.Write([]byte("INVITE,a,ToR,b,c,d,2013-12-30T15:00:01Z,f,g,h,i,j,k,l")) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) - - eR.conReqs <- struct{}{} - eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -//Test pairToRecord() error, where both methods of unpaired record object are INVITE -func TestFlatstoreProcessEvent2Error2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - //Create new logger - utils.Logger, err = utils.Newlogger(utils.MetaStdLog, utils.EmptyString) - if err != nil { - t.Error(err) - } - utils.Logger.SetLogLevel(7) - buf := new(bytes.Buffer) - log.SetOutput(buf) - //baToR - file.Write([]byte("INVITE,a,ToR,b,c,d,2013-12-30T15:00:01Z,f,g,h,i,j,k,l")) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - record := []string{"INVITE", "a", "ToR", "b", "c", "d", "2013-12-30T16:00:01Z", "f", "g", "h", "i", "j", "k", "l"} - pr := &fstRecord{method: utils.FstInvite, values: record, fileName: fname} - eR.cache = ltcache.NewCache(ltcache.UnlimitedCaching, 0, false, eR.dumpToFile) - eR.cache.Set("baToR:INVITE", pr, []string{"baToR"}) - eR.conReqs <- struct{}{} - eR.Config().Opts[utils.FstFailedCallsPrefixOpt] = "x" - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - errExpect := "[WARNING] Overwriting the INVITE method for record " - if rcv := buf.String(); !strings.Contains(rcv, errExpect) { - t.Errorf("\nExpected %v but \nreceived %v", errExpect, rcv) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } - buf.Reset() -} - -//Fields in template are empty in order to trigger SetFields() error -func TestFlatstoreProcessEventError2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - file.Write([]byte(`#ToR,OriginID,RequestType,Tenant,Category,Account,Subject,Destination,SetupTime,AnswerTime,Usage -,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m`)) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - - eR.Config().Fields = []*config.FCTemplate{ - {}, - } - - errExpect := `unsupported method: <"">` - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -//Test invalid filters in order to trigger Pass() function error -func TestFlatstoreProcessEventError3(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].Fields = []*config.FCTemplate{} - data := engine.NewInternalDB(nil, nil, true) - dm := engine.NewDataManager(data, cfg.CacheCfg(), nil) - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := engine.NewFilterS(cfg, nil, dm) - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - file.Write([]byte(`#ToR,OriginID,RequestType,Tenant,Category,Account,Subject,Destination,SetupTime,AnswerTime,Usage -BYE,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m`)) - file.Close() - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - cache: ltcache.NewCache(-1, 0, false, nil), - } - eR.conReqs <- struct{}{} - eR.cache.Set("OriginCDR1*voice:INVITE", &fstRecord{method: utils.FstInvite, values: []string{}}, []string{"OriginCDR1*voice"}) - // - eR.Config().Filters = []string{"Filter1"} - errExpect := "NOT_FOUND:Filter1" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - - // - eR.cache.Set("OriginCDR1*voice:INVITE", &fstRecord{method: utils.FstInvite, values: []string{}}, []string{"OriginCDR1*voice"}) - - eR.Config().Filters = []string{"*exists:~*req..Account:"} - errExpect = "Invalid fieldPath [ Account]" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestFlatstoreProcessEventDirError(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - filePath := "/tmp/TestFlatstoreProcessEvent/" - fname := "file1.csv" - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/ers/out/", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - errExpect := "open /tmp/TestFlatstoreProcessEvent/file1.csv: no such file or directory" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } -} - -func TestFlatstore(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - if err := eR.Serve(); err != nil { - t.Error(err) - } -} - -func TestFlatstoreServeDefault(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - filePath := "/tmp/flatstoreErs/out" - err := os.MkdirAll(filePath, 0777) - if err != nil { - t.Error(err) - } - for i := 1; i < 4; i++ { - if _, err := os.Create(path.Join(filePath, fmt.Sprintf("file%d.csv", i))); err != nil { - t.Error(err) - } - } - eR.Config().RunDelay = 1 * time.Millisecond - os.Create(path.Join(filePath, "file1.txt")) - go func() { - time.Sleep(20 * time.Millisecond) - close(eR.rdrExit) - }() - eR.serveDefault() - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestFileFlatstoreExit(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - eR := &FlatstoreER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/flatstoreErs/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - eR.Config().RunDelay = 1 * time.Millisecond - if err := eR.Serve(); err != nil { - t.Error(err) - } - eR.rdrExit <- struct{}{} -} - -func TestFlatstoreServeErrTimeDurationNeg1(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfgIdx := 0 - rdr, err := NewFlatstoreER(cfg, cfgIdx, nil, nil, nil, nil) - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } - rdr.Config().RunDelay = time.Duration(-1) - expected := "no such file or directory" - err = rdr.Serve() - if err == nil || err.Error() != expected { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", expected, err) - } -} -*/ diff --git a/ers/kafka.go b/ers/kafka.go index be8e5de73..68fa6128c 100644 --- a/ers/kafka.go +++ b/ers/kafka.go @@ -173,7 +173,7 @@ func (rdr *KafkaER) processMessage(msg []byte) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/libers.go b/ers/libers.go index 0e96d7a15..68d244a75 100644 --- a/ers/libers.go +++ b/ers/libers.go @@ -19,10 +19,14 @@ along with this program. If not, see package ers import ( - "encoding/csv" - "io" + "fmt" + "sort" "strings" + "time" + "github.com/cgrates/cgrates/agents" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -36,20 +40,74 @@ func getProcessOptions(opts map[string]interface{}) (proc map[string]interface{} return } -func newCSVReader(file io.Reader, opts map[string]interface{}, prfx string) (csvReader *csv.Reader, err error) { - csvReader = csv.NewReader(file) - var rowLength int64 - if rowLength, err = utils.IfaceAsTInt64(opts[prfx+utils.RowLengthOpt]); err != nil { - return +// mergePartialEvents will unite the events using the reader configuration +func mergePartialEvents(cgrEvs []*utils.CGREvent, cfg *config.EventReaderCfg, fltrS *engine.FilterS, dftTnt, dftTmz, rsrSep string) (cgrEv *utils.CGREvent, err error) { + cgrEv = cgrEvs[0] // by default there is at least one event + if len(cgrEvs) != 1 { // need to merge the incoming events + // prepare the field after which the events are ordered + ordFld := utils.IfaceAsString(cfg.Opts[utils.PartialOrderFieldOpt]) // safe as the checkConfigSanity forces this option to be populated + var ordPath config.RSRParsers + if ordPath, err = config.NewRSRParsers(ordFld, rsrSep); err != nil { // convert the option to rsrParsers + return nil, err + } + + // get the field as interface in a slice + fields := make([]interface{}, len(cgrEvs)) + for i, ev := range cgrEvs { + if fields[i], err = ordPath.ParseDataProviderWithInterfaces(ev.AsDataProvider()); err != nil { + return + } + if fldStr, castStr := fields[i].(string); castStr { // attempt converting string since deserialization fails here (ie: time.Time fields) + fields[i] = utils.StringToInterface(fldStr) + } + } + //sort CGREvents based on partialOrderFieldOpt + sort.Slice(cgrEvs, func(i, j int) bool { + gt, serr := utils.GreaterThan(fields[i], fields[j], true) + if serr != nil { // save the last non nil error + err = serr + } + return !gt + }) + if err != nil { // the fields are not comparable + return + } + + // compose the CGREvent from slice + cgrEv = &utils.CGREvent{ + Tenant: cgrEvs[0].Tenant, + ID: utils.UUIDSha1Prefix(), + Time: utils.TimePointer(time.Now()), + Event: make(map[string]interface{}), + APIOpts: make(map[string]interface{}), + } + for _, ev := range cgrEvs { // merge the maps + for key, value := range ev.Event { + cgrEv.Event[key] = value + } + for key, val := range ev.APIOpts { + cgrEv.APIOpts[key] = val + } + } } - csvReader.FieldsPerRecord = int(rowLength) - csvReader.Comment = utils.CommentChar - csvReader.Comma = utils.CSVSep - if fieldSep, has := opts[prfx+utils.FieldSepOpt]; has { - csvReader.Comma = rune(utils.IfaceAsString(fieldSep)[0]) - } - if val, has := opts[prfx+utils.LazyQuotes]; has { - csvReader.LazyQuotes, err = utils.IfaceAsBool(val) + if len(cfg.PartialCommitFields) != 0 { // apply the partial commit template + agReq := agents.NewAgentRequest( + utils.MapStorage(cgrEv.Event), nil, + nil, nil, cgrEv.APIOpts, cfg.Tenant, dftTnt, + utils.FirstNonEmpty(cfg.Timezone, dftTmz), + fltrS, nil) // create an AgentRequest + if err = agReq.SetFields(cfg.PartialCommitFields); err != nil { + utils.Logger.Warning( + fmt.Sprintf("<%s> processing partial event: <%s>, ignoring due to error: <%s>", + utils.ERs, utils.ToJSON(cgrEv), err.Error())) + return + } + if ev := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, + utils.NestingSep, agReq.Opts); ev != nil { // add the modified fields in the event + for k, v := range ev.Event { + cgrEv.Event[k] = v + } + } } return } diff --git a/ers/partial_csv_it_test.go b/ers/partial_csv_it_test.go index 3fefa8aaf..aa292edf5 100644 --- a/ers/partial_csv_it_test.go +++ b/ers/partial_csv_it_test.go @@ -65,9 +65,7 @@ var ( partCsvFileContent2 = `4986517174963,004986517174964,DE-National,04.07.2016 19:00:00,04.07.2016 18:58:55,0,15,Offpeak,0.003360,498651,partial` partCsvFileContent3 = `4986517174964,004986517174960,DE-National,04.07.2016 19:05:55,04.07.2016 19:05:55,0,23,Offpeak,0.003360,498651,partial` - eCacheDumpFile1 = `4986517174963_004986517174964_04.07.2016 18:58:55,1467658800,*rated,086517174963,+4986517174964,2016-07-04T18:58:55Z,2016-07-04T18:58:55Z,15s,-1.00000 -4986517174963_004986517174964_04.07.2016 18:58:55,1467658735,*rated,086517174963,+4986517174964,2016-07-04T18:58:55Z,2016-07-04T18:58:55Z,1m5s,-1.00000 -` + eCacheDumpFile1 = "4986517174963_004986517174964_04.07.2016 18:58:55,1467658735,*rated,086517174963,+4986517174964,04.07.2016 18:58:55,04.07.2016 18:58:55,65s\n" ) func TestPartReadFile(t *testing.T) { @@ -188,7 +186,7 @@ func testPartITVerifyFiles(t *testing.T) { } var fileName string for _, file := range filesInDir { // First file in directory is the one we need, harder to find it's name out of config - if strings.HasPrefix(file.Name(), "4986517174963_004986517174964") { + if strings.HasPrefix(file.Name(), "72533c7f80dde4cf7eb625eda75c93857995f8a8") { fileName = file.Name() break } @@ -224,710 +222,3 @@ func testPartITKillEngine(t *testing.T) { t.Error(err) } } - -/* -func TestNewPartialCSVFileER(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltr := &engine.FilterS{} - result, err := NewPartialCSVFileER(cfg, 0, nil, nil, fltr, nil) - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } - expected := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: result.(*PartialCSVFileER).cache, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: result.(*PartialCSVFileER).conReqs, - } - if !reflect.DeepEqual(result, expected) { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", expected, result) - } -} - -func TestNewPartialCSVFileERCase2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].SourcePath = "/" - fltr := &engine.FilterS{} - result, err := NewPartialCSVFileER(cfg, 0, nil, nil, fltr, nil) - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } - expected := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: result.(*PartialCSVFileER).cache, - rdrDir: "", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: result.(*PartialCSVFileER).conReqs, - } - - if !reflect.DeepEqual(result, expected) { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", expected, result) - } -} - -func TestNewPartialCSVFileERCase3(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].Opts[utils.PartialCSVCacheExpiryActionOpt] = utils.MetaDumpToFile - fltr := &engine.FilterS{} - result, err := NewPartialCSVFileER(cfg, 0, nil, nil, fltr, nil) - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } - expected := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: result.(*PartialCSVFileER).cache, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: result.(*PartialCSVFileER).conReqs, - } - - if !reflect.DeepEqual(result, expected) { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", expected, result) - } -} - -func TestPartialCSVConfig(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers = []*config.EventReaderCfg{ - { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - }, - } - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: nil, - } - expected := &config.EventReaderCfg{ - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - } - result := testStruct.Config() - if !reflect.DeepEqual(result, expected) { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", expected, result) - } -} - -func TestPartialCSVServe1(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers = []*config.EventReaderCfg{ - { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 0, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - }, - } - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: nil, - } - result := testStruct.Serve() - if result != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, result) - } -} - -func TestPartialCSVServe3(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers = []*config.EventReaderCfg{ - { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - }, - } - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: nil, - } - - err := testStruct.Serve() - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } -} - -func TestPartialCSVServe4(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers = []*config.EventReaderCfg{ - { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: 1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - }, - } - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: make(chan struct{}, 1), - conReqs: nil, - } - testStruct.rdrExit <- struct{}{} - err := testStruct.Serve() - if err != nil { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", nil, err) - } -} - -func TestPartialCSVProcessFile(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: make(chan struct{}, 1), - conReqs: nil, - } - err := testStruct.processFile("", "") - if err == nil || err.Error() != "open : no such file or directory" { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "open : no such file or directory", err) - } -} - -func TestPartialCSVProcessFile2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: make(chan struct{}, 1), - conReqs: make(chan struct{}, 1), - } - testStruct.conReqs <- struct{}{} - err := testStruct.processFile("", "") - if err == nil || err.Error() != "open : no such file or directory" { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "open : no such file or directory", err) - } -} - -func TestPartialCSVServe2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers = []*config.EventReaderCfg{ - { - ID: utils.MetaDefault, - Type: utils.MetaNone, - RunDelay: -1, - ConcurrentReqs: 1024, - SourcePath: "/var/spool/cgrates/ers/in", - ProcessedPath: "/var/spool/cgrates/ers/out", - Tenant: nil, - Timezone: utils.EmptyString, - Filters: []string{}, - Flags: utils.FlagsWithParams{}, - Fields: nil, - CacheDumpFields: nil, - Opts: make(map[string]interface{}), - }, - } - fltr := &engine.FilterS{} - testStruct := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltr, - cache: nil, - rdrDir: "/var/spool/cgrates/ers/in", - rdrEvents: nil, - rdrError: nil, - rdrExit: nil, - conReqs: nil, - } - - err := testStruct.Serve() - if err == nil || err.Error() != "no such file or directory" { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "no such file or directory", err) - } -} - -func TestPartialCSVServe5(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - filePath := "/tmp/partErs1/out" - err := os.MkdirAll(filePath, 0777) - if err != nil { - t.Error(err) - } - for i := 1; i < 4; i++ { - if _, err := os.Create(path.Join(filePath, fmt.Sprintf("file%d.csv", i))); err != nil { - t.Error(err) - } - } - eR.Config().RunDelay = 1 * time.Millisecond - if err := eR.Serve(); err != nil { - t.Error(err) - } - os.Create(path.Join(filePath, "file1.txt")) - eR.Config().RunDelay = 1 * time.Millisecond - if err := eR.Serve(); err != nil { - t.Error(err) - } -} - -func TestPartialCSVProcessEvent(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestPartialCSVProcessEvent/" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, "file1.csv")) - if err != nil { - t.Error(err) - } - file.Write([]byte(",a,ToR,b,c,d,e,f,g,h,i,j,k,l")) - file.Close() - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - fname := "file1.csv" - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } - value := []*utils.CGREvent{ - { - Tenant: "cgrates.org", - Event: map[string]interface{}{ - "Partial": true, - }, - }, - { - Tenant: "cgrates2.org", - Event: map[string]interface{}{ - "Partial": true, - }, - }, - } - eR.Config().ProcessedPath = "/tmp" - eR.dumpToFile("ID1", value) -} - -func TestPartialCSVProcessEventPrefix(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - cfg.ERsCfg().Readers[0].Opts[utils.HeaderDefineCharOpt] = ":" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestPartialCSVProcessEvent/" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, "file1.csv")) - if err != nil { - t.Error(err) - } - file.Write([]byte(`:Test,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m - :Test2,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m`)) - file.Close() - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/ers/out/", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - fname := "file1.csv" - if err := eR.processFile(filePath, fname); err != nil { - t.Error(err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestPartialCSVProcessEventError1(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := &engine.FilterS{} - filePath := "/tmp/TestPartialCSVProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, fname)) - if err != nil { - t.Error(err) - } - file.Write([]byte(`#ToR,OriginID,RequestType,Tenant,Category,Account,Subject,Destination,SetupTime,AnswerTime,Usage - ,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m`)) - file.Close() - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/ers/out/", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - - eR.Config().Fields = []*config.FCTemplate{ - {}, - } - - errExpect := "unsupported type: <>" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestPartialCSVProcessEventError2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - cfg.ERsCfg().Readers[0].Fields = []*config.FCTemplate{} - data := engine.NewInternalDB(nil, nil, true) - dm := engine.NewDataManager(data, cfg.CacheCfg(), nil) - cfg.ERsCfg().Readers[0].ProcessedPath = "" - fltrs := engine.NewFilterS(cfg, nil, dm) - filePath := "/tmp/TestPartialCSVProcessEvent/" - fname := "file1.csv" - if err := os.MkdirAll(filePath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(filePath, "file1.csv")) - if err != nil { - t.Error(err) - } - file.Write([]byte(`#ToR,OriginID,RequestType,Tenant,Category,Account,Subject,Destination,SetupTime,AnswerTime,Usage - ,,*voice,OriginCDR1,*prepaid,,cgrates.org,*call,1001,SUBJECT_TEST_1001,1002,2021-01-07 17:00:02 +0000 UTC,2021-01-07 17:00:04 +0000 UTC,1h2m`)) - file.Close() - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/ers/out/", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - - // - eR.Config().Filters = []string{"Filter1"} - errExpect := "NOT_FOUND:Filter1" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - - // - eR.Config().Filters = []string{"*exists:~*req..Account:"} - errExpect = "Invalid fieldPath [ Account]" - if err := eR.processFile(filePath, fname); err == nil || err.Error() != errExpect { - t.Errorf("Expected %v but received %v", errExpect, err) - } - if err := os.RemoveAll(filePath); err != nil { - t.Error(err) - } -} - -func TestPartialCSVDumpToFileErr1(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - var err error - utils.Logger, err = utils.Newlogger(utils.MetaStdLog, utils.EmptyString) - if err != nil { - t.Error(err) - } - utils.Logger.SetLogLevel(7) - buf := new(bytes.Buffer) - log.SetOutput(buf) - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - value := []*utils.CGREvent{ - { - Event: map[string]interface{}{ - "Partial": true, - }, - }, - } - //ProcessedPath is not declared in order to trigger the - //file creation error - eR.dumpToFile("ID1", value) - errExpect := "[ERROR] Failed creating /var/spool/cgrates/ers/out/.tmp." - if rcv := buf.String(); !strings.Contains(rcv, errExpect) { - t.Errorf("\nExpected %v but \nreceived %v", errExpect, rcv) - } - value = []*utils.CGREvent{ - { - Event: map[string]interface{}{ - //Value is false in order to stop - //further execution - "Partial": false, - }, - }, - } - eR.dumpToFile("ID1", value) - eR.postCDR("ID1", value) -} - -func TestPartialCSVDumpToFileErr2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - var err error - utils.Logger, err = utils.Newlogger(utils.MetaStdLog, utils.EmptyString) - if err != nil { - t.Error(err) - } - utils.Logger.SetLogLevel(7) - buf := new(bytes.Buffer) - log.SetOutput(buf) - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - value := []*utils.CGREvent{ - { - Event: map[string]interface{}{ - //Value of field is string in order to trigger - //the converting error - "Partial": "notBool", - }, - }, - } - eR.dumpToFile("ID1", value) - errExpect := `[WARNING] Converting Event : <{"Partial":"notBool"}> to cdr , ignoring due to error: ` - if rcv := buf.String(); !strings.Contains(rcv, errExpect) { - t.Errorf("\nExpected %v but \nreceived %v", errExpect, rcv) - } -} - -func TestPartialCSVDumpToFileErr3(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - var err error - utils.Logger, err = utils.Newlogger(utils.MetaStdLog, utils.EmptyString) - if err != nil { - t.Error(err) - } - utils.Logger.SetLogLevel(7) - buf := new(bytes.Buffer) - log.SetOutput(buf) - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - value := []*utils.CGREvent{ - { - Tenant: "cgrates.org", - Event: map[string]interface{}{ - "Partial": true, - }, - }, - //Added a second event in order to pass the length check - { - Tenant: "cgrates2.org", - Event: map[string]interface{}{ - "Partial": "notBool", - }, - }, - } - eR.Config().ProcessedPath = "/tmp" - eR.dumpToFile("ID1", value) - errExpect := `[WARNING] Converting Event : <{"Partial":"notBool"}> to cdr , ignoring due to error: ` - if rcv := buf.String(); !strings.Contains(rcv, errExpect) { - t.Errorf("\nExpected %v but \nreceived %v", errExpect, rcv) - } -} - -func TestPartialCSVPostCDR(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - fltrs := &engine.FilterS{} - eR := &PartialCSVFileER{ - cgrCfg: cfg, - cfgIdx: 0, - fltrS: fltrs, - rdrDir: "/tmp/partErs1/out", - rdrEvents: make(chan *erEvent, 1), - rdrError: make(chan error, 1), - rdrExit: make(chan struct{}), - conReqs: make(chan struct{}, 1), - } - eR.conReqs <- struct{}{} - - value := []*utils.CGREvent{ - { - Tenant: "cgrates.org", - Event: map[string]interface{}{ - "Partial": true, - }, - APIOpts: map[string]interface{}{ - "Opt1": "testOpt", - }, - }, - } - expEvent := &utils.CGREvent{ - Tenant: "cgrates.org", - Event: value[0].Event, - APIOpts: value[0].APIOpts, - } - eR.postCDR("ID1", nil) - eR.postCDR("ID1", value) - select { - case data := <-eR.rdrEvents: - expEvent.ID = data.cgrEvent.ID - expEvent.Time = data.cgrEvent.Time - if !reflect.DeepEqual(expEvent, data.cgrEvent) { - t.Errorf("\nExpected %v but \nreceived %v", expEvent, data.cgrEvent) - } - case <-time.After(50 * time.Millisecond): - t.Error("Time limit exceeded") - } -} -*/ diff --git a/ers/s3.go b/ers/s3.go index 5ba01694d..95da213c1 100644 --- a/ers/s3.go +++ b/ers/s3.go @@ -139,7 +139,7 @@ func (rdr *S3ER) processMessage(body []byte) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/sql.go b/ers/sql.go index a2b5f1e2b..9aa1c0916 100644 --- a/ers/sql.go +++ b/ers/sql.go @@ -255,7 +255,7 @@ func (rdr *SQLEventReader) processMessage(msg map[string]interface{}) (err error } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/ers/sqs.go b/ers/sqs.go index c964a7520..506eea522 100644 --- a/ers/sqs.go +++ b/ers/sqs.go @@ -127,7 +127,7 @@ func (rdr *SQSER) processMessage(body []byte) (err error) { } cgrEv := utils.NMAsCGREvent(agReq.CGRRequest, agReq.Tenant, utils.NestingSep, agReq.Opts) rdrEv := rdr.rdrEvents - if _, isPartial := cgrEv.APIOpts[partialOpt]; isPartial { + if _, isPartial := cgrEv.APIOpts[utils.PartialOpt]; isPartial { rdrEv = rdr.partialEvents } rdrEv <- &erEvent{ diff --git a/packages/debian/changelog b/packages/debian/changelog index 56e11b276..9189e8237 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -158,6 +158,8 @@ cgrates (0.11.0~dev) UNRELEASED; urgency=medium * [DataDB] Updated config options * [StorDB] Updated config options * [ERs] Refactored the reader options + * [ERs] Removed *flatstore and *partialcsv + * [ERs] Added *opts.*partial to control if the event is partial or not -- DanB Wed, 19 Feb 2020 13:25:52 +0200 diff --git a/utils/consts.go b/utils/consts.go index d84c82fca..ff01b3ade 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -2349,6 +2349,9 @@ const ( FieldsCfg = "fields" CacheDumpFieldsCfg = "cache_dump_fields" PartialCommitFieldsCfg = "partial_commit_fields" + PartialCacheTTLCfg = "partial_cache_ttl" + PartialCacheActionCfg = "partial_cache_action" + PartialPathCfg = "partial_path" ) // RegistrarCCfg @@ -2551,32 +2554,11 @@ const ( SQLTableNameOpt = "sqlTableName" // fileCSV - RowLengthOpt = "RowLength" - FieldSepOpt = "FieldSeparator" - LazyQuotes = "LazyQuotes" + CSVRowLengthOpt = "csvRowLength" + CSVFieldSepOpt = "csvFieldSeparator" + CSVLazyQuotes = "csvLazyQuotes" HeaderDefineCharOpt = "csvHeaderDefineChar" - // partialCSV - PartialCSVCacheExpiryActionOpt = "csvCacheExpiryAction" - PartialCSVRecordCacheOpt = "csvRecordCacheTTL" - - // flatStore - FlatstorePrfx = "fst" - OptsMethod = "*method" - FstInvite = "INVITE" - FstBye = "BYE" - FstAck = "ACK" - - MetaInvite = "*invite" - MetaBye = "*bye" - MetaAck = "*ack" - - FstFailedCallsPrefixOpt = "fstFailedCallsPrefix" - FstPartialRecordCacheOpt = "fstRecordCacheTTL" - FstMethodOpt = "fstMethod" - FstOriginIDOpt = "fstOriginID" - FstMadatoryACKOpt = "fstMadatoryACK" - // fileXML XMLRootPathOpt = "xmlRootPath" @@ -2600,19 +2582,12 @@ const ( KafkaMaxWait = "kafkaMaxWait" // partial - PartialOrderFieldOpt = "partialOrderField" - PartialCacheAction = "partialCacheAction" - PartialPathOpt = "partialPath" - PartialCSVFieldSepartor = "partialcsvFieldSeparator" -) + PartialOpt = "*partial" -var ( - // FstMethodToPrfx used for flatstore to convert the method in DP prefix - FstMethodToPrfx = map[string]string{ - FstInvite: MetaInvite, - FstBye: MetaBye, - FstAck: MetaAck, - } + PartialOrderFieldOpt = "partialOrderField" + PartialCacheActionOpt = "partialCacheAction" + PartialPathOpt = "partialPath" + PartialCSVFieldSepartorOpt = "partialcsvFieldSeparator" ) // Analyzers constants