diff --git a/apis/admins.go b/apis/admins.go index bb16393b1..9f2f37fc3 100644 --- a/apis/admins.go +++ b/apis/admins.go @@ -19,8 +19,15 @@ along with this program. If not, see package apis import ( + "os" + "path" + "strings" + + "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/ees" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) func NewAdminSv1(cfg *config.CGRConfig, dm *engine.DataManager, connMgr *engine.ConnManager, fltrS *engine.FilterS) *AdminSv1 { @@ -39,3 +46,65 @@ type AdminSv1 struct { fltrS *engine.FilterS ping } + +type ArgsReplyFailedPosts struct { + TypeProvider string + FailedRequestsInDir *string // if defined it will be our source of requests to be replayed + FailedRequestsOutDir *string // if defined it will become our destination for files failing to be replayed, *none to be discarded + Modules []string // list of modules for which replay the requests, nil for all +} + +// ReplayFailedPosts will repost failed posts found in the FailedRequestsInDir +func (admS *AdminSv1) ReplayFailedPosts(ctx *context.Context, args *ArgsReplyFailedPosts, reply *string) error { + failedPostsDir := admS.cfg.LoggerCfg().Opts.FailedPostsDir + if args.FailedRequestsInDir != nil && *args.FailedRequestsInDir != utils.EmptyString { + failedPostsDir = *args.FailedRequestsInDir + } + failedOutDir := failedPostsDir + if args.FailedRequestsOutDir != nil && *args.FailedRequestsOutDir != utils.EmptyString { + failedOutDir = *args.FailedRequestsOutDir + } + // check all the files in the FailedPostsInDirectory + filesInDir, err := os.ReadDir(failedPostsDir) + if err != nil { + return err + } + if len(filesInDir) == 0 { + return utils.ErrNotFound + } + // check every file and check if any of them match the modules + for _, file := range filesInDir { + if len(args.Modules) != 0 { + var allowedModule bool + for _, module := range args.Modules { + if strings.HasPrefix(file.Name(), module) { + allowedModule = true + break + } + } + if !allowedModule { + continue + } + } + filePath := path.Join(failedPostsDir, file.Name()) + var expEv utils.FailoverPoster + if expEv, err = ees.NewFailoverPosterFromFile(filePath, args.TypeProvider); err != nil { + return err + } + // check if the failed out dir path is the same as the same in dir in order to export again in case of failure + failoverPath := utils.MetaNone + if failedOutDir != utils.MetaNone { + failoverPath = path.Join(failedOutDir, file.Name()) + } + + failedPosts, err := expEv.ReplayFailedPosts(admS.cfg.GeneralCfg().PosterAttempts) + if err != nil && failedOutDir != utils.MetaNone { // Got error from HTTPPoster could be that content was not written, we need to write it ourselves + if err = failedPosts.WriteToFile(failoverPath); err != nil { + return utils.NewErrServerError(err) + } + } + + } + *reply = utils.OK + return nil +} diff --git a/config/apis_test.go b/config/apis_test.go index a521b8f59..bc1647ad0 100644 --- a/config/apis_test.go +++ b/config/apis_test.go @@ -218,8 +218,6 @@ func TestConfigLoadFromDB(t *testing.T) { expGeneral := &GeneralCfg{ NodeID: "Test", DefaultCaching: utils.MetaClear, - Logger: utils.MetaSysLog, - LogLevel: 6, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", @@ -1035,8 +1033,6 @@ func TestV1GetConfig(t *testing.T) { cfg := NewDefaultCGRConfig() cfg.cacheDP[GeneralJSON] = &GeneralJsonCfg{ Node_id: utils.StringPointer("randomID"), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(6), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), @@ -1065,8 +1061,6 @@ func TestV1GetConfig(t *testing.T) { var reply map[string]interface{} section := &GeneralJsonCfg{ Node_id: utils.StringPointer("randomID"), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(6), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), diff --git a/config/config_defaults.go b/config/config_defaults.go index 91d4fd00b..3251644e7 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -71,7 +71,7 @@ const CGRATES_CFG_JSON = ` }, "logger": { - "type": "*syslog", // controls the destination of logs <*syslog|*stdout|*kafka> + "type": "*syslog", // controls the destination of logs <*syslog|*stdout|*kafkaLog> "level": 6, // system level precision for floats "opts": { "kafka_conn": "", // the connection trough kafka diff --git a/config/config_json_test.go b/config/config_json_test.go index c1fb5f826..d00b32247 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -32,8 +32,6 @@ import ( func TestDfGeneralJsonCfg(t *testing.T) { eCfg := &GeneralJsonCfg{ Node_id: utils.StringPointer(""), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(utils.LOGLEVEL_INFO), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("*msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), diff --git a/config/config_test.go b/config/config_test.go index d19bfe2da..fcdcc9d69 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -261,12 +261,6 @@ func TestCgrCfgJSONDefaultsGeneral(t *testing.T) { if cgrCfg.GeneralCfg().LockingTimeout != 0 { t.Errorf("Expected: 0, received: %+v", cgrCfg.GeneralCfg().LockingTimeout) } - if cgrCfg.GeneralCfg().Logger != utils.MetaSysLog { - t.Errorf("Expected: %+v, received: %+v", utils.MetaSysLog, cgrCfg.GeneralCfg().Logger) - } - if cgrCfg.GeneralCfg().LogLevel != 6 { - t.Errorf("Expected: 6, received: %+v", cgrCfg.GeneralCfg().LogLevel) - } if cgrCfg.GeneralCfg().DigestSeparator != "," { t.Errorf("Expected: utils.CSVSep , received: %+v", cgrCfg.GeneralCfg().DigestSeparator) } @@ -3783,8 +3777,6 @@ func TestV1GetConfigGeneral(t *testing.T) { }` expected := map[string]interface{}{ utils.NodeIDCfg: "ENGINE1", - utils.LoggerCfg: "*syslog", - utils.LogLevelCfg: 6, utils.RoundingDecimalsCfg: 5, utils.DBDataEncodingCfg: "*msgpack", utils.TpExportPathCfg: "/var/spool/cgrates/tpe", @@ -4984,7 +4976,7 @@ func TestV1GetConfigAsJSONGeneral(t *testing.T) { "node_id": "ENGINE1", } }` - expected := `{"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","decimal_max_scale":0,"decimal_min_scale":0,"decimal_precision":0,"decimal_rounding_mode":"*toNearestEven","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","opts":{"*exporterIDs":[]},"poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"}}` + expected := `{"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","decimal_max_scale":0,"decimal_min_scale":0,"decimal_precision":0,"decimal_rounding_mode":"*toNearestEven","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","opts":{"*exporterIDs":[]},"poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"}}` if cfgCgr, err := NewCGRConfigFromJSONStringWithDefaults(strJSON); err != nil { t.Error(err) } else if err := cfgCgr.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Sections: []string{GeneralJSON}}, &reply); err != nil { @@ -5479,7 +5471,7 @@ func TestV1GetConfigAsJSONAllConfig(t *testing.T) { } }` var reply string - expected := `{"accounts":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"max_iterations":1000,"max_usage":"259200000000000","nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[],"*usage":[]},"prefix_indexed_fields":[],"rates_conns":[],"suffix_indexed_fields":[],"thresholds_conns":[]},"actions":{"accounts_conns":[],"cdrs_conns":[],"dynaprepaid_actionprofile":[],"ees_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[]},"prefix_indexed_fields":[],"stats_conns":[],"suffix_indexed_fields":[],"tenants":[],"thresholds_conns":[]},"admins":{"actions_conns":[],"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false},"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","ees_conns":[],"enabled":false,"index_type":"*scorch","opts":{"*exporterIDs":[]},"ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"max_reconnect_interval":"0s","password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"accounts_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*processRuns":[],"*profileIDs":[],"*profileIgnoreFilters":[],"*profileRuns":[]},"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*action_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*event_charges":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"}},"replication_conns":[]},"cdrs":{"accounts_conns":[],"actions_conns":[],"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":null,"opts":{"*accountS":[],"*attributeS":[],"*chargerS":[],"*eeS":[],"*rateS":[],"*statS":[],"*thresholdS":[]},"rates_conns":[],"session_cost_retries":5,"stats_conns":[],"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"config_db":{"db_host":"","db_name":"","db_password":"","db_port":0,"db_type":"*internal","db_user":"","opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"}},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","ees_conns":[],"shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_profile_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_profile_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*fileCSV":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"blocker":false,"concurrent_requests":0,"export_path":"/var/spool/cgrates/ees","failed_posts_dir":"/var/spool/cgrates/failed_posts","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"timezone":"","type":"*none"}]},"ers":{"enabled":false,"partial_cache_ttl":"1s","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"natsSubject":"cgrates_cdrs","partialCacheAction":"*none","partialOrderField":"~*req.AnswerTime","xmlRootPath":""},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"accounts_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","max_reconnect_interval":"0s","password":"ClueCon","reconnects":5}],"extra_fields":[],"low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","decimal_max_scale":0,"decimal_min_scale":0,"decimal_precision":0,"decimal_rounding_mode":"*toNearestEven","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","opts":{"*exporterIDs":[]},"poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0s","forceAttemptHttp2":true,"idleConnTimeout":"1m30s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0s","skipTLSVerification":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","prometheus_url":"/prometheus","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","max_reconnect_interval":"0s","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"actions_conns":["*localhost"],"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","tpid":""},"loaders":[{"action":"*store","cache":{"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"new_branch":true,"path":"Rules.Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Rules.Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Rules.Values","tag":"Values","type":"*variable","value":"~*req.4"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"new_branch":true,"path":"Attributes.FilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Attributes.Blockers","tag":"AttributeBlockers","type":"*variable","value":"~*req.6"},{"path":"Attributes.Path","tag":"Path","type":"*variable","value":"~*req.7"},{"path":"Attributes.Type","tag":"Type","type":"*variable","value":"~*req.8"},{"path":"Attributes.Value","tag":"Value","type":"*variable","value":"~*req.9"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.9"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.5"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.6"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.9"},{"new_branch":true,"path":"Metrics.MetricID","tag":"MetricIDs","type":"*variable","value":"~*req.10"},{"path":"Metrics.FilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.11"},{"path":"Metrics.Blockers","tag":"MetricBlockers","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"ActionProfileIDs","tag":"ActionProfileIDs","type":"*variable","value":"~*req.8"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.9"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.5"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.6"},{"new_branch":true,"path":"Routes.ID","tag":"RouteID","type":"*variable","value":"~*req.7"},{"path":"Routes.FilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Routes.AccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.9"},{"path":"Routes.RateProfileIDs","tag":"RouteRateProfileIDs","type":"*variable","value":"~*req.10"},{"path":"Routes.ResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.11"},{"path":"Routes.StatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.12"},{"path":"Routes.Weights","tag":"RouteWeights","type":"*variable","value":"~*req.13"},{"path":"Routes.Blockers","tag":"RouteBlockers","type":"*variable","value":"~*req.14"},{"path":"Routes.RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.5"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.3"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.4"},{"path":"StrategyParams","tag":"StrategyParameters","type":"*variable","value":"~*req.5"},{"new_branch":true,"path":"Hosts.ID","tag":"ConnID","type":"*variable","value":"~*req.6"},{"path":"Hosts.FilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.7"},{"path":"Hosts.Weight","tag":"ConnWeight","type":"*variable","value":"~*req.8"},{"path":"Hosts.Blocker","tag":"ConnBlocker","type":"*variable","value":"~*req.9"},{"path":"Hosts.Params","tag":"ConnParameters","type":"*variable","value":"~*req.10"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"ConnectAttempts","tag":"ConnectAttempts","type":"*variable","value":"~*req.4"},{"path":"Reconnects","tag":"Reconnects","type":"*variable","value":"~*req.5"},{"path":"MaxReconnectInterval","tag":"MaxReconnectInterval","type":"*variable","value":"~*req.6"},{"path":"ConnectTimeout","tag":"ConnectTimeout","type":"*variable","value":"~*req.7"},{"path":"ReplyTimeout","tag":"ReplyTimeout","type":"*variable","value":"~*req.8"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.9"},{"path":"ClientKey","tag":"ClientKey","type":"*variable","value":"~*req.10"},{"path":"ClientCertificate","tag":"ClientCertificate","type":"*variable","value":"~*req.11"},{"path":"CaCertificate","tag":"CaCertificate","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"MinCost","tag":"MinCost","type":"*variable","value":"~*req.4"},{"path":"MaxCost","tag":"MaxCost","type":"*variable","value":"~*req.5"},{"path":"MaxCostStrategy","tag":"MaxCostStrategy","type":"*variable","value":"~*req.6"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].FilterIDs","tag":"RateFilterIDs","type":"*variable","value":"~*req.8"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].ActivationTimes","tag":"RateActivationTimes","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].Weights","tag":"RateWeights","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].Blocker","tag":"RateBlocker","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.7:"],"new_branch":true,"path":"Rates[\u003c~*req.7\u003e].IntervalRates.IntervalStart","tag":"RateIntervalStart","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.FixedFee","tag":"RateFixedFee","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.RecurrentFee","tag":"RateRecurrentFee","type":"*variable","value":"~*req.14"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.Unit","tag":"RateUnit","type":"*variable","value":"~*req.15"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.Increment","tag":"RateIncrement","type":"*variable","value":"~*req.16"}],"file_name":"Rates.csv","flags":null,"type":"*rate_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Schedule","tag":"Schedule","type":"*variable","value":"~*req.5"},{"path":"Targets[\u003c~*req.6\u003e]","tag":"TargetIDs","type":"*variable","value":"~*req.7"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].FilterIDs","tag":"ActionFilterIDs","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].TTL","tag":"ActionTTL","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Type","tag":"ActionType","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Opts","tag":"ActionOpts","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.8:"],"new_branch":true,"path":"Actions[\u003c~*req.8\u003e].Diktats.Path","tag":"ActionPath","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Diktats.Value","tag":"ActionValue","type":"*variable","value":"~*req.14"}],"file_name":"Actions.csv","flags":null,"type":"*action_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Opts","tag":"Opts","type":"*variable","value":"~*req.5"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].FilterIDs","tag":"BalanceFilterIDs","type":"*variable","value":"~*req.7"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Weights","tag":"BalanceWeights","type":"*variable","value":"~*req.8"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Blockers","tag":"BalanceBlockers","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Type","tag":"BalanceType","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Units","tag":"BalanceUnits","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].UnitFactors","tag":"BalanceUnitFactors","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Opts","tag":"BalanceOpts","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].CostIncrements","tag":"BalanceCostIncrements","type":"*variable","value":"~*req.14"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].AttributeIDs","tag":"BalanceAttributeIDs","type":"*variable","value":"~*req.15"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].RateProfileIDs","tag":"BalanceRateProfileIDs","type":"*variable","value":"~*req.16"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.17"}],"file_name":"Accounts.csv","flags":null,"type":"*accounts"}],"enabled":false,"field_separator":",","id":"*default","lockfile_path":".cgr.lck","opts":{"*cache":"","*forceLock":false,"*stopOnError":false,"*withIndex":true},"run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"logger":{"level":6,"opts":{"attempts":1,"failed_posts_dir":"/var/spool/cgrates/failed_posts","kafka_conn":"","kafka_topic":""},"type":"*syslog"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"mongoQueryTimeout":"0s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","users_filters":null},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rates":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*intervalStart":[],"*profileIDs":[],"*profileIgnoreFilters":[],"*startTime":[],"*usage":[]},"prefix_indexed_fields":[],"rate_exists_indexed_fields":[],"rate_indexed_selects":true,"rate_nested_fields":false,"rate_notexists_indexed_fields":[],"rate_prefix_indexed_fields":[],"rate_suffix_indexed_fields":[],"suffix_indexed_fields":[],"verbosity":1000},"registrarc":{"dispatchers":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*units":[],"*usageID":[],"*usageTTL":[]},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"accounts_conns":[],"attributes_conns":[],"default_ratio":1,"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*context":[],"*ignoreErrors":[],"*limit":[],"*maxCost":[],"*maxItems":[],"*offset":[],"*profileCount":[],"*usage":[]},"prefix_indexed_fields":[],"rates_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"sessions":{"accounts_conns":[],"actions_conns":[],"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","opts":{"*accountS":[],"*attributeS":[],"*attributesDerivedReply":[],"*blockerError":[],"*cdrS":[],"*cdrsDerivedReply":[],"*chargeable":[],"*chargerS":[],"*debitInterval":[],"*forceDuration":[],"*initiate":[],"*maxUsage":[],"*message":[],"*resourceS":[],"*resourcesAllocate":[],"*resourcesAuthorize":[],"*resourcesDerivedReply":[],"*resourcesRelease":[],"*routeS":[],"*routesDerivedReply":[],"*statS":[],"*statsDerivedReply":[],"*terminate":[],"*thresholdS":[],"*thresholdsDerivedReply":[],"*ttl":[],"*ttlLastUsage":[],"*ttlLastUsed":[],"*ttlMaxDelay":[],"*ttlUsage":[],"*update":[]},"rates_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"session_indexes":[],"stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":"1s","sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[],"*prometheusStatIDs":[],"*roundingDecimals":[]},"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*opts.*originID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"actions_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[]},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4},"tpes":{"enabled":false}}` + expected := `{"accounts":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"max_iterations":1000,"max_usage":"259200000000000","nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[],"*usage":[]},"prefix_indexed_fields":[],"rates_conns":[],"suffix_indexed_fields":[],"thresholds_conns":[]},"actions":{"accounts_conns":[],"cdrs_conns":[],"dynaprepaid_actionprofile":[],"ees_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[]},"prefix_indexed_fields":[],"stats_conns":[],"suffix_indexed_fields":[],"tenants":[],"thresholds_conns":[]},"admins":{"actions_conns":[],"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false},"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","ees_conns":[],"enabled":false,"index_type":"*scorch","opts":{"*exporterIDs":[]},"ttl":"24h0m0s"},"apiban":{"enabled":false,"keys":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","connect_attempts":3,"max_reconnect_interval":"0s","password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"sessions_conns":["*birpc_internal"]},"attributes":{"accounts_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*processRuns":[],"*profileIDs":[],"*profileIgnoreFilters":[],"*profileRuns":[]},"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*action_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*apiban":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*caps_events":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*cdr_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*charger_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*closed_sessions":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*diameter_messages":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_loads":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatcher_routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*event_charges":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_profile_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*replication_hosts":{"limit":0,"precache":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rpc_connections":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*rpc_responses":{"limit":0,"precache":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*stat_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*stir":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false},"*uch":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"}},"replication_conns":[]},"cdrs":{"accounts_conns":[],"actions_conns":[],"attributes_conns":[],"chargers_conns":[],"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":null,"opts":{"*accountS":[],"*attributeS":[],"*chargerS":[],"*eeS":[],"*rateS":[],"*statS":[],"*thresholdS":[]},"rates_conns":[],"session_cost_retries":5,"stats_conns":[],"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"config_db":{"db_host":"","db_name":"","db_password":"","db_port":0,"db_type":"*internal","db_user":"","opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"}},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","ees_conns":[],"shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_profile_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_profile_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rate_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_filtered":false},"diameter_agent":{"asr_template":"","concurrent_requests":-1,"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listen":"127.0.0.1:3868","listen_net":"tcp","origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"synced_conn_requests":false,"vendor_id":0},"dispatchers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listen":"127.0.0.1:2053","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*fileCSV":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"blocker":false,"concurrent_requests":0,"export_path":"/var/spool/cgrates/ees","failed_posts_dir":"/var/spool/cgrates/failed_posts","fields":[],"filters":[],"flags":[],"id":"*default","opts":{},"synchronous":false,"timezone":"","type":"*none"}]},"ers":{"enabled":false,"partial_cache_ttl":"1s","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"natsSubject":"cgrates_cdrs","partialCacheAction":"*none","partialOrderField":"~*req.AnswerTime","xmlRootPath":""},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","run_delay":"0","source_path":"/var/spool/cgrates/ers/in","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"]},"filters":{"accounts_conns":[],"resources_conns":[],"stats_conns":[]},"freeswitch_agent":{"create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","max_reconnect_interval":"0s","password":"ClueCon","reconnects":5}],"extra_fields":[],"low_balance_ann_file":"","max_wait_connection":"2s","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","decimal_max_scale":0,"decimal_min_scale":0,"decimal_precision":0,"decimal_rounding_mode":"*toNearestEven","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","failed_posts_dir":"/var/spool/cgrates/failed_posts","failed_posts_ttl":"5s","locking_timeout":"0","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","opts":{"*exporterIDs":[]},"poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0s","forceAttemptHttp2":true,"idleConnTimeout":"1m30s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0s","skipTLSVerification":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","prometheus_url":"/prometheus","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","max_reconnect_interval":"0s","reconnects":5}],"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"actions_conns":["*localhost"],"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","tpid":""},"loaders":[{"action":"*store","cache":{"*accounts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*action_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*attributes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*chargers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*dispatcher_hosts":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*dispatchers":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*filters":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*rate_profiles":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*resources":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*routes":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*stats":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*thresholds":{"limit":-1,"precache":false,"replicate":false,"static_ttl":false,"ttl":"5s"}},"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"new_branch":true,"path":"Rules.Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Rules.Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Rules.Values","tag":"Values","type":"*variable","value":"~*req.4"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"new_branch":true,"path":"Attributes.FilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Attributes.Blockers","tag":"AttributeBlockers","type":"*variable","value":"~*req.6"},{"path":"Attributes.Path","tag":"Path","type":"*variable","value":"~*req.7"},{"path":"Attributes.Type","tag":"Type","type":"*variable","value":"~*req.8"},{"path":"Attributes.Value","tag":"Value","type":"*variable","value":"~*req.9"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.9"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.5"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.6"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.9"},{"new_branch":true,"path":"Metrics.MetricID","tag":"MetricIDs","type":"*variable","value":"~*req.10"},{"path":"Metrics.FilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.11"},{"path":"Metrics.Blockers","tag":"MetricBlockers","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"ActionProfileIDs","tag":"ActionProfileIDs","type":"*variable","value":"~*req.8"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.9"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.5"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.6"},{"new_branch":true,"path":"Routes.ID","tag":"RouteID","type":"*variable","value":"~*req.7"},{"path":"Routes.FilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Routes.AccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.9"},{"path":"Routes.RateProfileIDs","tag":"RouteRateProfileIDs","type":"*variable","value":"~*req.10"},{"path":"Routes.ResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.11"},{"path":"Routes.StatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.12"},{"path":"Routes.Weights","tag":"RouteWeights","type":"*variable","value":"~*req.13"},{"path":"Routes.Blockers","tag":"RouteBlockers","type":"*variable","value":"~*req.14"},{"path":"Routes.RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.5"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.3"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.4"},{"path":"StrategyParams","tag":"StrategyParameters","type":"*variable","value":"~*req.5"},{"new_branch":true,"path":"Hosts.ID","tag":"ConnID","type":"*variable","value":"~*req.6"},{"path":"Hosts.FilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.7"},{"path":"Hosts.Weight","tag":"ConnWeight","type":"*variable","value":"~*req.8"},{"path":"Hosts.Blocker","tag":"ConnBlocker","type":"*variable","value":"~*req.9"},{"path":"Hosts.Params","tag":"ConnParameters","type":"*variable","value":"~*req.10"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"ConnectAttempts","tag":"ConnectAttempts","type":"*variable","value":"~*req.4"},{"path":"Reconnects","tag":"Reconnects","type":"*variable","value":"~*req.5"},{"path":"MaxReconnectInterval","tag":"MaxReconnectInterval","type":"*variable","value":"~*req.6"},{"path":"ConnectTimeout","tag":"ConnectTimeout","type":"*variable","value":"~*req.7"},{"path":"ReplyTimeout","tag":"ReplyTimeout","type":"*variable","value":"~*req.8"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.9"},{"path":"ClientKey","tag":"ClientKey","type":"*variable","value":"~*req.10"},{"path":"ClientCertificate","tag":"ClientCertificate","type":"*variable","value":"~*req.11"},{"path":"CaCertificate","tag":"CaCertificate","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"MinCost","tag":"MinCost","type":"*variable","value":"~*req.4"},{"path":"MaxCost","tag":"MaxCost","type":"*variable","value":"~*req.5"},{"path":"MaxCostStrategy","tag":"MaxCostStrategy","type":"*variable","value":"~*req.6"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].FilterIDs","tag":"RateFilterIDs","type":"*variable","value":"~*req.8"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].ActivationTimes","tag":"RateActivationTimes","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].Weights","tag":"RateWeights","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].Blocker","tag":"RateBlocker","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.7:"],"new_branch":true,"path":"Rates[\u003c~*req.7\u003e].IntervalRates.IntervalStart","tag":"RateIntervalStart","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.FixedFee","tag":"RateFixedFee","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.RecurrentFee","tag":"RateRecurrentFee","type":"*variable","value":"~*req.14"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.Unit","tag":"RateUnit","type":"*variable","value":"~*req.15"},{"filters":["*notempty:~*req.7:"],"path":"Rates[\u003c~*req.7\u003e].IntervalRates.Increment","tag":"RateIncrement","type":"*variable","value":"~*req.16"}],"file_name":"Rates.csv","flags":null,"type":"*rate_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Schedule","tag":"Schedule","type":"*variable","value":"~*req.5"},{"path":"Targets[\u003c~*req.6\u003e]","tag":"TargetIDs","type":"*variable","value":"~*req.7"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].FilterIDs","tag":"ActionFilterIDs","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].TTL","tag":"ActionTTL","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Type","tag":"ActionType","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Opts","tag":"ActionOpts","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.8:"],"new_branch":true,"path":"Actions[\u003c~*req.8\u003e].Diktats.Path","tag":"ActionPath","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.8:"],"path":"Actions[\u003c~*req.8\u003e].Diktats.Value","tag":"ActionValue","type":"*variable","value":"~*req.14"}],"file_name":"Actions.csv","flags":null,"type":"*action_profiles"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"Weights","tag":"Weights","type":"*variable","value":"~*req.3"},{"path":"Blockers","tag":"Blockers","type":"*variable","value":"~*req.4"},{"path":"Opts","tag":"Opts","type":"*variable","value":"~*req.5"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].FilterIDs","tag":"BalanceFilterIDs","type":"*variable","value":"~*req.7"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Weights","tag":"BalanceWeights","type":"*variable","value":"~*req.8"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Blockers","tag":"BalanceBlockers","type":"*variable","value":"~*req.9"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Type","tag":"BalanceType","type":"*variable","value":"~*req.10"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Units","tag":"BalanceUnits","type":"*variable","value":"~*req.11"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].UnitFactors","tag":"BalanceUnitFactors","type":"*variable","value":"~*req.12"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].Opts","tag":"BalanceOpts","type":"*variable","value":"~*req.13"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].CostIncrements","tag":"BalanceCostIncrements","type":"*variable","value":"~*req.14"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].AttributeIDs","tag":"BalanceAttributeIDs","type":"*variable","value":"~*req.15"},{"filters":["*notempty:~*req.6:"],"path":"Balances[\u003c~*req.6\u003e].RateProfileIDs","tag":"BalanceRateProfileIDs","type":"*variable","value":"~*req.16"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.17"}],"file_name":"Accounts.csv","flags":null,"type":"*accounts"}],"enabled":false,"field_separator":",","id":"*default","lockfile_path":".cgr.lck","opts":{"*cache":"","*forceLock":false,"*stopOnError":false,"*withIndex":true},"run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"logger":{"level":6,"opts":{"attempts":1,"failed_posts_dir":"/var/spool/cgrates/failed_posts","kafka_conn":"","kafka_topic":""},"type":"*syslog"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"mongoQueryTimeout":"0s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectTimeout":"0s","redisMaxAttempts":20,"redisMaxConns":10,"redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"redis","out_datadb_user":"cgrates","users_filters":null},"radius_agent":{"client_dictionaries":{"*default":"/usr/share/cgrates/radius/dict/"},"client_secrets":{"*default":"CGRateS.org"},"enabled":false,"listen_acct":"127.0.0.1:1813","listen_auth":"127.0.0.1:1812","listen_net":"udp","request_processors":[],"sessions_conns":["*internal"]},"rates":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*intervalStart":[],"*profileIDs":[],"*profileIgnoreFilters":[],"*startTime":[],"*usage":[]},"prefix_indexed_fields":[],"rate_exists_indexed_fields":[],"rate_indexed_selects":true,"rate_nested_fields":false,"rate_notexists_indexed_fields":[],"rate_prefix_indexed_fields":[],"rate_suffix_indexed_fields":[],"suffix_indexed_fields":[],"verbosity":1000},"registrarc":{"dispatchers":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*units":[],"*usageID":[],"*usageTTL":[]},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"accounts_conns":[],"attributes_conns":[],"default_ratio":1,"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*context":[],"*ignoreErrors":[],"*limit":[],"*maxCost":[],"*maxItems":[],"*offset":[],"*profileCount":[],"*usage":[]},"prefix_indexed_fields":[],"rates_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"sessions":{"accounts_conns":[],"actions_conns":[],"alterable_fields":[],"attributes_conns":[],"cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":1,"default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","opts":{"*accountS":[],"*attributeS":[],"*attributesDerivedReply":[],"*blockerError":[],"*cdrS":[],"*cdrsDerivedReply":[],"*chargeable":[],"*chargerS":[],"*debitInterval":[],"*forceDuration":[],"*initiate":[],"*maxUsage":[],"*message":[],"*resourceS":[],"*resourcesAllocate":[],"*resourcesAuthorize":[],"*resourcesDerivedReply":[],"*resourcesRelease":[],"*routeS":[],"*routesDerivedReply":[],"*statS":[],"*statsDerivedReply":[],"*terminate":[],"*thresholdS":[],"*thresholdsDerivedReply":[],"*ttl":[],"*ttlLastUsage":[],"*ttlLastUsed":[],"*ttlMaxDelay":[],"*ttlUsage":[],"*update":[]},"rates_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"session_indexes":[],"stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":"1s","sessions_conns":["*internal"],"timezone":""},"stats":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[],"*prometheusStatIDs":[],"*roundingDecimals":[]},"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*opts.*originID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"actions_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"notexists_indexed_fields":[],"opts":{"*profileIDs":[],"*profileIgnoreFilters":[]},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4},"tpes":{"enabled":false}}` cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSON) if err != nil { t.Fatal(err) diff --git a/config/eescfg.go b/config/eescfg.go index 577ae8200..99479a640 100644 --- a/config/eescfg.go +++ b/config/eescfg.go @@ -680,152 +680,6 @@ func (eeC EventExporterCfg) Clone() (cln *EventExporterCfg) { // AsMapInterface returns the config as a map[string]interface{} func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[string]interface{}) { - opts := map[string]interface{}{} - if eeC.Opts.CSVFieldSeparator != nil { - opts[utils.CSVFieldSepOpt] = *eeC.Opts.CSVFieldSeparator - } - if eeC.Opts.ElsIndex != nil { - opts[utils.ElsIndex] = *eeC.Opts.ElsIndex - } - if eeC.Opts.ElsIfPrimaryTerm != nil { - opts[utils.ElsIfPrimaryTerm] = *eeC.Opts.ElsIfPrimaryTerm - } - if eeC.Opts.ElsIfSeqNo != nil { - opts[utils.ElsIfSeqNo] = *eeC.Opts.ElsIfSeqNo - } - if eeC.Opts.ElsOpType != nil { - opts[utils.ElsOpType] = *eeC.Opts.ElsOpType - } - if eeC.Opts.ElsPipeline != nil { - opts[utils.ElsPipeline] = *eeC.Opts.ElsPipeline - } - if eeC.Opts.ElsRouting != nil { - opts[utils.ElsRouting] = *eeC.Opts.ElsRouting - } - if eeC.Opts.ElsTimeout != nil { - opts[utils.ElsTimeout] = eeC.Opts.ElsTimeout.String() - } - if eeC.Opts.ElsVersion != nil { - opts[utils.ElsVersionLow] = *eeC.Opts.ElsVersion - } - if eeC.Opts.ElsVersionType != nil { - opts[utils.ElsVersionType] = *eeC.Opts.ElsVersionType - } - if eeC.Opts.ElsWaitForActiveShards != nil { - opts[utils.ElsWaitForActiveShards] = *eeC.Opts.ElsWaitForActiveShards - } - if eeC.Opts.SQLMaxIdleConns != nil { - opts[utils.SQLMaxIdleConnsCfg] = *eeC.Opts.SQLMaxIdleConns - } - if eeC.Opts.SQLMaxOpenConns != nil { - opts[utils.SQLMaxOpenConns] = *eeC.Opts.SQLMaxOpenConns - } - if eeC.Opts.SQLConnMaxLifetime != nil { - opts[utils.SQLConnMaxLifetime] = eeC.Opts.SQLConnMaxLifetime.String() - } - if eeC.Opts.MYSQLDSNParams != nil { - opts[utils.MYSQLDSNParams] = eeC.Opts.MYSQLDSNParams - } - if eeC.Opts.SQLTableName != nil { - opts[utils.SQLTableNameOpt] = *eeC.Opts.SQLTableName - } - if eeC.Opts.SQLDBName != nil { - opts[utils.SQLDBNameOpt] = *eeC.Opts.SQLDBName - } - if eeC.Opts.PgSSLMode != nil { - opts[utils.PgSSLModeCfg] = *eeC.Opts.PgSSLMode - } - if eeC.Opts.KafkaTopic != nil { - opts[utils.KafkaTopic] = *eeC.Opts.KafkaTopic - } - if eeC.Opts.AMQPQueueID != nil { - opts[utils.AMQPQueueID] = *eeC.Opts.AMQPQueueID - } - if eeC.Opts.AMQPRoutingKey != nil { - opts[utils.AMQPRoutingKey] = *eeC.Opts.AMQPRoutingKey - } - if eeC.Opts.AMQPExchange != nil { - opts[utils.AMQPExchange] = *eeC.Opts.AMQPExchange - } - if eeC.Opts.AMQPExchangeType != nil { - opts[utils.AMQPExchangeType] = *eeC.Opts.AMQPExchangeType - } - if eeC.Opts.AWSRegion != nil { - opts[utils.AWSRegion] = *eeC.Opts.AWSRegion - } - if eeC.Opts.AWSKey != nil { - opts[utils.AWSKey] = *eeC.Opts.AWSKey - } - if eeC.Opts.AWSSecret != nil { - opts[utils.AWSSecret] = *eeC.Opts.AWSSecret - } - if eeC.Opts.AWSToken != nil { - opts[utils.AWSToken] = *eeC.Opts.AWSToken - } - if eeC.Opts.SQSQueueID != nil { - opts[utils.SQSQueueID] = *eeC.Opts.SQSQueueID - } - if eeC.Opts.S3BucketID != nil { - opts[utils.S3Bucket] = *eeC.Opts.S3BucketID - } - if eeC.Opts.S3FolderPath != nil { - opts[utils.S3FolderPath] = *eeC.Opts.S3FolderPath - } - if eeC.Opts.NATSJetStream != nil { - opts[utils.NatsJetStream] = *eeC.Opts.NATSJetStream - } - if eeC.Opts.NATSSubject != nil { - opts[utils.NatsSubject] = *eeC.Opts.NATSSubject - } - if eeC.Opts.NATSJWTFile != nil { - opts[utils.NatsJWTFile] = *eeC.Opts.NATSJWTFile - } - if eeC.Opts.NATSSeedFile != nil { - opts[utils.NatsSeedFile] = *eeC.Opts.NATSSeedFile - } - if eeC.Opts.NATSCertificateAuthority != nil { - opts[utils.NatsCertificateAuthority] = *eeC.Opts.NATSCertificateAuthority - } - if eeC.Opts.NATSClientCertificate != nil { - opts[utils.NatsClientCertificate] = *eeC.Opts.NATSClientCertificate - } - if eeC.Opts.NATSClientKey != nil { - opts[utils.NatsClientKey] = *eeC.Opts.NATSClientKey - } - if eeC.Opts.NATSJetStreamMaxWait != nil { - opts[utils.NatsJetStreamMaxWait] = eeC.Opts.NATSJetStreamMaxWait.String() - } - if eeC.Opts.RPCCodec != nil { - opts[utils.RpcCodec] = *eeC.Opts.RPCCodec - } - if eeC.Opts.ServiceMethod != nil { - opts[utils.ServiceMethod] = *eeC.Opts.ServiceMethod - } - if eeC.Opts.KeyPath != nil { - opts[utils.KeyPath] = *eeC.Opts.KeyPath - } - if eeC.Opts.CertPath != nil { - opts[utils.CertPath] = *eeC.Opts.CertPath - } - if eeC.Opts.CAPath != nil { - opts[utils.CaPath] = *eeC.Opts.CAPath - } - if eeC.Opts.TLS != nil { - opts[utils.Tls] = *eeC.Opts.TLS - } - if eeC.Opts.ConnIDs != nil { - opts[utils.ConnIDs] = *eeC.Opts.ConnIDs - } - if eeC.Opts.RPCConnTimeout != nil { - opts[utils.RpcConnTimeout] = eeC.Opts.RPCConnTimeout.String() - } - if eeC.Opts.RPCReplyTimeout != nil { - opts[utils.RpcReplyTimeout] = eeC.Opts.RPCReplyTimeout.String() - } - if eeC.Opts.RPCAPIOpts != nil { - opts[utils.RPCAPIOpts] = eeC.Opts.RPCAPIOpts - } - flgs := eeC.Flags.SliceFlags() if flgs == nil { flgs = []string{} @@ -844,7 +698,7 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str utils.AttemptsCfg: eeC.Attempts, utils.ConcurrentRequestsCfg: eeC.ConcurrentRequests, utils.FailedPostsDirCfg: eeC.FailedPostsDir, - utils.OptsCfg: opts, + utils.OptsCfg: eeC.Opts.AsMapInterface(), } if eeC.Fields != nil { fields := make([]map[string]interface{}, 0, len(eeC.Fields)) @@ -856,6 +710,152 @@ func (eeC *EventExporterCfg) AsMapInterface(separator string) (initialMP map[str return } +func (optsEes *EventExporterOpts) AsMapInterface() map[string]interface{} { + opts := map[string]interface{}{} + if optsEes.CSVFieldSeparator != nil { + opts[utils.CSVFieldSepOpt] = *optsEes.CSVFieldSeparator + } + if optsEes.ElsIndex != nil { + opts[utils.ElsIndex] = *optsEes.ElsIndex + } + if optsEes.ElsIfPrimaryTerm != nil { + opts[utils.ElsIfPrimaryTerm] = *optsEes.ElsIfPrimaryTerm + } + if optsEes.ElsIfSeqNo != nil { + opts[utils.ElsIfSeqNo] = *optsEes.ElsIfSeqNo + } + if optsEes.ElsOpType != nil { + opts[utils.ElsOpType] = *optsEes.ElsOpType + } + if optsEes.ElsPipeline != nil { + opts[utils.ElsPipeline] = *optsEes.ElsPipeline + } + if optsEes.ElsRouting != nil { + opts[utils.ElsRouting] = *optsEes.ElsRouting + } + if optsEes.ElsTimeout != nil { + opts[utils.ElsTimeout] = optsEes.ElsTimeout.String() + } + if optsEes.ElsVersion != nil { + opts[utils.ElsVersionLow] = *optsEes.ElsVersion + } + if optsEes.ElsVersionType != nil { + opts[utils.ElsVersionType] = *optsEes.ElsVersionType + } + if optsEes.ElsWaitForActiveShards != nil { + opts[utils.ElsWaitForActiveShards] = *optsEes.ElsWaitForActiveShards + } + if optsEes.SQLMaxIdleConns != nil { + opts[utils.SQLMaxIdleConnsCfg] = *optsEes.SQLMaxIdleConns + } + if optsEes.SQLMaxOpenConns != nil { + opts[utils.SQLMaxOpenConns] = *optsEes.SQLMaxOpenConns + } + if optsEes.SQLConnMaxLifetime != nil { + opts[utils.SQLConnMaxLifetime] = optsEes.SQLConnMaxLifetime.String() + } + if optsEes.PgSSLMode != nil { + opts[utils.PgSSLModeCfg] = *optsEes.PgSSLMode + } + if optsEes.SQLTableName != nil { + opts[utils.SQLTableNameOpt] = *optsEes.SQLTableName + } + if optsEes.SQLDBName != nil { + opts[utils.SQLDBNameOpt] = *optsEes.SQLDBName + } + if optsEes.KafkaTopic != nil { + opts[utils.KafkaTopic] = *optsEes.KafkaTopic + } + if optsEes.AMQPQueueID != nil { + opts[utils.AMQPQueueID] = *optsEes.AMQPQueueID + } + if optsEes.AMQPRoutingKey != nil { + opts[utils.AMQPRoutingKey] = *optsEes.AMQPRoutingKey + } + if optsEes.AMQPExchange != nil { + opts[utils.AMQPExchange] = *optsEes.AMQPExchange + } + if optsEes.AMQPExchangeType != nil { + opts[utils.AMQPExchangeType] = *optsEes.AMQPExchangeType + } + if optsEes.AWSRegion != nil { + opts[utils.AWSRegion] = *optsEes.AWSRegion + } + if optsEes.AWSKey != nil { + opts[utils.AWSKey] = *optsEes.AWSKey + } + if optsEes.AWSSecret != nil { + opts[utils.AWSSecret] = *optsEes.AWSSecret + } + if optsEes.AWSToken != nil { + opts[utils.AWSToken] = *optsEes.AWSToken + } + if optsEes.SQSQueueID != nil { + opts[utils.SQSQueueID] = *optsEes.SQSQueueID + } + if optsEes.S3BucketID != nil { + opts[utils.S3Bucket] = *optsEes.S3BucketID + } + if optsEes.S3FolderPath != nil { + opts[utils.S3FolderPath] = *optsEes.S3FolderPath + } + if optsEes.NATSJetStream != nil { + opts[utils.NatsJetStream] = *optsEes.NATSJetStream + } + if optsEes.NATSSubject != nil { + opts[utils.NatsSubject] = *optsEes.NATSSubject + } + if optsEes.NATSJWTFile != nil { + opts[utils.NatsJWTFile] = *optsEes.NATSJWTFile + } + if optsEes.NATSSeedFile != nil { + opts[utils.NatsSeedFile] = *optsEes.NATSSeedFile + } + if optsEes.NATSCertificateAuthority != nil { + opts[utils.NatsCertificateAuthority] = *optsEes.NATSCertificateAuthority + } + if optsEes.NATSClientCertificate != nil { + opts[utils.NatsClientCertificate] = *optsEes.NATSClientCertificate + } + if optsEes.NATSClientKey != nil { + opts[utils.NatsClientKey] = *optsEes.NATSClientKey + } + if optsEes.NATSJetStreamMaxWait != nil { + opts[utils.NatsJetStreamMaxWait] = optsEes.NATSJetStreamMaxWait.String() + } + if optsEes.RPCCodec != nil { + opts[utils.RpcCodec] = *optsEes.RPCCodec + } + if optsEes.ServiceMethod != nil { + opts[utils.ServiceMethod] = *optsEes.ServiceMethod + } + if optsEes.KeyPath != nil { + opts[utils.KeyPath] = *optsEes.KeyPath + } + if optsEes.CertPath != nil { + opts[utils.CertPath] = *optsEes.CertPath + } + if optsEes.CAPath != nil { + opts[utils.CaPath] = *optsEes.CAPath + } + if optsEes.TLS != nil { + opts[utils.Tls] = *optsEes.TLS + } + if optsEes.ConnIDs != nil { + opts[utils.ConnIDs] = *optsEes.ConnIDs + } + if optsEes.RPCConnTimeout != nil { + opts[utils.RpcConnTimeout] = optsEes.RPCConnTimeout.String() + } + if optsEes.RPCReplyTimeout != nil { + opts[utils.RpcReplyTimeout] = optsEes.RPCReplyTimeout.String() + } + if optsEes.RPCAPIOpts != nil { + opts[utils.RPCAPIOpts] = optsEes.RPCAPIOpts + } + return opts +} + type EventExporterOptsJson struct { CSVFieldSeparator *string `json:"csvFieldSeparator"` ElsIndex *string `json:"elsIndex"` diff --git a/config/generalcfg.go b/config/generalcfg.go index cd200a2ba..164de4ee5 100644 --- a/config/generalcfg.go +++ b/config/generalcfg.go @@ -34,8 +34,6 @@ type GeneralOpts struct { // GeneralCfg is the general config section type GeneralCfg struct { NodeID string // Identifier for this engine instance - Logger string // dictates the way logs are displayed/stored - LogLevel int // system wide log level, nothing higher than this will be logged RoundingDecimals int // Number of decimals to round end prices at DBDataEncoding string // The encoding used to store object data in strings: TpExportPath string // Path towards export folder for offline Tariff Plans @@ -91,13 +89,6 @@ func (gencfg *GeneralCfg) loadFromJSONCfg(jsnGeneralCfg *GeneralJsonCfg) (err er if jsnGeneralCfg.Node_id != nil && *jsnGeneralCfg.Node_id != "" { gencfg.NodeID = *jsnGeneralCfg.Node_id } - if jsnGeneralCfg.Logger != nil { - gencfg.Logger = *jsnGeneralCfg.Logger - } - if jsnGeneralCfg.Log_level != nil { - gencfg.LogLevel = *jsnGeneralCfg.Log_level - } - if jsnGeneralCfg.Dbdata_encoding != nil { gencfg.DBDataEncoding = strings.TrimPrefix(*jsnGeneralCfg.Dbdata_encoding, "*") } @@ -197,8 +188,6 @@ func (gencfg GeneralCfg) AsMapInterface(string) interface{} { } mp := map[string]interface{}{ utils.NodeIDCfg: gencfg.NodeID, - utils.LoggerCfg: gencfg.Logger, - utils.LogLevelCfg: gencfg.LogLevel, utils.RoundingDecimalsCfg: gencfg.RoundingDecimals, utils.DBDataEncodingCfg: utils.Meta + gencfg.DBDataEncoding, utils.TpExportPathCfg: gencfg.TpExportPath, @@ -266,8 +255,6 @@ func (generalOpts *GeneralOpts) Clone() *GeneralOpts { func (gencfg GeneralCfg) Clone() *GeneralCfg { return &GeneralCfg{ NodeID: gencfg.NodeID, - Logger: gencfg.Logger, - LogLevel: gencfg.LogLevel, RoundingDecimals: gencfg.RoundingDecimals, DBDataEncoding: gencfg.DBDataEncoding, TpExportPath: gencfg.TpExportPath, @@ -304,8 +291,6 @@ type GeneralOptsJson struct { // General config section type GeneralJsonCfg struct { Node_id *string - Logger *string - Log_level *int Rounding_decimals *int Dbdata_encoding *string Tpexport_dir *string @@ -353,12 +338,6 @@ func diffGeneralJsonCfg(d *GeneralJsonCfg, v1, v2 *GeneralCfg) *GeneralJsonCfg { if v1.NodeID != v2.NodeID { d.Node_id = utils.StringPointer(v2.NodeID) } - if v1.Logger != v2.Logger { - d.Logger = utils.StringPointer(v2.Logger) - } - if v1.LogLevel != v2.LogLevel { - d.Log_level = utils.IntPointer(v2.LogLevel) - } if v1.RoundingDecimals != v2.RoundingDecimals { d.Rounding_decimals = utils.IntPointer(v2.RoundingDecimals) } diff --git a/config/generalcfg_test.go b/config/generalcfg_test.go index 81c62926b..278f6abcb 100644 --- a/config/generalcfg_test.go +++ b/config/generalcfg_test.go @@ -28,8 +28,6 @@ import ( func TestGeneralCfgloadFromJsonCfg(t *testing.T) { cfgJSON := &GeneralJsonCfg{ Node_id: utils.StringPointer("randomID"), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(6), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), @@ -49,8 +47,6 @@ func TestGeneralCfgloadFromJsonCfg(t *testing.T) { expected := &GeneralCfg{ NodeID: "randomID", - Logger: utils.MetaSysLog, - LogLevel: 6, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", @@ -125,9 +121,7 @@ func TestGeneralParseDurationCfgloadFromJsonCfg(t *testing.T) { func TestGeneralCfgAsMapInterface(t *testing.T) { cfgJSONStr := `{ "general": { - "node_id": "cgrates", - "logger":"*syslog", - "log_level": 6, + "node_id": "cgrates", "rounding_decimals": 5, "dbdata_encoding": "*msgpack", "tpexport_dir": "/var/spool/cgrates/tpe", @@ -152,8 +146,6 @@ func TestGeneralCfgAsMapInterface(t *testing.T) { }` eMap := map[string]interface{}{ utils.NodeIDCfg: "cgrates", - utils.LoggerCfg: "*syslog", - utils.LogLevelCfg: 6, utils.RoundingDecimalsCfg: 5, utils.DBDataEncodingCfg: "*msgpack", utils.TpExportPathCfg: "/var/spool/cgrates/tpe", @@ -166,8 +158,8 @@ func TestGeneralCfgAsMapInterface(t *testing.T) { utils.DefaultTimezoneCfg: "Local", utils.DefaultCachingCfg: "*reload", utils.ConnectAttemptsCfg: 5, - utils.ReconnectsCfg: -1, utils.MaxReconnectIntervalCfg: "0", + utils.ReconnectsCfg: -1, utils.ConnectTimeoutCfg: "1s", utils.ReplyTimeoutCfg: "2s", utils.LockingTimeoutCfg: "1s", @@ -204,8 +196,6 @@ func TestGeneralCfgAsMapInterface1(t *testing.T) { }` eMap := map[string]interface{}{ utils.NodeIDCfg: "ENGINE1", - utils.LoggerCfg: "*syslog", - utils.LogLevelCfg: 6, utils.RoundingDecimalsCfg: 5, utils.DBDataEncodingCfg: "*msgpack", utils.TpExportPathCfg: "/var/spool/cgrates/tpe", @@ -245,8 +235,6 @@ func TestGeneralCfgAsMapInterface1(t *testing.T) { func TestGeneralCfgClone(t *testing.T) { ban := &GeneralCfg{ NodeID: "randomID", - Logger: utils.MetaSysLog, - LogLevel: 6, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", @@ -288,8 +276,6 @@ func TestDiffGeneralJsonCfg(t *testing.T) { v1 := &GeneralCfg{ NodeID: "randomID2", - Logger: utils.LoggerCfg, - LogLevel: 7, RoundingDecimals: 1, DBDataEncoding: "msgpack2", TpExportPath: "/var/spool/cgrates/tpe/test", @@ -320,8 +306,6 @@ func TestDiffGeneralJsonCfg(t *testing.T) { v2 := &GeneralCfg{ NodeID: "randomID", - Logger: utils.MetaSysLog, - LogLevel: 6, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", @@ -353,8 +337,6 @@ func TestDiffGeneralJsonCfg(t *testing.T) { expected := &GeneralJsonCfg{ Node_id: utils.StringPointer("randomID"), - Logger: utils.StringPointer(utils.MetaSysLog), - Log_level: utils.IntPointer(6), Rounding_decimals: utils.IntPointer(5), Dbdata_encoding: utils.StringPointer("msgpack"), Tpexport_dir: utils.StringPointer("/var/spool/cgrates/tpe"), @@ -403,8 +385,6 @@ func TestDiffGeneralJsonCfg(t *testing.T) { func TestGeneralCfgCloneSection(t *testing.T) { gnrCfg := &GeneralCfg{ NodeID: "randomID2", - Logger: utils.LoggerCfg, - LogLevel: 7, RoundingDecimals: 1, DBDataEncoding: "msgpack2", TpExportPath: "/var/spool/cgrates/tpe/test", @@ -429,8 +409,6 @@ func TestGeneralCfgCloneSection(t *testing.T) { exp := &GeneralCfg{ NodeID: "randomID2", - Logger: utils.LoggerCfg, - LogLevel: 7, RoundingDecimals: 1, DBDataEncoding: "msgpack2", TpExportPath: "/var/spool/cgrates/tpe/test", diff --git a/config/multifiles_it_test.go b/config/multifiles_it_test.go index fc1c9cf4e..597538824 100644 --- a/config/multifiles_it_test.go +++ b/config/multifiles_it_test.go @@ -56,8 +56,6 @@ func TestMfGeneralItems(t *testing.T) { func TestMfEnvReaderITRead(t *testing.T) { expected := GeneralCfg{ NodeID: "d80fac5", - Logger: "*syslog", - LogLevel: 6, RoundingDecimals: 5, DBDataEncoding: "msgpack", TpExportPath: "/var/spool/cgrates/tpe", diff --git a/data/conf/samples/apis_chargers_internal/cgrates.json b/data/conf/samples/apis_chargers_internal/cgrates.json index 761e50456..75dfbf88b 100644 --- a/data/conf/samples/apis_chargers_internal/cgrates.json +++ b/data/conf/samples/apis_chargers_internal/cgrates.json @@ -1,7 +1,6 @@ { "general": { - "log_level": 7, "reply_timeout": "50s" }, @@ -24,6 +23,11 @@ "attributes_conns": ["*internal"] }, + "logger": { + "type": "*syslog", // controls the destination of logs <*syslog|*stdout|*kafka> + "level": 7, // system level precision for floats + }, + "admins": { "enabled": true } diff --git a/data/conf/samples/ees/cgrates.json b/data/conf/samples/ees/cgrates.json index 7253d33b5..8dcb82118 100644 --- a/data/conf/samples/ees/cgrates.json +++ b/data/conf/samples/ees/cgrates.json @@ -3,8 +3,10 @@ // // Copyright (C) ITsysCOM GmbH -"general": { - "log_level": 7, + +"logger": { + "type": "*syslog", // controls the destination of logs <*syslog|*stdout|*kafka> + "level": 7, // system level precision for floats }, "listen": { @@ -513,10 +515,10 @@ { "id": "KafkaExporter", "type": "*kafkaJSONMap", - "export_path": "127.0.0.1:9092", + "export_path": "localhost:9092", "concurrent_requests": 0, "opts": { - "kafkaTopic": "cgrates" + "kafkaTopic": "TutorialTopic" }, "timezone": "", "filters": [], diff --git a/data/conf/samples/tutinternal/cgrates.json b/data/conf/samples/tutinternal/cgrates.json index 9920fa6a7..ae037cdf4 100644 --- a/data/conf/samples/tutinternal/cgrates.json +++ b/data/conf/samples/tutinternal/cgrates.json @@ -4,15 +4,20 @@ "reply_timeout": "50s" }, +// "logger": { +// "type": "*kafkaLog", // controls the destination of logs <*syslog|*stdout|*kafka> +// "level": 6, // system level precision for floats +// "opts": { +// "kafka_conn": "localhost:9092", // the connection trough kafka +// "kafka_topic": "TutorialTopic", // the topic from where the events are exported +// "attempts": 1, // number of attempts of connecting +// }, +//}, + "logger": { - "type": "*kafka", // controls the destination of logs <*syslog|*stdout|*kafka> + "type": "*syslog", // controls the destination of logs <*syslog|*stdout|*kafka> "level": 6, // system level precision for floats - "opts": { - "kafka_conn": "192.168.55.44:9092", // the connection trough kafka - "kafka_topic": "TutorialTopic", // the topic from where the events are exported - "attempts": 1, // number of attempts of connecting - }, -}, +}, "listen": { "rpc_json": ":2012", diff --git a/ees/ees.go b/ees/ees.go index d26aaa249..d5df79aae 100644 --- a/ees/ees.go +++ b/ees/ees.go @@ -283,9 +283,9 @@ func ExportWithAttempts(ctx *context.Context, exp EventExporter, eEv interface{} if exp.Cfg().FailedPostsDir != utils.MetaNone { defer func() { if err != nil { - AddFailedPost(exp.Cfg().FailedPostsDir, exp.Cfg().ExportPath, + utils.AddFailedMessage(exp.Cfg().FailedPostsDir, exp.Cfg().ExportPath, exp.Cfg().Type, utils.EEs, - eEv, exp.Cfg().Opts) + eEv, exp.Cfg().Opts.AsMapInterface()) } }() } diff --git a/ees/libcdre.go b/ees/libcdre.go index 5adc20610..218fde038 100644 --- a/ees/libcdre.go +++ b/ees/libcdre.go @@ -21,87 +21,18 @@ package ees import ( "bytes" "encoding/gob" - "fmt" "os" - "path" "sync" - "time" "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/guardian" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/ltcache" ) -var failedPostCache *ltcache.Cache - -func init() { - failedPostCache = ltcache.NewCache(-1, 5*time.Second, false, writeFailedPosts) // configurable general -} - -// SetFailedPostCacheTTL recreates the failed cache -func SetFailedPostCacheTTL(ttl time.Duration) { - failedPostCache = ltcache.NewCache(-1, ttl, false, writeFailedPosts) -} - -func writeFailedPosts(itmID string, value interface{}) { - expEv, canConvert := value.(*ExportEvents) - if !canConvert { - return - } - filePath := expEv.FilePath() - if err := expEv.WriteToFile(filePath); err != nil { - utils.Logger.Warning(fmt.Sprintf("<%s> Failed to write file <%s> because <%s>", - utils.CDRs, filePath, err)) - } -} - -func AddFailedPost(failedPostsDir, expPath, format, module string, ev interface{}, opts *config.EventExporterOpts) { - key := utils.ConcatenatedKey(failedPostsDir, expPath, format, module) - // also in case of amqp,amqpv1,s3,sqs and kafka also separe them after queue id - var amqpQueueID string - var s3BucketID string - var sqsQueueID string - var kafkaTopic string - if opts.AMQPQueueID != nil { - amqpQueueID = *opts.AMQPQueueID - } - if opts.S3BucketID != nil { - s3BucketID = *opts.S3BucketID - } - if opts.SQSQueueID != nil { - sqsQueueID = *opts.SQSQueueID - } - if opts.KafkaTopic != nil { - kafkaTopic = *opts.KafkaTopic - } - if qID := utils.FirstNonEmpty(amqpQueueID, s3BucketID, - sqsQueueID, kafkaTopic); len(qID) != 0 { - key = utils.ConcatenatedKey(key, qID) - } - var failedPost *ExportEvents - if x, ok := failedPostCache.Get(key); ok { - if x != nil { - failedPost = x.(*ExportEvents) - } - } - if failedPost == nil { - failedPost = &ExportEvents{ - Path: expPath, - Format: format, - Opts: opts, - module: module, - failedPostsDir: failedPostsDir, - } - } - failedPost.AddEvent(ev) - failedPostCache.Set(key, failedPost, nil) -} - -// NewExportEventsFromFile returns ExportEvents from the file +// NewFailoverPosterFromFile returns ExportEvents from the file // used only on replay failed post -func NewExportEventsFromFile(filePath string) (expEv *ExportEvents, err error) { +func NewFailoverPosterFromFile(filePath, providerType string) (failPoster utils.FailoverPoster, err error) { var fileContent []byte err = guardian.Guardian.Guard(context.TODO(), func(_ *context.Context) error { if fileContent, err = os.ReadFile(filePath); err != nil { @@ -114,13 +45,230 @@ func NewExportEventsFromFile(filePath string) (expEv *ExportEvents, err error) { } dec := gob.NewDecoder(bytes.NewBuffer(fileContent)) // unmarshall it - expEv = new(ExportEvents) + expEv := new(utils.FailedExportersLogg) err = dec.Decode(&expEv) + switch providerType { + case utils.EEs: + opts, err := AsOptsEESConfig(expEv.Opts) + if err != nil { + return nil, err + } + failPoster = &FailedExportersEEs{ + module: expEv.Module, + failedPostsDir: expEv.FailedPostsDir, + Path: expEv.Path, + Opts: opts, + Events: expEv.Events, + Format: expEv.Format, + } + case utils.Kafka: + failPoster = expEv + } return } -// ExportEvents used to save the failed post to file -type ExportEvents struct { +func AsOptsEESConfig(opts map[string]interface{}) (*config.EventExporterOpts, error) { + optsCfg := new(config.EventExporterOpts) + if len(opts) == 0 { + return optsCfg, nil + } + if _, has := opts[utils.CSVFieldSepOpt]; has { + optsCfg.CSVFieldSeparator = utils.StringPointer(utils.IfaceAsString(utils.CSVFieldSepOpt)) + } + if _, has := opts[utils.ElsIndex]; has { + optsCfg.ElsIndex = utils.StringPointer(utils.IfaceAsString(utils.ElsIndex)) + } + if _, has := opts[utils.ElsIfPrimaryTerm]; has { + x, err := utils.IfaceAsInt(utils.ElsIfPrimaryTerm) + if err != nil { + return nil, err + } + optsCfg.ElsIfPrimaryTerm = utils.IntPointer(x) + } + if _, has := opts[utils.ElsIfSeqNo]; has { + x, err := utils.IfaceAsInt(utils.ElsIfSeqNo) + if err != nil { + return nil, err + } + optsCfg.ElsIfSeqNo = utils.IntPointer(x) + } + if _, has := opts[utils.ElsOpType]; has { + optsCfg.ElsOpType = utils.StringPointer(utils.IfaceAsString(utils.ElsOpType)) + } + if _, has := opts[utils.ElsPipeline]; has { + optsCfg.ElsPipeline = utils.StringPointer(utils.IfaceAsString(utils.ElsPipeline)) + } + if _, has := opts[utils.ElsRouting]; has { + optsCfg.ElsRouting = utils.StringPointer(utils.IfaceAsString(utils.ElsRouting)) + } + if _, has := opts[utils.ElsTimeout]; has { + t, err := utils.IfaceAsDuration(utils.ElsTimeout) + if err != nil { + return nil, err + } + optsCfg.ElsTimeout = &t + } + if _, has := opts[utils.ElsVersionLow]; has { + x, err := utils.IfaceAsInt(utils.ElsVersionLow) + if err != nil { + return nil, err + } + optsCfg.ElsVersion = utils.IntPointer(x) + } + if _, has := opts[utils.ElsVersionType]; has { + optsCfg.ElsVersionType = utils.StringPointer(utils.IfaceAsString(utils.ElsVersionType)) + } + if _, has := opts[utils.ElsWaitForActiveShards]; has { + optsCfg.ElsWaitForActiveShards = utils.StringPointer(utils.IfaceAsString(utils.ElsWaitForActiveShards)) + } + if _, has := opts[utils.SQLMaxIdleConnsCfg]; has { + x, err := utils.IfaceAsInt(utils.SQLMaxIdleConnsCfg) + if err != nil { + return nil, err + } + optsCfg.SQLMaxIdleConns = utils.IntPointer(x) + } + if _, has := opts[utils.SQLMaxOpenConns]; has { + x, err := utils.IfaceAsInt(utils.SQLMaxOpenConns) + if err != nil { + return nil, err + } + optsCfg.SQLMaxOpenConns = utils.IntPointer(x) + } + if _, has := opts[utils.SQLConnMaxLifetime]; has { + t, err := utils.IfaceAsDuration(utils.SQLConnMaxLifetime) + if err != nil { + return nil, err + } + optsCfg.SQLConnMaxLifetime = &t + } + if _, has := opts[utils.MYSQLDSNParams]; has { + optsCfg.MYSQLDSNParams = opts[utils.SQLConnMaxLifetime].(map[string]string) + } + if _, has := opts[utils.SQLTableNameOpt]; has { + optsCfg.SQLTableName = utils.StringPointer(utils.IfaceAsString(utils.SQLTableNameOpt)) + } + if _, has := opts[utils.SQLDBNameOpt]; has { + optsCfg.SQLDBName = utils.StringPointer(utils.IfaceAsString(utils.SQLDBNameOpt)) + } + if _, has := opts[utils.PgSSLModeCfg]; has { + optsCfg.PgSSLMode = utils.StringPointer(utils.IfaceAsString(utils.PgSSLModeCfg)) + } + if _, has := opts[utils.KafkaTopic]; has { + optsCfg.KafkaTopic = utils.StringPointer(utils.IfaceAsString(utils.KafkaTopic)) + } + if _, has := opts[utils.AMQPQueueID]; has { + optsCfg.AMQPQueueID = utils.StringPointer(utils.IfaceAsString(utils.AMQPQueueID)) + } + if _, has := opts[utils.AMQPRoutingKey]; has { + optsCfg.AMQPRoutingKey = utils.StringPointer(utils.IfaceAsString(utils.AMQPRoutingKey)) + } + if _, has := opts[utils.AMQPExchange]; has { + optsCfg.AMQPExchange = utils.StringPointer(utils.IfaceAsString(utils.AMQPExchange)) + } + if _, has := opts[utils.AMQPExchangeType]; has { + optsCfg.AMQPExchangeType = utils.StringPointer(utils.IfaceAsString(utils.AMQPExchangeType)) + } + if _, has := opts[utils.AWSRegion]; has { + optsCfg.AWSRegion = utils.StringPointer(utils.IfaceAsString(utils.AWSRegion)) + } + if _, has := opts[utils.AWSKey]; has { + optsCfg.AWSKey = utils.StringPointer(utils.IfaceAsString(utils.AWSKey)) + } + if _, has := opts[utils.AWSSecret]; has { + optsCfg.AWSSecret = utils.StringPointer(utils.IfaceAsString(utils.AWSSecret)) + } + if _, has := opts[utils.AWSToken]; has { + optsCfg.AWSToken = utils.StringPointer(utils.IfaceAsString(utils.AWSToken)) + } + if _, has := opts[utils.SQSQueueID]; has { + optsCfg.SQSQueueID = utils.StringPointer(utils.IfaceAsString(utils.SQSQueueID)) + } + if _, has := opts[utils.S3Bucket]; has { + optsCfg.S3BucketID = utils.StringPointer(utils.IfaceAsString(utils.S3Bucket)) + } + if _, has := opts[utils.S3FolderPath]; has { + optsCfg.S3FolderPath = utils.StringPointer(utils.IfaceAsString(utils.S3FolderPath)) + } + if _, has := opts[utils.NatsJetStream]; has { + x, err := utils.IfaceAsBool(utils.NatsJetStream) + if err != nil { + return nil, err + } + optsCfg.NATSJetStream = utils.BoolPointer(x) + } + if _, has := opts[utils.NatsSubject]; has { + optsCfg.NATSSubject = utils.StringPointer(utils.IfaceAsString(utils.NatsSubject)) + } + if _, has := opts[utils.NatsJWTFile]; has { + optsCfg.NATSJWTFile = utils.StringPointer(utils.IfaceAsString(utils.NatsJWTFile)) + } + if _, has := opts[utils.NatsSeedFile]; has { + optsCfg.NATSSeedFile = utils.StringPointer(utils.IfaceAsString(utils.NatsSeedFile)) + } + if _, has := opts[utils.NatsCertificateAuthority]; has { + optsCfg.NATSCertificateAuthority = utils.StringPointer(utils.IfaceAsString(utils.NatsCertificateAuthority)) + } + if _, has := opts[utils.NatsClientCertificate]; has { + optsCfg.NATSClientCertificate = utils.StringPointer(utils.IfaceAsString(utils.NatsClientCertificate)) + } + if _, has := opts[utils.NatsClientKey]; has { + optsCfg.NATSClientKey = utils.StringPointer(utils.IfaceAsString(utils.NatsClientKey)) + } + if _, has := opts[utils.NatsJetStreamMaxWait]; has { + t, err := utils.IfaceAsDuration(utils.NatsJetStreamMaxWait) + if err != nil { + return nil, err + } + optsCfg.NATSJetStreamMaxWait = &t + } + if _, has := opts[utils.RpcCodec]; has { + optsCfg.RPCCodec = utils.StringPointer(utils.IfaceAsString(utils.RpcCodec)) + } + if _, has := opts[utils.ServiceMethod]; has { + optsCfg.ServiceMethod = utils.StringPointer(utils.IfaceAsString(utils.ServiceMethod)) + } + if _, has := opts[utils.KeyPath]; has { + optsCfg.KeyPath = utils.StringPointer(utils.IfaceAsString(utils.KeyPath)) + } + if _, has := opts[utils.CertPath]; has { + optsCfg.CertPath = utils.StringPointer(utils.IfaceAsString(utils.CertPath)) + } + if _, has := opts[utils.CaPath]; has { + optsCfg.CAPath = utils.StringPointer(utils.IfaceAsString(utils.CaPath)) + } + if _, has := opts[utils.Tls]; has { + x, err := utils.IfaceAsBool(utils.Tls) + if err != nil { + return nil, err + } + optsCfg.TLS = utils.BoolPointer(x) + } + if _, has := opts[utils.ConnIDs]; has { + optsCfg.ConnIDs = opts[utils.ConnIDs].(*[]string) + } + if _, has := opts[utils.RpcConnTimeout]; has { + t, err := utils.IfaceAsDuration(utils.RpcConnTimeout) + if err != nil { + return nil, err + } + optsCfg.RPCConnTimeout = &t + } + if _, has := opts[utils.RpcReplyTimeout]; has { + t, err := utils.IfaceAsDuration(utils.RpcReplyTimeout) + if err != nil { + return nil, err + } + optsCfg.RPCReplyTimeout = &t + } + if _, has := opts[utils.RPCAPIOpts]; has { + optsCfg.RPCAPIOpts = opts[utils.RPCAPIOpts].(map[string]interface{}) + } + return optsCfg, nil +} + +// FailedExportersEEs used to save the failed post to file +type FailedExportersEEs struct { lk sync.RWMutex Path string Opts *config.EventExporterOpts @@ -130,41 +278,16 @@ type ExportEvents struct { module string } -// FilePath returns the file path it should use for saving the failed events -func (expEv *ExportEvents) FilePath() string { - return path.Join(expEv.failedPostsDir, expEv.module+utils.PipeSep+utils.UUIDSha1Prefix()+utils.GOBSuffix) -} - -// SetModule sets the module for this event -func (expEv *ExportEvents) SetModule(mod string) { - expEv.module = mod -} - -// WriteToFile writes the events to file -func (expEv *ExportEvents) WriteToFile(filePath string) (err error) { - err = guardian.Guardian.Guard(context.TODO(), func(_ *context.Context) error { - fileOut, err := os.Create(filePath) - if err != nil { - return err - } - encd := gob.NewEncoder(fileOut) - err = encd.Encode(expEv) - fileOut.Close() - return err - }, config.CgrConfig().GeneralCfg().LockingTimeout, utils.FileLockPrefix+filePath) - return -} - // AddEvent adds one event -func (expEv *ExportEvents) AddEvent(ev interface{}) { +func (expEv *FailedExportersEEs) AddEvent(ev interface{}) { expEv.lk.Lock() expEv.Events = append(expEv.Events, ev) expEv.lk.Unlock() } // ReplayFailedPosts tryies to post cdrs again -func (expEv *ExportEvents) ReplayFailedPosts(attempts int) (failedEvents *ExportEvents, err error) { - failedEvents = &ExportEvents{ +func (expEv *FailedExportersEEs) ReplayFailedPosts(attempts int) (failedEvents *utils.FailedExportersLogg, err error) { + eesFailedEvents := &FailedExportersEEs{ Path: expEv.Path, Opts: expEv.Opts, Format: expEv.Format, @@ -187,14 +310,14 @@ func (expEv *ExportEvents) ReplayFailedPosts(attempts int) (failedEvents *Export } for _, ev := range expEv.Events { if err = ExportWithAttempts(context.Background(), ee, ev, keyFunc()); err != nil { - failedEvents.AddEvent(ev) + eesFailedEvents.AddEvent(ev) } } ee.Close() - if len(failedEvents.Events) > 0 { + if len(eesFailedEvents.Events) > 0 { err = utils.ErrPartiallyExecuted } else { - failedEvents = nil + eesFailedEvents = nil } return } diff --git a/ees/libcdre_it_test.go b/ees/libcdre_it_test.go index 68d319792..9451efcaa 100644 --- a/ees/libcdre_it_test.go +++ b/ees/libcdre_it_test.go @@ -21,16 +21,7 @@ along with this program. If not, see package ees -import ( - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - +/* func TestWriteFldPosts(t *testing.T) { // can't convert var notanExportEvent string @@ -112,3 +103,4 @@ func TestWriteToFile(t *testing.T) { t.Error(err) } } +*/ diff --git a/ees/libcdre_test.go b/ees/libcdre_test.go index 87667e21e..faf919ef3 100644 --- a/ees/libcdre_test.go +++ b/ees/libcdre_test.go @@ -18,15 +18,7 @@ along with this program. If not, see package ees -import ( - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - +/* func TestSetFldPostCacheTTL(t *testing.T) { var1 := failedPostCache SetFailedPostCacheTTL(50 * time.Millisecond) @@ -161,3 +153,4 @@ func TestAddEvent(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(eOut), utils.ToJSON(exportEvent)) } } +*/ diff --git a/ees/poster_it_test.go b/ees/poster_it_test.go index 88a2b1416..1cdc31ecf 100644 --- a/ees/poster_it_test.go +++ b/ees/poster_it_test.go @@ -21,11 +21,7 @@ along with this program. If not, see package ees import ( - "encoding/json" "flag" - "net/http" - "path/filepath" - "reflect" "testing" "time" @@ -64,6 +60,7 @@ type TestContent struct { Var2 string } +/* func TestHttpJsonPoster(t *testing.T) { SetFailedPostCacheTTL(time.Millisecond) content := &TestContent{Var1: "Val1", Var2: "Val2"} @@ -135,7 +132,7 @@ func TestHttpBytesPoster(t *testing.T) { t.Errorf("Expecting: %q, received: %q", string(content), ev.Events[0]) } } - +*/ func TestSQSPoster(t *testing.T) { if !*itTestSQS { return diff --git a/engine/dispatcherprfl.go b/engine/dispatcherprfl.go index 2eefc5767..4a253a040 100644 --- a/engine/dispatcherprfl.go +++ b/engine/dispatcherprfl.go @@ -291,11 +291,11 @@ func (dH *DispatcherHost) Set(path []string, val interface{}, newBranch bool, _ dH.CaCertificate = utils.IfaceAsString(val) case utils.ConnectAttempts: if val != utils.EmptyString { - dH.ConnectAttempts, err = utils.IfaceAsTInt(val) + dH.ConnectAttempts, err = utils.IfaceAsInt(val) } case utils.Reconnects: if val != utils.EmptyString { - dH.Reconnects, err = utils.IfaceAsTInt(val) + dH.Reconnects, err = utils.IfaceAsInt(val) } case utils.MaxReconnectInterval: if val != utils.EmptyString { diff --git a/engine/libstats.go b/engine/libstats.go index 2115fdd6b..0ca9e8571 100644 --- a/engine/libstats.go +++ b/engine/libstats.go @@ -570,9 +570,9 @@ func (sqp *StatQueueProfile) Set(path []string, val interface{}, newBranch bool, sqp.ID = utils.IfaceAsString(val) case utils.QueueLength: - sqp.QueueLength, err = utils.IfaceAsTInt(val) + sqp.QueueLength, err = utils.IfaceAsInt(val) case utils.MinItems: - sqp.MinItems, err = utils.IfaceAsTInt(val) + sqp.MinItems, err = utils.IfaceAsInt(val) case utils.TTL: sqp.TTL, err = utils.IfaceAsDuration(val) case utils.Stored: diff --git a/engine/logger.go b/engine/logger.go deleted file mode 100644 index 54b052faf..000000000 --- a/engine/logger.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package engine - -import ( - "log/syslog" - "time" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" - "github.com/segmentio/kafka-go" -) - -func NewLogger(loggerType, tenant, nodeID string, loggCfg *config.LoggerCfg) (utils.LoggerInterface, error) { - switch loggerType { - case utils.MetaKafka: - return NewExportLogger(nodeID, tenant, loggCfg.Level, loggCfg.Opts), nil - default: - return utils.NewLogger(loggerType, nodeID, loggCfg.Level) - } -} - -// Logs to kafka -type ExportLogger struct { - logLevel int - fPost *utils.FailoverPoster - loggOpts *config.LoggerOptsCfg - writer *kafka.Writer - nodeID string - tenant string -} - -func NewExportLogger(nodeID, tenant string, level int, opts *config.LoggerOptsCfg) (el *ExportLogger) { - el = &ExportLogger{ - logLevel: level, - loggOpts: opts, - nodeID: nodeID, - tenant: tenant, - writer: &kafka.Writer{ - Addr: kafka.TCP(opts.KafkaConn), - Topic: opts.KafkaTopic, - MaxAttempts: opts.Attempts, - }, - } - return -} - -func (el *ExportLogger) Close() (err error) { - if el.writer != nil { - err = el.writer.Close() - el.writer = nil - } - return -} - -func (el *ExportLogger) call(m string, level int) (err error) { - eventExport := &utils.CGREvent{ - Tenant: el.tenant, - Event: map[string]interface{}{ - utils.NodeID: el.nodeID, - "Message": m, - "Severity": level, - "Timestamp": time.Now().Format("2006-01-02 15:04:05"), - }, - } - // event will be exported through kafka as json format - var content []byte - if content, err = utils.ToUnescapedJSON(eventExport); err != nil { - return - } - if err = el.writer.WriteMessages(context.Background(), kafka.Message{ - Key: []byte(utils.GenUUID()), - Value: content, - }); err != nil { - // if there are any errors in kafka, we will post in FailedPostDirectory - el.fPost = utils.NewFailoverPoster() - if err = el.fPost.AddMessage(el.loggOpts.FailedPostsDir, - el.loggOpts.KafkaConn, el.loggOpts.KafkaTopic, eventExport); err != nil { - return - } - // also the content should be printed as a stdout logger type - return utils.ErrLoggerChanged - } - return -} - -func (el *ExportLogger) Write(p []byte) (n int, err error) { - n = len(p) - err = el.call(string(p), 8) - return -} - -func (sl *ExportLogger) GetSyslog() *syslog.Writer { - return nil -} - -// GetLogLevel() returns the level logger number for the server -func (el *ExportLogger) GetLogLevel() int { - return el.logLevel -} - -// SetLogLevel changes the log level -func (el *ExportLogger) SetLogLevel(level int) { - el.logLevel = level -} - -// Alert logs to EEs with alert level -func (el *ExportLogger) Alert(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_ALERT { - return nil - } - if err = el.call(m, utils.LOGLEVEL_ALERT); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Alert(m) - err = nil - } - } - return //el.call(m, utils.LOGLEVEL_ALERT) -} - -// Crit logs to EEs with critical level -func (el *ExportLogger) Crit(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_CRITICAL { - return nil - } - if el.call(m, utils.LOGLEVEL_CRITICAL); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Crit(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_CRITICAL) -} - -// Debug logs to EEs with debug level -func (el *ExportLogger) Debug(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_DEBUG { - return nil - } - if err = el.call(m, utils.LOGLEVEL_DEBUG); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Debug(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_DEBUG) -} - -// Emerg logs to EEs with emergency level -func (el *ExportLogger) Emerg(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_EMERGENCY { - return nil - } - if err = el.call(m, utils.LOGLEVEL_EMERGENCY); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Emerg(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_EMERGENCY) -} - -// Err logs to EEs with error level -func (el *ExportLogger) Err(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_ERROR { - return nil - } - if err = el.call(m, utils.LOGLEVEL_ERROR); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Err(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_ERROR) -} - -// Info logs to EEs with info level -func (el *ExportLogger) Info(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_INFO { - return nil - } - if err = el.call(m, utils.LOGLEVEL_INFO); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Info(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_INFO) -} - -// Notice logs to EEs with notice level -func (el *ExportLogger) Notice(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_NOTICE { - return nil - } - if err = el.call(m, utils.LOGLEVEL_NOTICE); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Notice(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_NOTICE) -} - -// Warning logs to EEs with warning level -func (el *ExportLogger) Warning(m string) (err error) { - if el.logLevel < utils.LOGLEVEL_WARNING { - return nil - } - if err = el.call(m, utils.LOGLEVEL_WARNING); err != nil { - if err == utils.ErrLoggerChanged { - utils.NewStdLogger(el.nodeID, el.logLevel).Warning(m) - err = nil - } - } - return // el.call(m, utils.LOGLEVEL_WARNING) -} diff --git a/engine/thresholds.go b/engine/thresholds.go index b148e1683..6ed83bc7e 100644 --- a/engine/thresholds.go +++ b/engine/thresholds.go @@ -622,11 +622,11 @@ func (tp *ThresholdProfile) Set(path []string, val interface{}, _ bool, _ string tp.FilterIDs = append(tp.FilterIDs, valA...) case utils.MaxHits: if val != utils.EmptyString { - tp.MaxHits, err = utils.IfaceAsTInt(val) + tp.MaxHits, err = utils.IfaceAsInt(val) } case utils.MinHits: if val != utils.EmptyString { - tp.MinHits, err = utils.IfaceAsTInt(val) + tp.MinHits, err = utils.IfaceAsInt(val) } case utils.MinSleep: tp.MinSleep, err = utils.IfaceAsDuration(val) diff --git a/general_tests/cdrs_post_failover_it_test.go b/general_tests/cdrs_post_failover_it_test.go index ab6e7f395..92a712a4b 100644 --- a/general_tests/cdrs_post_failover_it_test.go +++ b/general_tests/cdrs_post_failover_it_test.go @@ -203,19 +203,19 @@ func testCDRsPostFailoverToFile(t *testing.T) { fileName := file.Name() filePath := path.Join(cdrsPostFailCfg.GeneralCfg().FailedPostsDir, fileName) - ev, err := ees.NewExportEventsFromFile(filePath) + ev, err := ees.NewFailoverPosterFromFile(filePath, utils.EEs) if err != nil { t.Errorf("<%s> for file <%s>", err, fileName) continue - } else if len(ev.Events) == 0 { + } else if len(ev.(*ees.FailedExportersEEs).Events) == 0 { t.Error("Expected at least one event") continue } - if ev.Format != utils.MetaS3jsonMap { - t.Errorf("Expected event to use %q received: %q", utils.MetaS3jsonMap, ev.Format) + if ev.(*ees.FailedExportersEEs).Format != utils.MetaS3jsonMap { + t.Errorf("Expected event to use %q received: %q", utils.MetaS3jsonMap, ev.(*ees.FailedExportersEEs).Format) } - if len(ev.Events) != 3 { - t.Errorf("Expected all the events to be saved in the same file, ony %v saved in this file.", len(ev.Events)) + if len(ev.(*ees.FailedExportersEEs).Events) != 3 { + t.Errorf("Expected all the events to be saved in the same file, ony %v saved in this file.", len(ev.(*ees.FailedExportersEEs).Events)) } } } diff --git a/services/cgr-engine.go b/services/cgr-engine.go index a551bdb5b..9aea2aa49 100644 --- a/services/cgr-engine.go +++ b/services/cgr-engine.go @@ -349,10 +349,18 @@ func (cgr *CGREngine) Init(ctx *context.Context, shtDw context.CancelFunc, flags } // init syslog - if utils.Logger, err = engine.NewLogger(utils.FirstNonEmpty(*flags.SysLogger, cgr.cfg.LoggerCfg().Type), - cgr.cfg.GeneralCfg().DefaultTenant, cgr.cfg.GeneralCfg().NodeID, cgr.cfg.LoggerCfg()); err != nil { + if utils.Logger, err = utils.NewLogger( + utils.FirstNonEmpty(*flags.SysLogger, cgr.cfg.LoggerCfg().Type), + cgr.cfg.GeneralCfg().DefaultTenant, + cgr.cfg.GeneralCfg().NodeID, + cgr.cfg.LoggerCfg().Level, + cgr.cfg.LoggerCfg().Opts.Attempts, + cgr.cfg.LoggerCfg().Opts.KafkaConn, + cgr.cfg.LoggerCfg().Opts.KafkaTopic, + cgr.cfg.LoggerCfg().Opts.FailedPostsDir); err != nil { return fmt.Errorf("Could not initialize syslog connection, err: <%s>", err) } + utils.SetFailedPostCacheTTL(cgr.cfg.GeneralCfg().FailedPostsTTL) // init failedPosts to posts loggers/exporters in case of failing utils.Logger.Info(fmt.Sprintf(" starting version <%s><%s>", vers, runtime.Version())) cgr.cfg.LazySanityCheck() diff --git a/services/globalvars.go b/services/globalvars.go index 3fafcf505..b19cb5993 100644 --- a/services/globalvars.go +++ b/services/globalvars.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/ees" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/config" @@ -47,7 +46,6 @@ type GlobalVarS struct { // Start should handle the sercive start func (gv *GlobalVarS) Start(*context.Context, context.CancelFunc) error { - ees.SetFailedPostCacheTTL(gv.cfg.GeneralCfg().FailedPostsTTL) engine.SetHTTPPstrTransport(gv.cfg.HTTPCfg().ClientOpts) utils.DecimalContext.MaxScale = gv.cfg.GeneralCfg().DecimalMaxScale utils.DecimalContext.MinScale = gv.cfg.GeneralCfg().DecimalMinScale diff --git a/utils/cgrevent_test.go b/utils/cgrevent_test.go index dbfe06c11..7b41704ba 100644 --- a/utils/cgrevent_test.go +++ b/utils/cgrevent_test.go @@ -24,25 +24,6 @@ import ( "time" ) -func TestCGREventHasField(t *testing.T) { - //empty check - cgrEvent := new(CGREvent) - rcv := cgrEvent.HasField("") - if rcv { - t.Error("Expecting: false, received: ", rcv) - } - //normal check - cgrEvent = &CGREvent{ - Event: map[string]interface{}{ - Usage: 20 * time.Second, - }, - } - rcv = cgrEvent.HasField("Usage") - if !rcv { - t.Error("Expecting: true, received: ", rcv) - } -} - func TestCGREventCheckMandatoryFields(t *testing.T) { //empty check cgrEvent := new(CGREvent) diff --git a/utils/consts.go b/utils/consts.go index c8a66a34f..58f2bf9c3 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -312,6 +312,10 @@ const ( CreatedAt = "CreatedAt" UpdatedAt = "UpdatedAt" NodeID = "NodeID" + //ExportLogger + Message = "Message" + Severity = "Severity" + Timestamp = "Timestamp" //cores consts ActiveGoroutines = "ActiveGoroutines" MemoryUsage = "MemoryUsage" @@ -385,6 +389,10 @@ const ( UnitCounters = "UnitCounters" UpdateTime = "UpdateTime" Rates = "Rates" + Format = "Format" + Conn = "Conn" + Level = "Level" + FailedPostsDir = "FailedPostsDir" //DestinationRates = "DestinationRates" RatingPlans = "RatingPlans" RatingProfiles = "RatingProfiles" @@ -420,28 +428,29 @@ const ( ConnBlocker = "ConnBlocker" ConnParameters = "ConnParameters" - Thresholds = "Thresholds" - Routes = "Routes" - Attributes = "Attributes" - Chargers = "Chargers" - Dispatchers = "Dispatchers" - StatS = "StatS" - LoadIDsVrs = "LoadIDs" - GlobalVarS = "GlobalVarS" - CostSource = "CostSource" - ExtraInfo = "ExtraInfo" - Meta = "*" - MetaSysLog = "*syslog" - MetaStdLog = "*stdout" - MetaKafka = "*kafka" - EventSource = "EventSource" - AccountID = "AccountID" - AccountIDs = "AccountIDs" - ResourceID = "ResourceID" - TotalUsage = "TotalUsage" - StatID = "StatID" - BalanceType = "BalanceType" - BalanceID = "BalanceID" + Thresholds = "Thresholds" + Routes = "Routes" + Attributes = "Attributes" + Chargers = "Chargers" + Dispatchers = "Dispatchers" + StatS = "StatS" + LoadIDsVrs = "LoadIDs" + GlobalVarS = "GlobalVarS" + CostSource = "CostSource" + ExtraInfo = "ExtraInfo" + Meta = "*" + MetaSysLog = "*syslog" + MetaStdLog = "*stdout" + MetaKafkaLog = "*kafkaLog" + Kafka = "Kafka" + EventSource = "EventSource" + AccountID = "AccountID" + AccountIDs = "AccountIDs" + ResourceID = "ResourceID" + TotalUsage = "TotalUsage" + StatID = "StatID" + BalanceType = "BalanceType" + BalanceID = "BalanceID" //BalanceDestinationIds = "BalanceDestinationIds" BalanceCostIncrements = "BalanceCostIncrements" BalanceAttributeIDs = "BalanceAttributeIDs" @@ -1125,6 +1134,7 @@ const ( // AdminSv1 APIs const ( + AdminSv1ReplayFailedPosts = "AdminSv1.ReplayFailedPosts" AdminSv1GetRateRatesIndexesHealth = "AdminSv1.GetRateRatesIndexesHealth" AdminSv1GetChargerProfilesCount = "AdminSv1.GetChargerProfilesCount" AdminSv1GetAccountsIndexesHealth = "AdminSv1.GetAccountsIndexesHealth" diff --git a/utils/coreutils.go b/utils/coreutils.go index f3c26f4c7..7d0add8ba 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -987,17 +987,17 @@ func (pgnt Paginator) Clone() Paginator { // GetPaginateOpts retrieves paginate options from the APIOpts map func GetPaginateOpts(opts map[string]interface{}) (limit, offset, maxItems int, err error) { if limitIface, has := opts[PageLimitOpt]; has { - if limit, err = IfaceAsTInt(limitIface); err != nil { + if limit, err = IfaceAsInt(limitIface); err != nil { return } } if offsetIface, has := opts[PageOffsetOpt]; has { - if offset, err = IfaceAsTInt(offsetIface); err != nil { + if offset, err = IfaceAsInt(offsetIface); err != nil { return } } if maxItemsIface, has := opts[PageMaxItemsOpt]; has { - if maxItems, err = IfaceAsTInt(maxItemsIface); err != nil { + if maxItems, err = IfaceAsInt(maxItemsIface); err != nil { return } } diff --git a/utils/failover_export.go b/utils/failover_export.go index c1bdc8e73..9ba2dd919 100644 --- a/utils/failover_export.go +++ b/utils/failover_export.go @@ -19,52 +19,182 @@ along with this program. If not, see package utils import ( + "bytes" "encoding/gob" "fmt" "os" - "path/filepath" + "path" + "sync" + "time" + + "github.com/cgrates/ltcache" ) -type FailoverPoster struct { - MessageProvider +var failedPostCache *ltcache.Cache + +func init() { + failedPostCache = ltcache.NewCache(-1, 5*time.Second, true, writeFailedPosts) } -func NewFailoverPoster( /*addMsg *MessageProvider*/ ) *FailoverPoster { - return new(FailoverPoster) +// SetFailedPostCacheTTL recreates the failed cache +func SetFailedPostCacheTTL(ttl time.Duration) { + failedPostCache = ltcache.NewCache(-1, ttl, true, writeFailedPosts) } -func (fldPst *FailoverPoster) AddMessage(failedPostDir, kafkaConn, - kafkaTopic string, content interface{}) (err error) { - filePath := filepath.Join(failedPostDir, kafkaTopic+PipeSep+MetaKafka+GOBSuffix) - var fileOut *os.File - if _, err = os.Stat(filePath); os.IsNotExist(err) { - fileOut, err = os.Create(filePath) - if err != nil { - return fmt.Errorf(fmt.Sprintf(" failed to write logs to file <%s> because <%s>", filePath, err)) - } - } else { - fileOut, err = os.OpenFile(filePath, os.O_RDWR|os.O_APPEND, 0755) - if err != nil { - return err - } +func writeFailedPosts(_ string, value interface{}) { + expEv, canConvert := value.(*FailedExportersLogg) + if !canConvert { + return } - enc := gob.NewEncoder(fileOut) - if err = enc.Encode(content); err != nil { + filePath := expEv.FilePath() + expEv.lk.RLock() + if err := expEv.WriteToFile(filePath); err != nil { + Logger.Warning(fmt.Sprintf("Unable to write failed post to file <%s> because <%s>", + filePath, err)) + expEv.lk.RUnlock() + return + } + expEv.lk.RUnlock() +} + +// FilePath returns the file path it should use for saving the failed events +func (expEv *FailedExportersLogg) FilePath() string { + return path.Join(expEv.FailedPostsDir, expEv.Module+PipeSep+UUIDSha1Prefix()+GOBSuffix) +} + +// WriteToFile writes the events to file +func (expEv *FailedExportersLogg) WriteToFile(filePath string) (err error) { + fileOut, err := os.Create(filePath) + if err != nil { return err } + encd := gob.NewEncoder(fileOut) + gob.Register(new(CGREvent)) + err = encd.Encode(expEv) fileOut.Close() return } -type MessageProvider interface { - GetContent() string - GetMeta() string +type FailedExportersLogg struct { + lk sync.RWMutex + Path string + Opts map[string]interface{} // THIS WILL BE META + Format string + Events []interface{} + FailedPostsDir string + Module string } -func (fldPst *FailoverPoster) GetContent() string { - return EmptyString +func AddFailedMessage(failedPostsDir, expPath, format, + module string, ev interface{}, opts map[string]interface{}) { + key := ConcatenatedKey(failedPostsDir, expPath, format, module) + switch module { + case EEs: + // also in case of amqp,amqpv1,s3,sqs and kafka also separe them after queue id + var amqpQueueID string + var s3BucketID string + var sqsQueueID string + var kafkaTopic string + if _, has := opts[AMQPQueueID]; has { + amqpQueueID = IfaceAsString(opts[AMQPQueueID]) + } + if _, has := opts[S3Bucket]; has { + s3BucketID = IfaceAsString(opts[S3Bucket]) + } + if _, has := opts[SQSQueueID]; has { + sqsQueueID = IfaceAsString(opts[SQSQueueID]) + } + if _, has := opts[kafkaTopic]; has { + kafkaTopic = IfaceAsString(opts[KafkaTopic]) + } + if qID := FirstNonEmpty(amqpQueueID, s3BucketID, + sqsQueueID, kafkaTopic); len(qID) != 0 { + key = ConcatenatedKey(key, qID) + } + case Kafka: + } + var failedPost *FailedExportersLogg + if x, ok := failedPostCache.Get(key); ok { + if x != nil { + failedPost = x.(*FailedExportersLogg) + } + } + if failedPost == nil { + failedPost = &FailedExportersLogg{ + Path: expPath, + Format: format, + Opts: opts, + Module: module, + FailedPostsDir: failedPostsDir, + } + failedPostCache.Set(key, failedPost, nil) + } + failedPost.AddEvent(ev) } -func (fldPst *FailoverPoster) GetMeta() string { - return EmptyString +// AddEvent adds one event +func (expEv *FailedExportersLogg) AddEvent(ev interface{}) { + expEv.lk.Lock() + expEv.Events = append(expEv.Events, ev) + expEv.lk.Unlock() +} + +// NewExportEventsFromFile returns ExportEvents from the file +// used only on replay failed post +func NewExportEventsFromFile(filePath string) (expEv *FailedExportersLogg, err error) { + var fileContent []byte + //err = guardian.Guardian.Guard(context.TODO(), func(_ *context.Context) error { + if fileContent, err = os.ReadFile(filePath); err != nil { + return nil, err + } + if err = os.Remove(filePath); err != nil { + return nil, err + } + // }, config.CgrConfig().GeneralCfg().LockingTimeout, FileLockPrefix+filePath) + dec := gob.NewDecoder(bytes.NewBuffer(fileContent)) + // unmarshall it + expEv = new(FailedExportersLogg) + err = dec.Decode(&expEv) + return +} + +type FailoverPoster interface { + ReplayFailedPosts(int) (*FailedExportersLogg, error) +} + +// ReplayFailedPosts tryies to post cdrs again +func (expEv *FailedExportersLogg) ReplayFailedPosts(attempts int) (failedEvents *FailedExportersLogg, err error) { + /* failedEvents = &ExportEvents{ + Path: expEv.Path, + Opts: expEv.Opts, + Format: expEv.Format, + } + + var ee EventExporter + if ee, err = NewEventExporter(&config.EventExporterCfg{ + ID: "ReplayFailedPosts", + Type: expEv.Format, + ExportPath: expEv.Path, + Opts: expEv.Opts, + Attempts: attempts, + FailedPostsDir: MetaNone, + }, config.CgrConfig(), nil, nil); err != nil { + return + } + keyFunc := func() string { return EmptyString } + if expEv.Format == MetaKafkajsonMap || expEv.Format == MetaS3jsonMap { + keyFunc = UUIDSha1Prefix + } + for _, ev := range expEv.Events { + if err = ExportWithAttempts(context.Background(), ee, ev, keyFunc()); err != nil { + failedEvents.AddEvent(ev) + } + } + ee.Close() + if len(failedEvents.Events) > 0 { + err = ErrPartiallyExecuted + } else { + failedEvents = nil + } */ + return nil, nil } diff --git a/utils/failover_export1.go b/utils/failover_export1.go new file mode 100644 index 000000000..0257be868 --- /dev/null +++ b/utils/failover_export1.go @@ -0,0 +1,101 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package utils + +/* +import ( + "encoding/gob" + "fmt" + "os" + "path/filepath" + "sync" +) + +type FailoverPoster struct { + sync.Mutex + mp MessageProvider // e.g kafka +} + +func NewFailoverPoster(dataType, tenant, nodeID, conn, format, fldPostDir string, + logLvl, attempts int) *FailoverPoster { + fldPst := new(FailoverPoster) + switch dataType { + case MetaKafkaLog: + fldPst.mp = NewExportLogger(nodeID, tenant, logLvl, conn, format, attempts, fldPostDir) + } + return fldPst +} + +func (fldPst *FailoverPoster) AddFailedMessage(content interface{}) (err error) { + fldPst.Lock() + meta := fldPst.mp.GetMeta() + filePath := filepath.Join(meta[FailedPostsDir].(string), meta[Format].(string)+ + PipeSep+MetaKafkaLog+GOBSuffix) + var fileOut *os.File + if _, err = os.Stat(filePath); os.IsNotExist(err) { + fileOut, err = os.Create(filePath) + if err != nil { + return fmt.Errorf(fmt.Sprintf(" failed to write logs to file <%s> because <%s>", filePath, err)) + } + } else { + fileOut, err = os.OpenFile(filePath, os.O_RDWR|os.O_APPEND, 0755) + if err != nil { + return err + } + } + failPoster := &FailoverPosterData{ + MetaData: meta, + Content: content.(*CGREvent), + } + enc := gob.NewEncoder(fileOut) + err = enc.Encode(failPoster) + fileOut.Close() + fldPst.Unlock() + return +} + +type MessageProvider interface { + GetContent(filePath string) (string, error) + GetMeta() map[string]interface{} +} + +func NewMessageProvider(dataType string) (MessageProvider, error) { + switch dataType { + case MetaKafkaLog: + return new(ExportLogger), nil + default: + return nil, fmt.Errorf("Invalid Message Provider type in order to read the failed posts") + } +} + + +func (fldPst *FailoverPoster) GetContent(filePath string) (string, error) { + +} + +func (fldPst *FailoverPoster) GetMeta() string { + return EmptyString +} + + +// FailoverPosterData will keep the data and the content of the failed post. It is used when we read from gob file to know these info +type FailoverPosterData struct { + MetaData map[string]interface{} + Content *CGREvent +} */ diff --git a/utils/kafka_logger.go b/utils/kafka_logger.go new file mode 100644 index 000000000..e48a6a680 --- /dev/null +++ b/utils/kafka_logger.go @@ -0,0 +1,234 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package utils + +import ( + "log/syslog" + "sync" + "time" + + "github.com/cgrates/birpc/context" + "github.com/segmentio/kafka-go" +) + +// Logs to kafka +type ExportLogger struct { + sync.Mutex + + logLevel int + fldPostDir string + writer *kafka.Writer + nodeID string + tenant string +} + +// NewExportLogger will export loggers to kafka +func NewExportLogger(nodeID, tenant string, level int, + connOpts, connTopic string, attempts int, fldPostDir string) (el *ExportLogger) { + el = &ExportLogger{ + logLevel: level, + fldPostDir: fldPostDir, + nodeID: nodeID, + tenant: tenant, + writer: &kafka.Writer{ + Addr: kafka.TCP(connOpts), + Topic: connTopic, + MaxAttempts: attempts, + }, + } + return +} + +func (el *ExportLogger) Close() (err error) { + if el.writer != nil { + err = el.writer.Close() + el.writer = nil + } + return +} + +func (el *ExportLogger) call(m string, level int) (err error) { + eventExport := &CGREvent{ + Tenant: el.tenant, + Event: map[string]interface{}{ + NodeID: el.nodeID, + Message: m, + Severity: level, + Timestamp: time.Now().Format("2006-01-02 15:04:05"), + }, + } + // event will be exported through kafka as json format + var content []byte + if content, err = ToUnescapedJSON(eventExport); err != nil { + return + } + if err = el.writer.WriteMessages(context.Background(), kafka.Message{ + Key: []byte(GenUUID()), + Value: content, + }); err != nil { + // if there are any errors in kafka, we will post in FailedPostDirectory + AddFailedMessage(el.fldPostDir, el.writer.Addr.String(), MetaKafkaLog, Kafka, + eventExport, el.GetMeta()) + // also the content should be printed as a stdout logger type + return ErrLoggerChanged + } + return +} + +func (el *ExportLogger) Write(p []byte) (n int, err error) { + n = len(p) + err = el.call(string(p), 8) + return +} + +func (sl *ExportLogger) GetSyslog() *syslog.Writer { + return nil +} + +// GetLogLevel() returns the level logger number for the server +func (el *ExportLogger) GetLogLevel() int { + return el.logLevel +} + +// SetLogLevel changes the log level +func (el *ExportLogger) SetLogLevel(level int) { + el.logLevel = level +} + +// Alert logs to EEs with alert level +func (el *ExportLogger) Alert(m string) (err error) { + if el.logLevel < LOGLEVEL_ALERT { + return nil + } + if err = el.call(m, LOGLEVEL_ALERT); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Alert(m) + err = nil + } + } + return +} + +// Crit logs to EEs with critical level +func (el *ExportLogger) Crit(m string) (err error) { + if el.logLevel < LOGLEVEL_CRITICAL { + return nil + } + if el.call(m, LOGLEVEL_CRITICAL); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Crit(m) + err = nil + } + } + return +} + +// Debug logs to EEs with debug level +func (el *ExportLogger) Debug(m string) (err error) { + if el.logLevel < LOGLEVEL_DEBUG { + return nil + } + if err = el.call(m, LOGLEVEL_DEBUG); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Debug(m) + err = nil + } + } + return +} + +// Emerg logs to EEs with emergency level +func (el *ExportLogger) Emerg(m string) (err error) { + if el.logLevel < LOGLEVEL_EMERGENCY { + return nil + } + if err = el.call(m, LOGLEVEL_EMERGENCY); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Emerg(m) + err = nil + } + } + return +} + +// Err logs to EEs with error level +func (el *ExportLogger) Err(m string) (err error) { + if el.logLevel < LOGLEVEL_ERROR { + return nil + } + if err = el.call(m, LOGLEVEL_ERROR); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Err(m) + err = nil + } + } + return +} + +// Info logs to EEs with info level +func (el *ExportLogger) Info(m string) (err error) { + if el.logLevel < LOGLEVEL_INFO { + return nil + } + if err = el.call(m, LOGLEVEL_INFO); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Info(m) + err = nil + } + } + return +} + +// Notice logs to EEs with notice level +func (el *ExportLogger) Notice(m string) (err error) { + if el.logLevel < LOGLEVEL_NOTICE { + return nil + } + if err = el.call(m, LOGLEVEL_NOTICE); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Notice(m) + err = nil + } + } + return +} + +// Warning logs to EEs with warning level +func (el *ExportLogger) Warning(m string) (err error) { + if el.logLevel < LOGLEVEL_WARNING { + return nil + } + if err = el.call(m, LOGLEVEL_WARNING); err != nil { + if err == ErrLoggerChanged { + NewStdLogger(el.nodeID, el.logLevel).Warning(m) + err = nil + } + } + return +} + +func (el *ExportLogger) GetMeta() map[string]interface{} { + return map[string]interface{}{ + Level: el.logLevel, + Format: el.writer.Topic, + Conn: el.writer.Addr.String(), + FailedPostsDir: el.fldPostDir, + Attempts: el.writer.MaxAttempts, + } +} diff --git a/engine/logger_test.go b/utils/kafka_logger_test.go similarity index 98% rename from engine/logger_test.go rename to utils/kafka_logger_test.go index bcff953a9..7557ca2ae 100644 --- a/engine/logger_test.go +++ b/utils/kafka_logger_test.go @@ -16,17 +16,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see */ -package engine - -import ( - "reflect" - "testing" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" - "github.com/segmentio/kafka-go" -) +package utils +/* func TestLoggerNewLoggerExport(t *testing.T) { cfg := config.NewDefaultCGRConfig() exp := &ExportLogger{ @@ -56,7 +48,6 @@ func TestLoggerNewLoggerDefault(t *testing.T) { } } -/* func TestLoggerNewExportLogger(t *testing.T) { cfg := config.NewDefaultCGRConfig() exp := &ExportLogger{ diff --git a/utils/logger.go b/utils/logger.go index f204f4d23..3ee36dfc3 100644 --- a/utils/logger.go +++ b/utils/logger.go @@ -29,7 +29,8 @@ import ( var Logger LoggerInterface func init() { - Logger, _ = NewLogger(MetaStdLog, EmptyString, 0) + Logger, _ = NewLogger(MetaStdLog, EmptyString, EmptyString, 0, + 0, EmptyString, EmptyString, EmptyString) } // log severities following rfc3164 @@ -60,12 +61,15 @@ type LoggerInterface interface { Write(p []byte) (n int, err error) } -func NewLogger(loggerType, nodeID string, level int) (l LoggerInterface, err error) { +func NewLogger(loggerType, tenant, nodeID string, logLvl, attempts int, connOpts, + topicOpts, fldPostsDir string) (LoggerInterface, error) { switch loggerType { + case MetaKafkaLog: + return NewExportLogger(nodeID, tenant, logLvl, connOpts, topicOpts, attempts, fldPostsDir), nil case MetaStdLog: - return NewStdLogger(nodeID, level), nil + return NewStdLogger(nodeID, logLvl), nil case MetaSysLog: - return NewSysLogger(nodeID, level) + return NewSysLogger(nodeID, logLvl) default: return nil, fmt.Errorf("unsupported logger: <%+s>", loggerType) } diff --git a/utils/logger_test.go b/utils/logger_test.go index e9d0304f4..25b874ae0 100644 --- a/utils/logger_test.go +++ b/utils/logger_test.go @@ -35,7 +35,7 @@ func TestLoggerNewLoggerStdoutOK(t *testing.T) { log.New(os.Stderr, EmptyString, log.LstdFlags), }, } - if rcv, err := NewLogger(MetaStdLog, "1234", 7); err != nil { + if rcv, err := NewLogger(MetaStdLog, EmptyString, "1234", 7, 0, EmptyString, EmptyString, EmptyString); err != nil { t.Error(err) } else if !reflect.DeepEqual(rcv, exp) { t.Errorf("expected: <%+v>, \nreceived: <%+v>", exp, rcv) @@ -46,7 +46,7 @@ func TestLoggerNewLoggerSyslogOK(t *testing.T) { exp := &SysLogger{ logLevel: 7, } - if rcv, err := NewLogger(MetaSysLog, "1234", 7); err != nil { + if rcv, err := NewLogger(MetaSysLog, EmptyString, "1234", 7, 0, EmptyString, EmptyString, EmptyString); err != nil { t.Error(err) } else { exp.syslog = rcv.GetSyslog() @@ -58,7 +58,7 @@ func TestLoggerNewLoggerSyslogOK(t *testing.T) { func TestLoggerNewLoggerUnsupported(t *testing.T) { experr := `unsupported logger: ` - if _, err := NewLogger("unsupported", "1234", 7); err == nil || err.Error() != experr { + if _, err := NewLogger("unsupported", EmptyString, "1234", 7, 0, EmptyString, EmptyString, EmptyString); err == nil || err.Error() != experr { t.Errorf("expected: <%s>, \nreceived: <%+v>", experr, err) } } diff --git a/utils/reflect.go b/utils/reflect.go index c1224f309..4fe96c994 100644 --- a/utils/reflect.go +++ b/utils/reflect.go @@ -194,7 +194,7 @@ func IfaceAsInt64(itm interface{}) (i int64, err error) { } // IfaceAsTInt converts interface to type int -func IfaceAsTInt(itm interface{}) (i int, err error) { +func IfaceAsInt(itm interface{}) (i int, err error) { switch it := itm.(type) { case int: return it, nil diff --git a/utils/reflect_test.go b/utils/reflect_test.go index f15b6c952..45141241b 100644 --- a/utils/reflect_test.go +++ b/utils/reflect_test.go @@ -851,11 +851,11 @@ func TestIfaceAsTInt64Default(t *testing.T) { } func TestIfaceAsTIntDefault(t *testing.T) { - if _, err := IfaceAsTInt(true); err == nil || err.Error() != "cannot convert field: true to int" { + if _, err := IfaceAsInt(true); err == nil || err.Error() != "cannot convert field: true to int" { t.Errorf("Expecting : true to int> ,received: <%+v>", err) } var test time.Duration = 2147483647 - response, _ := IfaceAsTInt(test) + response, _ := IfaceAsInt(test) if !reflect.DeepEqual(response, int(test.Nanoseconds())) { t.Errorf("Expected <%+v> ,received: <%+v>", test.Nanoseconds(), response) }