diff --git a/apier/v1/tp_it_test.go b/apier/v1/tp_it_test.go index e9d55b874..372b20ce9 100644 --- a/apier/v1/tp_it_test.go +++ b/apier/v1/tp_it_test.go @@ -124,7 +124,7 @@ func testTPExportTPToFolder(t *testing.T) { ExportPath: "/tmp/", ExportedFiles: []string{utils.RatingProfilesCsv, utils.RatingPlansCsv, utils.ActionsCsv, utils.AccountActionsCsv, utils.ChargersCsv, utils.ActionPlansCsv, utils.ResourcesCsv, utils.StatsCsv, utils.ThresholdsCsv, - utils.DestinationsCsv, utils.RatesCsv, utils.DestinationRatesCsv, utils.FiltersCsv, utils.RoutesCsv, utils.AttributesCsv}, + utils.DestinationsCsv, utils.RatesCsv, utils.DestinationRatesCsv, utils.FiltersCsv, utils.RoutesCsv, utils.AttributesCsv, utils.ActionTriggersCsv}, } sort.Strings(expectedTPStas.ExportedFiles) tpid := "TEST_TPID2" diff --git a/console/export_cdrs.go b/console/export_cdrs.go new file mode 100644 index 000000000..029dd3a76 --- /dev/null +++ b/console/export_cdrs.go @@ -0,0 +1,64 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package console + +import ( + "github.com/cgrates/cgrates/utils" +) + +func init() { + c := &CmdExportCdrs{ + name: "export_cdrs", + rpcMethod: utils.APIerSv1ExportCDRs, + } + commands[c.Name()] = c + c.CommandExecuter = &CommandExecuter{c} +} + +// Commander implementation +type CmdExportCdrs struct { + name string + rpcMethod string + rpcParams *utils.ArgExportCDRs + *CommandExecuter +} + +func (self *CmdExportCdrs) Name() string { + return self.name +} + +func (self *CmdExportCdrs) RpcMethod() string { + return self.rpcMethod +} + +func (self *CmdExportCdrs) RpcParams(reset bool) any { + if reset || self.rpcParams == nil { + self.rpcParams = new(utils.ArgExportCDRs) + } + return self.rpcParams +} + +func (self *CmdExportCdrs) PostprocessRpcParams() error { + return nil +} + +func (self *CmdExportCdrs) RpcResult() any { + var reply map[string]any + return &reply +} diff --git a/console/export_cdrs_test.go b/console/export_cdrs_test.go new file mode 100644 index 000000000..df58bb194 --- /dev/null +++ b/console/export_cdrs_test.go @@ -0,0 +1,53 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package console + +import ( + "reflect" + "strings" + "testing" + + v1 "github.com/cgrates/cgrates/apier/v1" + "github.com/cgrates/cgrates/utils" +) + +func TestCmdExportCdrs(t *testing.T) { + // commands map is initiated in init function + command := commands["export_cdrs"] + // verify if ApierSv1 object has method on it + m, ok := reflect.TypeOf(new(v1.APIerSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1]) + if !ok { + t.Fatal("method", command.RpcMethod(), " not found") + } + if m.Type.NumIn() != 4 { // expecting 4 inputs + t.Fatalf("invalid number of input parameters ") + } + // verify the type of input parameter + if ok := m.Type.In(2).AssignableTo(reflect.TypeOf(command.RpcParams(true))); !ok { + t.Fatalf("cannot assign input parameter") + } + // verify the type of output parameter + if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok { + t.Fatalf("cannot assign output parameter <%+v>", reflect.TypeOf(command.RpcResult())) + } + // for coverage purpose + if err := command.PostprocessRpcParams(); err != nil { + t.Fatal(err) + } +} diff --git a/data/tariffplans/tutorial/AccountActions.csv b/data/tariffplans/tutorial/AccountActions.csv index 9535788a8..8e2ce6684 100644 --- a/data/tariffplans/tutorial/AccountActions.csv +++ b/data/tariffplans/tutorial/AccountActions.csv @@ -1,4 +1,4 @@ #Tenant,Account,ActionPlanId,ActionTriggersId,AllowNegative,Disabled -cgrates.org,1001,AP_PACKAGE_10,,, -cgrates.org,1002,AP_PACKAGE_10,,, -cgrates.org,1003,AP_PACKAGE_10,,, \ No newline at end of file +cgrates.org,1001,AP_PACKAGE_10,STANDARD_TRIGGERS,, +cgrates.org,1002,AP_PACKAGE_10,STANDARD_TRIGGERS,, +cgrates.org,1003,AP_PACKAGE_10,STANDARD_TRIGGERS,, \ No newline at end of file diff --git a/data/tariffplans/tutorial/ActionTriggers.csv b/data/tariffplans/tutorial/ActionTriggers.csv new file mode 100644 index 000000000..8816dbc68 --- /dev/null +++ b/data/tariffplans/tutorial/ActionTriggers.csv @@ -0,0 +1,6 @@ +#Tag[0],UniqueId[1],ThresholdType[2],ThresholdValue[3],Recurrent[4],MinSleep[5],ExpiryTime[6],ActivationTime[7],BalanceTag[8],BalanceType[9],BalanceCategories[10],BalanceDestinationIds[11],BalanceRatingSubject[12],BalanceSharedGroup[13],BalanceExpiryTime[14],BalanceTimingIds[15],BalanceWeight[16],BalanceBlocker[17],BalanceDisabled[18],ActionsId[19],Weight[20] +STANDARD_TRIGGERS,,*min_balance,2,false,0,,,,*monetary,,,,,,,,,,ACT_LOG_WARNING,10 +STANDARD_TRIGGERS,,*max_event_counter,5,false,0,,,,*monetary,,FS_USERS,,,,,,,,ACT_LOG_WARNING,10 +STANDARD_TRIGGERS,,*max_balance,20,false,0,,,,*monetary,,,,,,,,,,ACT_LOG_WARNING,10 +STANDARD_TRIGGERS,,*max_balance,100,false,0,,,,*monetary,,,,,,,,,,DISABLE_AND_LOG,10 + diff --git a/data/tariffplans/tutorial/Actions.csv b/data/tariffplans/tutorial/Actions.csv index 256e83424..e6569cf46 100644 --- a/data/tariffplans/tutorial/Actions.csv +++ b/data/tariffplans/tutorial/Actions.csv @@ -1,3 +1,5 @@ #ActionsId[0],Action[1],ExtraParameters[2],Filter[3],BalanceId[4],BalanceType[5],Categories[6],DestinationIds[7],RatingSubject[8],SharedGroup[9],ExpiryTime[10],TimingIds[11],Units[12],BalanceWeight[13],BalanceBlocker[14],BalanceDisabled[15],Weight[16] ACT_TOPUP_RST_10,*topup_reset,,,test,*monetary,,*any,,,*unlimited,,10,10,false,false,10 -ACT_LOG_WARNING,*log,,,,,,,,,,,,,false,false,10 \ No newline at end of file +ACT_LOG_WARNING,*log,,,,,,,,,,,,,false,false,10 +DISABLE_AND_LOG,*log,,,,,,,,,,,,,false,false,10 +DISABLE_AND_LOG,*disable_account,,,,,,,,,,,,,false,false,10 \ No newline at end of file diff --git a/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json b/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json index 5c0214ea4..30616d43b 100644 --- a/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/asterisk_ari/cgrates/etc/cgrates/cgrates.json @@ -106,10 +106,42 @@ "string_indexed_fields": ["*req.Account"], }, +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "exporters": [ + { + "id": "tutorial_export", + "type": "*file_csv", + "export_path": "/tmp/", + "timezone": "UTC", + "synchronous": true, + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, "apiers": { "enabled": true, "scheduler_conns": ["*internal"], + "ees_conns": ["*localhost"], }, diff --git a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json index 0a39d7331..c97a3950f 100644 --- a/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/fs_evsock/cgrates/etc/cgrates/cgrates.json @@ -105,10 +105,42 @@ "string_indexed_fields": ["*req.Account"], }, +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "exporters": [ + { + "id": "tutorial_export", + "type": "*file_csv", + "export_path": "/tmp/", + "timezone": "UTC", + "synchronous": true, + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, "apiers": { "enabled": true, "scheduler_conns": ["*internal"], + "ees_conns": ["*localhost"], }, diff --git a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json index 577bb8d1f..9f4216f39 100644 --- a/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/kamevapi/cgrates/etc/cgrates/cgrates.json @@ -80,6 +80,38 @@ }, +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "exporters": [ + { + "id": "tutorial_export", + "type": "*file_csv", + "export_path": "/tmp/", + "timezone": "UTC", + "synchronous": true, + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, + "resources": { "enabled": true, "string_indexed_fields": ["*req.Account"], @@ -110,6 +142,7 @@ "apiers": { "enabled": true, "scheduler_conns": ["*internal"], + "ees_conns": ["*localhost"], }, diff --git a/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json b/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json index 1a1b742b8..5f33afe77 100644 --- a/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json +++ b/data/tutorials/osips/cgrates/etc/cgrates/cgrates.json @@ -96,10 +96,42 @@ "prefix_indexed_fields": ["*req.Destination"], }, +"ees": { + "enabled": true, + "attributes_conns":["*internal"], + "exporters": [ + { + "id": "tutorial_export", + "type": "*file_csv", + "export_path": "/tmp/", + "timezone": "UTC", + "synchronous": true, + "attempts": 1, + "field_separator": ",", + "fields":[ + {"tag": "CGRID", "path": "*exp.CGRID", "type": "*variable", "value": "~*req.CGRID"}, + {"tag": "RunID", "path": "*exp.RunID", "type": "*variable", "value": "~*req.RunID"}, + {"tag": "ToR", "path": "*exp.ToR", "type": "*variable", "value": "~*req.ToR"}, + {"tag": "OriginID", "path": "*exp.OriginID", "type": "*variable", "value": "~*req.OriginID"}, + {"tag": "RequestType", "path": "*exp.RequestType", "type": "*variable", "value": "~*req.RequestType"}, + {"tag": "Tenant", "path": "*exp.Tenant", "type": "*variable", "value": "~*req.Tenant"}, + {"tag": "Category", "path": "*exp.Category", "type": "*variable", "value": "~*req.Category"}, + {"tag": "Account", "path": "*exp.Account", "type": "*variable", "value": "~*req.Account"}, + {"tag": "Subject", "path": "*exp.Subject", "type": "*variable", "value": "~*req.Subject"}, + {"tag": "Destination", "path": "*exp.Destination", "type": "*variable", "value": "~*req.Destination"}, + {"tag": "SetupTime", "path": "*exp.SetupTime", "type": "*variable", "value": "~*req.SetupTime{*time_string:2006-01-02T15:04:05Z}" }, + {"tag": "AnswerTime", "path": "*exp.AnswerTime", "type": "*variable", "value": "~*req.AnswerTime{*time_string:2006-01-02T15:04:05Z}"}, + {"tag": "Usage", "path": "*exp.Usage", "type": "*variable", "value": "~*req.Usage{*duration_seconds}"}, + {"tag": "Cost", "path": "*exp.Cost", "type": "*variable", "value": "~*req.Cost{*round:4}"}, + ], + }, + ] +}, "apiers": { "enabled": true, "scheduler_conns": ["*localhost"], + "ees_conns": ["*localhost"], }, diff --git a/docs/cdrs.rst b/docs/cdrs.rst index b4b182f6e..8d69b4463 100644 --- a/docs/cdrs.rst +++ b/docs/cdrs.rst @@ -48,9 +48,30 @@ stats_conns Connections towards :ref:`StatS` component to compute stat metrics for CDR events. Empty to disable the functionality. online_cdr_exports - List of :ref:`CDRe` profiles which will be processed for each CDR event. Empty to disable online CDR exports. + List of :ref:`EEs` profiles which will be processed for each CDR event. Empty to disable online CDR exports. +Export types +------------ + +There are two types of exports with common configuration but different data sources: + + +Online exports +^^^^^^^^^^^^^^ + +Are real-time exports, triggered by the CDR event processed by :ref:`CDRs`, and take these events as data source. + +The *online exports* are enabled via *online_cdr_exports* :ref:`JSON configuration ` option within *cdrs*. + +You can control the templates which are to be executed via the filters which are applied for each export template individually. + + +Offline exports +^^^^^^^^^^^^^^^ + +Are exports which are triggered via `CGRateS RPC APIs `_ and they have as data source the CDRs stored within *StorDB*. + APIs logic ---------- @@ -79,7 +100,7 @@ Receives the CDR in the form of *CGRateS Event* together with processing flags a Will store the *CDR* to *StorDB*. Defaults to *store_cdrs* parameter within :ref:`JSON configuration `. If store process fails for one of the CDRs, an automated refund is performed for all derived. \*export - Will export the event matching export profiles. These profiles are defined within *cdre* section inside :ref:`JSON configuration `. Defaults to *true* if there is at least one *online_cdr_exports* profile configured within :ref:`JSON configuration `. + Will export the event matching export profiles. These profiles are defined within *ees* section inside :ref:`JSON configuration `. Defaults to *true* if there is at least one *online_cdr_exports* profile configured within :ref:`JSON configuration `. \*thresholds Will process the event with the :ref:`ThresholdS`, allowing us to execute actions based on filters set for matching profiles. Defaults to *true* if there are connections towards :ref:`ThresholdS` within :ref:`JSON configuration `. diff --git a/docs/cgr-console.rst b/docs/cgr-console.rst index 2edb28eed..1600dfef9 100644 --- a/docs/cgr-console.rst +++ b/docs/cgr-console.rst @@ -13,10 +13,18 @@ Configurable via command line arguments. Usage of cgr-console: -ca_path string path to CA for tls connection(only for self sign certificate) + -connect_attempts int + Connect attempts (default 3) + -connect_timeout int + Connect timeout in seconds (default 1) -crt_path string path to certificate for tls connection -key_path string path to key for tls connection + -max_reconnect_interval int + Maximum reconnect interval + -reconnects int + Reconnect attempts (default 3) -reply_timeout int Reply timeout in seconds (default 300) -rpc_encoding string @@ -31,4 +39,5 @@ Configurable via command line arguments. Prints the application version. + .. hint:: # cgr-console status \ No newline at end of file diff --git a/docs/cgr-engine.rst b/docs/cgr-engine.rst index 1a2ea2e9e..b536333ae 100644 --- a/docs/cgr-engine.rst +++ b/docs/cgr-engine.rst @@ -13,32 +13,37 @@ Able to read the configuration from either a local directory of *.json* files w $ cgr-engine -help Usage of cgr-engine: + -check_config + Verify the config without starting the engine -config_path string - Configuration directory path. (default "/etc/cgrates/") + Configuration directory path. (default "/etc/cgrates/") -cpuprof_dir string - write cpu profile to files + write cpu profile to files -httprof_path string - http address used for program profiling + http address used for program profiling -log_level int - Log level (0-emergency to 7-debug) (default -1) + Log level (0-emergency to 7-debug) (default -1) -logger string - logger <*syslog|*stdout> + logger <*syslog|*stdout> -memprof_dir string - write memory profile to file + write memory profile to file -memprof_interval duration - Time betwen memory profile saves (default 5s) + Time between memory profile saves (default 5s) -memprof_nrfiles int - Number of memory profile to write (default 1) + Number of memory profile to write (default 1) -node_id string - The node ID of the engine + The node ID of the engine -pid string - Write pid file + Write pid file + -preload string + LoaderIDs used to load the data before the engine starts -scheduled_shutdown string - shutdown the engine after this duration + shutdown the engine after this duration -singlecpu - Run on single CPU core + Run on single CPU core -version - Prints the application version. + Prints the application version. + .. hint:: $ cgr-engine -config_path=/etc/cgrates @@ -61,7 +66,7 @@ The components from the diagram can be found documented in the links bellow: sessions rals cdrs - cdre + ees attributes chargers resources diff --git a/docs/cgr-loader.rst b/docs/cgr-loader.rst index 85da28e22..fe17e118a 100644 --- a/docs/cgr-loader.rst +++ b/docs/cgr-loader.rst @@ -23,6 +23,8 @@ Customisable through the use of :ref:`JSON configuration ` or com CacheS component to contact for cache reloads, empty to disable automatic cache reloads (default "*localhost") -caching string Caching strategy used when loading TP + -caching_delay duration + Adds delay before cache reload -config_path string Configuration directory path. -datadb_host string @@ -34,7 +36,7 @@ Customisable through the use of :ref:`JSON configuration ` or com -datadb_port string The DataDb port to bind to. (default "6379") -datadb_type string - The type of the DataDB database <*redis|*mongo> (default "redis") + The type of the DataDB database <*redis|*mongo> (default "*redis") -datadb_user string The DataDb user to sign in as. (default "cgrates") -dbdata_encoding string @@ -51,20 +53,38 @@ Customisable through the use of :ref:`JSON configuration ` or com Load the tariff plan from storDb to dataDb -import_id string Uniquely identify an import/load, postpended to some automatic fields + -mongoConnScheme string + Scheme for MongoDB connection (default "mongodb") + -mongoQueryTimeout duration + The timeout for queries (default 10s) -path string The path to folder containing the data files (default "./") - -recursive - Loads data from folder recursive. + -redisCACertificate string + Path to the CA certificate + -redisClientCertificate string + Path to the client certificate + -redisClientKey string + Path to the client key + -redisCluster + Is the redis datadb a cluster + -redisClusterOndownDelay duration + The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state + -redisClusterSync duration + The sync interval for the redis cluster (default 5s) + -redisConnectAttempts int + The maximum amount of dial attempts (default 20) + -redisConnectTimeout duration + The amount of wait time until timeout for a connection attempt + -redisMaxConns int + The connection pool size (default 10) + -redisReadTimeout duration + The amount of wait time until timeout for reading operations -redisSentinel string The name of redis sentinel - -redisCluster bool - Is the redis datadb a cluster - -cluster_sync string - The sync interval for the redis cluster - -cluster_ondown_delay string - The delay before executing the commands if thredis cluster is in the CLUSTERDOWN state - -mongoQueryTimeout string - The timeout for queries + -redisTLS + Enable TLS when connecting to Redis + -redisWriteTimeout duration + The amount of wait time until timeout for writing operations -remove Will remove instead of adding data from DB -route_id string @@ -78,15 +98,17 @@ Customisable through the use of :ref:`JSON configuration ` or com -stordb_name string The name/number of the storDb to connect to. (default "cgrates") -stordb_passwd string - The storDb user's password. + The storDb user's password. (default "CGRateS.org") -stordb_port string The storDb port to bind to. (default "3306") -stordb_type string - The type of the storDb database <*mysql|*postgres|*mongo> (default "mysql") + The type of the storDb database <*mysql|*postgres|*mongo> (default "*mysql") -stordb_user string The storDb user to sign in as. (default "cgrates") + -tenant string + (default "cgrates.org") -timezone string - Timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> + Timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB> (default "Local") -to_stordb Import the tariff plan from files to storDb -tpid string @@ -94,4 +116,4 @@ Customisable through the use of :ref:`JSON configuration ` or com -verbose Enable detailed verbose logging output -version - Prints the application version. \ No newline at end of file + Prints the application version. diff --git a/docs/cgr-migrator.rst b/docs/cgr-migrator.rst index d05c8e38a..2a7070546 100644 --- a/docs/cgr-migrator.rst +++ b/docs/cgr-migrator.rst @@ -22,7 +22,7 @@ Customisable through the use of :ref:`JSON configuration ` or com -datadb_port string the DataDB port (default "6379") -datadb_type string - the type of the DataDB Database <*redis|*mongo> (default "redis") + the type of the DataDB Database <*redis|*mongo> (default "*redis") -datadb_user string the DataDB user (default "cgrates") -dbdata_encoding string @@ -31,11 +31,17 @@ Customisable through the use of :ref:`JSON configuration ` or com parse loaded data for consistency and errors, without storing it -exec string fire up automatic migration <*set_versions|*cost_details|*accounts|*actions|*action_triggers|*action_plans|*shared_groups|*filters|*stordb|*datadb> + -mongoConnScheme string + Scheme for MongoDB connection (default "mongodb") + -mongoQueryTimeout duration + The timeout for queries (default 10s) + -out_datadb_encoding string + the encoding used to store object Data in strings in move mode (default "*datadb") -out_datadb_host string output DataDB host to connect to (default "*datadb") -out_datadb_name string output DataDB name/number (default "*datadb") - -out_datadb_passwd string + -out_datadb_password string output DataDB password (default "*datadb") -out_datadb_port string output DataDB port (default "*datadb") @@ -43,15 +49,13 @@ Customisable through the use of :ref:`JSON configuration ` or com output DataDB type <*redis|*mongo> (default "*datadb") -out_datadb_user string output DataDB user (default "*datadb") - -out_dbdata_encoding string - the encoding used to store object Data in strings in move mode (default "*datadb") -out_redis_sentinel string the name of redis sentinel (default "*datadb") -out_stordb_host string output StorDB host (default "*stordb") -out_stordb_name string output StorDB name/number (default "*stordb") - -out_stordb_passwd string + -out_stordb_password string output StorDB password (default "*stordb") -out_stordb_port string output StorDB port (default "*stordb") @@ -59,26 +63,42 @@ Customisable through the use of :ref:`JSON configuration ` or com output StorDB type for move mode <*mysql|*postgres|*mongo> (default "*stordb") -out_stordb_user string output StorDB user (default "*stordb") + -redisCACertificate string + Path to the CA certificate + -redisClientCertificate string + Path to the client certificate + -redisClientKey string + Path to the client key + -redisCluster + Is the redis datadb a cluster + -redisClusterOndownDelay duration + The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state + -redisClusterSync duration + The sync interval for the redis cluster (default 5s) + -redisConnectAttempts int + The maximum amount of dial attempts (default 20) + -redisConnectTimeout duration + The amount of wait time until timeout for a connection attempt + -redisMaxConns int + The connection pool size (default 10) + -redisReadTimeout duration + The amount of wait time until timeout for reading operations -redisSentinel string the name of redis sentinel - -redisCluster bool - Is the redis datadb a cluster - -cluster_sync string - The sync interval for the redis cluster - -cluster_ondown_delay string - The delay before executing the commands if thredis cluster is in the CLUSTERDOWN state - -mongoQueryTimeout string - The timeout for queries + -redisTLS + Enable TLS when connecting to Redis + -redisWriteTimeout duration + The amount of wait time until timeout for writing operations -stordb_host string the StorDB host (default "127.0.0.1") -stordb_name string the name/number of the StorDB (default "cgrates") -stordb_passwd string - the StorDB password + the StorDB password (default "CGRateS.org") -stordb_port string the StorDB port (default "3306") -stordb_type string - the type of the StorDB Database <*mysql|*postgres|*mongo> (default "mysql") + the type of the StorDB Database <*mysql|*postgres|*mongo> (default "*mysql") -stordb_user string the StorDB user (default "cgrates") -verbose diff --git a/docs/cgr-tester.rst b/docs/cgr-tester.rst index eaad73e93..e62c61c83 100644 --- a/docs/cgr-tester.rst +++ b/docs/cgr-tester.rst @@ -9,10 +9,14 @@ Command line stress testing tool configurable via command line arguments. $ cgr-tester -h Usage of cgr-tester: + -calls int + run n number of calls (default 100) -category string The Record category to test. (default "call") -config_path string Configuration directory path. + -cps int + run n requests in parallel (default 100) -cpuprofile string write cpu profile to file -datadb_host string @@ -24,44 +28,70 @@ Command line stress testing tool configurable via command line arguments. -datadb_port string The DataDb port to bind to. (default "6379") -datadb_type string - The type of the DataDb database (default "redis") + The type of the DataDb database (default "*redis") -datadb_user string The DataDb user to sign in as. (default "cgrates") -dbdata_encoding string The encoding used to store object data in strings. (default "msgpack") -destination string The destination to use in queries. (default "1002") + -digits int + Number of digits Account and Destination will have (default 10) + -exec string + Pick what you want to test <*sessions|*cost> -file_path string read requests from file with path -json Use JSON RPC + -max_usage duration + Maximum usage a session can have (default 5s) -memprofile string write memory profile to this file - -parallel int - run n requests in parallel + -min_usage duration + Minimum usage a session can have (default 1s) + -mongoConnScheme string + Scheme for MongoDB connection (default "mongodb") + -mongoQueryTimeout duration + The timeout for queries (default 10s) -rater_address string Rater address for remote tests. Empty for internal rater. + -redisCluster + Is the redis datadb a cluster + -redisClusterOndownDelay duration + The delay before executing the commands if the redis cluster is in the CLUSTERDOWN state + -redisClusterSync duration + The sync interval for the redis cluster (default 5s) + -redisConnectAttempts int + The maximum amount of dial attempts (default 20) + -redisConnectTimeout duration + The amount of wait time until timeout for a connection attempt + -redisMaxConns int + The connection pool size (default 10) + -redisReadTimeout duration + The amount of wait time until timeout for reading operations -redisSentinel string The name of redis sentinel - -redisCluster bool - Is the redis datadb a cluster - -cluster_sync string - The sync interval for the redis cluster - -cluster_ondown_delay string - The delay before executing the commands if thredis cluster is in the CLUSTERDOWN state - -mongoQueryTimeout string - The timeout for queries + -redisWriteTimeout duration + The amount of wait time until timeout for writing operations -req_separator string separator for requests in file (default "\n\n") + -request_type string + Request type of the call (default "*rated") -runs int stress cycle number (default 100000) -subject string The rating subject to use in queries. (default "1001") -tenant string The type of record to use in queries. (default "cgrates.org") + -timeout duration + After last call, time out after this much duration (default 10s) -tor string The type of record to use in queries. (default "*voice") + -update_interval duration + Time duration added for each session update (default 1s) -usage string The duration to use in call simulation. (default "1m") + -verbose + Enable detailed verbose logging output -version Prints the application version. diff --git a/docs/cdre.rst b/docs/ees.rst similarity index 81% rename from docs/cdre.rst rename to docs/ees.rst index 4b5ed2ea1..2530635c2 100644 --- a/docs/cdre.rst +++ b/docs/ees.rst @@ -4,49 +4,31 @@ .. _Kafka: https://kafka.apache.org/ -.. _CDRe: +.. _EEs: -CDRe +EEs ==== -**CDRe** is an extension of :ref:`CDRs`, responsible for exporting the *CDR* events processed by :ref:`CDRs`. It is accessed via `CGRateS RPC APIs `_ and configured within *cdre* section inside :ref:`JSON configuration `. +**EventExporterService/EEs** is a subsystem designed to convert internal, already processed events into external ones and then export them to a defined destination. It is accessible via `CGRateS RPC APIs `_. -Export types ------------- +Configuration +------------- -There are two types of exports with common configuration but different data sources: +**EEs** is configured within **ees** section from :ref:`JSON configuration `. +Config params +^^^^^^^^^^^^^ -Online exports -^^^^^^^^^^^^^^ +Most of the parameters are explained in :ref:`JSON configuration `, hence we mention here only the ones where additional info is necessary or there will be particular implementation for *EventExporterService*. -Are real-time exports, triggered by the CDR event processed by :ref:`CDRs`, and take these events as data source. +One **exporters** instance includes the following parameters: -The *online exports* are enabled via *online_cdr_exports* :ref:`JSON configuration ` option within *cdrs*. +id + Exporter identificator, used mostly for debug. The id should be unique per each exporter since it can influence updating configuration from different *.json* configuration. -You can control the templates which are to be executed via the filters which are applied for each export template individually. - - -Offline exports -^^^^^^^^^^^^^^^ - -Are exports which are triggered via `CGRateS RPC APIs `_ and they have as data source the CDRs stored within *StorDB*. - - - -Parameters ----------- - -CDRe -^^^^ - -**CDRe** it is configured within **cdre** section from :ref:`JSON configuration `. - -One **export profile** includes the following parameters: - -export_format +type Specify the type of export which will run. Possible values are: **\*file_csv** @@ -82,6 +64,21 @@ export_format **\*kafka_json_map** Will post the CDR to an `Apache Kafka `_. The export content will be a JSON serialized hmap with fields defined within the *fields* section of the template. + **\*nats_json_map** + Exporter for publishing messages to NATS (Message Queue) in JSON format. + + **\*virt** + In-memory exporter. + + **\*els** + Exporter for Elasticsearch. + + **\*sql** + Exporter for generic content to *SQL* databases. Supported databases are: MySQL_, PostgreSQL_ and MSSQL_. + + **\*rpc** + Exporter for calling APIs through node connections. + export_path Specify the export path. It has special format depending of the export type. @@ -111,6 +108,20 @@ export_path Sample: *localhost:9092?topic=cgrates_cdrs* + **\*sql** + SQL URL with extra parameters. + + Sample: *mysql://cgrates:CGRateS.org@127.0.0.1:3306* + + **\*nats** + NATS URL. + + Sample: *nats://localhost:4222* + + **\*els** + Elasticsearch URL + + Sample: *http://localhost:9200* filters List of filters to pass for the export profile to execute. For the dynamic content (prefixed with *~*) following special variables are available: @@ -128,13 +139,7 @@ synchronous Block further exports until this one finishes. In case of *false* the control will be given to the next export template as soon as this one was started. attempts - Number of attempts before giving up on the export and writing the failed request to file. The failed request will be written to *failed_posts_dir* defined in *general* section. - -field_separator - Field separator to be used in some export types (ie. *\*file_csv*). - -attributes_context - The context used when sending the CDR event to :ref:`AttributeS` for modifications. If empty, there will be no event sent to :ref:`AttributeS`. + Number of attempts before giving up on the export and writing the failed request to file. The failed request will be written to *failed_posts_dir*. fields List of fields for the exported event. Not affecting templates like *\*http_json_cdr* or *\*amqp_json_cdr* with fixed content. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index d8e76af31..47530c40e 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -67,7 +67,7 @@ Regarding the SIP Servers, click on the tab corresponding to the choice you made .. code-block:: bash TOKEN=YOURSIGNALWIRETOKEN # Insert your SignalWire Personal Access Token here - apt-get update && apt-get install -y gnupg2 wget lsb-release + sudo apt-get update && apt-get install -y gnupg2 wget lsb-release wget --http-user=signalwire --http-password=$TOKEN -O /usr/share/keyrings/signalwire-freeswitch-repo.gpg https://freeswitch.signalwire.com/repo/deb/debian-release/signalwire-freeswitch-repo.gpg echo "machine freeswitch.signalwire.com login signalwire password $TOKEN" > /etc/apt/auth.conf chmod 600 /etc/apt/auth.conf @@ -75,7 +75,7 @@ Regarding the SIP Servers, click on the tab corresponding to the choice you made echo "deb-src [signed-by=/usr/share/keyrings/signalwire-freeswitch-repo.gpg] https://freeswitch.signalwire.com/repo/deb/debian-release/ `lsb_release -sc` main" >> /etc/apt/sources.list.d/freeswitch.list # If /etc/freeswitch does not exist, the standard vanilla configuration is deployed - apt-get update && apt-get install -y freeswitch-meta-all + sudo apt-get update && apt-get install -y freeswitch-meta-all .. group-tab:: Asterisk @@ -118,9 +118,9 @@ Regarding the SIP Servers, click on the tab corresponding to the choice you made .. code-block:: bash wget -O- http://deb.kamailio.org/kamailiodebkey.gpg | sudo apt-key add - - echo "deb http://deb.kamailio.org/kamailio56 bullseye main" > /etc/apt/sources.list.d/kamailio.list - apt-get update - apt-get install kamailio kamailio-extra-modules kamailio-json-modules + echo "deb http://deb.kamailio.org/kamailio57 bullseye main" > /etc/apt/sources.list.d/kamailio.list + sudo apt-get update + sudo apt-get install kamailio kamailio-extra-modules kamailio-json-modules .. group-tab:: OpenSIPS @@ -131,7 +131,7 @@ Regarding the SIP Servers, click on the tab corresponding to the choice you made curl https://apt.opensips.org/opensips-org.gpg -o /usr/share/keyrings/opensips-org.gpg echo "deb [signed-by=/usr/share/keyrings/opensips-org.gpg] https://apt.opensips.org bookworm 3.4-releases" >/etc/apt/sources.list.d/opensips.list echo "deb [signed-by=/usr/share/keyrings/opensips-org.gpg] https://apt.opensips.org bookworm cli-nightly" >/etc/apt/sources.list.d/opensips-cli.list - apt-get update + sudo apt-get update sudo apt-get install opensips opensips-mysql-module opensips-cgrates-module opensips-cli Configuration and initialization @@ -157,13 +157,13 @@ Regarding the SIP Servers, we have prepared custom configurations in advance, as .. code-block:: bash - /usr/share/cgrates/tutorials/fs_evsock/freeswitch/etc/init.d/freeswitch start + sudo /usr/share/cgrates/tutorials/fs_evsock/freeswitch/etc/init.d/freeswitch start To verify that FreeSWITCH_ is running, run the following command: .. code-block:: bash - fs_cli -x status + sudo fs_cli -x status .. group-tab:: Asterisk @@ -179,14 +179,14 @@ Regarding the SIP Servers, we have prepared custom configurations in advance, as .. code-block:: bash - /usr/share/cgrates/tutorials/asterisk_ari/asterisk/etc/init.d/asterisk start + sudo /usr/share/cgrates/tutorials/asterisk_ari/asterisk/etc/init.d/asterisk start To verify that Asterisk_ is running, run the following commands: .. code-block:: bash - asterisk -r -s /tmp/cgr_asterisk_ari/asterisk/run/asterisk.ctl + sudo asterisk -r -s /tmp/cgr_asterisk_ari/asterisk/run/asterisk.ctl ari show status .. group-tab:: Kamailio @@ -202,13 +202,13 @@ Regarding the SIP Servers, we have prepared custom configurations in advance, as .. code-block:: bash - /usr/share/cgrates/tutorials/kamevapi/kamailio/etc/init.d/kamailio start + sudo /usr/share/cgrates/tutorials/kamevapi/kamailio/etc/init.d/kamailio start To verify that Kamailio_ is running, run the following command: .. code-block:: bash - kamctl moni + sudo kamctl moni .. group-tab:: OpenSIPS @@ -347,7 +347,7 @@ Regarding the SIP Servers, we have prepared custom configurations in advance, as - **SupplierS**: selection of suppliers for each session (in case of OpenSIPS_, it will work in tandem with their DRouting module); - **StatS**: computing statistics in real-time regarding sessions and their charging; - **ThresholdS**: monitoring and reacting to events coming from above subsystems; - - **CDRe**: exporting rated CDRs from CGR StorDB (export path: */tmp*). + - **EEs**: exporting rated CDRs from CGR StorDB (export path: */tmp*). Just as with the SIP Servers, we have also prepared configurations and init scripts for CGRateS. And just as well, you can manage the CGRateS service using systemctl if you prefer. You can even start it using the cgr-engine binary, like so: @@ -365,19 +365,19 @@ Just as with the SIP Servers, we have also prepared configurations and init scri .. code-block:: bash - /usr/share/cgrates/tutorials/fs_evsock/cgrates/etc/init.d/cgrates start + sudo /usr/share/cgrates/tutorials/fs_evsock/cgrates/etc/init.d/cgrates start .. group-tab:: Asterisk .. code-block:: bash - /usr/share/cgrates/tutorials/asterisk_ari/cgrates/etc/init.d/cgrates start + sudo /usr/share/cgrates/tutorials/asterisk_ari/cgrates/etc/init.d/cgrates start .. group-tab:: Kamailio .. code-block:: bash - /usr/share/cgrates/tutorials/kamevapi/cgrates/etc/init.d/cgrates start + sudo /usr/share/cgrates/tutorials/kamevapi/cgrates/etc/init.d/cgrates start .. group-tab:: OpenSIPS @@ -439,7 +439,7 @@ Test calls 1001 -> 1002 ~~~~~~~~~~~~ -Since the user 1001 is marked as *prepaid* inside the telecom switch, calling between 1001 and 1002 should generate pre-auth and prepaid debits which can be checked with *get_account* command integrated within *cgr-console* tool. Charging will be done based on time of day as described in the tariff plan definition above. +Since the user 1001 is marked as *prepaid* inside the telecom switch, calling between 1001 and 1002 should generate pre-auth and prepaid debits which can be checked with *accounts* command integrated within *cgr-console* tool. Charging will be done based on time of day as described in the tariff plan definition above. .. note:: @@ -480,11 +480,11 @@ CDR Processing CDR Exporting ------------- -Once the CDRs are mediated, they are available to be exported. One can use available RPC APIs for that or directly call exports from console: +Once the CDRs are mediated, they are available to be exported. To export them, you first need to configure your EEs in configs (already done by the cgrates script from earlier). Important fields to populate are "id" (sample: tutorial_export), "type" (sample: *file_csv), "export_path" (sample: /tmp), and "fields" where you define all the data that you want to export. After that, you can use available RPC APIs or directly call export_cdrs from the console to export them: .. code-block:: bash - cgr-console 'cdrs_export ExportArgs={"ExportFormat":"*file_csv", "ExportPath":"/tmp"}' + cgr-console 'export_cdrs ExportArgs={"ExportFormat":"*file_csv", "ExportPath":"/tmp"}' Fraud detection