diff --git a/docs/cgrates_json.rst b/docs/cgrates_json.rst index cf75db2e8..160022ece 100644 --- a/docs/cgrates_json.rst +++ b/docs/cgrates_json.rst @@ -5,13 +5,19 @@ ================================== Has a *JSON* format with commented lines starting with *//*. -Organized into configuration sections which offers the advantage of being easily splitable. -All configuration options come with defaults and we have tried our best to choose the best ones for a minimum of efforts necessary when running. -Can be loaded from local folders or remotely using http transport. + +Organized into configuration sections which offers the advantage of being easily splitable. .. hint:: You can split the configuration into any number of *.json* files/directories since the :ref:cgr-engine loads recursively the complete configuration folder, alphabetically ordered +All configuration options come with defaults and we have tried our best to choose the best ones for a minimum of efforts necessary when running. + +Can be loaded from local folders or remotely using HTTP transport. + +The configuration can be loaded at start and reloaded at run time using APIs designed for that. This can be done either as *config pull* (reload from path) or as *config push* (the *JSON BLOB* is sent via API to the engine). + +.. hint:: You can reload from remote HTTP server as well. Below is the default configuration file which comes hardcoded into :ref:cgr-engine: diff --git a/docs/diamagent.rst b/docs/diamagent.rst index 4e526f5dd..0d1669cc9 100644 --- a/docs/diamagent.rst +++ b/docs/diamagent.rst @@ -11,12 +11,12 @@ Implements Diameter_ protocol in a standard agnostic manner, giving users the ab Used mostly in modern mobile networks (LTE/xG). -The **DiameterAgent** is configured via *diameter_agent* section within :ref:`configuration `. - Configuration ------------- +The **DiameterAgent** is configured within *diameter_agent* section from :ref:`JSON configuration `. + Sample config ^^^^^^^^^^^^^ @@ -26,147 +26,235 @@ With explanations in the comments: :: "diameter_agent": { - "enabled": false, // enables the diameter agent: - "listen": "127.0.0.1:3868", // address where to listen for diameter requests - "listen_net": "tcp", // transport type for diameter - "dictionaries_path": "/usr/share/cgrates/diameter/dict/", // path towards directory holding additional dictionaries to load - "sessions_conns": ["*internal"], - "origin_host": "CGR-DA", // diameter Origin-Host AVP used in replies - "origin_realm": "cgrates.org", // diameter Origin-Realm AVP used in replies - "vendor_id": 0, // diameter Vendor-Id AVP used in replies - "product_name": "CGRateS", // diameter Product-Name AVP used in replies - "concurrent_requests": -1, // limit the number of active requests processed by the server <-1|0-n> - "synced_conn_requests": false, // process one request at the time per connection - "asr_template": "*asr", // enable AbortSession message being sent to client - // forcing session disconnection from CGRateS side + "enabled": false, // enables the diameter agent: + "listen": "127.0.0.1:3868", // address where to listen for diameter requests + "listen_net": "tcp", // transport type for diameter + "dictionaries_path": "/usr/share/cgrates/diameter/dict/", // path towards directory + // holding additional dictionaries to load + "sessions_conns": ["*internal"], // connection towards SessionS + "origin_host": "CGR-DA", // diameter Origin-Host AVP used in replies + "origin_realm": "cgrates.org", // diameter Origin-Realm AVP used in replies + "vendor_id": 0, // diameter Vendor-Id AVP used in replies + "product_name": "CGRateS", // diameter Product-Name AVP used in replies + "concurrent_requests": -1, // limit the number of active requests processed by the server <-1|0-n> + "synced_conn_requests": false, // process one request at the time per connection + "asr_template": "*asr", // enable AbortSession message being sent to client + // forcing session disconnection from CGRateS side - "templates":{ // message templates which can be injected within request/replies - "*err": [ - {"tag": "SessionId", "field_id": "Session-Id", "type": "*composed", - "value": "~*req.Session-Id", "mandatory": true}, - {"tag": "OriginHost", "field_id": "Origin-Host", "type": "*composed", - "value": "~*vars.OriginHost", "mandatory": true}, - {"tag": "OriginRealm", "field_id": "Origin-Realm", "type": "*composed", - "value": "~*vars.OriginRealm", "mandatory": true}, + "templates":{ // message templates which can be injected within request/replies + "*err": [ // *err is used mostly in automatic diameter replies with errors + { + "tag": "SessionId", "field_id": "Session-Id", + "type": "*variable", "mandatory": true, + "value": "~*req.Session-Id" + }, + { + "tag": "OriginHost", "field_id": "Origin-Host", + "type": "*variable", "mandatory": true, + "value": "~*vars.OriginHost" + }, + { + "tag": "OriginRealm", "field_id": "Origin-Realm", + "type": "*variable", "mandatory": true, + "value": "~*vars.OriginRealm" + }, ], - "*cca": [ - {"tag": "SessionId", "field_id": "Session-Id", "type": "*composed", - "value": "~*req.Session-Id", "mandatory": true}, - {"tag": "ResultCode", "field_id": "Result-Code", "type": "*constant", - "value": "2001"}, - {"tag": "OriginHost", "field_id": "Origin-Host", "type": "*composed", - "value": "~*vars.OriginHost", "mandatory": true}, - {"tag": "OriginRealm", "field_id": "Origin-Realm", "type": "*composed", - "value": "~*vars.OriginRealm", "mandatory": true}, - {"tag": "AuthApplicationId", "field_id": "Auth-Application-Id", "type": "*composed", - "value": "~*vars.*appid", "mandatory": true}, - {"tag": "CCRequestType", "field_id": "CC-Request-Type", "type": "*composed", - "value": "~*req.CC-Request-Type", "mandatory": true}, - {"tag": "CCRequestNumber", "field_id": "CC-Request-Number", "type": "*composed", - "value": "~*req.CC-Request-Number", "mandatory": true}, + "*cca": [ // *cca is used into CallControlAnswer messages + { + "tag": "SessionId", "field_id": "Session-Id", + "type": "*composed", "mandatory": true, + "value": "~*req.Session-Id" + }, + { + "tag": "ResultCode", "field_id": "Result-Code", + "type": "*constant", "value": "2001"}, + { + "tag": "OriginHost", "field_id": "Origin-Host", + "type": "*variable", "mandatory": true, + "value": "~*vars.OriginHost" + }, + { + "tag": "OriginRealm", "field_id": "Origin-Realm", + "type": "*variable", "mandatory": true, + "value": "~*vars.OriginRealm" + }, + { + "tag": "AuthApplicationId", + "field_id": "Auth-Application-Id", + "type": "*variable", "mandatory": true, + "value": "~*vars.*appid" + }, + { + "tag": "CCRequestType", + "field_id": "CC-Request-Type", + "type": "*variable", "mandatory": true, + "value": "~*req.CC-Request-Type" + }, + { + "tag": "CCRequestNumber", + "field_id": "CC-Request-Number", + "type": "*variable", "mandatory": true, + "value": "~*req.CC-Request-Number" + }, ], - "*asr": [ - {"tag": "SessionId", "field_id": "Session-Id", "type": "*variable", - "value": "~*req.Session-Id", "mandatory": true}, - {"tag": "OriginHost", "field_id": "Origin-Host", "type": "*variable", - "value": "~*req.Destination-Host", "mandatory": true}, - {"tag": "OriginRealm", "field_id": "Origin-Realm", "type": "*variable", - "value": "~*req.Destination-Realm", "mandatory": true}, - {"tag": "DestinationRealm", "field_id": "Destination-Realm", "type": "*variable", - "value": "~*req.Origin-Realm", "mandatory": true}, - {"tag": "DestinationHost", "field_id": "Destination-Host", "type": "*variable", - "value": "~*req.Origin-Host", "mandatory": true}, - {"tag": "AuthApplicationId", "field_id": "Auth-Application-Id", "type": "*variable", - "value": "~*vars.*appid", "mandatory": true}, - {"tag": "UserName", "field_id": "User-Name", "type": "*variable", - "value": "~*req.User-Name", "mandatory": true}, - {"tag": "OriginStateID", "field_id": "Origin-State-Id", "type": "*constant", - "value": "1"}, + "*asr": [ // *asr is used to build AbortSessionRequest + { + "tag": "SessionId", "field_id": "Session-Id", + "type": "*variable", "mandatory": true, + "value": "~*req.Session-Id" + }, + { + "tag": "OriginHost", "field_id": "Origin-Host", + "type": "*variable", "mandatory": true, + "value": "~*req.Destination-Host" + }, + { + "tag": "OriginRealm", "field_id": "Origin-Realm", + "type": "*variable", "mandatory": true, + "value": "~*req.Destination-Realm" + }, + { + "tag": "DestinationRealm", + "field_id": "Destination-Realm", + "type": "*variable", "mandatory": true, + "value": "~*req.Origin-Realm" + }, + { + "tag": "DestinationHost", + "field_id": "Destination-Host", + "type": "*variable", "mandatory": true, + "value": "~*req.Origin-Host" + }, + { + "tag": "AuthApplicationId", + "field_id": "Auth-Application-Id", + "type": "*variable", "mandatory": true, + "value": "~*vars.*appid" + }, + { + "tag": "UserName", "field_id": "User-Name", + "type": "*variable", "mandatory": true, + "value": "~*req.User-Name" + }, + { + "tag": "OriginStateID", "field_id": "Origin-State-Id", + "type": "*constant", "value": "1" + } ] }, - "request_processors": [ // decision logic for message processing + "request_processors": [ // decision logic for message processing { - "id": "SMSes", // id is used for debug in logs (ie: using *log flag) - "filters": [ // list of filters to be applied on message for this processor to run + "id": "SMSes", // id is used for debug in logs (ie: using *log flag) + "filters": [ // list of filters to be applied on message for this processor to run "*string:~*vars.*cmd:CCR", "*string:~*req.CC-Request-Type:4", "*string:~*req.Service-Context-Id:LPP" ], - "flags": ["*event", "*accounts", "*cdrs"], // influence processing logic within CGRateS workflow - "request_fields":[ // data exchanged between Diameter and CGRateS + "flags": ["*event", "*accounts", "*cdrs"], // influence processing logic within CGRateS workflow + "request_fields":[ // data exchanged between Diameter and CGRateS { - "tag": "TOR", "field_id": "ToR", // tag is used in debug, field_id is the field on CGRateS side - "type": "*constant", "value": "*sms"} // type defines the method to provide the value + "tag": "TOR", // tag is used in debug, + "field_id": "ToR", // field_id is the field on CGRateS side + "type": "*constant", // type defines the method to provide the value + "value": "*sms"} { - "tag": "OriginID", "field_id": "OriginID", // OriginID will identify uniquely the session on CGRateS side - "type": "*variable", "mandatory": true, // it's value will be taken from Diameter AVP: - "value": "~*req.Multiple-Services-Credit-Control.Service-Identifier"// Multiple-Services-Credit-Control.Service-Identifier + "tag": "OriginID", // OriginID will identify uniquely + "field_id": "OriginID", // the session on CGRateS side + "type": "*variable", // it's value will be taken from Diameter AVP: + "mandatory": true, // Multiple-Services-Credit-Control.Service-Identifier + "value": "~*req.Multiple-Services-Credit-Control.Service-Identifier" }, { - "tag": "OriginHost", "field_id": "OriginHost", // OriginHost combined with OriginID is used by CGRateS to build the CGRID - "mandatory": true, "type": "*constant", "value": "0.0.0.0" + "tag": "OriginHost", // OriginHost combined with OriginID + "field_id": "OriginHost", // is used by CGRateS to build the CGRID + "mandatory": true, + "type": "*variable", // have the value out of special variable: *vars + "value": "*vars.OriginHost" }, { - "tag": "RequestType", "field_id": "RequestType",// RequestType tells SessionS which charging type to apply for the event - "type": "*constant", "value": "*prepaid" + "tag": "RequestType", // RequestType instructs SessionS + "field_id": "RequestType", // about charging type to apply for the event + "type": "*constant", + "value": "*prepaid" }, { - "tag": "Category", "field_id": "Category", // Category serves for ataching Account and RatingProfile to the request - "type": "*constant", "value": "sms" + "tag": "Category", // Category serves for ataching Account + "field_id": "Category", // and RatingProfile to the request + "type": "*constant", + "value": "sms" }, { - "tag": "Account", "field_id": "Account", // Account serves for ataching Account and RatingProfile to the request - "type": "*variable", "mandatory": true, // value is taken from a groupped AVP ( - "value": "~*req.Subscription-Id.Subscription-Id-Data[~Subscription-Id-Type(0)]" // where Subscription-Id-Type is 0) + "tag": "Account", // Account is required by charging + "field_id": "Account", + "type": "*variable", // value is taken dynamically from a group AVP + "mandatory": true, // where Subscription-Id-Type is 0 + "value": "~*req.Subscription-Id.Subscription-Id-Data[~Subscription-Id-Type(0)]" }, { - "tag": "Destination", "field_id": "Destination", // Destination is used for charging - "type": "*variable", "mandatory": true, // value from Diameter will be mediated before sent to CGRateS + "tag": "Destination", // Destination is used for charging + "field_id": "Destination", // value from Diameter will be mediated before sent to CGRateS + "type": "*variable", + "mandatory": true, "value": "~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:s/^\\+49(\\d+)/int${1}/:s/^0049(\\d+)/int${1}/:s/^49(\\d+)/int${1}/:s/^00(\\d+)/+${1}/:s/^[\\+]?(\\d+)/int${1}/:s/int(\\d+)/+49${1}/" }, { - "tag": "Destination", "field_id": "Destination", // Second Destination will overwrite the first in specific cases - "filters":[ // Only overwrite when filters are matching + "tag": "Destination", // Second Destination will overwrite the first if filter matches + "field_id": "Destination", + "filters":[ // Only overwrite when filters are matching "*notprefix:~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:49", - "*notprefix:~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:3958" + "*notprefix:~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:3312" ], - "type": "*variable", "mandatory": true, - "value": "~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:s/^[\\+]?(\\d+)/int${1}/:s/int(\\d+)/+00${1}/"}, + "type": "*variable", + "mandatory": true, + "value": "~*req.Service-Information.SMS-Information.Recipient-Info.Recipient-Address.Address-Data:s/^[\\+]?(\\d+)/int${1}/:s/int(\\d+)/+00${1}/" + }, { - "tag": "SetupTime", "field_id": "SetupTime", // SetupTime is used by charging + "tag": "SetupTime", // SetupTime is used by charging + "field_id": "SetupTime", "type": "*variable", - "value": "~*req.Event-Timestamp", "mandatory": true + "value": "~*req.Event-Timestamp", + "mandatory": true }, { - "tag": "AnswerTime", "field_id": "AnswerTime", // AnswerTime is used by charging - "type": "*variable", , "mandatory": true, "value": "~*req.Event-Timestamp" + "tag": "AnswerTime", // AnswerTime is used by charging + "field_id": "AnswerTime", + "type": "*variable", + "mandatory": true, + "value": "~*req.Event-Timestamp" }, { - "tag": "Usage", "field_id": "Usage", // Usage is used by charging - "type": "*variable", "mandatory": true, + "tag": "Usage", // Usage is used by charging + "field_id": "Usage", + "type": "*variable", + "mandatory": true, "value": "~*req.Multiple-Services-Credit-Control.Requested-Service-Unit.CC-Service-Specific-Units" }, { "tag": "Originator-SCCP-Address", // Originator-SCCP-Address is an extra field which we want in CDR "field_id": "Originator-SCCP-Address", // not used by CGRateS - "type": "*variable", "mandatory": true, + "type": "*variable", "mandatory": true, "value": "~*req.Service-Information.SMS-Information.Originator-SCCP-Address" }, ], "reply_fields":[ // fields which are sent back to DiameterClient { - "tag": "CCATemplate", // inject complete Template defined as *cca above - "type": "*template", "value": "*cca" + "tag": "CCATemplate", // inject complete Template defined as *cca above + "type": "*template", + "value": "*cca" }, { - "tag": "ResultCode", // Change the ResultCode if the reply received from CGRateS contains a 0 MaxUsage + "tag": "ResultCode", // Change the ResultCode if the reply received from CGRateS contains a 0 MaxUsage "filters": ["*eq:~*cgrep.MaxUsage:0"], - "field_id": "Result-Code", "blocker": true, // do not consider further fields if this one is processed - "type": "*constant", "value": "4012"}, - {"tag": "ResultCode", // Change the ResultCode AVP if there was an error received from CGRateS + "field_id": "Result-Code", + "blocker": true, // do not consider further fields if this one is processed + "type": "*constant", + "value": "4012"}, + {"tag": "ResultCode", // Change the ResultCode AVP if there was an error received from CGRateS "filters": ["*notempty:~*cgrep.Error:"], - "field_id": "Result-Code", "blocker": true, - "type": "*constant", "value": "5030"} + "field_id": "Result-Code", + "blocker": true, + "type": "*constant", + "value": "5030"} ] } @@ -255,7 +343,7 @@ flags The **auxiliary** flags only make sense in combination with **main** ones. - Implemented flags are (in order of priority, and not working in simultaneously unless specified): + Implemented flags are (in order of priority, and not working simultaneously unless specified): * **\*log** Logs the Diameter request/reply. Can be used together with other *main* actions. diff --git a/docs/ers.rst b/docs/ers.rst index a8ef1078b..5f119ef06 100644 --- a/docs/ers.rst +++ b/docs/ers.rst @@ -1,5 +1,196 @@ + +.. _MySQL: https://dev.mysql.com/ +.. _PostgreSQL: https://www.postgresql.org/ +.. _MSSQL: https://www.microsoft.com/en-us/sql-server/ +.. _Kamailio: https://www.kamailio.org/w/ +.. _OpenSIPS: https://opensips.org/ +.. _Kafka_: https://kafka.apache.org/ + +.. EventReaderService: + EventReaderService ================== -TBD \ No newline at end of file +**EventReaderService/ERs** is a subsystem designed to read events coming from external sources and convert them into internal ones. The converted events are then sent to other CGRateS subsystems, like *SessionS* where further processing logic is applied to them. + +The translation between external and internal events is done based on field mapping, defined in :ref:`json configuration `. + + +Configuration +------------- + +The **EventReaderService** is configured within *ers* section from :ref:`JSON configuration `. + + +Sample config +^^^^^^^^^^^^^ + +With explanations in the comments: + +:: + + "ers": { + "enabled": true, // enable the service + "sessions_conns": ["*internal"], // connection towards SessionS + "readers": [ // list of active readers + { + "id": "file_reader2", // file_reader2 reader + "run_delay": -1, // reading of events it is triggered outside of ERs + "field_separator": ";", // field separator definition + "type": "*file_csv", // type of reader, *file_csv can read .csv files + "flags": [ // influence processing logic within CGRateS workflow + "*cdrs", // *cdrs will create CDRs + "*log" // *log will log the events to syslog + ], + "source_path": "/tmp/ers2/in", // location of the files + "processed_path": "/tmp/ers2/out", // move the files here once processed + "content_fields":[ // mapping definition between line index in the file and CGRateS field + { + "tag": "OriginID", // OriginID together with OriginHost will + "field_id": "OriginID", // uniquely identify the session on CGRateS side + "type": "*variable", + "value": "~*req.0",q // take the content from line index 0 + "mandatory": true // in the request file + }, + { + "tag": "RequestType", // RequestType instructs SessionS + "field_id": "RequestType", // about charging type to apply for the event + "type": "*variable", + "value": "~*req.1", + "mandatory": true + }, + { + "tag": "Category", // Category serves for ataching Account + "field_id": "Category", // and RatingProfile to the request + "type": "*constant", + "value": "call", + "mandatory": true + }, + { + "tag": "Account", // Account is required by charging + "field_id": "Account", + "type": "*variable", + "value": "~*req.3", + "mandatory": true + }, + { + "tag": "Subject", // Subject is required by charging + "field_id": "Subject", + "type": "*variable", + "value": "~*req.3", + "mandatory": true + }, + { + "tag": "Destination", // Destination is required by charging + "field_id": "Destination", + "type": "*variable", + "value": "~*req.4:s/0([1-9]\\d+)/+49${1}/", + "mandatory": true // Additional mediation is performed on number format + }, + { + "tag": "AnswerTime", // AnswerTime is required by charging + "field_id": "AnswerTime", + "type": "*variable", + "value": "~*req.5", + "mandatory": true + }, + { + "tag": "Usage", // Usage is required by charging + "field_id": "Usage", + "type": "*variable", + "value": "~*req.6", + "mandatory": true + }, + { + "tag": "HDRExtra1", // HDRExtra1 is transparently stored into CDR + "field_id": "HDRExtra1", // as extra field not used by CGRateS + "type": "*composed", + "value": "~*req.6", + "mandatory": true + } + ], + } + ] + } + + + Config params +^^^^^^^^^^^^^ + +Most of the parameters are explained in :ref:`JSON configuration `, hence we mention here only the ones where additional info is necessary or there will be particular implementation for *EventReaderService*. + + +readers + List of reader profiles which ERs manages. Simultaneous readers of the same type are possible. + +id + Reader identificator, used mostly for debug. The id should be unique per each reader since it can influence updating configuration from different *.json* configuration. + +type + Reader type. Following types are implemented: + + **\*file_csv** + Reader for *comma separated* files. + + **\*partial_csv** + Reader for *comma separated* where content spans over multiple files. + + **\*flatstore** + Reader for Kamailio_/OpenSIPS_ *db_flatstore* files. + + **\*file_xml** + Reader for *.xml* formatted files. + + **\*file_fwv** + Reader for *fixed width value* formatted files. + + **\*kafka_json_map** + Reader for hashmaps within Kafka_ database. + + **\*sql** + Reader for generic content out of *SQL* databases. Supported databases are: MySQL_, PostgreSQL_ and MSSQL_. + +run_delay + Duration interval between consecutive reads from source. If 0 or less, *ERs* relies on external source (ie. Linux inotify for files) for starting the reading process. + +concurrent_requests + Limits the number of concurrent reads from source (ie: the number of simultaneously opened files). + +source_path + Path towards the events source + +processed_path + Optional path for moving the events source to after processing. + +xml_root_path + Used in case of XML content and will specify the prefix path applied to each xml element read. + +tenant + Will auto-populate the Tenant within the API calls sent to CGRateS. It has the form of a RSRField. If undefined, default one from *general* section will be used. + +timezone + Defines the timezone for source content which does not carry that information. If undefined, default one from *general* section will be used. + +filters + List of filters to pass for the reader to process the event. In case of file content without field name, the index will be passed instead of field source path. + +flags + Special tags enforcing the actions/verbs done on an event. There are two types of flags: **main** and **auxiliary**. + + There can be any number of flags or combination of those specified in the list however the flags have priority one against another and only some simultaneous combinations of *main* flags are possible. + + The **main** flags will select mostly the action taken on a request. + + The **auxiliary** flags only make sense in combination with **main** ones. + + Implemented flags are (in order of priority, and not working simultaneously unless specified): + + **\*none** + Disable transfering the request from *Reader* to *CGRateS* side. + + + + + +