From 5024944a2b66bb0275c5fea4331b1fd9cc7f82ca Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 29 Dec 2013 14:17:38 +0100 Subject: [PATCH] History basic documentation, FsCgr tests fixup for localtime compliance, SaveInterval config on history server parsed as time.Duration --- cdrs/fscdr_test.go | 2 +- cmd/cgr-engine/cgr-engine.go | 2 +- config/config.go | 13 +++++++----- config/config_test.go | 4 ++-- config/test_data.txt | 11 ++++++----- data/conf/cgrates.cfg | 18 +++++++++-------- docs/advanced.rst | 1 + docs/cdrclient.rst | 12 +++++++----- docs/history.rst | 38 ++++++++++++++++++++++++++++++++++++ docs/installation.rst | 3 ++- history/file_scribe.go | 7 ++----- 11 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 docs/history.rst diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 2e43bf73e..0fd986bc5 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -108,7 +108,7 @@ func TestFsCdrAsRatedCdr(t *testing.T) { expctRatedCdr := &utils.RatedCDR{CgrId: utils.FSCgrId("01df56f4-d99a-4ef6-b7fe-b924b2415b7f"), AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", CdrHost: "127.0.0.1", CdrSource: FS_CDR_SOURCE, ReqType: utils.RATED, Direction: "*out", Tenant: "ipbx.itsyscom.com", TOR: "call", Account: "dan", Subject: "dan", Destination: "+4986517174963", - AnswerTime: time.Date(2013, 8, 4, 11, 50, 56, 0, time.Local), Duration: time.Duration(4)*time.Second, + AnswerTime: time.Date(2013, 8, 4, 9, 50, 56, 0, time.UTC).Local(), Duration: time.Duration(4)*time.Second, ExtraFields: map[string]string{"effective_caller_id_number": "+4986517174960"}, MediationRunId: "wholesale_run", Cost: -1} if !reflect.DeepEqual(rtCdrOut, expctRatedCdr) { t.Errorf("Received: %v, expected: %v", rtCdrOut, expctRatedCdr) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 0f70faa05..b6820dcef 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -220,7 +220,7 @@ func startHistoryScribe() { var scribeServer history.Scribe if cfg.HistoryServerEnabled { - if scribeServer, err = history.NewFileScribe(cfg.HistoryPath, cfg.HistorySavePeriod); err != nil { + if scribeServer, err = history.NewFileScribe(cfg.HistoryPath, cfg.HistorySaveInterval); err != nil { engine.Logger.Crit(err.Error()) exitChan <- true return diff --git a/config/config.go b/config/config.go index 22f3fb3f7..f2497dd61 100644 --- a/config/config.go +++ b/config/config.go @@ -133,11 +133,11 @@ type CGRConfig struct { FreeswitchPass string // FS socket password FreeswitchReconnects int // number of times to attempt reconnect after connect fails HistoryAgentEnabled bool // Starts History as an agent: . - HistoryServerEnabled bool // Starts History as server: . HistoryServer string // Address where to reach the master history server: + HistoryServerEnabled bool // Starts History as server: . HistoryListen string // History server listening interface: HistoryPath string // Location on disk where to store history files. - HistorySavePeriod string // The timout duration between history writes + HistorySaveInterval time.Duration // The timout duration between history writes } func (self *CGRConfig) setDefaults() error { @@ -225,7 +225,7 @@ func (self *CGRConfig) setDefaults() error { self.HistoryServer = "127.0.0.1:2013" self.HistoryListen = "127.0.0.1:2013" self.HistoryPath = "/var/log/cgrates/history" - self.HistorySavePeriod = "1s" + self.HistorySaveInterval = time.Duration(1) * time.Second return nil } @@ -538,8 +538,11 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { if hasOpt = c.HasOption("history_server", "path"); hasOpt { cfg.HistoryPath, _ = c.GetString("history_server", "path") } - if hasOpt = c.HasOption("history_server", "save_period"); hasOpt { - cfg.HistorySavePeriod, _ = c.GetString("history_server", "save_period") + if hasOpt = c.HasOption("history_server", "save_interval"); hasOpt { + saveIntvlStr,_ := c.GetString("history_server", "save_interval") + if cfg.HistorySaveInterval, errParse = utils.ParseDurationWithSecs(saveIntvlStr); errParse != nil { + return nil, errParse + } } return cfg, nil } diff --git a/config/config_test.go b/config/config_test.go index acf1db745..a67c4fad7 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -130,7 +130,7 @@ func TestDefaults(t *testing.T) { eCfg.HistoryServerEnabled = false eCfg.HistoryListen = "127.0.0.1:2013" eCfg.HistoryPath = "/var/log/cgrates/history" - eCfg.HistorySavePeriod = "1s" + eCfg.HistorySaveInterval = time.Duration(1)*time.Second if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) @@ -251,7 +251,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.HistoryServerEnabled = true eCfg.HistoryListen = "test" eCfg.HistoryPath = "test" - eCfg.HistorySavePeriod = "test" + eCfg.HistorySaveInterval = time.Duration(99)*time.Second if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) t.Log(cfg) diff --git a/config/test_data.txt b/config/test_data.txt index 9bc43ddc2..a593f28f7 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -99,12 +99,13 @@ server = test # Adress where to connect to FreeSWITCH socket. passwd = test # FreeSWITCH socket password. reconnects = 99 # Number of attempts on connect failure. -[history_agent] -enabled = true # Starts History as a client: . -server = test # Address where to reach the master history server: - [history_server] enabled = true # Starts History service: . listen = test # Listening addres for history server: path = test # Location on disk where to store history files. -save_period = test # Timeout duration between saves +save_interval = 99 # Timeout duration between saves + +[history_agent] +enabled = true # Starts History as a client: . +server = test # Address where to reach the master history server: + diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index d457dbe43..5e6fe1033 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -34,7 +34,7 @@ [balancer] # enabled = false # Start Balancer service: . -# listen = 127.0.0.1:2012 # Balancer listen interface: . +# listen = 127.0.0.1:2012 # Balancer listen interface: <""|x.y.z.y:1234>. [rater] # enabled = false # Enable RaterCDRSExportPath service: . @@ -101,11 +101,13 @@ # passwd = ClueCon # FreeSWITCH socket password. # reconnects = 5 # Number of attempts on connect failure. -[history_agent] -#enabled = false # Starts History as a client: . -#server = 127.0.0.1:2013 # Address where to reach the master history server: - [history_server] -#enabled = false # Starts History service: . -#listen = 127.0.0.1:2013 # Listening addres for history server: -#path = /var/log/cgrates/history # Location on disk where to store history files. +# enabled = false # Starts History service: . +# listen = 127.0.0.1:2013 # Listening addres for history server: +# path = /var/log/cgrates/history # Location on disk where to store history files. +# save_interval = 1s # Interval to save changed cache into .git archive + +[history_agent] +# enabled = false # Starts History as a client: . +# server = 127.0.0.1:2013 # Address where to reach the master history server: + diff --git a/docs/advanced.rst b/docs/advanced.rst index d558bbfc7..29a085432 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -8,6 +8,7 @@ cdrserver cdrclient cdrexporter + history ratinglogic diff --git a/docs/cdrclient.rst b/docs/cdrclient.rst index 552e451b3..a75144787 100644 --- a/docs/cdrclient.rst +++ b/docs/cdrclient.rst @@ -3,19 +3,21 @@ CDR Client (cdrc) It's role is to gather offline CDRs and post them to CDR Server(CDRS) component. -Controlled within *cdrc* section of the configuration file. - Part of the *cgr-engine*, can be started on a remote server as standalone component. +Controlled within *cdrc* section of the configuration file. + Has two modes of operation: + - Automated: CDR file processing is triggered on file creation/move. -- Manual: CDR file processing will be triggered at configured time interval (delay/sleep between processes). +- Periodic: CDR file processing will be triggered at configured time interval (delay/sleep between processes) and it will be performed on all files present in the folder (IN) at run time. Principles behind functionality: + - Monitor/process a CDR folder (IN) as outlined above. -- Read every file in the folder, extract the information based on configuration and post it via configured mechanism to CDRS. +- For every file processed, extract the information based on configuration and post it via configured mechanism to CDRS. - The fields extracted out of each CDR row are the same ones depicted in the CDRS documentation (following primary and extra fields concept). -- Once the file processing completes, move it in it's original format in another folder (OUT) in order to avoid re-processing. Here it worths mentioning the auto-detection of duplicated CDRs at server side based on accid and host fields. +- Once the file processing completes, move it in it's original format in another folder (OUT) in order to avoid re-processing. Here it's worth mentioning the auto-detection of duplicated CDRs at server side based on accid and host fields. For the moment we support processing CDRs in the following formats: diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 000000000..c1f80a65f --- /dev/null +++ b/docs/history.rst @@ -0,0 +1,38 @@ +Rates history +============= + +Enhances CGRateS with ability to archive rates modifications. + +Ability to scale by using server-agents approach. +In a distributed environment, there will be a single server (which can be backed up using technologies such as Linux-HA) and more agents sending the modifications to be archived. + +History-Server +-------------- + +Part of the *cgr-engine*. + +Controlled within *history_server* section of the configuration file. + +Stores rates archive in a .git folder, hence making the rating changes available for analysis via any git browser tool (eg: gitg in linux). + +Functionality: + +- On startup reads the rates archive out of .git folder and caches the data. +- When receiving rates information from the agents it will recompile the rates cache. +- Based on configured save interval it will dump the rating cache (if changed) into the .git archive. +- Archives the following rating data: + - Destinations inside *destinations.json* file. + - Rating plans inside *rating_plans.json* file. + - Rating profiles inside *rating_profiles.json* file. + +History-Agent +------------- + +Integrated in the rates loader components. + +Part of *cgr-engine* and *cgr-loader*. + +Enabled via *history_agent* configuration section within *cgr-engine* and *history_server* command line parameter in case of *cgr-loader*. + +Sends the complete rating data loaded into ratingDb to *history_server* for archiving. + diff --git a/docs/installation.rst b/docs/installation.rst index fca70df2d..cda1ee8fa 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -32,7 +32,7 @@ Since on Debian we use Daemontools_ to control the CGRateS another way to check 3.2. Using source ----------------- -After the go environment is installed_ and configured_ issue the following commands: +After the go environment is installed_ (at least go1.2) and configured_ issue the following commands: :: go get github.com/cgrates/cgrates @@ -42,6 +42,7 @@ This command will install the trunk version of CGRateS together with all the nec .. _installed: http://golang.org/doc/install .. _configured: http://golang.org/doc/code.html + 3.3. Post-install ----------------- CGRateS needs at minimum one external database where to keep it's main data as well as logs of it's operation. diff --git a/history/file_scribe.go b/history/file_scribe.go index 7944ff2ba..a5b5dd09c 100644 --- a/history/file_scribe.go +++ b/history/file_scribe.go @@ -49,17 +49,14 @@ type FileScribe struct { savePeriod time.Duration } -func NewFileScribe(fileRoot string, savePeriod string) (*FileScribe, error) { +func NewFileScribe(fileRoot string, saveInterval time.Duration) (*FileScribe, error) { // looking for git gitCommand, err := exec.LookPath("git") if err != nil { return nil, errors.New("Please install git: " + err.Error()) } - s := &FileScribe{fileRoot: fileRoot, gitCommand: gitCommand} + s := &FileScribe{fileRoot: fileRoot, gitCommand: gitCommand, savePeriod: saveInterval} s.loopChecker = make(chan int) - if s.savePeriod, err = time.ParseDuration(savePeriod); err != nil { - return nil, err - } s.gitInit() if err := s.load(DESTINATIONS_FILE); err != nil { return nil, err