From 063c29400eed9a45c391274bb3fd626f2849fe86 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 13 Nov 2018 11:50:58 +0200 Subject: [PATCH] Fixes for cpu and memory profiling --- cmd/cgr-engine/cgr-engine.go | 108 +++++++++++++++++++++------------ cmd/cgr-engine/registration.go | 57 ----------------- 2 files changed, 70 insertions(+), 95 deletions(-) delete mode 100644 cmd/cgr-engine/registration.go diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index c47b51ff9..c6ba5cebc 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -23,10 +23,13 @@ import ( "fmt" "log" "os" + "os/signal" + "path" "runtime" "runtime/pprof" "strconv" "strings" + "syscall" "time" "github.com/cgrates/cgrates/agents" @@ -62,12 +65,10 @@ var ( cfgDir = flag.String("config_dir", utils.CONFIG_DIR, "Configuration directory path.") version = flag.Bool("version", false, "Prints the application version.") pidFile = flag.String("pid", "", "Write pid file") - cpupath = flag.String("cpupath", "", "write cpu profile to files") - cputimeout = flag.Duration("cputimeout", 5*time.Second, "Time betwen cpu profile saves") - cpuNoFiles = flag.Int("cpuNoFiles", 5, "Number of cpu profile to write") - mempath = flag.String("mempath", "", "write memory profile to file") - memtimeout = flag.Duration("memtimeout", 5*time.Second, "Time betwen memory profile saves") - memNoFiles = flag.Int("memNoFiles", 5, "Number of memory profile to write") + cpuProfDir = flag.String("cpuprof_dir", "", "write cpu profile to files") + memProfDir = flag.String("memprof_dir", "", "write memory profile to file") + memProfInterval = flag.Duration("memprof_interval", 5*time.Second, "Time betwen memory profile saves") + memProfNrFiles = flag.Int("memprof_nrfiles", 1, "Number of memory profile to write") scheduledShutdown = flag.String("scheduled_shutdown", "", "shutdown the engine after this duration") singlecpu = flag.Bool("singlecpu", false, "Run on single CPU core") syslogger = flag.String("logger", "", "logger <*syslog|*stdout>") @@ -1295,38 +1296,58 @@ func schedCDRsConns(internalCDRSChan chan rpcclient.RpcClientConnection, exitCha } engine.SetSchedCdrsConns(cdrsConn) } -func memprofiling(mempath string, timeout time.Duration, noFile int) { - for i := 1; ; i++ { - time.Sleep(timeout) - f, err := os.Create(fmt.Sprintf("%smem%v.prof", mempath, i)) - if err != nil { - log.Fatal("could not create memory profile file: ", err) - } - runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - log.Fatal("could not write memory profile: ", err) - } + +func memProfFile(memProfPath string) bool { + f, err := os.Create(memProfPath) + if err != nil { + utils.Logger.Crit(fmt.Sprintf("could not create memory profile file: %s", err)) + return false + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + utils.Logger.Crit(fmt.Sprintf("could not write memory profile: %s", err)) f.Close() - if i%noFile == 0 { + return false + } + f.Close() + return true +} + +func memProfiling(memProfDir string, interval time.Duration, nrFiles int, exitChan chan bool) { + for i := 1; ; i++ { + time.Sleep(interval) + memPath := path.Join(memProfDir, fmt.Sprintf("mem%v.prof", i)) + if !memProfFile(memPath) { + exitChan <- true + } + if i%nrFiles == 0 { i = 0 // reset the counting } } } -func cpuprofiling(cpupath string, timeout time.Duration, noFile int) { - for i := 1; ; i++ { - f, err := os.Create(fmt.Sprintf("%scpu%v.prof", cpupath, i)) - if err != nil { - log.Fatal(err) - } - pprof.StartCPUProfile(f) - time.Sleep(timeout) //wait to profile - pprof.StopCPUProfile() - f.Close() - if i%noFile == 0 { - i = 0 // reset the counting - } + +func cpuProfiling(cpuProfDir string, exitChan chan bool, stopChan, doneChan chan struct{}) { + cpuPath := path.Join(cpuProfDir, "cpu.prof") + f, err := os.Create(cpuPath) + if err != nil { + utils.Logger.Crit(fmt.Sprintf("could not create cpu profile file: %s", err)) + exitChan <- true + return } + pprof.StartCPUProfile(f) + <-stopChan + pprof.StopCPUProfile() + f.Close() + doneChan <- struct{}{} } + +func shutdownSingnalHandler(exitChan chan bool) { + c := make(chan os.Signal) + signal.Notify(c, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) + <-c + exitChan <- true +} + func main() { flag.Parse() if *version { @@ -1339,14 +1360,19 @@ func main() { if *singlecpu { runtime.GOMAXPROCS(1) // Having multiple cpus may slow down computing due to CPU management, to be reviewed in future Go releases } - exitChan := make(chan bool) - if *mempath != "" { - go memprofiling(*mempath, *memtimeout, *memNoFiles) + exitChan := make(chan bool) + go shutdownSingnalHandler(exitChan) + + if *memProfDir != "" { + go memProfiling(*memProfDir, *memProfInterval, *memProfNrFiles, exitChan) } - if *cpupath != "" { - go cpuprofiling(*cpupath, *cputimeout, *cpuNoFiles) + cpuProfChanStop := make(chan struct{}) + cpuProfChanDone := make(chan struct{}) + if *cpuProfDir != "" { + go cpuProfiling(*cpuProfDir, exitChan, cpuProfChanStop, cpuProfChanDone) } + if *scheduledShutdown != "" { shutdownDur, err := utils.ParseDurationWithNanosecs(*scheduledShutdown) if err != nil { @@ -1380,6 +1406,7 @@ func main() { lgLevel = *logLevel } utils.Logger.SetLogLevel(lgLevel) + var loadDb engine.LoadStorage var cdrDb engine.CdrStorage var dm *engine.DataManager @@ -1514,8 +1541,6 @@ func main() { // Start FreeSWITCHAgent if cfg.FsAgentCfg().Enabled { go startFsAgent(internalSMGChan, exitChan) - // close all sessions on shutdown - go shutdownSessionmanagerSingnalHandler(exitChan) } // Start SM-Kamailio @@ -1607,6 +1632,13 @@ func main() { internalSMGChan, internalDispatcherSChan, internalAnalyzerSChan, exitChan) <-exitChan + if *cpuProfDir != "" { // wait to end cpuProfiling + cpuProfChanStop <- struct{}{} + <-cpuProfChanDone + } + if *memProfDir != "" { // write last memory profiling + memProfFile(path.Join(*memProfDir, "mem_final.prof")) + } if *pidFile != "" { if err := os.Remove(*pidFile); err != nil { utils.Logger.Warning("Could not remove pid file: " + err.Error()) diff --git a/cmd/cgr-engine/registration.go b/cmd/cgr-engine/registration.go deleted file mode 100644 index a65348da0..000000000 --- a/cmd/cgr-engine/registration.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package main - -import ( - "fmt" - "os" - "os/signal" - "syscall" - - "github.com/cgrates/cgrates/utils" - "github.com/cgrates/rpcclient" -) - -/* -Listens for the SIGTERM, SIGINT, SIGQUIT system signals and closes the storage before exiting. -*/ -func stopRaterSignalHandler(internalCdrStatSChan chan rpcclient.RpcClientConnection, exitChan chan bool) { - c := make(chan os.Signal) - signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) - sig := <-c - - utils.Logger.Info(fmt.Sprintf("Caught signal %v", sig)) - var dummyInt int - select { - case cdrStats := <-internalCdrStatSChan: - cdrStats.Call("CDRStatsV1.Stop", dummyInt, &dummyInt) - default: - } - exitChan <- true -} - -/* -Listens for the SIGTERM, SIGINT, SIGQUIT system signals and shuts down the session manager. -*/ -func shutdownSessionmanagerSingnalHandler(exitChan chan bool) { - c := make(chan os.Signal) - signal.Notify(c, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) - <-c - exitChan <- true -}