diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index b8bf7407d..9cfe4d2f2 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -347,12 +347,12 @@ func main() { shdWg.Add(1) stopMemProf = make(chan struct{}) go cores.MemProfiling(*memProfDir, *memProfInterval, *memProfNrFiles, shdWg, stopMemProf, shdChan) + defer func() { + if cS == nil { + close(stopMemProf) + } + }() } - defer func() { - if stopMemProf != nil { - cS.StopMemoryProfiling() - } - }() var cpuProfileFile io.Closer if *cpuProfDir != utils.EmptyString { @@ -361,17 +361,17 @@ func main() { if err != nil { return } + defer func() { + if cS != nil { + cS.StopCPUProfiling() + return + } + if cpuProfileFile != nil { + pprof.StopCPUProfile() + cpuProfileFile.Close() + } + }() } - defer func() { - if cS != nil { - cS.StopCPUProfiling() - return - } - if cpuProfileFile != nil { - pprof.StopCPUProfile() - cpuProfileFile.Close() - } - }() if *scheduledShutdown != utils.EmptyString { shutdownDur, err := utils.ParseDurationWithNanosecs(*scheduledShutdown) diff --git a/cores/core.go b/cores/core.go index a8f22a591..85e43582e 100644 --- a/cores/core.go +++ b/cores/core.go @@ -57,20 +57,75 @@ type CoreService struct { stopMemPrf chan struct{} shdChan *utils.SyncedChan fileCPU io.Closer - fileMem io.Closer fileMx sync.Mutex } // Shutdown is called to shutdown the service func (cS *CoreService) Shutdown() { utils.Logger.Info(fmt.Sprintf("<%s> shutdown initialized", utils.CoreS)) - if cS.stopMemPrf != nil { - close(cS.stopMemPrf) - } + cS.StopChanMemProf() utils.Logger.Info(fmt.Sprintf("<%s> shutdown complete", utils.CoreS)) return } +// StopChanMemProf will stop the MemoryProfiling Channel in order to create +// the final MemoryProfiling when CoreS subsystem will stop. +func (cS *CoreService) StopChanMemProf() bool { + if cS.stopMemPrf != nil { + close(cS.stopMemPrf) + cS.stopMemPrf = nil + return true + } + return false +} + +func StartCPUProfiling(path string) (file io.WriteCloser, err error) { + file, err = os.Create(path) + if err != nil { + return nil, fmt.Errorf("could not create CPU profile: %v", err) + } + err = pprof.StartCPUProfile(file) + return +} + +func MemProfFile(memProfPath string) bool { + f, err := os.Create(memProfPath) + if err != nil { + utils.Logger.Crit(fmt.Sprintf("could not create memory profile file: %s", err)) + return false + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + utils.Logger.Crit(fmt.Sprintf("could not write memory profile: %s", err)) + f.Close() + return false + } + f.Close() + return true +} + +func MemProfiling(memProfDir string, interval time.Duration, nrFiles int, shdWg *sync.WaitGroup, stopChan chan struct{}, shdChan *utils.SyncedChan) { + tm := time.NewTimer(interval) + for i := 1; ; i++ { + select { + case <-stopChan: + tm.Stop() + shdWg.Done() + return + case <-tm.C: + } + if !MemProfFile(path.Join(memProfDir, fmt.Sprintf("mem%v.prof", i))) { + shdChan.CloseOnce() + shdWg.Done() + return + } + if i%nrFiles == 0 { + i = 0 // reset the counting + } + tm.Reset(interval) + } +} + // Status returns the status of the engine func (cS *CoreService) Status(arg *utils.TenantWithAPIOpts, reply *map[string]interface{}) (err error) { memstats := new(runtime.MemStats) @@ -116,24 +171,22 @@ func (cS *CoreService) StopCPUProfiling() (err error) { return fmt.Errorf(" cannot stop because CPUProfiling is not active") } -func StartCPUProfiling(path string) (file io.WriteCloser, err error) { - file, err = os.Create(path) - if err != nil { - return nil, fmt.Errorf("could not create CPU profile: %v", err) - } - err = pprof.StartCPUProfile(file) - return -} - // StartMemoryProfiling is used to start MemoryProfiling in the given path func (cS *CoreService) StartMemoryProfiling(args *utils.MemoryPrf) (err error) { if args.DirPath == utils.EmptyString { return utils.NewErrMandatoryIeMissing("Path") } - cS.shdWg.Add(1) - if cS.stopMemPrf == nil { - cS.stopMemPrf = make(chan struct{}) + if cS.stopMemPrf != nil { + return errors.New("Memory Profiling already started") } + if args.Interval <= 0 { + args.Interval = 5 * time.Second + } + if args.NrFiles == 0 { + args.NrFiles = 1 + } + cS.shdWg.Add(1) + cS.stopMemPrf = make(chan struct{}) go MemProfiling(args.DirPath, args.Interval, args.NrFiles, cS.shdWg, cS.stopMemPrf, cS.shdChan) return } @@ -147,42 +200,3 @@ func (cS *CoreService) StopMemoryProfiling() (err error) { cS.stopMemPrf = nil return } - -func MemProfFile(memProfPath string) bool { - f, err := os.Create(memProfPath) - if err != nil { - utils.Logger.Crit(fmt.Sprintf("could not create memory profile file: %s", err)) - return false - } - runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - utils.Logger.Crit(fmt.Sprintf("could not write memory profile: %s", err)) - f.Close() - return false - } - f.Close() - return true -} - -func MemProfiling(memProfDir string, interval time.Duration, nrFiles int, shdWg *sync.WaitGroup, stopChan chan struct{}, shdChan *utils.SyncedChan) { - tm := time.NewTimer(interval) - for i := 1; ; i++ { - select { - case <-stopChan: - tm.Stop() - shdWg.Done() - return - case <-tm.C: - } - memPath := path.Join(memProfDir, fmt.Sprintf("mem%v.prof", i)) - if !MemProfFile(memPath) { - shdChan.CloseOnce() - shdWg.Done() - return - } - if i%nrFiles == 0 { - i = 0 // reset the counting - } - tm.Reset(interval) - } -}