diff --git a/apier/v1/core.go b/apier/v1/core.go index feaeea27a..7af7b9216 100644 --- a/apier/v1/core.go +++ b/apier/v1/core.go @@ -74,3 +74,12 @@ func (cS *CoreSv1) StopCPUProfiling(_ string, reply *string) error { *reply = utils.OK return nil } + +// StartMemoryProfiling is used to start MemoryProfiling in the given path +func (cS *CoreSv1) StartMemoryProfiling(args *utils.MemoryPrf, reply *string) error { + if err := cS.cS.StartMemoryProfiling(args); err != nil { + return err + } + *reply = utils.OK + return nil +} diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index f737439b1..94a702650 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -257,45 +257,6 @@ func writePid() { } } -func memProfFile(memProfPath string) bool { - f, err := os.Create(memProfPath) - if err != nil { - utils.Logger.Crit(fmt.Sprintf("could not create memory profile file: %s", err)) - return false - } - runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - utils.Logger.Crit(fmt.Sprintf("could not write memory profile: %s", err)) - f.Close() - return false - } - f.Close() - return true -} - -func memProfiling(memProfDir string, interval time.Duration, nrFiles int, shdWg *sync.WaitGroup, shdChan *utils.SyncedChan) { - tm := time.NewTimer(interval) - for i := 1; ; i++ { - select { - case <-shdChan.Done(): - tm.Stop() - shdWg.Done() - return - case <-tm.C: - } - memPath := path.Join(memProfDir, fmt.Sprintf("mem%v.prof", i)) - if !memProfFile(memPath) { - shdChan.CloseOnce() - shdWg.Done() - return - } - if i%nrFiles == 0 { - i = 0 // reset the counting - } - tm.Reset(interval) - } -} - func singnalHandler(shdWg *sync.WaitGroup, shdChan *utils.SyncedChan) { shutdownSignal := make(chan os.Signal, 1) reloadSignal := make(chan os.Signal, 1) @@ -382,7 +343,7 @@ func main() { if *memProfDir != utils.EmptyString { shdWg.Add(1) - go memProfiling(*memProfDir, *memProfInterval, *memProfNrFiles, shdWg, shdChan) + go cores.MemProfiling(*memProfDir, *memProfInterval, *memProfNrFiles, shdWg, shdChan) } var cpuProfileFile io.Closer var cS *cores.CoreService @@ -723,7 +684,7 @@ func main() { } if *memProfDir != utils.EmptyString { // write last memory profiling - memProfFile(path.Join(*memProfDir, utils.MemProfFileCgr)) + cores.MemProfFile(path.Join(*memProfDir, utils.MemProfFileCgr)) } if *pidFile != utils.EmptyString { if err := os.Remove(*pidFile); err != nil { diff --git a/cores/core.go b/cores/core.go index 4e39f4652..82ea742dd 100644 --- a/cores/core.go +++ b/cores/core.go @@ -22,9 +22,11 @@ import ( "fmt" "io" "os" + "path" "runtime" "runtime/pprof" "sync" + "time" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -39,14 +41,15 @@ func NewCoreService(cfg *config.CGRConfig, caps *engine.Caps, file io.Closer, st return &CoreService{ cfg: cfg, CapsStats: st, - file: file, + fileCPU: file, } } type CoreService struct { cfg *config.CGRConfig CapsStats *engine.CapsStats - file io.Closer + fileCPU io.Closer + fileMem io.Closer fileMx sync.Mutex } @@ -79,13 +82,13 @@ func (cS *CoreService) Status(arg *utils.TenantWithAPIOpts, reply *map[string]in func (cS *CoreService) StartCPUProfiling(argPath string) (err error) { cS.fileMx.Lock() defer cS.fileMx.Unlock() - if cS.file != nil { + if cS.fileCPU != nil { return fmt.Errorf("CPU profiling already started") } if argPath == utils.EmptyString { return utils.NewErrMandatoryIeMissing("Path") } - cS.file, err = StartCPUProfiling(argPath) + cS.fileCPU, err = StartCPUProfiling(argPath) return } @@ -93,10 +96,10 @@ func (cS *CoreService) StartCPUProfiling(argPath string) (err error) { func (cS *CoreService) StopCPUProfiling() (err error) { cS.fileMx.Lock() defer cS.fileMx.Unlock() - if cS.file != nil { + if cS.fileCPU != nil { pprof.StopCPUProfile() - err = cS.file.Close() - cS.file = nil + err = cS.fileCPU.Close() + cS.fileCPU = nil return } return fmt.Errorf(" cannot stop because CPUProfiling is not active") @@ -110,3 +113,52 @@ func StartCPUProfiling(path string) (file io.WriteCloser, err error) { err = pprof.StartCPUProfile(file) return } + +func (cS *CoreService) StartMemoryProfiling(args *utils.MemoryPrf) (err error) { + if args.DirPath == utils.EmptyString { + return utils.NewErrMandatoryIeMissing("Path") + } + shdWg := new(sync.WaitGroup) + shdChan := utils.NewSyncedChan() + go MemProfiling(args.DirPath, args.Interval, args.NrFiles, shdWg, shdChan) + return +} + +func MemProfFile(memProfPath string) bool { + f, err := os.Create(memProfPath) + if err != nil { + utils.Logger.Crit(fmt.Sprintf("could not create memory profile file: %s", err)) + return false + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + utils.Logger.Crit(fmt.Sprintf("could not write memory profile: %s", err)) + f.Close() + return false + } + f.Close() + return true +} + +func MemProfiling(memProfDir string, interval time.Duration, nrFiles int, shdWg *sync.WaitGroup, shdChan *utils.SyncedChan) { + tm := time.NewTimer(interval) + for i := 1; ; i++ { + select { + case <-shdChan.Done(): + tm.Stop() + shdWg.Done() + return + case <-tm.C: + } + memPath := path.Join(memProfDir, fmt.Sprintf("mem%v.prof", i)) + if !MemProfFile(memPath) { + shdChan.CloseOnce() + shdWg.Done() + return + } + if i%nrFiles == 0 { + i = 0 // reset the counting + } + tm.Reset(interval) + } +} diff --git a/utils/coreutils.go b/utils/coreutils.go index 0d188c30b..c2e3cf330 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -741,6 +741,12 @@ type TenantWithAPIOpts struct { APIOpts map[string]interface{} } +type MemoryPrf struct { + DirPath string + Interval time.Duration + NrFiles int +} + type TenantID struct { Tenant string ID string