Added integration tests for agents services

agents/fsagent.go

@@ -358,7 +358,8 @@ func (fsa *FSsessions) disconnectSession(connIdx int, uuid, redirectNr, notify s
 // Shutdown stops all connected fsock connections
 func (fsa *FSsessions) Shutdown() (err error) {
 	for connIdx, fSock := range fsa.conns {
-		if !fSock.Connected() {
+		if fSock == nil ||
+			!fSock.Connected() {
 			utils.Logger.Err(fmt.Sprintf("<%s> Cannot shutdown sessions, fsock not connected for connection index: %v", utils.FreeSWITCHAgent, connIdx))
 			continue
 		}

apier/v1/preload_it_test.go

@@ -21,7 +21,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package v1
 
 import (
-	"fmt"
 	"io/ioutil"
 	"net/rpc"
 	"net/rpc/jsonrpc"
@@ -97,11 +96,11 @@ func testPreloadITStartEngine(t *testing.T) {
 	}
 	fib := utils.Fib()
 	var connected bool
-	for i := 0; i < 200; i++ {
+	for i := 0; i < 25; i++ {
 		time.Sleep(time.Duration(fib()) * time.Millisecond)
 		if _, err := jsonrpc.Dial(utils.TCP, preloadCfg.ListenCfg().RPCJSONListen); err != nil {
-			utils.Logger.Warning(fmt.Sprintf("Error <%s> when opening test connection to: <%s>",
-				err.Error(), preloadCfg.ListenCfg().RPCJSONListen))
+			t.Logf("Error <%s> when opening test connection to: <%s>",
+				err.Error(), preloadCfg.ListenCfg().RPCJSONListen)
 		} else {
 			connected = true
 			break

services/asteriskagent_it_test.go

@@ -0,0 +1,88 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package services
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/cores"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/servmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+func TestAsteriskAgentReload(t *testing.T) {
+	cfg, err := config.NewDefaultCGRConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.SessionSCfg().Enabled = true
+	utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
+	utils.Logger.SetLogLevel(7)
+	filterSChan := make(chan *engine.FilterS, 1)
+	filterSChan <- nil
+	engineShutdown := make(chan struct{}, 1)
+	chS := engine.NewCacheS(cfg, nil)
+
+	cacheSChan := make(chan rpcclient.ClientConnector, 1)
+	cacheSChan <- chS
+
+	server := cores.NewServer(nil)
+	srvMngr := servmanager.NewServiceManager(cfg, engineShutdown)
+	db := NewDataDBService(cfg, nil)
+	anz := NewAnalyzerService(cfg, server, engineShutdown, make(chan rpcclient.ClientConnector, 1))
+	sS := NewSessionService(cfg, db, server, make(chan rpcclient.ClientConnector, 1),
+		engineShutdown, nil, nil, anz)
+	srv := NewAsteriskAgent(cfg, engineShutdown, nil)
+	engine.NewConnManager(cfg, nil)
+	srvMngr.AddServices(srv, sS,
+		NewLoaderService(cfg, db, filterSChan, server, make(chan rpcclient.ClientConnector, 1), nil, anz), db)
+	if err = srvMngr.StartServices(); err != nil {
+		t.Fatal(err)
+	}
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	var reply string
+	if err := cfg.V1ReloadConfigFromPath(&config.ConfigReloadWithOpts{
+		Path:    path.Join("/usr", "share", "cgrates", "tutorial_tests", "asterisk_ari", "cgrates", "etc", "cgrates"),
+		Section: config.AsteriskAgentJSN,
+	}, &reply); err != nil {
+		t.Fatal(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expecting OK ,received %s", reply)
+	}
+	time.Sleep(10 * time.Millisecond) //need to switch to gorutine
+	if !srv.IsRunning() {
+		t.Errorf("Expected service to be running")
+	}
+	cfg.AsteriskAgentCfg().Enabled = false
+	cfg.GetReloadChan(config.AsteriskAgentJSN) <- struct{}{}
+	time.Sleep(10 * time.Millisecond)
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	close(engineShutdown)
+	srvMngr.ShutdownServices(10 * time.Millisecond)
+}

services/diameteragent.go

@@ -76,13 +76,13 @@ func (da *DiameterAgent) Start() (err error) {
 	da.lnet = da.cfg.DiameterAgentCfg().ListenNet
 	da.laddr = da.cfg.DiameterAgentCfg().Listen
 	da.stopChan = make(chan struct{})
-	go func() {
-		if err = da.da.ListenAndServe(da.stopChan); err != nil {
+	go func(d *agents.DiameterAgent) {
+		if err = d.ListenAndServe(da.stopChan); err != nil {
 			utils.Logger.Err(fmt.Sprintf("<%s> error: %s!",
 				utils.DiameterAgent, err))
 			close(da.exitChan)
 		}
-	}()
+	}(da.da)
 	return
 }
 

services/diameteragent_it_test.go

@@ -0,0 +1,88 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package services
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/cores"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/servmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+func TestDiameterAgentReload(t *testing.T) {
+	cfg, err := config.NewDefaultCGRConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.SessionSCfg().Enabled = true
+	utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
+	utils.Logger.SetLogLevel(7)
+	filterSChan := make(chan *engine.FilterS, 1)
+	filterSChan <- nil
+	engineShutdown := make(chan struct{}, 1)
+	chS := engine.NewCacheS(cfg, nil)
+
+	cacheSChan := make(chan rpcclient.ClientConnector, 1)
+	cacheSChan <- chS
+
+	server := cores.NewServer(nil)
+	srvMngr := servmanager.NewServiceManager(cfg, engineShutdown)
+	db := NewDataDBService(cfg, nil)
+	anz := NewAnalyzerService(cfg, server, engineShutdown, make(chan rpcclient.ClientConnector, 1))
+	sS := NewSessionService(cfg, db, server, make(chan rpcclient.ClientConnector, 1),
+		engineShutdown, nil, nil, anz)
+	srv := NewDiameterAgent(cfg, filterSChan, engineShutdown, nil)
+	engine.NewConnManager(cfg, nil)
+	srvMngr.AddServices(srv, sS,
+		NewLoaderService(cfg, db, filterSChan, server, make(chan rpcclient.ClientConnector, 1), nil, anz), db)
+	if err = srvMngr.StartServices(); err != nil {
+		t.Fatal(err)
+	}
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	var reply string
+	if err := cfg.V1ReloadConfigFromPath(&config.ConfigReloadWithOpts{
+		Path:    path.Join("/usr", "share", "cgrates", "conf", "samples", "diamagent_mysql"),
+		Section: config.DA_JSN,
+	}, &reply); err != nil {
+		t.Fatal(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expecting OK ,received %s", reply)
+	}
+	time.Sleep(10 * time.Millisecond) //need to switch to gorutine
+	if !srv.IsRunning() {
+		t.Errorf("Expected service to be running")
+	}
+	cfg.DiameterAgentCfg().Enabled = false
+	cfg.GetReloadChan(config.DA_JSN) <- struct{}{}
+	time.Sleep(10 * time.Millisecond)
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	close(engineShutdown)
+	srvMngr.ShutdownServices(10 * time.Millisecond)
+}

services/freeswitchagent.go

@@ -61,12 +61,12 @@ func (fS *FreeswitchAgent) Start() (err error) {
 
 	fS.fS = agents.NewFSsessions(fS.cfg.FsAgentCfg(), fS.cfg.GeneralCfg().DefaultTimezone, fS.connMgr)
 
-	go func() {
-		if err := fS.fS.Connect(); err != nil {
+	go func(f *agents.FSsessions) {
+		if err := f.Connect(); err != nil {
 			utils.Logger.Err(fmt.Sprintf("<%s> error: %s!", utils.FreeSWITCHAgent, err))
 			close(fS.exitChan) // stop the engine here
 		}
-	}()
+	}(fS.fS)
 	return
 }
 
@@ -78,12 +78,12 @@ func (fS *FreeswitchAgent) Reload() (err error) {
 	fS.Lock()
 	defer fS.Unlock()
 	fS.fS.Reload()
-	go func() {
+	go func(f *agents.FSsessions) {
 		if err := fS.fS.Connect(); err != nil {
 			utils.Logger.Err(fmt.Sprintf("<%s> error: %s!", utils.FreeSWITCHAgent, err))
 			close(fS.exitChan) // stop the engine here
 		}
-	}()
+	}(fS.fS)
 	return
 }
 

services/freeswitchagent_it_test.go

@@ -0,0 +1,93 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package services
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/cores"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/servmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+func TestFreeSwitchAgentReload(t *testing.T) {
+	cfg, err := config.NewDefaultCGRConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.SessionSCfg().Enabled = true
+	utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
+	utils.Logger.SetLogLevel(7)
+	filterSChan := make(chan *engine.FilterS, 1)
+	filterSChan <- nil
+	engineShutdown := make(chan struct{}, 1)
+	chS := engine.NewCacheS(cfg, nil)
+
+	cacheSChan := make(chan rpcclient.ClientConnector, 1)
+	cacheSChan <- chS
+
+	server := cores.NewServer(nil)
+	srvMngr := servmanager.NewServiceManager(cfg, engineShutdown)
+	db := NewDataDBService(cfg, nil)
+	anz := NewAnalyzerService(cfg, server, engineShutdown, make(chan rpcclient.ClientConnector, 1))
+	sS := NewSessionService(cfg, db, server, make(chan rpcclient.ClientConnector, 1),
+		engineShutdown, nil, nil, anz)
+	srv := NewFreeswitchAgent(cfg, engineShutdown, nil)
+	engine.NewConnManager(cfg, nil)
+	srvMngr.AddServices(srv, sS,
+		NewLoaderService(cfg, db, filterSChan, server, make(chan rpcclient.ClientConnector, 1), nil, anz), db)
+	if err = srvMngr.StartServices(); err != nil {
+		t.Fatal(err)
+	}
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	var reply string
+	if err := cfg.V1ReloadConfigFromPath(&config.ConfigReloadWithOpts{
+		Path:    path.Join("/usr", "share", "cgrates", "tutorial_tests", "fs_evsock", "cgrates", "etc", "cgrates"),
+		Section: config.FreeSWITCHAgentJSN,
+	}, &reply); err != nil {
+		t.Fatal(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expecting OK ,received %s", reply)
+	}
+	time.Sleep(10 * time.Millisecond) //need to switch to gorutine
+	if !srv.IsRunning() {
+		t.Errorf("Expected service to be running")
+	}
+	cfg.FsAgentCfg().Enabled = false
+	cfg.GetReloadChan(config.FreeSWITCHAgentJSN) <- struct{}{}
+	time.Sleep(10 * time.Millisecond)
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	select {
+	case <-engineShutdown:
+		// if the chanel was closed by the connect error (we did not start the freeswitch for this tests)
+	default:
+		close(engineShutdown)
+	}
+	srvMngr.ShutdownServices(10 * time.Millisecond)
+}

services/kamailioagent.go

@@ -63,15 +63,15 @@ func (kam *KamailioAgent) Start() (err error) {
 	kam.kam = agents.NewKamailioAgent(kam.cfg.KamAgentCfg(), kam.connMgr,
 		utils.FirstNonEmpty(kam.cfg.KamAgentCfg().Timezone, kam.cfg.GeneralCfg().DefaultTimezone))
 
-	go func() {
-		if err = kam.kam.Connect(); err != nil {
+	go func(k *agents.KamailioAgent) {
+		if err = k.Connect(); err != nil {
 			if strings.Contains(err.Error(), "use of closed network connection") { // if closed by us do not log
 				return
 			}
 			utils.Logger.Err(fmt.Sprintf("<%s> error: %s", utils.KamailioAgent, err))
 			close(kam.exitChan)
 		}
-	}()
+	}(kam.kam)
 	return
 }
 
@@ -84,15 +84,15 @@ func (kam *KamailioAgent) Reload() (err error) {
 	kam.Lock()
 	defer kam.Unlock()
 	kam.kam.Reload()
-	go func() {
-		if err = kam.kam.Connect(); err != nil {
+	go func(k *agents.KamailioAgent) {
+		if err = k.Connect(); err != nil {
 			if strings.Contains(err.Error(), "use of closed network connection") { // if closed by us do not log
 				return
 			}
 			utils.Logger.Err(fmt.Sprintf("<%s> error: %s", utils.KamailioAgent, err))
 			close(kam.exitChan)
 		}
-	}()
+	}(kam.kam)
 	return
 }
 

services/kamailioagent_it_test.go

@@ -0,0 +1,93 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package services
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/cores"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/servmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+func TestKamailioAgentReload(t *testing.T) {
+	cfg, err := config.NewDefaultCGRConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.SessionSCfg().Enabled = true
+	utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
+	utils.Logger.SetLogLevel(7)
+	filterSChan := make(chan *engine.FilterS, 1)
+	filterSChan <- nil
+	engineShutdown := make(chan struct{}, 1)
+	chS := engine.NewCacheS(cfg, nil)
+
+	cacheSChan := make(chan rpcclient.ClientConnector, 1)
+	cacheSChan <- chS
+
+	server := cores.NewServer(nil)
+	srvMngr := servmanager.NewServiceManager(cfg, engineShutdown)
+	db := NewDataDBService(cfg, nil)
+	anz := NewAnalyzerService(cfg, server, engineShutdown, make(chan rpcclient.ClientConnector, 1))
+	sS := NewSessionService(cfg, db, server, make(chan rpcclient.ClientConnector, 1),
+		engineShutdown, nil, nil, anz)
+	srv := NewKamailioAgent(cfg, engineShutdown, nil)
+	engine.NewConnManager(cfg, nil)
+	srvMngr.AddServices(srv, sS,
+		NewLoaderService(cfg, db, filterSChan, server, make(chan rpcclient.ClientConnector, 1), nil, anz), db)
+	if err = srvMngr.StartServices(); err != nil {
+		t.Fatal(err)
+	}
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	var reply string
+	if err := cfg.V1ReloadConfigFromPath(&config.ConfigReloadWithOpts{
+		Path:    path.Join("/usr", "share", "cgrates", "tutorial_tests", "kamevapi", "cgrates", "etc", "cgrates"),
+		Section: config.KamailioAgentJSN,
+	}, &reply); err != nil {
+		t.Fatal(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expecting OK ,received %s", reply)
+	}
+	time.Sleep(10 * time.Millisecond) //need to switch to gorutine
+	if !srv.IsRunning() {
+		t.Errorf("Expected service to be running")
+	}
+	cfg.KamAgentCfg().Enabled = false
+	cfg.GetReloadChan(config.KamailioAgentJSN) <- struct{}{}
+	time.Sleep(10 * time.Millisecond)
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	select {
+	case <-engineShutdown:
+		// if the chanel was closed by the connect error (we did not start the kamailio for this tests)
+	default:
+		close(engineShutdown)
+	}
+	srvMngr.ShutdownServices(10 * time.Millisecond)
+}

services/radiusagent.go

@@ -77,12 +77,12 @@ func (rad *RadiusAgent) Start() (err error) {
 		return
 	}
 	rad.stopChan = make(chan struct{})
-	go func() {
-		if err = rad.rad.ListenAndServe(rad.stopChan); err != nil {
+	go func(r *agents.RadiusAgent) {
+		if err = r.ListenAndServe(rad.stopChan); err != nil {
 			utils.Logger.Err(fmt.Sprintf("<%s> error: <%s>", utils.RadiusAgent, err.Error()))
 			close(rad.exitChan)
 		}
-	}()
+	}(rad.rad)
 	return
 }
 

services/radiusagent_it_test.go

@@ -0,0 +1,88 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package services
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/cores"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/servmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+func TestRadiusAgentReload(t *testing.T) {
+	cfg, err := config.NewDefaultCGRConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.SessionSCfg().Enabled = true
+	utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
+	utils.Logger.SetLogLevel(7)
+	filterSChan := make(chan *engine.FilterS, 1)
+	filterSChan <- nil
+	engineShutdown := make(chan struct{}, 1)
+	chS := engine.NewCacheS(cfg, nil)
+
+	cacheSChan := make(chan rpcclient.ClientConnector, 1)
+	cacheSChan <- chS
+
+	server := cores.NewServer(nil)
+	srvMngr := servmanager.NewServiceManager(cfg, engineShutdown)
+	db := NewDataDBService(cfg, nil)
+	anz := NewAnalyzerService(cfg, server, engineShutdown, make(chan rpcclient.ClientConnector, 1))
+	sS := NewSessionService(cfg, db, server, make(chan rpcclient.ClientConnector, 1),
+		engineShutdown, nil, nil, anz)
+	srv := NewRadiusAgent(cfg, filterSChan, engineShutdown, nil)
+	engine.NewConnManager(cfg, nil)
+	srvMngr.AddServices(srv, sS,
+		NewLoaderService(cfg, db, filterSChan, server, make(chan rpcclient.ClientConnector, 1), nil, anz), db)
+	if err = srvMngr.StartServices(); err != nil {
+		t.Fatal(err)
+	}
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	var reply string
+	if err := cfg.V1ReloadConfigFromPath(&config.ConfigReloadWithOpts{
+		Path:    path.Join("/usr", "share", "cgrates", "conf", "samples", "radagent_mysql"),
+		Section: config.RA_JSN,
+	}, &reply); err != nil {
+		t.Fatal(err)
+	} else if reply != utils.OK {
+		t.Errorf("Expecting OK ,received %s", reply)
+	}
+	time.Sleep(10 * time.Millisecond) //need to switch to gorutine
+	if !srv.IsRunning() {
+		t.Errorf("Expected service to be running")
+	}
+	cfg.RadiusAgentCfg().Enabled = false
+	cfg.GetReloadChan(config.RA_JSN) <- struct{}{}
+	time.Sleep(10 * time.Millisecond)
+	if srv.IsRunning() {
+		t.Errorf("Expected service to be down")
+	}
+	close(engineShutdown)
+	srvMngr.ShutdownServices(10 * time.Millisecond)
+}