Update test helpers

- added support for dynamic configuration for dbs
- ConfigJSON field can now be used to partially overwrite an existing
configuration
- extra cgr-engine flags can now be passed to the Run method
- added default db configs for mongo/internal
- implemented helper to load using cgr-loader

engine/libtest.go

@@ -22,6 +22,7 @@ package engine
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -331,7 +332,8 @@ func NewRPCClient(t *testing.T, cfg *config.ListenCfg) *birpc.Client {
 // required for running integration tests.
 type TestEngine struct {
 	ConfigPath     string            // path to the main configuration file
-	ConfigJSON     string            // configuration JSON content (used if ConfigPath is empty)
+	ConfigJSON     string            // JSON cfg content (standalone/overwrites static configs)
+	DBCfg          DBCfg             // custom db settings for dynamic setup (overrides static config)
 	LogBuffer      io.Writer         // captures log output of the test environment
 	PreserveDataDB bool              // prevents automatic data_db flush when set
 	PreserveStorDB bool              // prevents automatic stor_db flush when set
@@ -343,53 +345,115 @@ type TestEngine struct {
 	PreStartHook func(*testing.T, *config.CGRConfig)
 }
 
-// Run initializes a cgr-engine instance for testing, loads tariff plans (if available) and returns
-// an RPC client and the CGRConfig object. It calls t.Fatal on any setup failure.
-func (ng TestEngine) Run(t *testing.T) (*birpc.Client, *config.CGRConfig) {
+// Run initializes a cgr-engine instance for testing. It calls t.Fatal on any setup failure.
+func (ng TestEngine) Run(t *testing.T, extraFlags ...string) (*birpc.Client, *config.CGRConfig) {
 	t.Helper()
-
-	// Parse config files.
-	var cfgPath string
-	switch {
-	case ng.ConfigJSON != "":
-		cfgPath = t.TempDir()
-		filePath := filepath.Join(cfgPath, "cgrates.json")
-		if err := os.WriteFile(filePath, []byte(ng.ConfigJSON), 0644); err != nil {
-			t.Fatal(err)
-		}
-	case ng.ConfigPath != "":
-		cfgPath = ng.ConfigPath
-	default:
-		t.Fatal("missing config source")
-	}
-	cfg, err := config.NewCGRConfigFromPath(cfgPath)
-	if err != nil {
-		t.Fatalf("could not init config from path %s: %v", cfgPath, err)
-	}
-
+	cfg := parseCfg(t, ng.ConfigPath, ng.ConfigJSON, ng.DBCfg)
 	flushDBs(t, cfg, !ng.PreserveDataDB, !ng.PreserveStorDB)
-
 	if ng.PreStartHook != nil {
 		ng.PreStartHook(t, cfg)
 	}
-
 	startEngine(t, cfg, ng.LogBuffer)
 	client := NewRPCClient(t, cfg.ListenCfg())
-
-	var customTpPath string
-	if len(ng.TpFiles) != 0 {
-		customTpPath = t.TempDir()
-	}
-	LoadCSVs(t, client, ng.TpPath, customTpPath, ng.TpFiles)
-
+	LoadCSVs(t, client, ng.TpPath, ng.TpFiles)
 	return client, cfg
 }
 
-// LoadCSVs loads tariff plan data from CSV files into the service. It handles directory creation and file
-// writing for custom paths, and loads data from the specified paths using the provided RPC client.
-func LoadCSVs(t *testing.T, client *birpc.Client, tpPath, customTpPath string, csvFiles map[string]string) {
+// DBParams contains database connection parameters.
+type DBParams struct {
+	Type     *string `json:"db_type,omitempty"`
+	Host     *string `json:"db_host,omitempty"`
+	Port     *int    `json:"db_port,omitempty"`
+	Name     *string `json:"db_name,omitempty"`
+	User     *string `json:"db_user,omitempty"`
+	Password *string `json:"db_password,omitempty"`
+}
+
+// DBCfg holds the configurations for data_db and/or stor_db.
+type DBCfg struct {
+	DataDB *DBParams `json:"data_db,omitempty"`
+	StorDB *DBParams `json:"stor_db,omitempty"`
+}
+
+// parseCfg initializes and returns a CGRConfig. It handles both static and
+// dynamic configs, including custom DB settings. For dynamic configs, it
+// creates temporary configuration files in a new directory.
+func parseCfg(t *testing.T, cfgPath, cfgJSON string, dbCfg DBCfg) (cfg *config.CGRConfig) {
 	t.Helper()
+	if cfgPath == "" && cfgJSON == "" {
+		t.Fatal("missing config source")
+	}
+
+	// Defer CGRConfig constructor to avoid repetition.
+	// cfg (named return) will be set by the deferred function.
+	// cfgPath is guaranteed non-empty on successful return.
+	defer func() {
+		t.Helper()
+		var err error
+		cfg, err = config.NewCGRConfigFromPath(cfgPath)
+		if err != nil {
+			t.Fatalf("could not init config from path %s: %v", cfgPath, err)
+		}
+	}()
+
+	hasCustomDBConfig := dbCfg.DataDB != nil || dbCfg.StorDB != nil
+	if cfgPath != "" && cfgJSON == "" && !hasCustomDBConfig {
+		// Config file already exists and is static; no need for
+		// further processing.
+		return
+	}
+
+	// Reaching this point means the configuration is at least partially dynamic.
+
+	tmp := t.TempDir()
+	if cfgPath != "" {
+		// An existing configuration directory is specified. Since
+		// configuration is not completely static, it's better to copy
+		// its contents to the temporary directory instead.
+		if err := os.CopyFS(tmp, os.DirFS(cfgPath)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cfgPath = tmp
+
+	if hasCustomDBConfig {
+		// Create a new JSON configuration file based on the DBConfigs object.
+		b, err := json.Marshal(dbCfg)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dbFilePath := filepath.Join(cfgPath, "zzz_dynamic_db.json")
+		if err := os.WriteFile(dbFilePath, b, 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if cfgJSON != "" {
+		// A JSON configuration string has been passed to the object.
+		// It can be standalone or used to overwrite sections from an
+		// existing configuration file. In case it's the latter, ensure
+		// the file is processed towards the end.
+		filePath := filepath.Join(cfgPath, "zzz_dynamic_cgrates.json")
+		if err := os.WriteFile(filePath, []byte(cfgJSON), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	return
+}
+
+func LoadCSVsWithCGRLoader(t *testing.T, cfgPath, tpPath string, logBuffer io.Writer, csvFiles map[string]string, extraFlags ...string) {
+	t.Helper()
+
+	if tpPath == "" && len(csvFiles) == 0 {
+		return // nothing to load
+	}
+
 	paths := make([]string, 0, 2)
+	var customTpPath string
+	if len(csvFiles) != 0 {
+		customTpPath = t.TempDir()
+	}
 	if customTpPath != "" {
 		for fileName, content := range csvFiles {
 			filePath := path.Join(customTpPath, fileName)
@@ -402,11 +466,49 @@ func LoadCSVs(t *testing.T, client *birpc.Client, tpPath, customTpPath string, c
 	if tpPath != "" {
 		paths = append(paths, tpPath)
 	}
-	if len(paths) == 0 {
-		return
+
+	for _, path := range paths {
+		flags := []string{"-config_path", cfgPath, "-path", path}
+		flags = append(flags, extraFlags...)
+		loader := exec.Command("cgr-loader", flags...)
+		if logBuffer != nil {
+			loader.Stdout = logBuffer
+			loader.Stderr = logBuffer
+		}
+		if err := loader.Run(); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// LoadCSVs loads tariff plan data from CSV files into the service. It handles directory creation and file
+// writing for custom paths, and loads data from the specified paths using the provided RPC client.
+func LoadCSVs(t *testing.T, client *birpc.Client, tpPath string, csvFiles map[string]string) {
+	t.Helper()
+
+	if tpPath == "" && len(csvFiles) == 0 {
+		return // nothing to load
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	paths := make([]string, 0, 2)
+	var customTpPath string
+	if len(csvFiles) != 0 {
+		customTpPath = t.TempDir()
+	}
+	if customTpPath != "" {
+		for fileName, content := range csvFiles {
+			filePath := path.Join(customTpPath, fileName)
+			if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
+				t.Fatalf("could not write to file %s: %v", filePath, err)
+			}
+		}
+		paths = append(paths, customTpPath)
+	}
+	if tpPath != "" {
+		paths = append(paths, tpPath)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
 	defer cancel()
 	WaitForService(t, ctx, client, utils.APIerSv1)
 
@@ -437,17 +539,20 @@ func flushDBs(t *testing.T, cfg *config.CGRConfig, flushDataDB, flushStorDB bool
 
 // startEngine starts the CGR engine process with the provided configuration. It writes engine logs to the
 // provided logBuffer (if any).
-func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer) {
+func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer, extraFlags ...string) {
 	t.Helper()
 	binPath, err := exec.LookPath("cgr-engine")
 	if err != nil {
 		t.Fatal(err)
 	}
-	engine := exec.Command(
-		binPath,
-		"-config_path", cfg.ConfigPath,
-		"-logger", utils.MetaStdLog,
-	)
+	flags := []string{"-config_path", cfg.ConfigPath}
+	if logBuffer != nil {
+		flags = append(flags, "-logger", utils.MetaStdLog)
+	}
+	if len(extraFlags) != 0 {
+		flags = append(flags, extraFlags...)
+	}
+	engine := exec.Command(binPath, flags...)
 	if logBuffer != nil {
 		engine.Stdout = logBuffer
 		engine.Stderr = logBuffer
@@ -460,15 +565,15 @@ func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer) {
 			t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err)
 		}
 	})
-	fib := utils.FibDuration(time.Millisecond, 0)
+	backoff := utils.FibDuration(time.Millisecond, 0)
 	for i := 0; i < 16; i++ {
-		time.Sleep(fib())
-		if _, err := jsonrpc.Dial(utils.TCP, cfg.ListenCfg().RPCJSONListen); err == nil {
+		time.Sleep(backoff())
+		if _, err = jsonrpc.Dial(utils.TCP, cfg.ListenCfg().RPCJSONListen); err == nil {
 			break
 		}
 	}
 	if err != nil {
-		t.Fatalf("starting cgr-engine on port %s failed: %v", cfg.ListenCfg().RPCJSONListen, err)
+		t.Fatalf("failed to start cgr-engine: %v", err)
 	}
 }
 
@@ -490,3 +595,29 @@ func WaitForService(t *testing.T, ctx *context.Context, client *birpc.Client, se
 		}
 	}
 }
+
+// Default DB configurations. For Redis/MySQL, it's missing because
+// it's the default.
+var (
+	InternalDBCfg = DBCfg{
+		DataDB: &DBParams{
+			Type: utils.StringPointer(utils.MetaInternal),
+		},
+		StorDB: &DBParams{
+			Type: utils.StringPointer(utils.MetaInternal),
+		},
+	}
+	MongoDBCfg = DBCfg{
+		DataDB: &DBParams{
+			Type: utils.StringPointer(utils.MetaMongo),
+			Port: utils.IntPointer(27017),
+			Name: utils.StringPointer("10"),
+		},
+		StorDB: &DBParams{
+			Type:     utils.StringPointer(utils.MetaMongo),
+			Port:     utils.IntPointer(27017),
+			Name:     utils.StringPointer("cgrates"),
+			Password: utils.StringPointer(""),
+		},
+	}
+)