From 531affc8aed592669abc1752a1c760dd76dc22fb Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Wed, 16 Oct 2024 11:23:02 +0300 Subject: [PATCH] Update test helpers - added support for dynamic configuration for dbs - ConfigJSON field can now be used to partially overwrite an existing configuration - extra cgr-engine flags can now be passed to the Run method - added default db configs for mongo/internal - implemented helper to load using cgr-loader --- engine/libtest.go | 229 ++++++++++++++---- general_tests/balance_timings_it_test.go | 6 +- .../reproc_cdrs_for_stats_it_test.go | 4 +- .../session_bkup_interval_it_test.go | 2 +- 4 files changed, 185 insertions(+), 56 deletions(-) diff --git a/engine/libtest.go b/engine/libtest.go index 7130f6e9a..7ed86363e 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -22,6 +22,7 @@ package engine import ( "bytes" + "encoding/json" "fmt" "io" "os" @@ -331,7 +332,8 @@ func NewRPCClient(t *testing.T, cfg *config.ListenCfg) *birpc.Client { // required for running integration tests. type TestEngine struct { ConfigPath string // path to the main configuration file - ConfigJSON string // configuration JSON content (used if ConfigPath is empty) + ConfigJSON string // JSON cfg content (standalone/overwrites static configs) + DBCfg DBCfg // custom db settings for dynamic setup (overrides static config) LogBuffer io.Writer // captures log output of the test environment PreserveDataDB bool // prevents automatic data_db flush when set PreserveStorDB bool // prevents automatic stor_db flush when set @@ -343,53 +345,115 @@ type TestEngine struct { PreStartHook func(*testing.T, *config.CGRConfig) } -// Run initializes a cgr-engine instance for testing, loads tariff plans (if available) and returns -// an RPC client and the CGRConfig object. It calls t.Fatal on any setup failure. -func (ng TestEngine) Run(t *testing.T) (*birpc.Client, *config.CGRConfig) { +// Run initializes a cgr-engine instance for testing. It calls t.Fatal on any setup failure. +func (ng TestEngine) Run(t *testing.T, extraFlags ...string) (*birpc.Client, *config.CGRConfig) { t.Helper() - - // Parse config files. - var cfgPath string - switch { - case ng.ConfigJSON != "": - cfgPath = t.TempDir() - filePath := filepath.Join(cfgPath, "cgrates.json") - if err := os.WriteFile(filePath, []byte(ng.ConfigJSON), 0644); err != nil { - t.Fatal(err) - } - case ng.ConfigPath != "": - cfgPath = ng.ConfigPath - default: - t.Fatal("missing config source") - } - cfg, err := config.NewCGRConfigFromPath(cfgPath) - if err != nil { - t.Fatalf("could not init config from path %s: %v", cfgPath, err) - } - + cfg := parseCfg(t, ng.ConfigPath, ng.ConfigJSON, ng.DBCfg) flushDBs(t, cfg, !ng.PreserveDataDB, !ng.PreserveStorDB) - if ng.PreStartHook != nil { ng.PreStartHook(t, cfg) } - startEngine(t, cfg, ng.LogBuffer) client := NewRPCClient(t, cfg.ListenCfg()) - - var customTpPath string - if len(ng.TpFiles) != 0 { - customTpPath = t.TempDir() - } - LoadCSVs(t, client, ng.TpPath, customTpPath, ng.TpFiles) - + LoadCSVs(t, client, ng.TpPath, ng.TpFiles) return client, cfg } -// LoadCSVs loads tariff plan data from CSV files into the service. It handles directory creation and file -// writing for custom paths, and loads data from the specified paths using the provided RPC client. -func LoadCSVs(t *testing.T, client *birpc.Client, tpPath, customTpPath string, csvFiles map[string]string) { +// DBParams contains database connection parameters. +type DBParams struct { + Type *string `json:"db_type,omitempty"` + Host *string `json:"db_host,omitempty"` + Port *int `json:"db_port,omitempty"` + Name *string `json:"db_name,omitempty"` + User *string `json:"db_user,omitempty"` + Password *string `json:"db_password,omitempty"` +} + +// DBCfg holds the configurations for data_db and/or stor_db. +type DBCfg struct { + DataDB *DBParams `json:"data_db,omitempty"` + StorDB *DBParams `json:"stor_db,omitempty"` +} + +// parseCfg initializes and returns a CGRConfig. It handles both static and +// dynamic configs, including custom DB settings. For dynamic configs, it +// creates temporary configuration files in a new directory. +func parseCfg(t *testing.T, cfgPath, cfgJSON string, dbCfg DBCfg) (cfg *config.CGRConfig) { t.Helper() + if cfgPath == "" && cfgJSON == "" { + t.Fatal("missing config source") + } + + // Defer CGRConfig constructor to avoid repetition. + // cfg (named return) will be set by the deferred function. + // cfgPath is guaranteed non-empty on successful return. + defer func() { + t.Helper() + var err error + cfg, err = config.NewCGRConfigFromPath(cfgPath) + if err != nil { + t.Fatalf("could not init config from path %s: %v", cfgPath, err) + } + }() + + hasCustomDBConfig := dbCfg.DataDB != nil || dbCfg.StorDB != nil + if cfgPath != "" && cfgJSON == "" && !hasCustomDBConfig { + // Config file already exists and is static; no need for + // further processing. + return + } + + // Reaching this point means the configuration is at least partially dynamic. + + tmp := t.TempDir() + if cfgPath != "" { + // An existing configuration directory is specified. Since + // configuration is not completely static, it's better to copy + // its contents to the temporary directory instead. + if err := os.CopyFS(tmp, os.DirFS(cfgPath)); err != nil { + t.Fatal(err) + } + } + cfgPath = tmp + + if hasCustomDBConfig { + // Create a new JSON configuration file based on the DBConfigs object. + b, err := json.Marshal(dbCfg) + if err != nil { + t.Fatal(err) + } + dbFilePath := filepath.Join(cfgPath, "zzz_dynamic_db.json") + if err := os.WriteFile(dbFilePath, b, 0644); err != nil { + t.Fatal(err) + } + } + + if cfgJSON != "" { + // A JSON configuration string has been passed to the object. + // It can be standalone or used to overwrite sections from an + // existing configuration file. In case it's the latter, ensure + // the file is processed towards the end. + filePath := filepath.Join(cfgPath, "zzz_dynamic_cgrates.json") + if err := os.WriteFile(filePath, []byte(cfgJSON), 0644); err != nil { + t.Fatal(err) + } + } + + return +} + +func LoadCSVsWithCGRLoader(t *testing.T, cfgPath, tpPath string, logBuffer io.Writer, csvFiles map[string]string, extraFlags ...string) { + t.Helper() + + if tpPath == "" && len(csvFiles) == 0 { + return // nothing to load + } + paths := make([]string, 0, 2) + var customTpPath string + if len(csvFiles) != 0 { + customTpPath = t.TempDir() + } if customTpPath != "" { for fileName, content := range csvFiles { filePath := path.Join(customTpPath, fileName) @@ -402,11 +466,49 @@ func LoadCSVs(t *testing.T, client *birpc.Client, tpPath, customTpPath string, c if tpPath != "" { paths = append(paths, tpPath) } - if len(paths) == 0 { - return + + for _, path := range paths { + flags := []string{"-config_path", cfgPath, "-path", path} + flags = append(flags, extraFlags...) + loader := exec.Command("cgr-loader", flags...) + if logBuffer != nil { + loader.Stdout = logBuffer + loader.Stderr = logBuffer + } + if err := loader.Run(); err != nil { + t.Fatal(err) + } + } +} + +// LoadCSVs loads tariff plan data from CSV files into the service. It handles directory creation and file +// writing for custom paths, and loads data from the specified paths using the provided RPC client. +func LoadCSVs(t *testing.T, client *birpc.Client, tpPath string, csvFiles map[string]string) { + t.Helper() + + if tpPath == "" && len(csvFiles) == 0 { + return // nothing to load } - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + paths := make([]string, 0, 2) + var customTpPath string + if len(csvFiles) != 0 { + customTpPath = t.TempDir() + } + if customTpPath != "" { + for fileName, content := range csvFiles { + filePath := path.Join(customTpPath, fileName) + if err := os.WriteFile(filePath, []byte(content), 0644); err != nil { + t.Fatalf("could not write to file %s: %v", filePath, err) + } + } + paths = append(paths, customTpPath) + } + if tpPath != "" { + paths = append(paths, tpPath) + } + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() WaitForService(t, ctx, client, utils.APIerSv1) @@ -437,17 +539,20 @@ func flushDBs(t *testing.T, cfg *config.CGRConfig, flushDataDB, flushStorDB bool // startEngine starts the CGR engine process with the provided configuration. It writes engine logs to the // provided logBuffer (if any). -func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer) { +func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer, extraFlags ...string) { t.Helper() binPath, err := exec.LookPath("cgr-engine") if err != nil { t.Fatal(err) } - engine := exec.Command( - binPath, - "-config_path", cfg.ConfigPath, - "-logger", utils.MetaStdLog, - ) + flags := []string{"-config_path", cfg.ConfigPath} + if logBuffer != nil { + flags = append(flags, "-logger", utils.MetaStdLog) + } + if len(extraFlags) != 0 { + flags = append(flags, extraFlags...) + } + engine := exec.Command(binPath, flags...) if logBuffer != nil { engine.Stdout = logBuffer engine.Stderr = logBuffer @@ -460,15 +565,15 @@ func startEngine(t *testing.T, cfg *config.CGRConfig, logBuffer io.Writer) { t.Errorf("failed to kill cgr-engine process (%d): %v", engine.Process.Pid, err) } }) - fib := utils.FibDuration(time.Millisecond, 0) + backoff := utils.FibDuration(time.Millisecond, 0) for i := 0; i < 16; i++ { - time.Sleep(fib()) - if _, err := jsonrpc.Dial(utils.TCP, cfg.ListenCfg().RPCJSONListen); err == nil { + time.Sleep(backoff()) + if _, err = jsonrpc.Dial(utils.TCP, cfg.ListenCfg().RPCJSONListen); err == nil { break } } if err != nil { - t.Fatalf("starting cgr-engine on port %s failed: %v", cfg.ListenCfg().RPCJSONListen, err) + t.Fatalf("failed to start cgr-engine: %v", err) } } @@ -490,3 +595,29 @@ func WaitForService(t *testing.T, ctx *context.Context, client *birpc.Client, se } } } + +// Default DB configurations. For Redis/MySQL, it's missing because +// it's the default. +var ( + InternalDBCfg = DBCfg{ + DataDB: &DBParams{ + Type: utils.StringPointer(utils.MetaInternal), + }, + StorDB: &DBParams{ + Type: utils.StringPointer(utils.MetaInternal), + }, + } + MongoDBCfg = DBCfg{ + DataDB: &DBParams{ + Type: utils.StringPointer(utils.MetaMongo), + Port: utils.IntPointer(27017), + Name: utils.StringPointer("10"), + }, + StorDB: &DBParams{ + Type: utils.StringPointer(utils.MetaMongo), + Port: utils.IntPointer(27017), + Name: utils.StringPointer("cgrates"), + Password: utils.StringPointer(""), + }, + } +) diff --git a/general_tests/balance_timings_it_test.go b/general_tests/balance_timings_it_test.go index f9823c149..041957058 100644 --- a/general_tests/balance_timings_it_test.go +++ b/general_tests/balance_timings_it_test.go @@ -436,8 +436,7 @@ cgrates.org,call,1001,2014-01-14T00:00:00Z,RP_1001,`, t.Errorf("Unexpected reply returned: %s", reply) } // LoadTPFromFolder - customTpPath := t.TempDir() - engine.LoadCSVs(t, client, "", customTpPath, tpFiles) + engine.LoadCSVs(t, client, "", tpFiles) attrsEA := &utils.AttrExecuteAction{Tenant: "cgrates.org", Account: "1001", ActionsId: "ACT_TOPUP_RST_10"} if err := client.Call(context.Background(), utils.APIerSv1ExecuteAction, attrsEA, &reply); err != nil { t.Errorf("APIerSv1ExecuteAction failed unexpectedly: %v", err) @@ -721,8 +720,7 @@ cgrates.org,call,1001,2014-01-14T00:00:00Z,RP_1001,`, t.Errorf("Unexpected reply returned: %s", reply) } // LoadTPFromFolder - customTpPath := t.TempDir() - engine.LoadCSVs(t, client, "", customTpPath, tpFiles) + engine.LoadCSVs(t, client, "", tpFiles) attrsEA := &utils.AttrExecuteAction{Tenant: "cgrates.org", Account: "1001", ActionsId: "ACT_TOPUP_RST_10"} if err := client.Call(context.Background(), utils.APIerSv1ExecuteAction, attrsEA, &reply); err != nil { t.Errorf("APIerSv1ExecuteAction failed unexpectedly: %v", err) diff --git a/general_tests/reproc_cdrs_for_stats_it_test.go b/general_tests/reproc_cdrs_for_stats_it_test.go index a301af47e..5afd18a74 100644 --- a/general_tests/reproc_cdrs_for_stats_it_test.go +++ b/general_tests/reproc_cdrs_for_stats_it_test.go @@ -122,7 +122,7 @@ func testRpcdrsRpcConn(t *testing.T) { } func testRpcdrsLoadTP(t *testing.T) { - engine.LoadCSVs(t, rpcdrsRpc, path.Join(*utils.DataDir, "tariffplans", "reratecdrs"), "", nil) + engine.LoadCSVs(t, rpcdrsRpc, path.Join(*utils.DataDir, "tariffplans", "reratecdrs"), nil) } func testRpcdrsSetBalance(t *testing.T) { @@ -367,7 +367,7 @@ cgrates.org,STAT_AGG,,2014-07-29T15:00:00Z,0,-1,0,*tcd;*tcc;*sum#1,,false,false, } rpcdrsRpc = engine.NewRPCClient(t, rpcdrsCfg.ListenCfg()) - engine.LoadCSVs(t, rpcdrsRpc, "", t.TempDir(), tpFiles) + engine.LoadCSVs(t, rpcdrsRpc, "", tpFiles) } diff --git a/general_tests/session_bkup_interval_it_test.go b/general_tests/session_bkup_interval_it_test.go index 278b4442f..0c5218573 100644 --- a/general_tests/session_bkup_interval_it_test.go +++ b/general_tests/session_bkup_interval_it_test.go @@ -136,7 +136,7 @@ RP_ANY,DR_ANY_20CNT,*any,10`, cgrates.org,call,*any,2014-01-14T00:00:00Z,RP_ANY,`, } - engine.LoadCSVs(t, sBkupRPC, "", t.TempDir(), tpFiles) + engine.LoadCSVs(t, sBkupRPC, "", tpFiles) time.Sleep(time.Duration(*utils.WaitRater) * time.Millisecond) }