Revise APIerSv1.ReplayFailedPosts API + tests

- renamed parameter type: ArgsReplyFailedPosts -> ReplayFailedPostsParams
- renamed param fields:
  - FailedRequestsInDir -> SourcePath
  - FailedRequestsOutDir -> FailedPath
- changed param fields types from *string to string
- used the SourcePath and FailedPath params directly instead of creating separate variables
- used filepath.WalkDir instead of reading the directory and looping over the entries
- used slices.ContainsFunc to check if the file belongs to any module (if 1+ is specified)
- used filepath.Join instead of path.Join
- used the path provided by WalkFunc instead of building the file paths ourselves
- made error returns more descriptive
- added logs for directories/files that are skipped
- paths that cannot be accessed are skipped after logging the error
This commit is contained in:
ionutboangiu
2024-06-05 20:45:31 +03:00
committed by Dan Christian Bogos
parent 89f97d45e1
commit b7dacfe8a6
10 changed files with 286 additions and 82 deletions

View File

@@ -96,7 +96,7 @@ func AddFailedPost(failedPostsDir, expPath, format string, ev any, opts *config.
if failedPost == nil {
failedPost = &ExportEvents{
Path: expPath,
Format: format,
Type: format,
Opts: opts,
failedPostsDir: failedPostsDir,
}
@@ -129,7 +129,7 @@ type ExportEvents struct {
lk sync.RWMutex
Path string
Opts *config.EventExporterOpts
Format string
Type string
Events []any
failedPostsDir string
}
@@ -162,32 +162,34 @@ func (expEv *ExportEvents) AddEvent(ev any) {
// ReplayFailedPosts tryies to post cdrs again
func (expEv *ExportEvents) ReplayFailedPosts(attempts int) (failedEvents *ExportEvents, err error) {
failedEvents = &ExportEvents{
Path: expEv.Path,
Opts: expEv.Opts,
Format: expEv.Format,
}
eeCfg := config.NewEventExporterCfg("ReplayFailedPosts", expEv.Format, expEv.Path, utils.MetaNone,
eeCfg := config.NewEventExporterCfg("ReplayFailedPosts", expEv.Type, expEv.Path, utils.MetaNone,
attempts, expEv.Opts)
var ee EventExporter
if ee, err = NewEventExporter(eeCfg, config.CgrConfig(), nil, nil); err != nil {
return
}
keyFunc := func() string { return utils.EmptyString }
if expEv.Format == utils.MetaKafkajsonMap || expEv.Format == utils.MetaS3jsonMap {
if expEv.Type == utils.MetaKafkajsonMap || expEv.Type == utils.MetaS3jsonMap {
keyFunc = utils.UUIDSha1Prefix
}
failedEvents = &ExportEvents{
Path: expEv.Path,
Opts: expEv.Opts,
Type: expEv.Type,
}
for _, ev := range expEv.Events {
if err = ExportWithAttempts(ee, ev, keyFunc()); err != nil {
failedEvents.AddEvent(ev)
}
}
ee.Close()
if len(failedEvents.Events) > 0 {
err = utils.ErrPartiallyExecuted
} else {
failedEvents = nil
switch len(failedEvents.Events) {
case 0: // none failed to be replayed
return nil, nil
case len(expEv.Events): // all failed, return last encountered error
return failedEvents, err
default:
return failedEvents, utils.ErrPartiallyExecuted
}
return
}

View File

@@ -81,7 +81,7 @@ func TestWriteToFile(t *testing.T) {
exportEvent = &ExportEvents{
Events: []any{"something1", "something2"},
Path: "path",
Format: "test",
Type: "test",
}
filePath = "/tmp/engine/libcdre_test/writeToFile2.txt"
if err := exportEvent.WriteToFile(filePath); err != nil {

View File

@@ -62,7 +62,7 @@ func TestAddFldPost(t *testing.T) {
}
eOut := &ExportEvents{
Path: "path1",
Format: "format1",
Type: "format1",
Events: []any{"1"},
Opts: &config.EventExporterOpts{
AMQP: &config.AMQPOpts{},
@@ -110,7 +110,7 @@ func TestAddFldPost(t *testing.T) {
}
eOut = &ExportEvents{
Path: "path1",
Format: "format1",
Type: "format1",
Events: []any{"1", "2"},
Opts: &config.EventExporterOpts{
AMQP: &config.AMQPOpts{},
@@ -138,7 +138,7 @@ func TestAddFldPost(t *testing.T) {
}
eOut = &ExportEvents{
Path: "path2",
Format: "format2",
Type: "format2",
Events: []any{"3"},
Opts: &config.EventExporterOpts{
Els: &config.ElsOpts{},