/* Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments Copyright (C) ITsysCOM GmbH This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see */ package ers import ( "bytes" "io" "reflect" "testing" "time" "github.com/aws/aws-sdk-go/service/s3" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) func TestS3ERServe(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr, err := NewS3ER(cfg, 0, nil, nil, nil, nil, nil) if err != nil { t.Error(err) } rdr.Config().RunDelay = 1 * time.Millisecond if err := rdr.Serve(); err != nil { t.Error(err) } } func TestS3ERServe2(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: nil, rdrEvents: nil, rdrExit: nil, rdrErr: nil, cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } if err := rdr.Serve(); err != nil { t.Error(err) } } func TestS3ERProcessMessage(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } expEvent := &utils.CGREvent{ Tenant: "cgrates.org", Event: map[string]any{ utils.CGRID: "testCgrId", }, APIOpts: map[string]any{}, } body := []byte(`{"CGRID":"testCgrId"}`) rdr.Config().Fields = []*config.FCTemplate{ { Tag: "CGRID", Type: utils.MetaConstant, Value: config.NewRSRParsersMustCompile("testCgrId", utils.InfieldSep), Path: "*cgreq.CGRID", }, } rdr.Config().Fields[0].ComputePath() if err := rdr.processMessage(body); err != nil { t.Error(err) } select { case data := <-rdr.rdrEvents: expEvent.ID = data.cgrEvent.ID expEvent.Time = data.cgrEvent.Time if !reflect.DeepEqual(data.cgrEvent, expEvent) { t.Errorf("Expected %v but received %v", utils.ToJSON(expEvent), utils.ToJSON(data.cgrEvent)) } case <-time.After(50 * time.Millisecond): t.Error("Time limit exceeded") } } func TestS3ERProcessMessageError1(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } rdr.Config().Fields = []*config.FCTemplate{ {}, } body := []byte(`{"CGRID":"testCgrId"}`) errExpect := "unsupported type: <>" if err := rdr.processMessage(body); err == nil || err.Error() != errExpect { t.Errorf("Expected %v but received %v", errExpect, err) } } func TestS3ERProcessMessageError2(t *testing.T) { cfg := config.NewDefaultCGRConfig() data, err := engine.NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items) if err != nil { t.Error(err) } dm := engine.NewDataManager(data, cfg.CacheCfg(), nil) cfg.ERsCfg().Readers[0].ProcessedPath = "" fltrs := engine.NewFilterS(cfg, nil, dm) rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: fltrs, rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } body := []byte(`{"CGRID":"testCgrId"}`) rdr.Config().Filters = []string{"Filter1"} errExpect := "NOT_FOUND:Filter1" if err := rdr.processMessage(body); err == nil || err.Error() != errExpect { t.Errorf("Expected %v but received %v", errExpect, err) } // rdr.Config().Filters = []string{"*exists:~*req..Account:"} if err := rdr.processMessage(body); err != nil { t.Error(err) } } func TestS3ERProcessMessageError3(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } body := []byte("invalid_format") errExpect := "invalid character 'i' looking for beginning of value" if err := rdr.processMessage(body); err == nil || err.Error() != errExpect { t.Errorf("Expected %v but received %v", errExpect, err) } } func TestS3ERParseOpts(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } opts := &config.EventReaderOpts{ AWS: &config.AWSROpts{ S3BucketID: utils.StringPointer("QueueID"), Region: utils.StringPointer("AWSRegion"), Key: utils.StringPointer("AWSKey"), Secret: utils.StringPointer("AWSSecret"), Token: utils.StringPointer("AWSToken"), }, } rdr.parseOpts(opts) if rdr.bucket != *opts.AWS.S3BucketID || rdr.awsRegion != *opts.AWS.Region || rdr.awsID != *opts.AWS.Key || rdr.awsKey != *opts.AWS.Secret || rdr.awsToken != *opts.AWS.Token { t.Error("Fields do not corespond") } rdr.Config().Opts = &config.EventReaderOpts{} rdr.Config().ProcessedPath = utils.EmptyString } func TestS3ERIsClosed(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } if rcv := rdr.isClosed(); rcv != false { t.Errorf("Expected %v but received %v", false, true) } rdr.rdrExit <- struct{}{} if rcv := rdr.isClosed(); rcv != true { t.Errorf("Expected %v but received %v", true, false) } } type s3ClientMock struct { ListObjectsV2PagesF func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error GetObjectF func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) DeleteObjectF func(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) } func (s *s3ClientMock) ListObjectsV2Pages(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { if s.ListObjectsV2PagesF != nil { return s.ListObjectsV2PagesF(input, fn) } return utils.ErrNotFound } func (s *s3ClientMock) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { if s.GetObjectF != nil { return s.GetObjectF(input) } return nil, utils.ErrNotImplemented } func (s *s3ClientMock) DeleteObject(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { // return nil, nil if s.DeleteObjectF != nil { return s.DeleteObjectF(input) } return nil, utils.ErrInvalidPath } func TestS3ERReadLoop(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } listObjects := func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { obj := &s3.ListObjectsV2Output{ Contents: []*s3.Object{ { Key: utils.StringPointer("Key"), }, }, } fn(obj, false) return nil } scv := &s3ClientMock{ ListObjectsV2PagesF: listObjects, } if err := rdr.readLoop(scv); err != nil { t.Error(err) } } func TestS3ERReadLoopIsClosed(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } listObjects := func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { return nil } scv := &s3ClientMock{ ListObjectsV2PagesF: listObjects, } rdr.rdrExit <- struct{}{} if err := rdr.readLoop(scv); err != nil { t.Error(err) } } func TestS3ERReadMsg(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: nil, awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } // rdr.poster = engine.NewS3Poster(rdr.Config().SourcePath, 1, make(map[string]any)) rdr.Config().SourcePath = rdr.awsRegion rdr.Config().ConcurrentReqs = -1 rdr.Config().Fields = []*config.FCTemplate{ { Tag: "Tor", Type: utils.MetaConstant, Value: config.NewRSRParsersMustCompile("*voice", utils.InfieldSep), Path: "*cgreq.ToR", }, } rdr.Config().Fields[0].ComputePath() listObjects := func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { return nil } getObject := func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{Body: io.NopCloser(bytes.NewBuffer([]byte(`{"key":"value"}`)))}, nil } deleteObject := func(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { return nil, nil } scv := &s3ClientMock{ ListObjectsV2PagesF: listObjects, GetObjectF: getObject, DeleteObjectF: deleteObject, } if err := rdr.readMsg(scv, "AWSKey"); err != nil { t.Error(err) } } func TestS3ERReadMsgError1(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: make(chan struct{}, 1), awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } rdr.Config().ConcurrentReqs = 1 listObjects := func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { return nil } getObject := func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{Body: io.NopCloser(bytes.NewBuffer([]byte(`{"key":"value"}`)))}, nil } deleteObject := func(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { return nil, nil } scv := &s3ClientMock{ ListObjectsV2PagesF: listObjects, GetObjectF: getObject, DeleteObjectF: deleteObject, } errExp := "NOT_FOUND:ToR" if err := rdr.readMsg(scv, "AWSKey"); err == nil || err.Error() != errExp { t.Errorf("Expected %v but received %v", errExp, err) } } func TestS3ERReadMsgError2(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: make(chan struct{}, 1), awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } rdr.Config().ConcurrentReqs = 1 scv := &s3ClientMock{} rdr.rdrExit <- struct{}{} if err := rdr.readMsg(scv, "AWSKey"); err != nil { t.Error(err) } } func TestS3ERReadMsgError3(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: make(chan struct{}, 1), awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } rdr.Config().ConcurrentReqs = -1 scv := &s3ClientMock{} errExp := "NOT_IMPLEMENTED" if err := rdr.readMsg(scv, "AWSKey"); err == nil || err.Error() != errExp { t.Errorf("Expected %v but received %v", errExp, err) } } func TestS3ERReadMsgError4(t *testing.T) { cfg := config.NewDefaultCGRConfig() rdr := &S3ER{ cgrCfg: cfg, cfgIdx: 0, fltrS: new(engine.FilterS), rdrEvents: make(chan *erEvent, 1), rdrExit: make(chan struct{}, 1), rdrErr: make(chan error, 1), cap: make(chan struct{}, 1), awsRegion: "us-east-2", awsID: "AWSId", awsKey: "AWSAccessKeyId", awsToken: "", bucket: "cgrates_cdrs", session: nil, } rdr.Config().SourcePath = rdr.awsRegion rdr.Config().ConcurrentReqs = -1 rdr.Config().Fields = []*config.FCTemplate{ { Tag: "Tor", Type: utils.MetaConstant, Value: config.NewRSRParsersMustCompile("*voice", utils.InfieldSep), Path: "*cgreq.ToR", }, } rdr.Config().Fields[0].ComputePath() listObjects := func(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error { return nil } getObject := func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{Body: io.NopCloser(bytes.NewBuffer([]byte(`{"key":"value"}`)))}, nil } deleteObject := func(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { return nil, utils.ErrInvalidPath } scv := &s3ClientMock{ ListObjectsV2PagesF: listObjects, GetObjectF: getObject, DeleteObjectF: deleteObject, } errExp := "INVALID_PATH" if err := rdr.readMsg(scv, "AWSKey"); err == nil || err.Error() != errExp { t.Errorf("Expected %v but received %v", errExp, err) } }