//go:build integration
// +build integration

/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>
*/
package loaders

import (
	"encoding/csv"
	"fmt"
	"sort"

	"io"
	"os"
	"path"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/cgrates/birpc"
	"github.com/cgrates/birpc/context"
	"github.com/cgrates/cgrates/config"
	"github.com/cgrates/cgrates/engine"
	"github.com/cgrates/cgrates/utils"
	"github.com/cgrates/rpcclient"
)

var (
	loaderCfgPath    string
	loaderCfgDIR     string //run tests for specific configuration
	loaderCfg        *config.CGRConfig
	loaderRPC        *birpc.Client
	customAttributes = "12012000001\t12018209998\n12012000002\t15512580598\n12012000007\t19085199998\n12012000008\t18622784999\n12012000010\t17329440866\n12012000011\t18623689800\n12012000012\t19082050951\n12012000014\t17329440866\n12012000015\t12018209999\n12012000031\t12018209999\n12012000032\t19082050951\n12012000033\t12018209998\n12012000034\t12018209998\n"

	sTestsLoader = []func(t *testing.T){
		testLoaderMakeFolders,
		testLoaderInitCfg,
		testLoaderResetDataDB,
		testLoaderStartEngine,
		testLoaderRPCConn,
		testLoaderPopulateData,
		testLoadFromFilesCsvActionProfile,
		testLoadFromFilesCsvActionProfileOpenError,
		testProcessFolderRemoveContent,
		testLoadFromFilesCsvActionProfileLockFolderError,
		testLoaderMoveFiles,
		testLoaderMoveFilesMatchingFiles,
		testLoaderMoveFilesRenameError,
		testProcessFile,
		testProcessFileLockFolder,
		testProcessFileUnableToOpen,
		testProcessFileAllFilesPresent,
		testProcessFileRenameError,
		testAllFilesPresentEmptyCSV,
		testIsFolderLocked,
		testNewLockFolder,
		testNewLockFolderNotFound,
		testNewIsFolderLock,
		testNewIsFolderUnlock,
		testLoaderLoadAttributes,
		testLoaderVerifyOutDir,
		testLoaderCheckAttributes,
		testLoaderResetDataDB,
		testLoaderPopulateDataWithoutMoving,
		testLoaderLoadAttributesWithoutMoving,
		testLoaderVerifyOutDirWithoutMoving,
		testLoaderCheckAttributes,
		testLoaderResetDataDB,
		testLoaderPopulateDataWithSubpath,
		testLoaderLoadAttributesWithSubpath,
		testLoaderVerifyOutDirWithSubpath,
		testLoaderCheckAttributes,
		testLoaderResetDataDB,
		testLoaderPopulateDataWithSubpathWithMove,
		testLoaderLoadAttributesWithoutSubpathWithMove,
		testLoaderVerifyOutDirWithSubpathWithMove,
		testLoaderCheckAttributes,
		testLoaderResetDataDB,
		testLoaderPopulateDataForTemplateLoader,
		testLoaderLoadAttributesForTemplateLoader,
		testLoaderVerifyOutDirForTemplateLoader,
		testLoaderCheckAttributes,
		testLoaderResetDataDB,
		testLoaderPopulateDataForCustomSep,
		testLoaderCheckForCustomSep,
		testLoaderVerifyOutDirForCustomSep,
		testLoaderKillEngine,
	}
)

//Test start here
func TestLoaderIT(t *testing.T) {
	switch *dbType {
	case utils.MetaInternal:
		loaderCfgDIR = "tutinternal"
	case utils.MetaMySQL:
		loaderCfgDIR = "tutmysql"
	case utils.MetaMongo:
		loaderCfgDIR = "tutmongo"
	case utils.MetaPostgres:
		t.SkipNow()
	default:
		t.Fatal("Unknown Database type")
	}

	for _, stest := range sTestsLoader {
		t.Run(loaderCfgDIR, stest)
	}
}

func testLoaderInitCfg(t *testing.T) {
	var err error
	loaderCfgPath = path.Join(*dataDir, "conf", "samples", "loaders", loaderCfgDIR)
	loaderCfg, err = config.NewCGRConfigFromPath(context.Background(), loaderCfgPath)
	if err != nil {
		t.Fatal(err)
	}
}

func testLoaderMakeFolders(t *testing.T) {
	// active the loaders here
	for _, dir := range loaderPaths {
		if err := os.RemoveAll(dir); err != nil {
			t.Fatal("Error removing folder: ", dir, err)
		}
		if err := os.MkdirAll(dir, 0755); err != nil {
			t.Fatal("Error creating folder: ", dir, err)
		}
	}
}

// Wipe out the cdr database
func testLoaderResetDataDB(t *testing.T) {
	if err := engine.InitDataDB(loaderCfg); err != nil {
		t.Fatal(err)
	}
	engine.Cache.Clear(nil)
}

// Start CGR Engine
func testLoaderStartEngine(t *testing.T) {
	if _, err := engine.StopStartEngine(loaderCfgPath, 100); err != nil {
		t.Fatal(err)
	}
}

// Connect rpc client to rater
func testLoaderRPCConn(t *testing.T) {
	var err error
	loaderRPC, err = newRPCClient(loaderCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed
	if err != nil {
		t.Fatal(err)
	}
}

func testLoaderPopulateData(t *testing.T) {
	fileName := utils.AttributesCsv
	tmpFilePath := path.Join("/tmp", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/In", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
}

func testLoaderLoadAttributes(t *testing.T) {
	var reply string
	if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
		&ArgsProcessFolder{LoaderID: "CustomLoader"}, &reply); err != nil {
		t.Error(err)
	}
}

func testLoaderVerifyOutDir(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	if outContent1, err := os.ReadFile(path.Join("/tmp/Out", utils.AttributesCsv)); err != nil {
		t.Error(err)
	} else if engine.AttributesCSVContent != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
	}
}

func testLoaderCheckAttributes(t *testing.T) {
	eAttrPrf := &engine.APIAttributeProfile{
		Tenant:    "cgrates.org",
		ID:        "ALS1",
		FilterIDs: []string{"*string:~*req.Account:1001", "*string:~*opts.*context:con1", "*string:~*opts.*context:con2|con3"},
		Attributes: []*engine.ExternalAttribute{
			{
				FilterIDs: []string{"*string:~*req.Field1:Initial"},
				Path:      utils.MetaReq + utils.NestingSep + "Field1",
				Type:      utils.MetaVariable,
				Value:     "Sub1",
			},
			{
				FilterIDs: []string{},
				Path:      utils.MetaReq + utils.NestingSep + "Field2",
				Type:      utils.MetaVariable,
				Value:     "Sub2",
			}},
		Blocker: true,
		Weight:  20,
	}
	if *encoding == utils.MetaGOB { // gob threats empty slices as nil values
		eAttrPrf.Attributes[1].FilterIDs = nil
	}
	var reply *engine.APIAttributeProfile
	if err := loaderRPC.Call(context.Background(), utils.AdminSv1GetAttributeProfile,
		&utils.TenantIDWithAPIOpts{
			TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"},
		}, &reply); err != nil {
		t.Fatal(err)
	}
	sort.Strings(reply.FilterIDs)
	sort.Strings(eAttrPrf.FilterIDs)
	if !reflect.DeepEqual(eAttrPrf, reply) {
		t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply))
	}
}

func testLoaderPopulateDataWithoutMoving(t *testing.T) {
	fileName := utils.AttributesCsv
	tmpFilePath := path.Join("/tmp/", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/LoaderIn", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
}

func testLoaderLoadAttributesWithoutMoving(t *testing.T) {
	var reply string
	if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
		&ArgsProcessFolder{LoaderID: "WithoutMoveToOut"}, &reply); err != nil {
		t.Error(err)
	}
}

func testLoaderVerifyOutDirWithoutMoving(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	// we expect that after the LoaderS process the file leave in in the input folder
	if outContent1, err := os.ReadFile(path.Join("/tmp/LoaderIn", utils.AttributesCsv)); err != nil {
		t.Error(err)
	} else if engine.AttributesCSVContent != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
	}
}

func testLoaderPopulateDataWithSubpath(t *testing.T) {
	fileName := utils.AttributesCsv
	tmpFilePath := path.Join("/tmp/", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.MkdirAll("/tmp/SubpathWithoutMove/folder1", 0755); err != nil {
		t.Fatal("Error creating folder: /tmp/SubpathWithoutMove/folder1", err)
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathWithoutMove/folder1", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
}

func testLoaderLoadAttributesWithSubpath(t *testing.T) {
	var reply string
	if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
		&ArgsProcessFolder{LoaderID: "SubpathLoaderWithoutMove"}, &reply); err != nil {
		t.Error(err)
	}
}

func testLoaderVerifyOutDirWithSubpath(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	// we expect that after the LoaderS process the file leave in in the input folder
	if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathWithoutMove/folder1", utils.AttributesCsv)); err != nil {
		t.Error(err)
	} else if engine.AttributesCSVContent != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
	}
}

func testLoaderPopulateDataWithSubpathWithMove(t *testing.T) {
	fileName := utils.AttributesCsv
	tmpFilePath := path.Join("/tmp/", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.MkdirAll("/tmp/SubpathLoaderWithMove/folder1", 0755); err != nil {
		t.Fatal("Error creating folder: /tmp/SubpathLoaderWithMove/folder1", err)
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathLoaderWithMove/folder1", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
}

func testLoaderLoadAttributesWithoutSubpathWithMove(t *testing.T) {
	var reply string
	if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
		&ArgsProcessFolder{LoaderID: "SubpathLoaderWithMove"}, &reply); err != nil {
		t.Error(err)
	}
}

func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathOut/folder1", utils.AttributesCsv)); err != nil {
		t.Error(err)
	} else if engine.AttributesCSVContent != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
	}
}

func testLoaderPopulateDataForTemplateLoader(t *testing.T) {
	fileName := utils.AttributesCsv
	tmpFilePath := path.Join("/tmp/", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(engine.AttributesCSVContent), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.MkdirAll("/tmp/templateLoaderIn", 0755); err != nil {
		t.Fatal("Error creating folder: /tmp/templateLoaderIn", err)
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/templateLoaderIn", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
}

func testLoaderLoadAttributesForTemplateLoader(t *testing.T) {
	var reply string
	if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
		&ArgsProcessFolder{LoaderID: "LoaderWithTemplate"}, &reply); err != nil {
		t.Error(err)
	}
}

func testLoaderVerifyOutDirForTemplateLoader(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	if outContent1, err := os.ReadFile(path.Join("/tmp/templateLoaderOut", utils.AttributesCsv)); err != nil {
		t.Error(err)
	} else if engine.AttributesCSVContent != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", engine.AttributesCSVContent, string(outContent1))
	}
}

func testLoaderKillEngine(t *testing.T) {
	if err := engine.KillEngine(100); err != nil {
		t.Error(err)
	}
}

func testLoaderPopulateDataForCustomSep(t *testing.T) {
	fileName := utils.Attributes
	tmpFilePath := path.Join("/tmp/", fileName)
	if err := os.WriteFile(tmpFilePath, []byte(customAttributes), 0777); err != nil {
		t.Fatal(err.Error())
	}
	if err := os.MkdirAll("/tmp/customSepLoaderIn", 0755); err != nil {
		t.Fatal("Error creating folder: /tmp/customSepLoaderIn", err)
	}
	if err := os.Rename(tmpFilePath, path.Join("/tmp/customSepLoaderIn", fileName)); err != nil {
		t.Fatal("Error moving file to processing directory: ", err)
	}
	time.Sleep(100 * time.Millisecond)
}

func testLoaderCheckForCustomSep(t *testing.T) {
	eAttrPrf := &engine.APIAttributeProfile{
		Tenant:    "cgrates.org",
		ID:        "ATTR_12012000001",
		FilterIDs: []string{"*string:~*req.Destination:12012000001"},
		Attributes: []*engine.ExternalAttribute{
			{
				FilterIDs: []string{},
				Path:      "*req.Destination",
				Type:      utils.MetaConstant,
				Value:     "12018209998",
			},
		},
	}
	if *encoding == utils.MetaGOB { // gob threats empty slices as nil values
		eAttrPrf.Attributes[0].FilterIDs = nil
	}
	var reply *engine.APIAttributeProfile
	if err := loaderRPC.Call(context.Background(), utils.AdminSv1GetAttributeProfile,
		&utils.TenantIDWithAPIOpts{
			TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_12012000001"},
		}, &reply); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(eAttrPrf, reply) {
		t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply))
	}
}

func testLoaderVerifyOutDirForCustomSep(t *testing.T) {
	time.Sleep(100 * time.Millisecond)
	if outContent1, err := os.ReadFile(path.Join("/tmp/customSepLoaderOut", utils.Attributes)); err != nil {
		t.Error(err)
	} else if customAttributes != string(outContent1) {
		t.Errorf("Expecting: %q, received: %q", customAttributes, string(outContent1))
	}
}

func testLoadFromFilesCsvActionProfile(t *testing.T) {
	flPath := "/tmp/TestLoadFromFilesCsvActionProfile"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	newFile, err := os.Create(path.Join(flPath, "ActionProfiles.csv"))
	if err != nil {
		t.Error(err)
	}
	content := `#Tenant[0],ID[1]
cgrates.org,SET_ACTPROFILE_3
`
	newFile.Write([]byte(content))
	newFile.Close()

	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "TestRemoveActionProfileContent",
		bufLoaderData: make(map[string][]LoaderData),
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		tpInDir:       flPath,
		tpOutDir:      utils.EmptyString,
		lockFilepath:  "ActionProfiles.lks",
		rdrTypes:      []string{utils.MetaActionProfiles},
		fieldSep:      ",",
		timezone:      "UTC",
	}
	ldr.dataTpls = map[string][]*config.FCTemplate{
		utils.MetaActionProfiles: {
			{Tag: "TenantID",
				Path:      "Tenant",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
				Mandatory: true},
			{Tag: "ProfileID",
				Path:      "ID",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
				Mandatory: true},
		},
	}

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaActionProfiles: {
			utils.ActionProfilesCsv: nil,
		},
	}
	if err := ldr.ProcessFolder(context.TODO(), utils.EmptyString, utils.MetaStore, true); err != nil {
		t.Error(err)
	}
	expACtPrf := &engine.ActionProfile{
		Tenant:    "cgrates.org",
		ID:        "SET_ACTPROFILE_3",
		FilterIDs: []string{},
		Targets:   map[string]utils.StringSet{},
		Actions:   []*engine.APAction{},
	}
	if rcv, err := ldr.dm.GetActionProfile(context.TODO(), expACtPrf.Tenant, expACtPrf.ID,
		true, true, utils.NonTransactional); err != nil {
		t.Error(err)
	} else if !reflect.DeepEqual(expACtPrf, rcv) {
		t.Errorf("Expected %+v, received %+v", utils.ToJSON(expACtPrf), utils.ToJSON(rcv))
	}

	// checking the error by adding a caching method
	ldr.connMgr = engine.NewConnManager(config.NewDefaultCGRConfig())
	ldr.cacheConns = []string{utils.MetaInternal}
	rdr := io.NopCloser(strings.NewReader(string(content)))
	csvRdr := csv.NewReader(rdr)
	csvRdr.Comment = '#'
	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaActionProfiles: {
			utils.ActionProfilesCsv: &openedCSVFile{
				fileName: utils.ActionProfilesCsv,
				rdr:      rdr,
				csvRdr:   csvRdr,
			},
		},
	}
	expected := "UNSUPPORTED_SERVICE_METHOD"
	if err := ldr.ProcessFolder(context.TODO(), utils.MetaReload, utils.MetaStore, true); err == nil || err.Error() != expected {
		t.Error(err)
	}

	if err = os.RemoveAll(flPath); err != nil {
		t.Fatal(err)
	}
}

func testLoadFromFilesCsvActionProfileOpenError(t *testing.T) {
	pathL := "/tmp/testLoadFromFilesCsvActionProfileOpenError/"
	if err := os.MkdirAll(pathL, 0777); err != nil {
		t.Error(err)
	}

	if _, err := os.Create(path.Join(pathL, "WrongFileName")); err != nil {
		t.Error(err)
	}
	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "TestRemoveActionProfileContent",
		bufLoaderData: make(map[string][]LoaderData),
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		tpInDir:       "/tmp/testLoadFromFilesCsvActionProfileOpenError",
		rdrTypes:      []string{utils.MetaActionProfiles},
		timezone:      "UTC",
		fieldSep:      utils.InfieldSep,
	}
	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaActionProfiles: {
			utils.ActionProfilesCsv: &openedCSVFile{
				fileName: utils.ActionProfilesCsv,
			},
		},
	}
	expectedErr := "open /tmp/testLoadFromFilesCsvActionProfileOpenError/ActionProfiles.csv: no such file or directory"
	if err := ldr.ProcessFolder(context.TODO(), utils.EmptyString, utils.MetaStore, true); err == nil || err.Error() != expectedErr {
		t.Errorf("Expected %+v, received %+v", expectedErr, err)
	}

	//if stopOnError is on true, the error is avoided,but instead will get a logger.warning message
	if err := ldr.ProcessFolder(context.TODO(), utils.EmptyString, utils.MetaStore, false); err != nil {
		t.Error(err)
	}

	if err := os.RemoveAll(pathL); err != nil {
		t.Error(err)
	}
}

func testProcessFolderRemoveContent(t *testing.T) {
	flPath := "/tmp/TestLoadFromFilesCsvActionProfile"
	content := `#Tenant[0],ID[1]
cgrates.org,SET_ACTPROFILE_3`
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	newFile, err := os.Create(path.Join(flPath, "ActionProfiles.csv"))
	if err != nil {
		t.Error(err)
	}
	newFile.Write([]byte(content))
	newFile.Close()

	ldr := &Loader{
		ldrID:         "TestRemoveActionProfileContent",
		bufLoaderData: make(map[string][]LoaderData),
		dm:            engine.NewDataManager(engine.NewInternalDB(nil, nil, true), config.CgrConfig().CacheCfg(), nil),
		tpInDir:       "/tmp/TestLoadFromFilesCsvActionProfile",
		tpOutDir:      utils.EmptyString,
		rdrTypes:      []string{utils.MetaActionProfiles},
		lockFilepath:  "ActionProfiles.csv",
		fieldSep:      ",",
		timezone:      "UTC",

		dataTpls: map[string][]*config.FCTemplate{
			utils.MetaActionProfiles: {
				{Tag: "TenantID",
					Path:      "Tenant",
					Type:      utils.MetaComposed,
					Value:     config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
					Mandatory: true},
				{Tag: "ProfileID",
					Path:      "ID",
					Type:      utils.MetaComposed,
					Value:     config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
					Mandatory: true},
			},
		},
		connMgr: engine.NewConnManager(config.NewDefaultCGRConfig()),
	}

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaActionProfiles: {
			utils.ActionProfilesCsv: {},
		},
	}
	expACtPrf := &engine.ActionProfile{
		Tenant:    "cgrates.org",
		ID:        "SET_ACTPROFILE_3",
		FilterIDs: []string{},
		Targets:   map[string]utils.StringSet{},
		Actions:   []*engine.APAction{},
	}
	if err := ldr.dm.SetActionProfile(context.TODO(), expACtPrf, true); err != nil {
		t.Error(err)
	}
	if err := ldr.ProcessFolder(context.TODO(), utils.EmptyString, utils.MetaRemove, true); err != nil {
		t.Error(err)
	}
	//nothing to get from database
	if _, err := ldr.dm.GetActionProfile(context.TODO(), expACtPrf.Tenant, expACtPrf.ID,
		true, true, utils.NonTransactional); err != utils.ErrNotFound {
		t.Error(err)
	}

	//checking the error by adding a caching method
	ldr.cacheConns = []string{utils.MetaInternal}
	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaActionProfiles: {
			utils.ActionProfilesCsv: {},
		},
	}
	if err := ldr.dm.SetActionProfile(context.TODO(), expACtPrf, true); err != nil {
		t.Error(err)
	}
	if err := ldr.ProcessFolder(context.TODO(), utils.MetaReload, utils.MetaRemove, true); err != rpcclient.ErrUnsupporteServiceMethod {
		t.Error(err)
	}
}

func testLoadFromFilesCsvActionProfileLockFolderError(t *testing.T) {
	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "TestRemoveActionProfileContent",
		bufLoaderData: make(map[string][]LoaderData),
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		timezone:      "UTC",
		lockFilepath:  "/tmp/test/.lck",
	}
	expectedErr := "open /tmp/test/.lck: no such file or directory"
	if err := ldr.ProcessFolder(context.TODO(), utils.EmptyString, utils.MetaStore, true); err == nil || err.Error() != expectedErr {
		t.Errorf("Expected %+v, received %+v", expectedErr, err)
	}
}

func testLoaderMoveFiles(t *testing.T) {
	flPath := "/tmp/TestLoadFromFilesCsvActionProfile"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	newFile, err := os.Create(path.Join(flPath, "ActionProfiles.csv"))
	if err != nil {
		t.Error(err)
	}
	newFile.Close()

	ldr := &Loader{
		tpInDir:  flPath,
		tpOutDir: "/tmp",
	}
	if err := ldr.moveFiles(); err != nil {
		t.Error(err)
	}

	if err := os.Remove("/tmp/ActionProfiles.csv"); err != nil {
		t.Error(err)
	} else if err := os.Remove("/tmp/TestLoadFromFilesCsvActionProfile"); err != nil {
		t.Error(err)
	}
}

func testLoaderMoveFilesMatchingFiles(t *testing.T) {
	flPath := "/tmp/TestLoaderMoveFilesMatchingFiles"
	ldr := &Loader{
		tpInDir:      flPath,
		tpOutDir:     "/tmp",
		lockFilepath: "ActionProfiles.csv",
	}
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	newFile, err := os.Create(path.Join(flPath, "ActionProfiles.csv"))
	if err != nil {
		t.Error(err)
	}
	newFile.Close()

	if err := ldr.moveFiles(); err != nil {
		t.Error(err)
	}

	if err := os.Remove(path.Join(flPath, "ActionProfiles.csv")); err != nil {
		t.Error(err)
	} else if err := os.Remove(flPath); err != nil {
		t.Error(err)
	}
}

func testLoaderMoveFilesRenameError(t *testing.T) {
	flPath := "/tmp/testLoaderMoveFilesRenameError"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	ldr := &Loader{
		tpInDir:      flPath,
		tpOutDir:     flPath,
		lockFilepath: "ActionProfiles.lks",
	}
	filepath := path.Join(flPath, "ActionProfiles")
	if err := os.MkdirAll(filepath, 0777); err != nil {
		t.Error(err)
	}

	expected := fmt.Sprintf("rename %s %s: file exists", filepath, filepath)
	if err := ldr.moveFiles(); err == nil || err.Error() != expected {
		t.Errorf("Expected %+v, received %+v", expected, err)
	}

	if err := os.Remove(filepath); err != nil {
		t.Error(err)
	}
}

func testProcessFile(t *testing.T) {
	flPath := "/tmp/testProcessFile"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}
	file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
	file.Close()

	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "testProcessFile",
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		fieldSep:      utils.FieldsSep,
		tpInDir:       flPath,
		tpOutDir:      "/tmp",
		lockFilepath:  "/tmp/testProcessFile/.lck",
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}
	ldr.dataTpls = map[string][]*config.FCTemplate{
		utils.MetaResources: {
			{Tag: "Tenant",
				Path:      "Tenant",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
				Mandatory: true},
			{Tag: "ID",
				Path:      "ID",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
				Mandatory: true},
		},
	}

	//loader file is empty (loaderType will be empty)
	if err := ldr.processFile("unusedValue", utils.ResourcesCsv); err != nil {
		t.Error(err)
	}

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			utils.ResourcesCsv: {
				rdr:    io.NopCloser(nil),
				csvRdr: csv.NewReader(nil),
			},
		},
	}

	expRes := &engine.ResourceProfile{
		Tenant:       "cgrates.org",
		ID:           "NewRes1",
		FilterIDs:    []string{},
		ThresholdIDs: []string{},
	}

	//successfully processed the file
	if err := ldr.processFile("unusedValue", utils.ResourcesCsv); err != nil {
		t.Error(err)
	}

	//get ResourceProfile and compare
	if rcv, err := ldr.dm.GetResourceProfile(context.TODO(), expRes.Tenant, expRes.ID, true, true, utils.NonTransactional); err != nil {
		t.Error(err)
	} else if !reflect.DeepEqual(rcv, expRes) {
		t.Errorf("Expected %+v, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv))
	}

	if err := ldr.dm.RemoveResourceProfile(context.TODO(), expRes.Tenant, expRes.ID, true); err != nil {
		t.Error(err)
	}

	file, err = os.Create(path.Join(flPath, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}
	file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
	file.Close()

	//cannot move file when tpOutDir is empty
	ldr.tpOutDir = utils.EmptyString
	if err := ldr.processFile("unusedValue", utils.ResourcesCsv); err != nil {
		t.Error(err)
	}

	if err := os.Remove(path.Join("/tmp", utils.ResourcesCsv)); err != nil {
		t.Error(err)
	} else if err := os.RemoveAll(flPath); err != nil {
		t.Error(err)
	}
}

func testProcessFileAllFilesPresent(t *testing.T) {
	flPath := "/tmp/testProcessFile"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	file, err := os.Create(path.Join(flPath, "inexistent.csv"))
	if err != nil {
		t.Error(err)
	}

	file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
	file.Close()

	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "testProcessFile",
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		fieldSep:      utils.FieldsSep,
		tpInDir:       flPath,
		tpOutDir:      "/tmp",
		lockFilepath:  utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}
	ldr.dataTpls = map[string][]*config.FCTemplate{
		utils.MetaResources: {
			{Tag: "Tenant",
				Path:      "Tenant",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.0", utils.FieldsSep),
				Mandatory: true},
			{Tag: "ID",
				Path:      "ID",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.1", utils.FieldsSep),
				Mandatory: true},
		},
	}

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			"inexistent.csv":    nil,
			utils.AttributesCsv: nil,
		},
	}

	if err := ldr.processFile("unusedValue", "inexistent.csv"); err != nil {
		t.Error(err)
	}

	if err := os.Remove(path.Join(flPath, "inexistent.csv")); err != nil {
		t.Error(err)
	} else if err := os.Remove(flPath); err != nil {
		t.Error(err)
	}
}

func testProcessFileLockFolder(t *testing.T) {
	flPath := "/tmp/testProcessFileLockFolder"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}
	_, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}

	ldr := &Loader{
		ldrID:        "testProcessFileLockFolder",
		tpInDir:      flPath,
		tpOutDir:     "/tmp",
		lockFilepath: "/tmp/test/.cgr.lck",
		fieldSep:     utils.InfieldSep,
	}

	resCsv := `
#Tenant[0],ID[1]
cgrates.org,NewRes1
`
	rdr := io.NopCloser(strings.NewReader(resCsv))

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			utils.ResourcesCsv: &openedCSVFile{
				fileName: utils.ResourcesCsv,
				rdr:      rdr,
			},
		},
	}

	//unable to lock the folder, because lockFileName is missing
	expected := "open /tmp/test/.cgr.lck: no such file or directory"
	if err := ldr.processFile("unusedValue", utils.ResourcesCsv); err == nil || err.Error() != expected {
		t.Errorf("Expected %+v, received %+v", expected, err)
	}

	if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
		t.Error(err)
	} else if err := os.Remove(flPath); err != nil {
		t.Error(err)
	}
}

func testProcessFileUnableToOpen(t *testing.T) {
	flPath := "/tmp/testProcessFileUnableToOpen"
	if err := os.MkdirAll(flPath, 0777); err != nil {
		t.Error(err)
	}

	ldr := &Loader{
		ldrID:        "testProcessFile",
		tpInDir:      flPath,
		fieldSep:     ",",
		lockFilepath: utils.MetaResources,
	}
	resCsv := `
#Tenant[0],ID[1]
cgrates.org,NewRes1
`
	rdr := io.NopCloser(strings.NewReader(resCsv))

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			`resources`: &openedCSVFile{
				fileName: utils.ResourcesCsv,
				rdr:      rdr,
			},
		},
	}

	//unable to lock the folder, because lockFileName is missing
	expected := "open /tmp/testProcessFileUnableToOpen/resources: no such file or directory"
	if err := ldr.processFile("unusedValue", `resources`); err == nil || err.Error() != expected {
		t.Errorf("Expected %+v, received %+v", expected, err)
	}

	if err := os.Remove(flPath); err != nil {
		t.Error(err)
	}
}

func testProcessFileRenameError(t *testing.T) {
	flPath1 := "/tmp/testProcessFileLockFolder"
	if err := os.MkdirAll(flPath1, 0777); err != nil {
		t.Error(err)
	}
	data := engine.NewInternalDB(nil, nil, true)
	ldr := &Loader{
		ldrID:         "testProcessFileRenameError",
		dm:            engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
		fieldSep:      utils.FieldsSep,
		tpInDir:       flPath1,
		tpOutDir:      "INEXISTING_FILE",
		lockFilepath:  utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}
	ldr.dataTpls = map[string][]*config.FCTemplate{
		utils.MetaResources: {
			{Tag: "Tenant",
				Path:      "Tenant",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
				Mandatory: true},
			{Tag: "ID",
				Path:      "ID",
				Type:      utils.MetaComposed,
				Value:     config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
				Mandatory: true},
		},
	}

	// 	resCsv := `
	// #Tenant[0],ID[1]
	// cgrates.org,NewRes1
	// `
	// 	rdr := io.NopCloser(strings.NewReader(resCsv))

	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			utils.ResourcesCsv: &openedCSVFile{
				rdr:    io.NopCloser(nil),
				csvRdr: csv.NewReader(nil),
			},
		},
	}

	file, err := os.Create(path.Join(flPath1, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}
	file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
	file.Close()

	expected := "rename /tmp/testProcessFileLockFolder/Resources.csv INEXISTING_FILE/Resources.csv: no such file or directory"
	if err := ldr.processFile("unusedValue", utils.ResourcesCsv); err == nil || err.Error() != expected {
		t.Errorf("Expected %+v, received %+v", expected, err)
	}

	if err := os.RemoveAll(flPath1); err != nil {
		t.Error(err)
	}
}

func testAllFilesPresentEmptyCSV(t *testing.T) {
	ldr := &Loader{
		ldrID:         "testProcessFileRenameError",
		lockFilepath:  utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}
	ldr.rdrs = map[string]map[string]*openedCSVFile{
		utils.MetaResources: {
			utils.ResourcesCsv: nil,
		},
	}
	if rcv := ldr.allFilesPresent(utils.MetaResources); rcv {
		t.Errorf("Expecting false")
	}
}

func testIsFolderLocked(t *testing.T) {
	flPath := "/tmp/testIsFolderLocked"
	ldr := &Loader{
		ldrID:         "TestLoadAndRemoveResources",
		tpInDir:       flPath,
		lockFilepath:  utils.EmptyString,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}
	expected := "stat /\x00: invalid argument"
	if _, err := ldr.isFolderLocked(); err != nil {
		t.Errorf("Expected %+v, received %+v", expected, err)
	}
}

func testNewLockFolder(t *testing.T) {
	pathL := "/tmp/testNewLockFolder/"
	if err := os.MkdirAll(pathL, 0777); err != nil {
		t.Error(err)
	}

	_, err := os.Create(path.Join(pathL, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}

	ldr := &Loader{
		ldrID:         "testNewLockFolder",
		tpInDir:       "",
		lockFilepath:  pathL + utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}

	if err := ldr.lockFolder(); err != nil {
		t.Error(err)
	}
	if err := os.RemoveAll(pathL); err != nil {
		t.Error(err)
	}
}

func testNewLockFolderNotFound(t *testing.T) {
	pathL := "/tmp/testNewLockFolder/"
	ldr := &Loader{
		ldrID:         "testNewLockFolder",
		tpInDir:       "",
		lockFilepath:  pathL + ".lck",
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}

	errExpect := "open /tmp/testNewLockFolder/.lck: no such file or directory"
	if err := ldr.lockFolder(); err == nil || err.Error() != errExpect {
		t.Error(err)
	}
}

func testNewIsFolderLock(t *testing.T) {
	pathL := "/tmp/testNewLockFolder/"
	if err := os.MkdirAll(pathL, 0777); err != nil {
		t.Error(err)
	}

	_, err := os.Create(path.Join(pathL, utils.ResourcesCsv))
	if err != nil {
		t.Error(err)
	}

	ldr := &Loader{
		ldrID:         "testNewLockFolder",
		tpInDir:       "",
		lockFilepath:  pathL + utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}

	if err := ldr.lockFolder(); err != nil {
		t.Error(err)
	}

	isLocked, err := ldr.isFolderLocked()
	if !isLocked {
		t.Error("Expected the file to be locked")
	}

	if err := os.RemoveAll(pathL); err != nil {
		t.Error(err)
	}
}

func testNewIsFolderUnlock(t *testing.T) {
	pathL := "/tmp/testNewLockFolder/"

	ldr := &Loader{
		ldrID:         "testNewLockFolder",
		tpInDir:       "",
		lockFilepath:  pathL + utils.ResourcesCsv,
		bufLoaderData: make(map[string][]LoaderData),
		timezone:      "UTC",
	}

	errExpect := "open /tmp/testNewLockFolder/Resources.csv: no such file or directory"
	if err := ldr.lockFolder(); err == nil || err.Error() != errExpect {
		t.Error(err)
	}

	isUnlocked, _ := ldr.isFolderLocked()
	if isUnlocked == true {
		t.Error("Expected the file to be unlocked")
	}
}