uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-14 20:59:53 +05:00
Code Issues Packages Projects Releases Wiki Activity

Almost done making lock file configurable in loaders

Browse Source
This commit is contained in:
nickolasdaniel
2021-09-23 17:14:35 +03:00
committed by Dan Christian Bogos
parent 269c48c1b8
commit bebaef2454
6 changed files with 146 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/config_test.go
View File

@@ -2097,7 +2097,7 @@ func TestLoaderConfig(t *testing.T) {
Enabled: false,
ID: utils.MetaDefault,
Tenant: ten,
LockFilePath: ".cgr.lck",
LockFilePath: "/var/spool/cgrates/loader/in/.cgr.lck",
CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
FieldSeparator: ",",
TpInDir: "/var/spool/cgrates/loader/in",

6
config/configsanity.go
View File

@@ -21,6 +21,7 @@ package config
import (
"fmt"
"os"
"path"
"strings"
"github.com/cgrates/cgrates/utils"
@@ -102,6 +103,11 @@ func (cfg *CGRConfig) checkConfigSanity() error {
return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpOutDir)
}
}
if ldrSCfg.LockFilePath != utils.EmptyString { // tpOutDir support empty string for no moving files after process
if _, err := os.Stat(path.Dir(ldrSCfg.LockFilePath)); err != nil && os.IsNotExist(err) {
return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.LockFilePath)
}
}
for _, data := range ldrSCfg.Data {
if !posibleLoaderTypes.Has(data.Type) {
return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type)

19
config/loaderscfg.go
View File

@@ -19,6 +19,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package config
import (
"os"
"path"
"path/filepath"
"time"
"github.com/cgrates/birpc/context"
@@ -159,9 +162,6 @@ func (l *LoaderSCfg) loadFromJSONCfg(jsnCfg *LoaderJsonCfg, msgTemplates map[str
return
}
}
if jsnCfg.Lockfile_path != nil {
l.LockFilePath = *jsnCfg.Lockfile_path
}
if jsnCfg.Caches_conns != nil {
l.CacheSConns = updateInternalConns(*jsnCfg.Caches_conns, utils.MetaCaches)
}
@@ -174,6 +174,19 @@ func (l *LoaderSCfg) loadFromJSONCfg(jsnCfg *LoaderJsonCfg, msgTemplates map[str
if jsnCfg.Tp_out_dir != nil {
l.TpOutDir = *jsnCfg.Tp_out_dir
}
if jsnCfg.Lockfile_path != nil {
// Check if path is relative, case in which "tpIn" folder should be prepended
l.LockFilePath = *jsnCfg.Lockfile_path
if !filepath.IsAbs(l.LockFilePath) {
l.LockFilePath = path.Join(l.TpInDir, l.LockFilePath)
}
file, err := os.Stat(l.LockFilePath)
if err == nil && file.IsDir() {
l.LockFilePath = path.Join(l.LockFilePath, l.ID+".lck")
}
}
if jsnCfg.Data != nil {
for _, jsnLoCfg := range *jsnCfg.Data {
if jsnLoCfg == nil {

74
config/loaderscfg_test.go
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>
package config
import (
"path"
"reflect"
"testing"
"time"
@@ -2834,3 +2835,76 @@ func TestDiffLoadersJsonCfg(t *testing.T) {
t.Errorf("Expected %v \n but received \n %v", utils.ToJSON(expected), utils.ToJSON(rcv))
}
}
func TestLockFolderRelativePath(t *testing.T) {
ldr := &LoaderSCfg{
TpInDir: "/var/spool/cgrates/loader/in/",
TpOutDir: "/var/spool/cgrates/loader/out/",
LockFilePath: utils.ResourcesCsv,
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer(utils.ResourcesCsv),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join(ldr.TpInDir, ldr.LockFilePath)
if err = ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}
func TestLockFolderNonRelativePath(t *testing.T) {
ldr := &LoaderSCfg{
TpInDir: "/var/spool/cgrates/loader/in/",
TpOutDir: "/var/spool/cgrates/loader/out/",
LockFilePath: utils.ResourcesCsv,
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer(path.Join("/tmp/", utils.ResourcesCsv)),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join("/tmp/", utils.ResourcesCsv)
if err = ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}
func TestLockFolderIsDir(t *testing.T) {
ldr := &LoaderSCfg{
LockFilePath: "test",
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer("/tmp"),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join("/tmp", *jsonCfg.ID+".lck")
if err = ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}

14
loaders/loader.go
View File

@@ -149,8 +149,9 @@ func (ldr *Loader) ProcessFolder(ctx *context.Context, caching, loadOption strin
// lockFolder will attempt to lock the folder by creating the lock file
func (ldr *Loader) lockFolder() (err error) {
if _, err = os.Stat(ldr.lockFilepath); err != nil && os.IsNotExist(err) {
return fmt.Errorf("file: %v not found", ldr.lockFilepath)
// If the path is an empty string, we should not be locking
if ldr.lockFilepath == utils.EmptyString {
return
}
_, err = os.OpenFile(ldr.lockFilepath,
os.O_RDONLY|os.O_CREATE, 0644)
@@ -158,6 +159,11 @@ func (ldr *Loader) lockFolder() (err error) {
}
func (ldr *Loader) unlockFolder() (err error) {
// If the path is an empty string, we should not be locking
if ldr.lockFilepath == utils.EmptyString {
return
}
if _, err = os.Stat(ldr.lockFilepath); err == nil {
return os.Remove(ldr.lockFilepath)
}
@@ -165,6 +171,10 @@ func (ldr *Loader) unlockFolder() (err error) {
}
func (ldr *Loader) isFolderLocked() (locked bool, err error) {
// If the path is an empty string, we should not be locking
if ldr.lockFilepath == utils.EmptyString {
return
}
if _, err = os.Stat(ldr.lockFilepath); err == nil {
return true, nil
}

37
loaders/loader_test.go
View File

@@ -5152,3 +5152,40 @@ func TestStoreLoadedDataDispatcherHosts(t *testing.T) {
t.Error(err)
}
}
func TestLockFolderEmptyString(t *testing.T) {
ldr := &Loader{
enabled: true,
tenant: "cgrates.org",
dryRun: false,
ldrID: "ldrid",
tpInDir: "/var/spool/cgrates/loader/in",
tpOutDir: "/var/spool/cgrates/loader/out",
lockFilepath: utils.EmptyString,
fieldSep: utils.InfieldSep,
}
if err := ldr.lockFolder(); err != nil {
t.Error(err)
}
}
// func TestLockFolderIsDir(t *testing.T) {
// ldr := &Loader{
// enabled: true,
// tenant: "cgrates.org",
// dryRun: false,
// ldrID: "ldrid",
// tpInDir: "/var/spool/cgrates/loader/in",
// tpOutDir: "/var/spool/cgrates/loader/out",
// lockFilepath: "/tmp",
// fieldSep: utils.InfieldSep,
// }
// expPath := path.Join(ldr.lockFilepath, ldr.ldrID+".lck")
// ldr.lockFolder()
// if ldr.lockFilepath != expPath {
// t.Errorf("Expected %v \n but received \n %v", expPath, ldr.lockFilepath)
// }
// }