uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 10:06:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

remove loaders subsystem

Browse Source
This commit is contained in:
gezimbll
2026-01-23 16:40:36 +01:00
committed by Dan Christian Bogos
parent 16370dbe53
commit 3153bc8378
68 changed files with 44 additions and 13021 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
apier/v1/loaders.go
View File

@@ -1,43 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package v1
import (
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/loaders"
)
func NewLoaderSv1(ldrS *loaders.LoaderService) *LoaderSv1 {
return &LoaderSv1{ldrS: ldrS}
}
// Exports RPC from LoaderService
type LoaderSv1 struct {
ldrS *loaders.LoaderService
}
func (ldrSv1 *LoaderSv1) Load(ctx *context.Context, args *loaders.ArgsProcessFolder,
rply *string) error {
return ldrSv1.ldrS.V1Load(ctx, args, rply)
}
func (ldrSv1 *LoaderSv1) Remove(ctx *context.Context, args *loaders.ArgsProcessFolder,
rply *string) error {
return ldrSv1.ldrS.V1Remove(ctx, args, rply)
}

36
apier/v1/loaders_test.go
View File

@@ -1,36 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package v1
import (
"testing"
"github.com/cgrates/cgrates/loaders"
)
func TestNewLoaderSv1(t *testing.T) {
mockLoaderService := &loaders.LoaderService{}
result := NewLoaderSv1(mockLoaderService)
if result == nil {
t.Errorf("Expected non-nil result, but got nil")
}
if result.ldrS != mockLoaderService {
t.Errorf("Expected ldrS to be %v, but got %v", mockLoaderService, result.ldrS)
}
}

184
apier/v1/preload_it_test.go
View File

@@ -1,184 +0,0 @@
//go:build integration
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package v1
import (
"os"
"os/exec"
"path"
"reflect"
"sort"
"testing"
"time"
"github.com/cgrates/birpc"
"github.com/cgrates/birpc/context"
"github.com/cgrates/birpc/jsonrpc"
"github.com/cgrates/cgrates/utils"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
)
var (
preloadCfgPath string
preloadCfgDIR string
preloadCfg *config.CGRConfig
preloadRPC *birpc.Client
preloadTests = []func(t *testing.T){
testCreateDirs,
testPreloadITInitConfig,
testPreloadITStartEngine,
testPreloadITRpcConn,
testPreloadITVerifyAttributes,
testCleanupFiles,
testPreloadITKillEngine,
}
)
func TestPreload(t *testing.T) {
preloadCfgDIR = "tutinternal"
for _, test := range preloadTests {
t.Run(preloadCfgDIR, test)
}
}
func testCreateDirs(t *testing.T) {
for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut",
"/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} {
if err := os.RemoveAll(dir); err != nil {
t.Fatal("Error removing folder: ", dir, err)
}
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal("Error creating folder: ", dir, err)
}
}
if err := os.WriteFile(path.Join("/tmp/In", utils.AttributesCsv), []byte(`
#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight
cgrates.org,ALS1,con1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*string:~*req.Field1:Initial,*req.Field1,*variable,Sub1,true,20
cgrates.org,ALS1,con2;con3,,,,*req.Field2,*variable,Sub2,true,20
`), 0644); err != nil {
t.Fatal(err.Error())
}
}
func testPreloadITInitConfig(t *testing.T) {
var err error
preloadCfgPath = path.Join(*utils.DataDir, "conf", "samples", "loaders", preloadCfgDIR)
if preloadCfg, err = config.NewCGRConfigFromPath(preloadCfgPath); err != nil {
t.Fatal("Got config error: ", err.Error())
}
}
func testPreloadITStartEngine(t *testing.T) {
enginePath, err := exec.LookPath("cgr-engine")
if err != nil {
t.Error(err)
}
engine := exec.Command(enginePath, "-config_path", preloadCfgPath, "-preload", "CustomLoader")
if err := engine.Start(); err != nil {
t.Error(err)
}
fib := utils.FibDuration(time.Millisecond, 0)
var connected bool
for i := 0; i < 25; i++ {
time.Sleep(fib())
if _, err := jsonrpc.Dial(utils.TCP, preloadCfg.ListenCfg().RPCJSONListen); err != nil {
t.Logf("Error <%s> when opening test connection to: <%s>",
err.Error(), preloadCfg.ListenCfg().RPCJSONListen)
} else {
connected = true
break
}
}
if !connected {
t.Errorf("engine did not open port <%s>", preloadCfg.ListenCfg().RPCJSONListen)
}
time.Sleep(100 * time.Millisecond)
}
func testPreloadITRpcConn(t *testing.T) {
var err error
preloadRPC, err = newRPCClient(preloadCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed
if err != nil {
t.Fatal("Could not connect to rater: ", err.Error())
}
}
func testPreloadITVerifyAttributes(t *testing.T) {
eAttrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "ALS1",
FilterIDs: []string{"*string:~*req.Account:1001"},
Contexts: []string{"con1", "con2", "con3"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)},
Attributes: []*engine.Attribute{
{
FilterIDs: []string{"*string:~*req.Field1:Initial"},
Path: "*req.Field1",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep),
},
{
FilterIDs: []string{},
Path: "*req.Field2",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep),
},
},
Blocker: true,
Weight: 20.0,
}
var reply *engine.AttributeProfile
if err := preloadRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile,
utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"}}, &reply); err != nil {
t.Fatal(err)
}
reply.Compile()
sort.Strings(reply.Contexts)
if !reflect.DeepEqual(eAttrPrf, reply) {
eAttrPrf.Attributes[1].FilterIDs = nil
if !reflect.DeepEqual(eAttrPrf, reply) {
t.Errorf("Expecting : %+v,\n received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply))
}
}
}
func testCleanupFiles(t *testing.T) {
for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} {
if err := os.RemoveAll(dir); err != nil {
t.Fatal("Error removing folder: ", dir, err)
}
}
}
func testPreloadITKillEngine(t *testing.T) {
if err := engine.KillEngine(*utils.WaitRater); err != nil {
t.Error(err)
}
}

48
cmd/cgr-engine/cgr-engine.go
View File

@@ -28,7 +28,6 @@ import (
"runtime"
"runtime/pprof"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -36,7 +35,6 @@ import (
"github.com/cgrates/birpc"
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/cores"
"github.com/cgrates/cgrates/loaders"
"github.com/cgrates/cgrates/registrarc"
v1 "github.com/cgrates/cgrates/apier/v1"
@@ -64,7 +62,6 @@ var (
syslogger = cgrEngineFlags.String(utils.LoggerCfg, utils.EmptyString, "Logger type <*syslog|*stdout>")
nodeID = cgrEngineFlags.String(utils.NodeIDCfg, utils.EmptyString, "Node ID of the engine")
logLevel = cgrEngineFlags.Int(utils.LogLevelCfg, -1, "Log level (0=emergency to 7=debug)")
preload = cgrEngineFlags.String(utils.PreloadCgr, utils.EmptyString, "Loader IDs used to load data before engine starts")
setVersions = cgrEngineFlags.Bool(utils.SetVersionsCgr, false, "Overwrite database versions (equivalent to cgr-migrator -exec=*set_versions)")
cfg *config.CGRConfig
@@ -145,7 +142,7 @@ func startRPC(server *cores.Server, internalRaterChan,
internalCdrSChan, internalRsChan, internalIPsChan, internalStatSChan,
internalAttrSChan, internalChargerSChan, internalThdSChan, internalTrendSChan, internalSuplSChan,
internalSMGChan, internalAnalyzerSChan, internalDispatcherSChan,
internalLoaderSChan, internalRALsv1Chan, internalCacheSChan,
internalRALsv1Chan, internalCacheSChan,
internalEEsChan, internalERsChan chan birpc.ClientConnector,
shdChan *utils.SyncedChan) {
if !cfg.DispatcherSCfg().Enabled {
@@ -174,8 +171,6 @@ func startRPC(server *cores.Server, internalRaterChan,
internalSuplSChan <- splS
case analyzerS := <-internalAnalyzerSChan:
internalAnalyzerSChan <- analyzerS
case loaderS := <-internalLoaderSChan:
internalLoaderSChan <- loaderS
case ralS := <-internalRALsv1Chan:
internalRALsv1Chan <- ralS
case chS := <-internalCacheSChan: // added in order to start the RPC before precaching is done
@@ -300,32 +295,6 @@ func singnalHandler(shdWg *sync.WaitGroup, shdChan *utils.SyncedChan) {
}
}
func runPreload(loader *services.LoaderService, internalLoaderSChan chan birpc.ClientConnector,
shdChan *utils.SyncedChan) {
if !cfg.LoaderCfg().Enabled() {
utils.Logger.Err(fmt.Sprintf("<%s> not enabled but required by preload mechanism", utils.LoaderS))
shdChan.CloseOnce()
return
}
ldrs := <-internalLoaderSChan
internalLoaderSChan <- ldrs
var reply string
for _, loaderID := range strings.Split(*preload, utils.FieldsSep) {
if err := loader.GetLoaderS().V1Load(context.TODO(),
&loaders.ArgsProcessFolder{
ForceLock: true, // force lock will unlock the file in case is locked and return error
LoaderID: loaderID,
StopOnError: true,
}, &reply); err != nil {
utils.Logger.Err(fmt.Sprintf("<%s> preload failed on loadID <%s> , err: <%s>", utils.LoaderS, loaderID, err.Error()))
shdChan.CloseOnce()
return
}
}
}
func main() {
cgrEngineFlags.Parse(os.Args[1:])
vers, err := utils.GetCGRVersion()
@@ -443,7 +412,6 @@ func main() {
internalResponderChan := make(chan birpc.ClientConnector, 1)
internalAPIerSv1Chan := make(chan birpc.ClientConnector, 1)
internalAPIerSv2Chan := make(chan birpc.ClientConnector, 1)
internalLoaderSChan := make(chan birpc.ClientConnector, 1)
internalEEsChan := make(chan birpc.ClientConnector, 1)
internalERsChan := make(chan birpc.ClientConnector, 1)
@@ -457,7 +425,6 @@ func main() {
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs): internalCDRServerChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaChargers): internalChargerSChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaGuardian): internalGuardianSChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaLoaders): internalLoaderSChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaResources): internalResourceSChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaIPs): internalIPsChan,
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaResponder): internalResponderChan,
@@ -498,7 +465,6 @@ func main() {
utils.GlobalVarS: new(sync.WaitGroup),
utils.HTTPAgent: new(sync.WaitGroup),
utils.KamailioAgent: new(sync.WaitGroup),
utils.LoaderS: new(sync.WaitGroup),
utils.RadiusAgent: new(sync.WaitGroup),
utils.RALService: new(sync.WaitGroup),
utils.ResourceS: new(sync.WaitGroup),
@@ -617,9 +583,6 @@ func main() {
smg := services.NewSessionService(cfg, dmService, server, internalSessionSChan, shdChan, connManager, anz, srvDep)
ldrs := services.NewLoaderService(cfg, dmService, filterSChan, server,
internalLoaderSChan, connManager, anz, srvDep)
srvManager.AddServices(gvService, attrS, chrS, tS, stS, trS, rnS, reS, ips, routeS, schS, rals,
apiSv1, apiSv2, cdrS, smg, coreS,
services.NewDNSAgent(cfg, filterSChan, shdChan, connManager, caps, srvDep),
@@ -630,7 +593,7 @@ func main() {
services.NewDiameterAgent(cfg, filterSChan, shdChan, connManager, caps, srvDep), // partial reload
services.NewHTTPAgent(cfg, filterSChan, server, connManager, srvDep), // no reload
services.NewPrometheusAgent(cfg, connManager, server, srvDep),
ldrs, anz, dspS, dspH, dmService, storDBService,
anz, dspS, dspH, dmService, storDBService,
services.NewEventExporterService(cfg, filterSChan,
connManager, server, internalEEsChan, anz, srvDep),
services.NewEventReaderService(cfg, dmService, filterSChan,
@@ -659,7 +622,6 @@ func main() {
engine.IntRPC.AddInternalRPCClient(utils.CDRsV2, internalCDRServerChan)
engine.IntRPC.AddInternalRPCClient(utils.ChargerSv1, internalChargerSChan)
engine.IntRPC.AddInternalRPCClient(utils.GuardianSv1, internalGuardianSChan)
engine.IntRPC.AddInternalRPCClient(utils.LoaderSv1, internalLoaderSChan)
engine.IntRPC.AddInternalRPCClient(utils.ResourceSv1, internalResourceSChan)
engine.IntRPC.AddInternalRPCClient(utils.IPsV1, internalIPsChan)
engine.IntRPC.AddInternalRPCClient(utils.Responder, internalResponderChan)
@@ -683,16 +645,12 @@ func main() {
log.Fatal(err)
}
if *preload != utils.EmptyString {
runPreload(ldrs, internalLoaderSChan, shdChan)
}
// Serve rpc connections
go startRPC(server, internalResponderChan, internalCDRServerChan,
internalResourceSChan, internalIPsChan, internalStatSChan,
internalAttributeSChan, internalChargerSChan, internalThresholdSChan,
internalTrendSChan, internalRouteSChan, internalSessionSChan, internalAnalyzerSChan,
internalDispatcherSChan, internalLoaderSChan, internalRALsChan,
internalDispatcherSChan, internalRALsChan,
internalCacheSChan, internalEEsChan, internalERsChan, shdChan)
if *memProfDir != utils.EmptyString {

7
cmd/cgr-engine/cgr-engine_flags_test.go
View File

@@ -133,13 +133,6 @@ func TestCgrEngineFlags(t *testing.T) {
defaultVal: -1,
want: 7,
},
{
name: "preload",
flags: []string{"-preload", "TestPreloadID"},
flagVar: preload,
defaultVal: "",
want: "TestPreloadID",
},
{
name: "setVersions",
flags: []string{"-set_versions"},

41
config/config.go
View File

@@ -46,7 +46,6 @@ var (
dfltKamConnConfig *KamConnCfg // Default Kamailio Connection configuration
dfltRemoteHost *RemoteHost
dfltAstConnCfg *AsteriskConnCfg
dfltLoaderConfig *LoaderSCfg
)
func newDbDefaults() dbDefaults {
@@ -171,7 +170,6 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) {
cfg.migratorCgrCfg.OutDataDBOpts = &DataDBOpts{}
cfg.migratorCgrCfg.OutStorDBOpts = &StorDBOpts{}
cfg.mailerCfg = new(MailerCfg)
cfg.loaderCfg = make(LoaderSCfgs, 0)
cfg.apier = new(ApierCfg)
cfg.ersCfg = new(ERsCfg)
cfg.eesCfg = &EEsCfg{
@@ -230,7 +228,6 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) {
dfltFsConnConfig = cfg.fsAgentCfg.EventSocketConns[0] // We leave it crashing here on purpose if no Connection defaults defined
dfltKamConnConfig = cfg.kamAgentCfg.EvapiConns[0]
dfltAstConnCfg = cfg.asteriskAgentCfg.AsteriskConns[0]
dfltLoaderConfig = cfg.loaderCfg[0].Clone()
dfltRemoteHost = new(RemoteHost)
*dfltRemoteHost = *cfg.rpcConns[utils.MetaLocalHost].Conns[0]
err = cfg.checkConfigSanity()
@@ -288,7 +285,6 @@ type CGRConfig struct {
dfltEvRdr *EventReaderCfg // default event reader
dfltEvExp *EventExporterCfg // default event exporter
loaderCfg LoaderSCfgs // LoaderS configs
httpAgentCfg HTTPAgentCfgs // HttpAgent configs
rldChans map[string]chan struct{} // index here the channels used for reloads
@@ -374,7 +370,7 @@ func (cfg *CGRConfig) loadFromJSONCfg(jsnCfg *CgrJsonCfg) (err error) {
cfg.loadAsteriskAgentCfg, cfg.loadDiameterAgentCfg, cfg.loadRadiusAgentCfg,
cfg.loadDNSAgentCfg, cfg.loadHTTPAgentCfg, cfg.loadPrometheusAgentCfg, cfg.loadAttributeSCfg,
cfg.loadChargerSCfg, cfg.loadResourceSCfg, cfg.loadStatSCfg, cfg.loadTrendSCfg,
cfg.loadRankingSCfg, cfg.loadThresholdSCfg, cfg.loadRouteSCfg, cfg.loadLoaderSCfg,
cfg.loadRankingSCfg, cfg.loadThresholdSCfg, cfg.loadRouteSCfg,
cfg.loadMailerCfg, cfg.loadSureTaxCfg, cfg.loadDispatcherSCfg,
cfg.loadLoaderCgrCfg, cfg.loadMigratorCgrCfg, cfg.loadTLSCgrCfg,
cfg.loadAnalyzerCgrCfg, cfg.loadApierCfg, cfg.loadErsCfg, cfg.loadEesCfg,
@@ -690,23 +686,6 @@ func (cfg *CGRConfig) loadRouteSCfg(jsnCfg *CgrJsonCfg) (err error) {
return cfg.routeSCfg.loadFromJSONCfg(jsnRouteSCfg)
}
// loadLoaderSCfg loads the LoaderS section of the configuration
func (cfg *CGRConfig) loadLoaderSCfg(jsnCfg *CgrJsonCfg) (err error) {
var jsnLoaderCfg []*LoaderJsonCfg
if jsnLoaderCfg, err = jsnCfg.LoaderJsonCfg(); err != nil {
return
}
// cfg.loaderCfg = make(LoaderSCfgs, len(jsnLoaderCfg))
for _, profile := range jsnLoaderCfg {
loadSCfgp := NewDfltLoaderSCfg()
if err = loadSCfgp.loadFromJSONCfg(profile, cfg.templates, cfg.generalCfg.RSRSep); err != nil {
return
}
cfg.loaderCfg = append(cfg.loaderCfg, loadSCfgp) // use append so the loaderS profile to be loaded from multiple files
}
return
}
// loadMailerCfg loads the Mailer section of the configuration
func (cfg *CGRConfig) loadMailerCfg(jsnCfg *CgrJsonCfg) (err error) {
var jsnMailerCfg *MailerJsonCfg
@@ -1019,13 +998,6 @@ func (cfg *CGRConfig) CacheCfg() *CacheCfg {
return cfg.cacheCfg
}
// LoaderCfg returns the Loader Service
func (cfg *CGRConfig) LoaderCfg() LoaderSCfgs {
cfg.lks[LoaderJson].Lock()
defer cfg.lks[LoaderJson].Unlock()
return cfg.loaderCfg
}
// LoaderCgrCfg returns the config for cgr-loader
func (cfg *CGRConfig) LoaderCgrCfg() *LoaderCgrCfg {
cfg.lks[CgrLoaderCfgJson].Lock()
@@ -1311,7 +1283,6 @@ func (cfg *CGRConfig) getLoadFunctions() map[string]func(*CgrJsonCfg) error {
RANKINGS_JSON: cfg.loadRankingSCfg,
THRESHOLDS_JSON: cfg.loadThresholdSCfg,
RouteSJson: cfg.loadRouteSCfg,
LoaderJson: cfg.loadLoaderSCfg,
MAILER_JSN: cfg.loadMailerCfg,
SURETAX_JSON: cfg.loadSureTaxCfg,
CgrLoaderCfgJson: cfg.loadLoaderCgrCfg,
@@ -1487,7 +1458,7 @@ func (cfg *CGRConfig) reloadSections(sections ...string) {
subsystemsThatNeedDataDB := utils.NewStringSet([]string{DATADB_JSN, SCHEDULER_JSN,
RALS_JSN, CDRS_JSN, SessionSJson, ATTRIBUTE_JSN,
ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON,
RouteSJson, LoaderJson, DispatcherSJson, ApierS, IPsJSON,
RouteSJson, DispatcherSJson, ApierS, IPsJSON,
})
subsystemsThatNeedStorDB := utils.NewStringSet([]string{STORDB_JSN, RALS_JSN, CDRS_JSN, ApierS})
needsDataDB := false
@@ -1572,8 +1543,6 @@ func (cfg *CGRConfig) reloadSections(sections ...string) {
cfg.rldChans[RouteSJson] <- struct{}{}
case JanusAgentJson:
cfg.rldChans[JanusAgentJson] <- struct{}{}
case LoaderJson:
cfg.rldChans[LoaderJson] <- struct{}{}
case DispatcherSJson:
cfg.rldChans[DispatcherSJson] <- struct{}{}
case AnalyzerCfgJson:
@@ -1595,7 +1564,6 @@ func (cfg *CGRConfig) reloadSections(sections ...string) {
// AsMapInterface returns the config as a map[string]any
func (cfg *CGRConfig) AsMapInterface(separator string) (mp map[string]any) {
return map[string]any{
LoaderJson: cfg.loaderCfg.AsMapInterface(separator),
HttpAgentJson: cfg.httpAgentCfg.AsMapInterface(separator),
RPCConnsJsonName: cfg.rpcConns.AsMapInterface(),
GENERAL_JSN: cfg.generalCfg.AsMapInterface(),
@@ -1778,8 +1746,6 @@ func (cfg *CGRConfig) V1GetConfig(ctx *context.Context, args *SectionWithAPIOpts
mp = cfg.DispatcherSCfg().AsMapInterface()
case RegistrarCJson:
mp = cfg.RegistrarCCfg().AsMapInterface()
case LoaderJson:
mp = cfg.LoaderCfg().AsMapInterface(cfg.GeneralCfg().RSRSep)
case CgrLoaderCfgJson:
mp = cfg.LoaderCgrCfg().AsMapInterface()
case CgrMigratorCfgJson:
@@ -1954,8 +1920,6 @@ func (cfg *CGRConfig) V1GetConfigAsJSON(ctx *context.Context, args *SectionWithA
mp = cfg.DispatcherSCfg().AsMapInterface()
case RegistrarCJson:
mp = cfg.RegistrarCCfg().AsMapInterface()
case LoaderJson:
mp = cfg.LoaderCfg().AsMapInterface(cfg.GeneralCfg().RSRSep)
case CgrLoaderCfgJson:
mp = cfg.LoaderCgrCfg().AsMapInterface()
case CgrMigratorCfgJson:
@@ -2042,7 +2006,6 @@ func (cfg *CGRConfig) Clone() (cln *CGRConfig) {
dfltEvRdr: cfg.dfltEvRdr.Clone(),
dfltEvExp: cfg.dfltEvExp.Clone(),
loaderCfg: cfg.loaderCfg.Clone(),
httpAgentCfg: cfg.httpAgentCfg.Clone(),
rpcConns: cfg.rpcConns.Clone(),
templates: cfg.templates.Clone(),

190
config/config_defaults.go
View File

@@ -1003,196 +1003,6 @@ const CGRATES_CFG_JSON = `
},
},
"loaders": [ // LoaderS config
{
"id": "*default", // identifier of the Loader
"enabled": false, // starts as service: <true|false>.
"tenant": "", // tenant used in filterS.Pass
"dry_run": false, // do not send the CDRs to CDRS, just parse them
"run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together
"lockfile_path": ".cgr.lck", // Filename containing concurrency lock in case of delayed processing
"caches_conns": ["*internal"],
"field_separator": ",", // separator used in case of csv files
"tp_in_dir": "/var/spool/cgrates/loader/in", // absolute path towards the directory where the TPs are stored
"tp_out_dir": "/var/spool/cgrates/loader/out", // absolute path towards the directory where processed TPs will be moved
"data":[ // data profiles to load
{
"type": "*attributes", // data source type
"file_name": "Attributes.csv", // file name in the tp_in_dir
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
{
"type": "*filters", // data source type
"file_name": "Filters.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.2"},
{"tag": "Element", "path": "Element", "type": "*variable", "value": "~*req.3"},
{"tag": "Values", "path": "Values", "type": "*variable", "value": "~*req.4"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.5"},
],
},
{
"type": "*resources", // data source type
"file_name": "Resources.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "TTL", "path": "UsageTTL", "type": "*variable", "value": "~*req.4"},
{"tag": "Limit", "path": "Limit", "type": "*variable", "value": "~*req.5"},
{"tag": "AllocationMessage", "path": "AllocationMessage", "type": "*variable", "value": "~*req.6"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.7"},
{"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.8"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.9"},
{"tag": "ThresholdIDs", "path": "ThresholdIDs", "type": "*variable", "value": "~*req.10"},
],
},
{
"type": "*ips", // data source type
"file_name": "IPs.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "TTL", "path": "TTL", "type": "*variable", "value": "~*req.4"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.5"},
{"tag": "AddressPool", "path": "AddressPool", "type": "*variable", "value": "~*req.6"},
{"tag": "Allocation", "path": "Allocation", "type": "*variable", "value": "~*req.7"},
{"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.8"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.9"}
]
},
{
"type": "*stats", // data source type
"file_name": "Stats.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "QueueLength", "path": "QueueLength", "type": "*variable", "value": "~*req.4"},
{"tag": "TTL", "path": "TTL", "type": "*variable", "value": "~*req.5"},
{"tag": "MinItems", "path": "MinItems", "type": "*variable", "value": "~*req.6"},
{"tag": "MetricIDs", "path": "MetricIDs", "type": "*variable", "value": "~*req.7"},
{"tag": "MetricFilterIDs", "path": "MetricFilterIDs", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.10"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.11"},
{"tag": "ThresholdIDs", "path": "ThresholdIDs", "type": "*variable", "value": "~*req.12"},
],
},
{
"type": "*thresholds", // data source type
"file_name": "Thresholds.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "MaxHits", "path": "MaxHits", "type": "*variable", "value": "~*req.4"},
{"tag": "MinHits", "path": "MinHits", "type": "*variable", "value": "~*req.5"},
{"tag": "MinSleep", "path": "MinSleep", "type": "*variable", "value": "~*req.6"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.7"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.8"},
{"tag": "ActionIDs", "path": "ActionIDs", "type": "*variable", "value": "~*req.9"},
{"tag": "Async", "path": "Async", "type": "*variable", "value": "~*req.10"},
],
},
{
"type": "*routes", // data source type
"file_name": "Routes.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "Sorting", "path": "Sorting", "type": "*variable", "value": "~*req.4"},
{"tag": "SortingParameters", "path": "SortingParameters", "type": "*variable", "value": "~*req.5"},
{"tag": "RouteID", "path": "RouteID", "type": "*variable", "value": "~*req.6"},
{"tag": "RouteFilterIDs", "path": "RouteFilterIDs", "type": "*variable", "value": "~*req.7"},
{"tag": "RouteAccountIDs", "path": "RouteAccountIDs", "type": "*variable", "value": "~*req.8"},
{"tag": "RouteRatingPlanIDs", "path": "RouteRatingPlanIDs", "type": "*variable", "value": "~*req.9"},
{"tag": "RouteResourceIDs", "path": "RouteResourceIDs", "type": "*variable", "value": "~*req.10"},
{"tag": "RouteStatIDs", "path": "RouteStatIDs", "type": "*variable", "value": "~*req.11"},
{"tag": "RouteWeight", "path": "RouteWeight", "type": "*variable", "value": "~*req.12"},
{"tag": "RouteBlocker", "path": "RouteBlocker", "type": "*variable", "value": "~*req.13"},
{"tag": "RouteParameters", "path": "RouteParameters", "type": "*variable", "value": "~*req.14"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.15"},
],
},
{
"type": "*chargers", // data source type
"file_name": "Chargers.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"},
{"tag": "RunID", "path": "RunID", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeIDs", "path": "AttributeIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.6"},
],
},
{
"type": "*dispatchers", // data source type
"file_name": "DispatcherProfiles.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "Strategy", "path": "Strategy", "type": "*variable", "value": "~*req.5"},
{"tag": "StrategyParameters", "path": "StrategyParameters", "type": "*variable", "value": "~*req.6"},
{"tag": "ConnID", "path": "ConnID", "type": "*variable", "value": "~*req.7"},
{"tag": "ConnFilterIDs", "path": "ConnFilterIDs", "type": "*variable", "value": "~*req.8"},
{"tag": "ConnWeight", "path": "ConnWeight", "type": "*variable", "value": "~*req.9"},
{"tag": "ConnBlocker", "path": "ConnBlocker", "type": "*variable", "value": "~*req.10"},
{"tag": "ConnParameters", "path": "ConnParameters", "type": "*variable", "value": "~*req.11"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.12"},
],
},
{
"type": "*dispatcher_hosts", // data source type
"file_name": "DispatcherHosts.csv", // file name in the tp_in_dir
"fields": [
{"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Address", "path": "Address", "type": "*variable", "value": "~*req.2"},
{"tag": "Transport", "path": "Transport", "type": "*variable", "value": "~*req.3"},
{"tag": "ConnectAttempts", "path": "ConnectAttempts", "type": "*variable", "value":"~*req.4"},
{"tag": "Reconnects", "path": "Reconnects", "type": "*variable", "value":"~*req.5"},
{"tag": "MaxReconnectInterval", "path": "MaxReconnectInterval", "type": "*variable", "value":"~*req.6"},
{"tag": "ConnectTimeout", "path": "ConnectTimeout", "type": "*variable", "value":"~*req.7"},
{"tag": "ReplyTimeout", "path": "ReplyTimeout", "type": "*variable", "value":"~*req.8"},
{"tag": "TLS", "path": "TLS", "type": "*variable", "value": "~*req.9"},
{"tag": "ClientKey", "path": "ClientKey", "type": "*variable", "value":"~*req.10"},
{"tag": "ClientCertificate", "path": "ClientCertificate", "type": "*variable", "value":"~*req.11"},
{"tag": "CaCertificate", "path": "CaCertificate", "type": "*variable", "value":"~*req.12"},
],
},
],
},
],
"mailer": {
"server": "localhost", // the server to use when sending emails out
"auth_user": "cgrates", // authenticate to email server using this user

15
config/config_json.go
View File

@@ -48,7 +48,6 @@ const (
TRENDS_JSON = "trends"
RANKINGS_JSON = "rankings"
RouteSJson = "routes"
LoaderJson = "loaders"
MAILER_JSN = "mailer"
SURETAX_JSON = "suretax"
DispatcherSJson = "dispatchers"
@@ -77,7 +76,7 @@ var (
sortedCfgSections = []string{GENERAL_JSN, RPCConnsJsonName, DATADB_JSN, STORDB_JSN, LISTEN_JSN, TlsCfgJson, HTTP_JSN, SCHEDULER_JSN,
CACHE_JSN, FilterSjsn, RALS_JSN, CDRS_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN,
DA_JSN, RA_JSN, HttpAgentJson, DNSAgentJson, PrometheusAgentJSON, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, TRENDS_JSON, RANKINGS_JSON,
THRESHOLDS_JSON, RouteSJson, LoaderJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, JanusAgentJson,
THRESHOLDS_JSON, RouteSJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, JanusAgentJson,
AnalyzerCfgJson, ApierS, EEsJson, SIPAgentJson, RegistrarCJson, TemplatesJson, ConfigSJson, APIBanCfgJson, SentryPeerCfgJson, CoreSCfgJson, IPsJSON}
)
@@ -433,18 +432,6 @@ func (jsnCfg CgrJsonCfg) RouteSJsonCfg() (*RouteSJsonCfg, error) {
return cfg, nil
}
func (jsnCfg CgrJsonCfg) LoaderJsonCfg() ([]*LoaderJsonCfg, error) {
rawCfg, hasKey := jsnCfg[LoaderJson]
if !hasKey {
return nil, nil
}
cfg := make([]*LoaderJsonCfg, 0)
if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
return nil, err
}
return cfg, nil
}
func (jsnCfg CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) {
rawCfg, hasKey := jsnCfg[MAILER_JSN]
if !hasKey {

563
config/config_json_test.go
View File

@@ -1379,569 +1379,6 @@ func TestDfRouteSJsonCfg(t *testing.T) {
}
}
func TestDfLoaderJsonCfg(t *testing.T) {
eCfg := []*LoaderJsonCfg{
{
ID: utils.StringPointer(utils.MetaDefault),
Enabled: utils.BoolPointer(false),
Tenant: utils.StringPointer(""),
Dry_run: utils.BoolPointer(false),
Run_delay: utils.StringPointer("0"),
Lockfile_path: utils.StringPointer(".cgr.lck"),
Caches_conns: &[]string{utils.MetaInternal},
Field_separator: utils.StringPointer(","),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out"),
Data: &[]*LoaderJsonDataType{
{
Type: utils.StringPointer(utils.MetaAttributes),
File_name: utils.StringPointer(utils.AttributesCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer("TenantID"),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("ProfileID"),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("Contexts"),
Path: utils.StringPointer(utils.Contexts),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer(utils.FilterIDs),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("AttributeFilterIDs"),
Path: utils.StringPointer("AttributeFilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("Path"),
Path: utils.StringPointer(utils.Path),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("Type"),
Path: utils.StringPointer("Type"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("Value"),
Path: utils.StringPointer("Value"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("Blocker"),
Path: utils.StringPointer("Blocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer(utils.Weight),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
},
},
{
Type: utils.StringPointer(utils.MetaFilters),
File_name: utils.StringPointer(utils.FiltersCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("Type"),
Path: utils.StringPointer("Type"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("Element"),
Path: utils.StringPointer("Element"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("Values"),
Path: utils.StringPointer("Values"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
},
},
{
Type: utils.StringPointer(utils.MetaResources),
File_name: utils.StringPointer(utils.ResourcesCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("TTL"),
Path: utils.StringPointer("UsageTTL"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("Limit"),
Path: utils.StringPointer("Limit"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("AllocationMessage"),
Path: utils.StringPointer("AllocationMessage"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("Blocker"),
Path: utils.StringPointer("Blocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("Stored"),
Path: utils.StringPointer("Stored"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("ThresholdIDs"),
Path: utils.StringPointer("ThresholdIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
},
},
{
Type: utils.StringPointer(utils.MetaIPs),
File_name: utils.StringPointer(utils.IPsCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("TTL"),
Path: utils.StringPointer("TTL"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("Type"),
Path: utils.StringPointer("Type"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer(utils.AddressPool),
Path: utils.StringPointer(utils.AddressPool),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer(utils.Allocation),
Path: utils.StringPointer(utils.Allocation),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("Stored"),
Path: utils.StringPointer("Stored"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
},
},
{
Type: utils.StringPointer(utils.MetaStats),
File_name: utils.StringPointer(utils.StatsCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("QueueLength"),
Path: utils.StringPointer("QueueLength"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("TTL"),
Path: utils.StringPointer("TTL"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("MinItems"),
Path: utils.StringPointer("MinItems"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("MetricIDs"),
Path: utils.StringPointer("MetricIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("MetricFilterIDs"),
Path: utils.StringPointer("MetricFilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("Blocker"),
Path: utils.StringPointer("Blocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("Stored"),
Path: utils.StringPointer("Stored"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.11")},
{Tag: utils.StringPointer("ThresholdIDs"),
Path: utils.StringPointer("ThresholdIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.12")},
},
},
{
Type: utils.StringPointer(utils.MetaThresholds),
File_name: utils.StringPointer(utils.ThresholdsCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("MaxHits"),
Path: utils.StringPointer("MaxHits"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("MinHits"),
Path: utils.StringPointer("MinHits"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("MinSleep"),
Path: utils.StringPointer("MinSleep"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("Blocker"),
Path: utils.StringPointer("Blocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("ActionIDs"),
Path: utils.StringPointer("ActionIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("Async"),
Path: utils.StringPointer("Async"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
},
},
{
Type: utils.StringPointer(utils.MetaRoutes),
File_name: utils.StringPointer(utils.RoutesCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("Sorting"),
Path: utils.StringPointer("Sorting"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("SortingParameters"),
Path: utils.StringPointer("SortingParameters"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("RouteID"),
Path: utils.StringPointer("RouteID"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("RouteFilterIDs"),
Path: utils.StringPointer("RouteFilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("RouteAccountIDs"),
Path: utils.StringPointer("RouteAccountIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("RouteRatingPlanIDs"),
Path: utils.StringPointer("RouteRatingPlanIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("RouteResourceIDs"),
Path: utils.StringPointer("RouteResourceIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
{Tag: utils.StringPointer("RouteStatIDs"),
Path: utils.StringPointer("RouteStatIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.11")},
{Tag: utils.StringPointer("RouteWeight"),
Path: utils.StringPointer("RouteWeight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.12")},
{Tag: utils.StringPointer("RouteBlocker"),
Path: utils.StringPointer("RouteBlocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.13")},
{Tag: utils.StringPointer("RouteParameters"),
Path: utils.StringPointer("RouteParameters"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.14")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.15")},
},
},
{
Type: utils.StringPointer(utils.MetaChargers),
File_name: utils.StringPointer(utils.ChargersCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("RunID"),
Path: utils.StringPointer("RunID"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("AttributeIDs"),
Path: utils.StringPointer("AttributeIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
},
},
{
Type: utils.StringPointer(utils.MetaDispatchers),
File_name: utils.StringPointer(utils.DispatcherProfilesCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("Contexts"),
Path: utils.StringPointer("Contexts"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("FilterIDs"),
Path: utils.StringPointer("FilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("ActivationInterval"),
Path: utils.StringPointer("ActivationInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4")},
{Tag: utils.StringPointer("Strategy"),
Path: utils.StringPointer("Strategy"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5")},
{Tag: utils.StringPointer("StrategyParameters"),
Path: utils.StringPointer("StrategyParameters"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6")},
{Tag: utils.StringPointer("ConnID"),
Path: utils.StringPointer("ConnID"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7")},
{Tag: utils.StringPointer("ConnFilterIDs"),
Path: utils.StringPointer("ConnFilterIDs"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8")},
{Tag: utils.StringPointer("ConnWeight"),
Path: utils.StringPointer("ConnWeight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9")},
{Tag: utils.StringPointer("ConnBlocker"),
Path: utils.StringPointer("ConnBlocker"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10")},
{Tag: utils.StringPointer("ConnParameters"),
Path: utils.StringPointer("ConnParameters"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.11")},
{Tag: utils.StringPointer("Weight"),
Path: utils.StringPointer("Weight"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.12")},
},
},
{
Type: utils.StringPointer(utils.MetaDispatcherHosts),
File_name: utils.StringPointer(utils.DispatcherHostsCsv),
Fields: &[]*FcTemplateJsonCfg{
{Tag: utils.StringPointer(utils.Tenant),
Path: utils.StringPointer(utils.Tenant),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.0"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer(utils.ID),
Path: utils.StringPointer(utils.ID),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.1"),
Mandatory: utils.BoolPointer(true)},
{Tag: utils.StringPointer("Address"),
Path: utils.StringPointer("Address"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.2")},
{Tag: utils.StringPointer("Transport"),
Path: utils.StringPointer("Transport"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.3")},
{Tag: utils.StringPointer("ConnectAttempts"),
Path: utils.StringPointer("ConnectAttempts"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.4"),
},
{Tag: utils.StringPointer("Reconnects"),
Path: utils.StringPointer("Reconnects"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.5"),
},
{Tag: utils.StringPointer("MaxReconnectInterval"),
Path: utils.StringPointer("MaxReconnectInterval"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.6"),
},
{Tag: utils.StringPointer("ConnectTimeout"),
Path: utils.StringPointer("ConnectTimeout"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.7"),
},
{Tag: utils.StringPointer("ReplyTimeout"),
Path: utils.StringPointer("ReplyTimeout"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.8"),
},
{Tag: utils.StringPointer("TLS"),
Path: utils.StringPointer("TLS"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.9"),
},
{Tag: utils.StringPointer("ClientKey"),
Path: utils.StringPointer("ClientKey"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.10"),
},
{Tag: utils.StringPointer("ClientCertificate"),
Path: utils.StringPointer("ClientCertificate"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.11"),
},
{Tag: utils.StringPointer("CaCertificate"),
Path: utils.StringPointer("CaCertificate"),
Type: utils.StringPointer(utils.MetaVariable),
Value: utils.StringPointer("~*req.12"),
},
},
},
},
},
}
dfCgrJSONCfg, err := NewCgrJsonCfgFromBytes([]byte(CGRATES_CFG_JSON))
if err != nil {
t.Error(err)
}
if cfg, err := dfCgrJSONCfg.LoaderJsonCfg(); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(eCfg, cfg) {
t.Errorf("Expecting: %s \n but received \n Received: %s ",
utils.ToJSON(eCfg), utils.ToJSON(cfg))
}
}
func TestDfMailerJsonCfg(t *testing.T) {
eCfg := &MailerJsonCfg{
Server: utils.StringPointer("localhost"),

798
config/config_test.go
View File

File diff suppressed because one or more lines are too long

46
config/configsanity.go
View File

@@ -22,7 +22,6 @@ import (
"fmt"
"math"
"os"
"path"
"slices"
"strings"
@@ -119,51 +118,6 @@ func (cfg *CGRConfig) checkConfigSanity() error {
}
}
}
// Loaders sanity checks
for _, ldrSCfg := range cfg.loaderCfg {
if !ldrSCfg.Enabled {
continue
}
if _, err := os.Stat(ldrSCfg.TpInDir); err != nil && os.IsNotExist(err) { // if loader is enabled tpInDir must exist
return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpInDir)
}
if ldrSCfg.TpOutDir != utils.EmptyString { // tpOutDir support empty string for no moving files after process
if _, err := os.Stat(ldrSCfg.TpOutDir); err != nil && os.IsNotExist(err) {
return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpOutDir)
}
}
if ldrSCfg.LockFilePath != utils.EmptyString { // tpOutDir support empty string for no moving files after process
pathL := ldrSCfg.GetLockFilePath()
if _, err := os.Stat(path.Dir(pathL)); err != nil && os.IsNotExist(err) {
return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, pathL)
}
}
for _, data := range ldrSCfg.Data {
if !possibleLoaderTypes.Has(data.Type) {
return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type)
}
for _, field := range data.Fields {
if field.Type != utils.MetaComposed && field.Type != utils.MetaString && field.Type != utils.MetaVariable {
return fmt.Errorf("<%s> invalid field type %s for %s at %s", utils.LoaderS, field.Type, data.Type, field.Tag)
}
if field.Path == utils.EmptyString {
return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, utils.NewErrMandatoryIeMissing(utils.Path), data.Type, field.Tag)
}
if err := utils.IsPathValidForExporters(field.Path); err != nil {
return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, field.Path, utils.Path)
}
for _, val := range field.Value {
if err := utils.IsPathValidForExporters(val.path); err != nil {
return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, val.path, utils.Values)
}
}
if err := utils.CheckInLineFilter(field.Filters); err != nil {
return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, field.Filters, utils.Filters)
}
}
}
}
// SessionS checks
if cfg.sessionSCfg.Enabled {
if cfg.sessionSCfg.TerminateAttempts < 1 {

110
config/configsanity_test.go
View File

@@ -138,116 +138,6 @@ func TestConfigSanityCDRServer(t *testing.T) {
}
}
func TestConfigSanityLoaders(t *testing.T) {
cfg := NewDefaultCGRConfig()
cfg.loaderCfg = LoaderSCfgs{
&LoaderSCfg{
Enabled: true,
TpInDir: "/not/exist",
Data: []*LoaderDataType{{
Type: "strsdfing",
}},
},
}
expected := "<LoaderS> nonexistent folder: /not/exist"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg = LoaderSCfgs{
&LoaderSCfg{
Enabled: true,
TpInDir: "/",
TpOutDir: "/not/exist",
Data: []*LoaderDataType{{
Type: "strsdfing",
}},
},
}
expected = "<LoaderS> nonexistent folder: /not/exist"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg = LoaderSCfgs{
&LoaderSCfg{
Enabled: true,
TpInDir: "/",
TpOutDir: "/",
Data: []*LoaderDataType{{
Type: "wrongtype",
}},
},
}
expected = "<LoaderS> unsupported data type wrongtype"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg = LoaderSCfgs{
&LoaderSCfg{
Enabled: true,
TpInDir: "/",
TpOutDir: "/",
Data: []*LoaderDataType{{
Type: utils.MetaStats,
Fields: []*FCTemplate{{
Type: utils.MetaStats,
Tag: "test1",
}},
}},
},
}
expected = "<LoaderS> invalid field type *stats for *stats at test1"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg = LoaderSCfgs{
&LoaderSCfg{
Enabled: true,
TpInDir: "/",
TpOutDir: "/",
Data: []*LoaderDataType{{
Type: utils.MetaStats,
Fields: []*FCTemplate{{
Type: utils.MetaComposed,
Tag: "test1",
Path: utils.EmptyString,
}},
}},
},
}
expected = "<LoaderS> MANDATORY_IE_MISSING: [Path] for *stats at test1"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg[0].Data[0].Fields[0].Path = "~req."
expected = "<LoaderS> Empty field path for ~req. at Path"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg[0].Data[0].Fields[0].Path = "~*req.Destination"
cfg.loaderCfg[0].Data[0].Fields[0].Value = RSRParsers{{}}
cfg.loaderCfg[0].Data[0].Fields[0].Value[0].path = "~req."
expected = "<LoaderS> Empty field path for ~req. at Values"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg[0].Data[0].Fields[0].Value[0].path = "*req.Destination"
cfg.loaderCfg[0].Data[0].Fields[0].Filters = make([]string, 1)
cfg.loaderCfg[0].Data[0].Fields[0].Filters = []string{"*string:~*req..Field"}
expected = "<LoaderS> inline parse error for string: <*string:~*req..Field> for [*string:~*req..Field] at Filters"
if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected {
t.Errorf("Expecting: %+q received: %+q", expected, err)
}
cfg.loaderCfg[0].Data[0].Fields[0].Filters = []string{"~req.Valid.Field"}
}
func TestConfigSanitySessionS(t *testing.T) {
cfg := NewDefaultCGRConfig()
cfg.sessionSCfg = &SessionSCfg{

21
config/libconfig_json.go
View File

@@ -800,27 +800,6 @@ type RouteSJsonCfg struct {
Opts *RoutesOptsJson
}
type LoaderJsonDataType struct {
Type *string
File_name *string
Flags *[]string
Fields *[]*FcTemplateJsonCfg
}
type LoaderJsonCfg struct {
ID *string
Enabled *bool
Tenant *string
Dry_run *bool
Run_delay *string
Lockfile_path *string
Caches_conns *[]string
Field_separator *string
Tp_in_dir *string
Tp_out_dir *string
Data *[]*LoaderJsonDataType
}
// Mailer config section
type MailerJsonCfg struct {
Server *string

277
config/loaderscfg.go
View File

@@ -1,277 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package config
import (
"os"
"path"
"path/filepath"
"time"
"github.com/cgrates/cgrates/utils"
)
// NewDfltLoaderSCfg returns the first cached default value for a LoaderSCfg connection
func NewDfltLoaderSCfg() *LoaderSCfg {
if dfltLoaderConfig == nil {
return new(LoaderSCfg)
}
dfltVal := *dfltLoaderConfig
return &dfltVal
}
// LoaderSCfgs to export some methods for LoaderS profiles
type LoaderSCfgs []*LoaderSCfg
// AsMapInterface returns the config as a map[string]any
func (ldrs LoaderSCfgs) AsMapInterface(separator string) (loaderCfg []map[string]any) {
loaderCfg = make([]map[string]any, len(ldrs))
for i, item := range ldrs {
loaderCfg[i] = item.AsMapInterface(separator)
}
return
}
// Enabled returns true if Loader Service is enabled
func (ldrs LoaderSCfgs) Enabled() bool {
for _, ldr := range ldrs {
if ldr.Enabled {
return true
}
}
return false
}
// Clone itself into a new LoaderSCfgs
func (ldrs LoaderSCfgs) Clone() (cln LoaderSCfgs) {
cln = make(LoaderSCfgs, len(ldrs))
for i, ldr := range ldrs {
cln[i] = ldr.Clone()
}
return
}
// LoaderSCfg the config for a loader
type LoaderSCfg struct {
ID string
Enabled bool
Tenant string
DryRun bool
RunDelay time.Duration
LockFilePath string
CacheSConns []string
FieldSeparator string
TpInDir string
TpOutDir string
Data []*LoaderDataType
}
// LoaderDataType the template for profile loading
type LoaderDataType struct {
Type string
Filename string
Flags utils.FlagsWithParams
Fields []*FCTemplate
}
func (lData *LoaderDataType) loadFromJSONCfg(jsnCfg *LoaderJsonDataType, msgTemplates map[string][]*FCTemplate, separator string) (err error) {
if jsnCfg == nil {
return nil
}
if jsnCfg.Type != nil {
lData.Type = *jsnCfg.Type
}
if jsnCfg.File_name != nil {
lData.Filename = *jsnCfg.File_name
}
if jsnCfg.Flags != nil {
lData.Flags = utils.FlagsWithParamsFromSlice(*jsnCfg.Flags)
}
if jsnCfg.Fields != nil {
if lData.Fields, err = FCTemplatesFromFCTemplatesJSONCfg(*jsnCfg.Fields, separator); err != nil {
return
}
if tpls, err := InflateTemplates(lData.Fields, msgTemplates); err != nil {
return err
} else if tpls != nil {
lData.Fields = tpls
}
}
return nil
}
func (l *LoaderSCfg) loadFromJSONCfg(jsnCfg *LoaderJsonCfg, msgTemplates map[string][]*FCTemplate, separator string) (err error) {
if jsnCfg == nil {
return nil
}
if jsnCfg.ID != nil {
l.ID = *jsnCfg.ID
}
if jsnCfg.Enabled != nil {
l.Enabled = *jsnCfg.Enabled
}
if jsnCfg.Tenant != nil {
l.Tenant = *jsnCfg.Tenant
}
if jsnCfg.Dry_run != nil {
l.DryRun = *jsnCfg.Dry_run
}
if jsnCfg.Run_delay != nil {
if l.RunDelay, err = utils.ParseDurationWithNanosecs(*jsnCfg.Run_delay); err != nil {
return
}
}
if jsnCfg.Caches_conns != nil {
l.CacheSConns = make([]string, len(*jsnCfg.Caches_conns))
for idx, connID := range *jsnCfg.Caches_conns {
// if we have the connection internal we change the name so we can have internal rpc for each subsystem
if connID == utils.MetaInternal {
l.CacheSConns[idx] = utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)
} else {
l.CacheSConns[idx] = connID
}
}
}
if jsnCfg.Field_separator != nil {
l.FieldSeparator = *jsnCfg.Field_separator
}
if jsnCfg.Tp_in_dir != nil {
l.TpInDir = *jsnCfg.Tp_in_dir
}
if jsnCfg.Tp_out_dir != nil {
l.TpOutDir = *jsnCfg.Tp_out_dir
}
if jsnCfg.Lockfile_path != nil {
l.LockFilePath = *jsnCfg.Lockfile_path
}
if jsnCfg.Data != nil {
data := make([]*LoaderDataType, len(*jsnCfg.Data))
for idx, jsnLoCfg := range *jsnCfg.Data {
data[idx] = new(LoaderDataType)
if err := data[idx].loadFromJSONCfg(jsnLoCfg, msgTemplates, separator); err != nil {
return err
}
}
l.Data = data
}
return nil
}
func (l LoaderSCfg) GetLockFilePath() (pathL string) {
pathL = l.LockFilePath
if !filepath.IsAbs(pathL) {
pathL = path.Join(l.TpInDir, pathL)
}
if file, err := os.Stat(pathL); err == nil && file.IsDir() {
pathL = path.Join(pathL, l.ID+".lck")
}
return
}
// Clone itself into a new LoaderDataType
func (lData LoaderDataType) Clone() (cln *LoaderDataType) {
cln = &LoaderDataType{
Type: lData.Type,
Filename: lData.Filename,
Flags: lData.Flags.Clone(),
Fields: make([]*FCTemplate, len(lData.Fields)),
}
for idx, val := range lData.Fields {
cln.Fields[idx] = val.Clone()
}
return
}
// Clone itself into a new LoadersConfig
func (l LoaderSCfg) Clone() (cln *LoaderSCfg) {
cln = &LoaderSCfg{
ID: l.ID,
Enabled: l.Enabled,
Tenant: l.Tenant,
DryRun: l.DryRun,
RunDelay: l.RunDelay,
LockFilePath: l.LockFilePath,
CacheSConns: make([]string, len(l.CacheSConns)),
FieldSeparator: l.FieldSeparator,
TpInDir: l.TpInDir,
TpOutDir: l.TpOutDir,
Data: make([]*LoaderDataType, len(l.Data)),
}
copy(cln.CacheSConns, l.CacheSConns)
for idx, fld := range l.Data {
cln.Data[idx] = fld.Clone()
}
return
}
// AsMapInterface returns the config as a map[string]any
func (lData *LoaderDataType) AsMapInterface(separator string) (initialMP map[string]any) {
initialMP = map[string]any{
utils.TypeCf: lData.Type,
utils.FilenameCfg: lData.Filename,
utils.FlagsCfg: lData.Flags.SliceFlags(),
}
fields := make([]map[string]any, len(lData.Fields))
for i, item := range lData.Fields {
fields[i] = item.AsMapInterface(separator)
}
initialMP[utils.FieldsCfg] = fields
return
}
// AsMapInterface returns the config as a map[string]any
func (l *LoaderSCfg) AsMapInterface(separator string) (initialMP map[string]any) {
initialMP = map[string]any{
utils.IDCfg: l.ID,
utils.TenantCfg: l.Tenant,
utils.EnabledCfg: l.Enabled,
utils.DryRunCfg: l.DryRun,
utils.LockFilePathCfg: l.LockFilePath,
utils.FieldSepCfg: l.FieldSeparator,
utils.TpInDirCfg: l.TpInDir,
utils.TpOutDirCfg: l.TpOutDir,
utils.RunDelayCfg: "0",
}
if l.Data != nil {
data := make([]map[string]any, len(l.Data))
for i, item := range l.Data {
data[i] = item.AsMapInterface(separator)
}
initialMP[utils.DataCfg] = data
}
if l.RunDelay != 0 {
initialMP[utils.RunDelayCfg] = l.RunDelay.String()
}
if l.CacheSConns != nil {
cacheSConns := make([]string, len(l.CacheSConns))
for i, item := range l.CacheSConns {
cacheSConns[i] = item
if item == utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches) {
cacheSConns[i] = utils.MetaInternal
}
}
initialMP[utils.CachesConnsCfg] = cacheSConns
}
return
}

586
config/loaderscfg_test.go
View File

@@ -1,586 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package config
import (
"os"
"path"
"reflect"
"testing"
"time"
"github.com/cgrates/cgrates/utils"
)
func TestLoaderSCfgloadFromJsonCfgCase1(t *testing.T) {
cfgJSONStr := `{
"loaders": [
{
"id": "*default",
"enabled": true,
"tenant": "cgrates.org",
"lockfile_path": ".cgr.lck",
"caches_conns": ["*internal","*conn1"],
"field_separator": ",",
"tp_in_dir": "/var/spool/cgrates/loader/in",
"tp_out_dir": "/var/spool/cgrates/loader/out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"flags": [],
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*composed", "value": "~req.0", "mandatory": true,"layout": "2006-01-02T15:04:05Z07:00"},
],
},
],
},
],
}`
val, err := NewRSRParsers("~req.0", utils.InfieldSep)
if err != nil {
t.Error(err)
}
ten := "cgrates.org"
expected := LoaderSCfgs{
{
Enabled: true,
ID: utils.MetaDefault,
Tenant: ten,
LockFilePath: ".cgr.lck",
CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "*conn1"},
FieldSeparator: ",",
TpInDir: "/var/spool/cgrates/loader/in",
TpOutDir: "/var/spool/cgrates/loader/out",
Data: []*LoaderDataType{
{
Type: "*attributes",
Filename: "Attributes.csv",
Flags: utils.FlagsWithParams{},
Fields: []*FCTemplate{
{
Tag: "TenantID",
Path: "Tenant",
pathSlice: []string{"Tenant"},
Type: utils.MetaComposed,
Value: val,
Mandatory: true,
Layout: time.RFC3339,
},
},
},
},
},
}
newCfg := new(CGRConfig)
newCfg.generalCfg = new(GeneralCfg)
newCfg.generalCfg.RSRSep = ";"
if jsonCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil {
t.Error(err)
} else if err = newCfg.loadLoaderSCfg(jsonCfg); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(expected, newCfg.loaderCfg) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected), utils.ToJSON(newCfg.loaderCfg))
}
}
// func TestLoaderSCfgloadFromJsonCfgCase2(t *testing.T) {
// cfgJSON := &LoaderJsonCfg{
// Tenant: utils.StringPointer("a{*"),
// }
// expected := "invalid converter terminator in rule: <a{*>"
// jsonCfg := NewDefaultCGRConfig()
// if err = jsonCfg.loaderCfg[0].loadFromJSONCfg(nil, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err != nil {
// t.Error(err)
// } else if err = jsonCfg.loaderCfg[0].loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected {
// t.Errorf("Expected %+v, received %+v", expected, err)
// }
// }
func TestLoaderSCfgloadFromJsonCfgCase3(t *testing.T) {
cfg := &LoaderJsonCfg{
Data: &[]*LoaderJsonDataType{
{
Fields: &[]*FcTemplateJsonCfg{
{
Value: utils.StringPointer("a{*"),
},
},
},
},
}
expected := "invalid converter terminator in rule: <a{*>"
jsonCfg := NewDefaultCGRConfig()
if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestLoaderSCfgloadFromJsonCfgCase4(t *testing.T) {
cfg := &LoaderJsonCfg{
Data: &[]*LoaderJsonDataType{
{
Fields: &[]*FcTemplateJsonCfg{
{
Type: utils.StringPointer(utils.MetaTemplate),
},
},
},
},
}
expected := "no template with id: <>"
jsonCfg := NewDefaultCGRConfig()
if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestLoaderSCfgloadFromJsonCfgCase5(t *testing.T) {
cfg := &LoaderJsonCfg{
Data: &[]*LoaderJsonDataType{
{
Fields: &[]*FcTemplateJsonCfg{
{
Tag: utils.StringPointer("randomTag"),
Path: utils.StringPointer("randomPath"),
Type: utils.StringPointer(utils.MetaTemplate),
Value: utils.StringPointer("randomTemplate"),
},
},
},
},
}
expectedFields := LoaderSCfgs{
{
Data: []*LoaderDataType{
{
Fields: []*FCTemplate{
{
Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaVariable,
Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true,
},
},
},
},
},
}
msgTemplates := map[string][]*FCTemplate{
"randomTemplate": {
{
Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaVariable,
Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true,
},
},
}
jsonCfg := NewDefaultCGRConfig()
if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, msgTemplates, jsonCfg.generalCfg.RSRSep); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(jsonCfg.loaderCfg[0].Data[0].Fields[0], expectedFields[0].Data[0].Fields[0]) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedFields[0].Data[0].Fields[0]), utils.ToJSON(jsonCfg.loaderCfg[0].Data[0].Fields[0]))
}
if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(nil, msgTemplates, jsonCfg.generalCfg.RSRSep); err != nil {
t.Error(err)
}
}
func TestLoaderSCfgloadFromJsonCfgCase6(t *testing.T) {
cfg := &LoaderJsonCfg{
Data: &[]*LoaderJsonDataType{nil},
}
jsonCfg := NewDefaultCGRConfig()
if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err != nil {
t.Error(err)
}
}
func TestEnabledCase1(t *testing.T) {
jsonCfg := NewDefaultCGRConfig()
if enabled := jsonCfg.loaderCfg.Enabled(); enabled {
t.Errorf("Expected %+v", enabled)
}
}
func TestEnabledCase2(t *testing.T) {
cfgJSONStr := `{
"loaders": [
{
"enabled": true,
},
],
}`
if jsonCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil {
t.Error(err)
} else if enabled := jsonCfg.loaderCfg.Enabled(); !enabled {
t.Errorf("Expected %+v", enabled)
}
}
func TestLoaderCfgloadFromJsonCfg(t *testing.T) {
cfgJSONStr := `{
"loaders": [
{
"enabled": true,
"run_delay": "1sa",
},
],
}`
expected := "time: unknown unit \"sa\" in duration \"1sa\""
if _, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err == nil || err.Error() != expected {
t.Errorf("Expected error: %s ,received: %v", expected, err)
}
}
func TestLoaderCfgAsMapInterfaceCase1(t *testing.T) {
cfgJSONStr := `{
"loaders": [
{
"id": "*default",
"enabled": false,
"tenant": "~*req.Destination1",
"dry_run": false,
"run_delay": "0",
"lockfile_path": ".cgr.lck",
"caches_conns": ["*internal:*caches"],
"field_separator": ",",
"tp_in_dir": "/var/spool/cgrates/loader/in",
"tp_out_dir": "/var/spool/cgrates/loader/out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
],
},
],
},
],
}`
eMap := []map[string]any{
{
utils.IDCfg: "*default",
utils.EnabledCfg: false,
utils.TenantCfg: "~*req.Destination1",
utils.DryRunCfg: false,
utils.RunDelayCfg: "0",
utils.LockFilePathCfg: ".cgr.lck",
utils.CachesConnsCfg: []string{utils.MetaInternal},
utils.FieldSepCfg: ",",
utils.TpInDirCfg: "/var/spool/cgrates/loader/in",
utils.TpOutDirCfg: "/var/spool/cgrates/loader/out",
utils.DataCfg: []map[string]any{
{
utils.TypeCfg: "*attributes",
utils.FilenameCfg: "Attributes.csv",
utils.FieldsCfg: []map[string]any{
{
utils.TagCfg: "TenantID",
utils.PathCfg: "Tenant",
utils.TypeCfg: "*variable",
utils.ValueCfg: "~*req.0",
utils.MandatoryCfg: true,
}, {
utils.TagCfg: "ProfileID",
utils.PathCfg: "ID",
utils.TypeCfg: "*variable",
utils.ValueCfg: "~*req.1",
utils.MandatoryCfg: true,
},
},
},
},
},
}
if cfgCgr, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil {
t.Error(err)
} else {
rcv := cgrCfg.loaderCfg.AsMapInterface(cfgCgr.generalCfg.RSRSep)
if !reflect.DeepEqual(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0],
rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0]) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0]),
utils.ToJSON(rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0]))
} else if !reflect.DeepEqual(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1],
rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1]) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1]),
utils.ToJSON(rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1]))
} else if !reflect.DeepEqual(eMap[0][utils.CachesConnsCfg], rcv[0][utils.CachesConnsCfg]) {
t.Errorf("Expected %+v, received %+v", eMap[0][utils.CachesConnsCfg], rcv[0][utils.CachesConnsCfg])
} else if !reflect.DeepEqual(eMap[0][utils.TpInDirCfg], rcv[0][utils.TpInDirCfg]) {
t.Errorf("Expected %+v, received %+v", eMap[0][utils.TpInDirCfg], rcv[0][utils.TpInDirCfg])
} else if !reflect.DeepEqual(eMap[0][utils.LockFilePathCfg], rcv[0][utils.LockFilePathCfg]) {
t.Errorf("Expected %+v, received %+v", eMap[0][utils.LockFilePathCfg], rcv[0][utils.LockFilePathCfg])
}
}
}
func TestLoaderCfgAsMapInterfaceCase2(t *testing.T) {
cfgJSONStr := `{
"loaders": [
{
"id": "*default",
"enabled": false,
"tenant": "~*req.Destination1",
"dry_run": false,
"run_delay": "1",
"lockfile_path": ".cgr.lck",
"caches_conns": ["*conn1"],
"field_separator": ",",
"tp_in_dir": "/var/spool/cgrates/loader/in",
"tp_out_dir": "/var/spool/cgrates/loader/out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
],
},
],
},
],
}`
eMap := []map[string]any{
{
utils.IDCfg: "*default",
utils.EnabledCfg: false,
utils.TenantCfg: "~*req.Destination1",
utils.DryRunCfg: false,
utils.RunDelayCfg: "0",
utils.LockFilePathCfg: ".cgr.lck",
utils.CachesConnsCfg: []string{"*conn1"},
utils.FieldSepCfg: ",",
utils.TpInDirCfg: "/var/spool/cgrates/loader/in",
utils.TpOutDirCfg: "/var/spool/cgrates/loader/out",
utils.DataCfg: []map[string]any{
{
utils.TypeCfg: "*attributes",
utils.FilenameCfg: "Attributes.csv",
utils.FieldsCfg: []map[string]any{
{
utils.TagCfg: "TenantID",
utils.PathCfg: "Tenant",
utils.TypeCfg: "*variable",
utils.ValueCfg: "~*req.0",
utils.MandatoryCfg: true,
}, {
utils.TagCfg: "ProfileID",
utils.PathCfg: "ID",
utils.TypeCfg: "*variable",
utils.ValueCfg: "~*req.1",
utils.MandatoryCfg: true,
},
},
},
},
},
}
if jsonCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil {
t.Error(err)
} else if rcv := jsonCfg.loaderCfg.AsMapInterface(jsonCfg.generalCfg.RSRSep); !reflect.DeepEqual(rcv[0][utils.Tenant], eMap[0][utils.Tenant]) {
t.Errorf("Expected %+v, received %+v", rcv[0][utils.Tenant], eMap[0][utils.Tenant])
}
}
func TestLoaderSCfgsClone(t *testing.T) {
ban := LoaderSCfgs{{
Enabled: true,
ID: utils.MetaDefault,
Tenant: "cgrate.org",
LockFilePath: ".cgr.lck",
CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "*conn1"},
FieldSeparator: ",",
TpInDir: "/var/spool/cgrates/loader/in",
TpOutDir: "/var/spool/cgrates/loader/out",
Data: []*LoaderDataType{{
Type: "*attributes",
Filename: "Attributes.csv",
Flags: utils.FlagsWithParams{},
Fields: []*FCTemplate{
{
Tag: "TenantID",
Path: "Tenant",
pathSlice: []string{"Tenant"},
Type: utils.MetaComposed,
Value: NewRSRParsersMustCompile("cgrate.org", utils.InfieldSep),
Mandatory: true,
Layout: time.RFC3339,
},
}},
},
}}
rcv := ban.Clone()
if !reflect.DeepEqual(ban, rcv) {
t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(ban), utils.ToJSON(rcv))
}
if rcv[0].CacheSConns[1] = ""; ban[0].CacheSConns[1] != "*conn1" {
t.Errorf("Expected clone to not modify the cloned")
}
if rcv[0].Data[0].Type = ""; ban[0].Data[0].Type != "*attributes" {
t.Errorf("Expected clone to not modify the cloned")
}
}
func TestLockFolderRelativePath(t *testing.T) {
ldr := &LoaderSCfg{
TpInDir: "/var/spool/cgrates/loader/in/",
TpOutDir: "/var/spool/cgrates/loader/out/",
LockFilePath: utils.ResourcesCsv,
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer(utils.ResourcesCsv),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join(ldr.LockFilePath)
if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}
func TestLockFolderNonRelativePath(t *testing.T) {
ldr := &LoaderSCfg{
TpInDir: "/var/spool/cgrates/loader/in/",
TpOutDir: "/var/spool/cgrates/loader/out/",
LockFilePath: utils.ResourcesCsv,
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer(path.Join("/tmp/", utils.ResourcesCsv)),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join("/tmp/", utils.ResourcesCsv)
if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}
func TestLockFolderIsDir(t *testing.T) {
ldr := &LoaderSCfg{
LockFilePath: "test",
}
jsonCfg := &LoaderJsonCfg{
ID: utils.StringPointer("loaderid"),
Enabled: utils.BoolPointer(true),
Tenant: utils.StringPointer("cgrates.org"),
Dry_run: utils.BoolPointer(false),
Lockfile_path: utils.StringPointer("/tmp"),
Field_separator: utils.StringPointer(utils.InfieldSep),
Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"),
Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"),
}
expPath := path.Join("/tmp")
if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil {
t.Error(err)
} else if ldr.LockFilePath != expPath {
t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath)
}
}
func TestLockGetLockFilePath(t *testing.T) {
l := LoaderSCfg{
LockFilePath: "folder",
TpInDir: "/dir",
ID: "4"}
expected := "/dir/folder"
if val := l.GetLockFilePath(); val != expected {
t.Error(val)
}
}
func TestLoadersCfgGetLockFilePath(t *testing.T) {
str := "test"
bl := true
l := LoaderSCfg{
ID: str,
Enabled: bl,
Tenant: str,
DryRun: bl,
RunDelay: 1 * time.Second,
LockFilePath: str,
CacheSConns: []string{str},
FieldSeparator: str,
TpInDir: str,
TpOutDir: str,
Data: []*LoaderDataType{},
}
rcv := l.GetLockFilePath()
if rcv != "test/test" {
t.Error(rcv)
}
}
func TestGetLockFilePath(t *testing.T) {
loader := LoaderSCfg{
LockFilePath: "/cgrates/cgrates/lockfile.lck",
}
if loader.GetLockFilePath() != "/cgrates/cgrates/lockfile.lck" {
t.Error("Expected /cgrates/cgrates/lockfile.lck")
}
loader = LoaderSCfg{
LockFilePath: "relative.lck",
TpInDir: "/base/dir",
}
if loader.GetLockFilePath() != "/base/dir/relative.lck" {
t.Error("Expected /base/dir/relative.lck")
}
tmpDir := "/tmp/cgrates_cgrates"
_ = os.Mkdir(tmpDir, 0755)
defer os.Remove(tmpDir)
loader = LoaderSCfg{
LockFilePath: tmpDir,
ID: "loader123",
}
expected := "/tmp/cgrates_cgrates/loader123.lck"
if loader.GetLockFilePath() != expected {
t.Errorf("Expected %v, but got %v", expected, loader.GetLockFilePath())
}
}

65
console/loader_load.go
View File

@@ -1,65 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package console
import (
"github.com/cgrates/cgrates/loaders"
"github.com/cgrates/cgrates/utils"
)
func init() {
c := &CmdLoaderLoad{
name: "loader_load",
rpcMethod: utils.LoaderSv1Load,
rpcParams: &loaders.ArgsProcessFolder{},
}
commands[c.Name()] = c
c.CommandExecuter = &CommandExecuter{c}
}
type CmdLoaderLoad struct {
name string
rpcMethod string
rpcParams *loaders.ArgsProcessFolder
*CommandExecuter
}
func (self *CmdLoaderLoad) Name() string {
return self.name
}
func (self *CmdLoaderLoad) RpcMethod() string {
return self.rpcMethod
}
func (self *CmdLoaderLoad) RpcParams(reset bool) any {
if reset || self.rpcParams == nil {
self.rpcParams = &loaders.ArgsProcessFolder{}
}
return self.rpcParams
}
func (self *CmdLoaderLoad) PostprocessRpcParams() error {
return nil
}
func (self *CmdLoaderLoad) RpcResult() any {
var s string
return &s
}

54
console/loader_load_test.go
View File

@@ -1,54 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package console
import (
"reflect"
"strings"
"testing"
v1 "github.com/cgrates/cgrates/apier/v1"
"github.com/cgrates/cgrates/utils"
)
func TestCmdLoaderLoad(t *testing.T) {
// commands map is initiated in init function
command := commands["loader_load"]
// verify if ApierSv1 object has method on it
m, ok := reflect.TypeOf(new(v1.LoaderSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1])
if !ok {
t.Fatal("method not found")
}
if m.Type.NumIn() != 4 { // expecting 4 inputs
t.Fatalf("invalid number of input parameters ")
}
// verify the type of input parameter
if ok := m.Type.In(2).AssignableTo(reflect.TypeOf(command.RpcParams(true))); !ok {
t.Fatalf("cannot assign input parameter")
}
// verify the type of output parameter
if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok {
t.Fatalf("cannot assign output parameter")
}
// for coverage purpose
if err := command.PostprocessRpcParams(); err != nil {
t.Fatal(err)
}
}

65
console/loader_remove.go
View File

@@ -1,65 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package console
import (
"github.com/cgrates/cgrates/loaders"
"github.com/cgrates/cgrates/utils"
)
func init() {
c := &CmdLoaderRemove{
name: "loader_remove",
rpcMethod: utils.LoaderSv1Remove,
rpcParams: &loaders.ArgsProcessFolder{},
}
commands[c.Name()] = c
c.CommandExecuter = &CommandExecuter{c}
}
type CmdLoaderRemove struct {
name string
rpcMethod string
rpcParams *loaders.ArgsProcessFolder
*CommandExecuter
}
func (self *CmdLoaderRemove) Name() string {
return self.name
}
func (self *CmdLoaderRemove) RpcMethod() string {
return self.rpcMethod
}
func (self *CmdLoaderRemove) RpcParams(reset bool) any {
if reset || self.rpcParams == nil {
self.rpcParams = &loaders.ArgsProcessFolder{}
}
return self.rpcParams
}
func (self *CmdLoaderRemove) PostprocessRpcParams() error {
return nil
}
func (self *CmdLoaderRemove) RpcResult() any {
var s string
return &s
}

54
console/loader_remove_test.go
View File

@@ -1,54 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package console
import (
"reflect"
"strings"
"testing"
v1 "github.com/cgrates/cgrates/apier/v1"
"github.com/cgrates/cgrates/utils"
)
func TestCmdLoaderRemove(t *testing.T) {
// commands map is initiated in init function
command := commands["loader_remove"]
// verify if ApierSv1 object has method on it
m, ok := reflect.TypeOf(new(v1.LoaderSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1])
if !ok {
t.Fatal("method not found")
}
if m.Type.NumIn() != 4 { // expecting 4 inputs
t.Fatalf("invalid number of input parameters ")
}
// verify the type of input parameter
if ok := m.Type.In(2).AssignableTo(reflect.TypeOf(command.RpcParams(true))); !ok {
t.Fatalf("cannot assign input parameter")
}
// verify the type of output parameter
if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok {
t.Fatalf("cannot assign output parameter")
}
// for coverage purpose
if err := command.PostprocessRpcParams(); err != nil {
t.Fatal(err)
}
}

2
console/ping.go
View File

@@ -65,8 +65,6 @@ func (self *CmdApierPing) RpcMethod() string {
return utils.ThresholdSv1Ping
case utils.SessionsLow:
return utils.SessionSv1Ping
case utils.LoaderSLow:
return utils.LoaderSv1Ping
case utils.DispatcherSLow:
return utils.DispatcherSv1Ping
case utils.AnalyzerSLow:

39
console/ping_test.go
View File

@@ -303,45 +303,6 @@ func TestCmdPingSessionsLow(t *testing.T) {
}
}
func TestCmdPingLoaderSLow(t *testing.T) {
// commands map is initiated in init function
command := commands["ping"]
castCommand, canCast := command.(*CmdApierPing)
if !canCast {
t.Fatalf("cannot cast")
}
castCommand.item = utils.LoaderSLow
result2 := command.RpcMethod()
if !reflect.DeepEqual(result2, utils.LoaderSv1Ping) {
t.Errorf("Expected <%+v>, Received <%+v>", utils.LoaderSv1Ping, result2)
}
srv, err := engine.NewService(&v1.LoaderSv1{})
if err != nil {
t.Fatal(err)
}
mType, ok := srv.Methods["Ping"]
if !ok {
t.Fatal("method not found")
}
m := mType.Method
if m.Type.NumIn() != 4 { // expecting 4 inputs
t.Fatalf("invalid number of input parameters ")
}
// for coverage purpose
result := command.RpcParams(true)
if !reflect.DeepEqual(result, new(StringWrapper)) {
t.Errorf("Expected <%T>, Received <%T>", new(StringWrapper), result)
}
// verify the type of output parameter
if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok {
t.Fatalf("cannot assign output parameter")
}
// for coverage purpose
if err := command.PostprocessRpcParams(); err != nil {
t.Fatal(err)
}
}
func TestCmdPingDispatcherSLow(t *testing.T) {
// commands map is initiated in init function
command := commands["ping"]

164
data/conf/samples/loaders/loaders_console_mongo/cgrates.json
View File

@@ -1,164 +0,0 @@
{
// CGRateS Configuration file
"general": {
"log_level": 7,
"reply_timeout": "30s",
},
"listen": {
"rpc_json": ":2012",
"rpc_gob": ":2013",
"http": ":2080",
},
"data_db": {
"db_type": "mongo",
"db_name": "10",
"db_port": 27017,
},
"stor_db": {
"db_type": "mongo",
"db_name": "cgrates",
"db_port": 27017,
"db_password": "",
},
"rals": {
"enabled": true,
"thresholds_conns": ["*internal"],
},
"schedulers": {
"enabled": true,
"cdrs_conns": ["*localhost"],
},
"cdrs": {
"enabled": true,
},
"chargers": {
"enabled": true,
"attributes_conns": ["*internal"],
},
"resources": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"]
},
"stats": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"],
},
"thresholds": {
"enabled": true,
"store_interval": "1s",
},
"routes": {
"enabled": true,
"stats_conns": ["*localhost"],
"resources_conns": ["*localhost"],
},
"attributes": { // Attribute service
"enabled": true, // starts Attribute service: <true|false>.
},
"loaders": [
{
"id": "CustomLoader",
"enabled": true,
"dry_run": false,
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/In",
"tp_out_dir": "/tmp/Out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
],
"sessions": {
"enabled": true,
"rals_conns": ["*internal"],
"cdrs_conns": ["*internal"],
"chargers_conns": ["*internal"],
},
"migrator": {
"out_datadb_type": "mongo",
"out_datadb_port": "27017",
"out_datadb_name": "10",
"out_stordb_type": "mongo",
"out_stordb_port": "27017",
"out_stordb_name": "cgrates",
"users_filters":["Account"],
},
"templates": {
"attrTemplateLoader": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
"apiers": {
"enabled": true,
"scheduler_conns": ["*internal"],
},
}

161
data/conf/samples/loaders/loaders_console_mysql/cgrates.json
View File

@@ -1,161 +0,0 @@
{
// CGRateS Configuration file
//
"general": {
"log_level": 7,
"reply_timeout": "50s",
},
"listen": {
"rpc_json": ":2012",
"rpc_gob": ":2013",
"http": ":2080",
},
"data_db": { // database used to store runtime data (eg: accounts, cdr stats)
"db_type": "redis", // data_db type: <redis|mongo>
"db_port": 6379, // data_db port to reach the database
"db_name": "10", // data_db database name to connect to
},
"stor_db": {
"db_password": "CGRateS.org",
},
"rals": {
"enabled": true,
"thresholds_conns": ["*internal"],
},
"schedulers": {
"enabled": true,
"cdrs_conns": ["*internal"],
},
"cdrs": {
"enabled": true,
"chargers_conns":["*internal"],
},
"loaders": [
{
"id": "CustomLoader",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/In",
"tp_out_dir": "/tmp/Out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
],
"attributes": {
"enabled": true,
"prefix_indexed_fields":["*req.Destination"],
},
"chargers": {
"enabled": true,
"attributes_conns": ["*internal"],
},
"resources": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"]
},
"stats": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"],
},
"thresholds": {
"enabled": true,
"store_interval": "1s",
},
"routes": {
"enabled": true,
"prefix_indexed_fields":["*req.Destination"],
"stats_conns": ["*internal"],
"resources_conns": ["*internal"],
},
"sessions": {
"enabled": true,
"routes_conns": ["*internal"],
"resources_conns": ["*internal"],
"attributes_conns": ["*internal"],
"rals_conns": ["*internal"],
"cdrs_conns": ["*internal"],
"chargers_conns": ["*internal"],
},
"migrator":{
"out_stordb_password": "CGRateS.org",
"users_filters":["Account"],
},
"templates": {
"attrTemplateLoader": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
"apiers": {
"enabled": true,
"scheduler_conns": ["*internal"],
},
}

256
data/conf/samples/loaders/tutinternal/cgrates.json
View File

@@ -1,256 +0,0 @@
{
// CGRateS Configuration file
//
"general": {
"log_level": 7,
"reply_timeout": "50s",
},
"listen": {
"rpc_json": ":2012",
"rpc_gob": ":2013",
"http": ":2080",
},
"data_db": {
"db_type": "*internal",
},
"stor_db": {
"db_type": "*internal"
},
"rals": {
"enabled": true,
},
"schedulers": {
"enabled": true,
"cdrs_conns": ["*internal"],
},
"cdrs": {
"enabled": true,
"chargers_conns":["*internal"],
},
"loaders": [
{
"id": "CustomLoader",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/In",
"tp_out_dir": "/tmp/Out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "WithoutMoveToOut",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/LoaderIn",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithoutMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathWithoutMove",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathLoaderWithMove",
"tp_out_dir": "/tmp/SubpathOut",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "LoaderWithTemplate",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/templateLoaderIn",
"tp_out_dir": "/tmp/templateLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
],
},
],
},
{
"id": "CustomSep",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "-1",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": "\t",
"tp_in_dir": "/tmp/customSepLoaderIn",
"tp_out_dir": "/tmp/customSepLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"},
],
},
],
}
],
"attributes": {
"enabled": true,
"prefix_indexed_fields":["*req.Destination"],
},
"chargers": {
"enabled": true,
"attributes_conns": ["*internal"],
},
"templates": {
"attrTemplateLoader": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
"apiers": {
"enabled": true,
"scheduler_conns": ["*internal"],
},
}

304
data/conf/samples/loaders/tutmongo/cgrates.json
View File

@@ -1,304 +0,0 @@
{
// CGRateS Configuration file
"general": {
"log_level": 7,
"reply_timeout": "30s",
},
"listen": {
"rpc_json": ":2012",
"rpc_gob": ":2013",
"http": ":2080",
},
"data_db": {
"db_type": "mongo",
"db_name": "10",
"db_port": 27017,
},
"stor_db": {
"db_type": "mongo",
"db_name": "cgrates",
"db_port": 27017,
"db_password": "",
},
"rals": {
"enabled": true,
"thresholds_conns": ["*internal"],
},
"schedulers": {
"enabled": true,
"cdrs_conns": ["*localhost"],
},
"cdrs": {
"enabled": true,
},
"chargers": {
"enabled": true,
"attributes_conns": ["*internal"],
},
"resources": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"]
},
"stats": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"],
},
"thresholds": {
"enabled": true,
"store_interval": "1s",
},
"routes": {
"enabled": true,
"stats_conns": ["*localhost"],
"resources_conns": ["*localhost"],
},
"attributes": { // Attribute service
"enabled": true, // starts Attribute service: <true|false>.
},
"loaders": [
{
"id": "CustomLoader",
"enabled": true,
"dry_run": false,
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/In",
"tp_out_dir": "/tmp/Out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "WithoutMoveToOut",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/LoaderIn",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithoutMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathWithoutMove",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathLoaderWithMove",
"tp_out_dir": "/tmp/SubpathOut",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "LoaderWithTemplate",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/templateLoaderIn",
"tp_out_dir": "/tmp/templateLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
],
},
],
},
{
"id": "CustomSep",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "-1",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": "\t",
"tp_in_dir": "/tmp/customSepLoaderIn",
"tp_out_dir": "/tmp/customSepLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"},
],
},
],
}
],
"sessions": {
"enabled": true,
"rals_conns": ["*internal"],
"cdrs_conns": ["*internal"],
"chargers_conns": ["*internal"],
},
"migrator": {
"out_datadb_type": "mongo",
"out_datadb_port": "27017",
"out_datadb_name": "10",
"out_stordb_type": "mongo",
"out_stordb_port": "27017",
"out_stordb_name": "cgrates",
"users_filters":["Account"],
},
"templates": {
"attrTemplateLoader": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
"apiers": {
"enabled": true,
"scheduler_conns": ["*internal"],
},
}

304
data/conf/samples/loaders/tutmysql/cgrates.json
View File

@@ -1,304 +0,0 @@
{
// CGRateS Configuration file
//
"general": {
"log_level": 7,
"reply_timeout": "50s",
},
"listen": {
"rpc_json": ":2012",
"rpc_gob": ":2013",
"http": ":2080",
},
"data_db": { // database used to store runtime data (eg: accounts, cdr stats)
"db_type": "redis", // data_db type: <redis|mongo>
"db_port": 6379, // data_db port to reach the database
"db_name": "10", // data_db database name to connect to
},
"stor_db": {
"db_password": "CGRateS.org",
},
"rals": {
"enabled": true,
"thresholds_conns": ["*internal"],
},
"schedulers": {
"enabled": true,
"cdrs_conns": ["*internal"],
},
"cdrs": {
"enabled": true,
"chargers_conns":["*internal"],
},
"loaders": [
{
"id": "CustomLoader",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/In",
"tp_out_dir": "/tmp/Out",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "WithoutMoveToOut",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/LoaderIn",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithoutMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathWithoutMove",
"tp_out_dir": "",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "SubpathLoaderWithMove",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/SubpathLoaderWithMove",
"tp_out_dir": "/tmp/SubpathOut",
"data":[
{
"type": "*attributes",
"file_name": "folder1/Attributes.csv",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
],
},
{
"id": "LoaderWithTemplate",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "0",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": ",",
"tp_in_dir": "/tmp/templateLoaderIn",
"tp_out_dir": "/tmp/templateLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes.csv",
"fields": [
{"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"}
],
},
],
},
{
"id": "CustomSep",
"enabled": true,
"dry_run": false,
"tenant": "cgrates.org",
"run_delay": "-1",
"lockfile_path": ".cgr.lock",
"caches_conns": ["*internal"],
"field_separator": "\t",
"tp_in_dir": "/tmp/customSepLoaderIn",
"tp_out_dir": "/tmp/customSepLoaderOut",
"data":[
{
"type": "*attributes",
"file_name": "Attributes",
"fields": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"},
],
},
],
}
],
"attributes": {
"enabled": true,
"prefix_indexed_fields":["*req.Destination"],
},
"chargers": {
"enabled": true,
"attributes_conns": ["*internal"],
},
"resources": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"]
},
"stats": {
"enabled": true,
"store_interval": "1s",
"thresholds_conns": ["*internal"],
},
"thresholds": {
"enabled": true,
"store_interval": "1s",
},
"routes": {
"enabled": true,
"prefix_indexed_fields":["*req.Destination"],
"stats_conns": ["*internal"],
"resources_conns": ["*internal"],
},
"sessions": {
"enabled": true,
"routes_conns": ["*internal"],
"resources_conns": ["*internal"],
"attributes_conns": ["*internal"],
"rals_conns": ["*internal"],
"cdrs_conns": ["*internal"],
"chargers_conns": ["*internal"],
},
"migrator":{
"out_stordb_password": "CGRateS.org",
"users_filters":["Account"],
},
"templates": {
"attrTemplateLoader": [
{"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true},
{"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true},
{"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"},
{"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"},
{"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"},
{"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"},
{"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"},
{"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"},
{"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"},
{"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"},
{"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"},
],
},
"apiers": {
"enabled": true,
"scheduler_conns": ["*internal"],
},
}

2
integration_test.sh
View File

@@ -8,7 +8,7 @@
# Example:
# ./integration_test.sh -dbtype=*mysql -rpc=*gob
packages=("agents" "apier/v1" "apier/v2" "cmd/cgr-loader" "dispatchers" "engine" "ers" "general_tests" "loaders" "registrarc" "sessions")
packages=("agents" "apier/v1" "apier/v2" "cmd/cgr-loader" "dispatchers" "engine" "ers" "general_tests" "registrarc" "sessions")
dbtypes=("*internal" "*mysql" "*mongo" "*postgres")
# Tests that are independent of the dbtype flag and run only once

739
loaders/lib_test.go
View File

@@ -1,739 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"encoding/csv"
"io"
"strings"
"testing"
"github.com/cgrates/birpc"
"github.com/cgrates/birpc/context"
"github.com/cgrates/rpcclient"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut",
"/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"}
type testMockCacheConn struct {
calls map[string]func(arg any, rply any) error
}
func (s *testMockCacheConn) Call(ctx *context.Context, method string, arg any, rply any) error {
if call, has := s.calls[method]; !has {
return rpcclient.ErrUnsupporteServiceMethod
} else {
return call(arg, rply)
}
}
func TestProcessContentCallsRemoveItems(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1RemoveItems: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestProcessContentCallsRemoveItems",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_ID
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
if err := ldr.processContent(utils.MetaAttributes, utils.MetaRemove); err != nil {
t.Error(err)
}
// Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.processContent(utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
// Calling the method again while caching method is invalid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
expected = "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.processContent(utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestProcessContentCallsClear(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestProcessContentCallsClear",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_ID
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
if err := ldr.processContent(utils.MetaAttributes, utils.MetaClear); err != nil {
t.Error(err)
}
//inexisting method(*none) of cache and reinitialized the reader will do nothing
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
if err := ldr.processContent(utils.MetaAttributes, utils.MetaNone); err != nil {
t.Error(err)
}
// Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.processContent(utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestRemoveContentCallsReload(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1ReloadCache: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestRemoveContentCallsReload",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_2
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "MOCK_RELOAD_2",
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaReload); err != nil {
t.Error(err)
}
//Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
//set and remove again from database
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaReload); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestRemoveContentCallsLoad(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1LoadCache: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestRemoveContentCallsReload",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_3
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "MOCK_RELOAD_3",
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaLoad); err != nil {
t.Error(err)
}
//Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
//set and remove again from database
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaLoad); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func TestRemoveContentCallsRemove(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1RemoveItems: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestRemoveContentCallsReload",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_4
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "MOCK_RELOAD_4",
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaRemove); err != nil {
t.Error(err)
}
//Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
//set and remove again from database
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
//inexisting method(*none) of cache and reinitialized the reader will do nothing
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaNone); err != nil {
t.Error(err)
}
}
func TestRemoveContentCallsClear(t *testing.T) {
// Clear cache because connManager sets the internal connection in cache
engine.Cache.Clear([]string{utils.CacheRPCConnections})
sMock := &testMockCacheConn{
calls: map[string]func(arg any, rply any) error{
utils.CacheSv1Clear: func(arg any, rply any) error {
prply, can := rply.(*string)
if !can {
t.Errorf("Wrong argument type : %T", rply)
return nil
}
*prply = utils.OK
return nil
},
},
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
internalCacheSChan := make(chan birpc.ClientConnector, 1)
internalCacheSChan <- sMock
ldr := &Loader{
ldrID: "TestRemoveContentCallsReload",
bufLoaderData: make(map[string][]LoaderData),
connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{
utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan,
}),
cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)},
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaAttributes: {
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
attributeCsv := `
#Tenant[0],ID[1]
cgrates.org,MOCK_RELOAD_3
`
rdr := io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv := csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "MOCK_RELOAD_3",
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaClear); err != nil {
t.Error(err)
}
//Calling the method again while cacheConnsID is not valid
ldr.cacheConns = []string{utils.MetaInternal}
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
//set and remove again from database
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
expected := "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.removeContent(utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
// Calling the method again while caching method is invalid
rdr = io.NopCloser(strings.NewReader(attributeCsv))
rdrCsv = csv.NewReader(rdr)
rdrCsv.Comment = '#'
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaAttributes: {
utils.AttributesCsv: &openedCSVFile{
fileName: utils.AttributesCsv,
rdr: rdr,
csvRdr: rdrCsv,
},
},
}
if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil {
t.Error(err)
}
expected = "UNSUPPORTED_SERVICE_METHOD"
if err := ldr.removeContent(utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}

166
loaders/libloader.go
View File

@@ -1,166 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"fmt"
"strconv"
"strings"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
type LoaderData map[string]any
func (ld LoaderData) TenantID() string {
return utils.ConcatenatedKey(utils.IfaceAsString(ld[utils.Tenant]),
utils.IfaceAsString(ld[utils.ID]))
}
func (ld LoaderData) GetRateIDs() ([]string, error) {
if _, has := ld[utils.RateIDs]; !has {
return nil, fmt.Errorf("cannot find RateIDs in <%+v>", ld)
}
if rateIDs := ld[utils.RateIDs].(string); len(rateIDs) != 0 {
return strings.Split(rateIDs, utils.InfieldSep), nil
}
return []string{}, nil
}
// UpdateFromCSV will update LoaderData with data received from fileName,
// contained in record and processed with cfgTpl
func (ld LoaderData) UpdateFromCSV(fileName string, record []string,
cfgTpl []*config.FCTemplate, tnt string, filterS *engine.FilterS) (err error) {
csvProvider := newCsvProvider(record, fileName)
tenant := tnt
for _, cfgFld := range cfgTpl {
// Make sure filters are matching
if len(cfgFld.Filters) != 0 {
if pass, err := filterS.Pass(tenant,
cfgFld.Filters, csvProvider); err != nil {
return err
} else if !pass {
continue // Not passes filters, ignore this CDR
}
}
out, err := cfgFld.Value.ParseDataProvider(csvProvider)
if err != nil {
return err
}
switch cfgFld.Type {
case utils.MetaComposed:
if _, has := ld[cfgFld.Path]; !has {
ld[cfgFld.Path] = out
} else if valOrig, canCast := ld[cfgFld.Path].(string); canCast {
valOrig += out
ld[cfgFld.Path] = valOrig
}
case utils.MetaVariable:
ld[cfgFld.Path] = out
case utils.MetaString:
if _, has := ld[cfgFld.Path]; !has {
ld[cfgFld.Path] = out
}
}
}
return
}
// newCsvProvider constructs a DataProvider
func newCsvProvider(record []string, fileName string) (dP utils.DataProvider) {
return &csvProvider{
req: record,
fileName: fileName,
cache: utils.MapStorage{},
cfg: config.CgrConfig().GetDataProvider(),
}
}
// csvProvider implements utils.DataProvider so we can pass it to filters
type csvProvider struct {
req []string
fileName string
cache utils.MapStorage
cfg utils.DataProvider
}
// String is part of utils.DataProvider interface
// when called, it will display the already parsed values out of cache
func (cP *csvProvider) String() string {
return utils.ToJSON(cP)
}
// FieldAsInterface is part of utils.DataProvider interface
func (cP *csvProvider) FieldAsInterface(fldPath []string) (data any, err error) {
if data, err = cP.cache.FieldAsInterface(fldPath); err == nil ||
err != utils.ErrNotFound { // item found in cache
return
}
err = nil // cancel previous err
switch {
case strings.HasPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart):
fileName := strings.TrimPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart)
hasSelEnd := false
for _, val := range fldPath[1:] {
if hasSelEnd = strings.HasSuffix(val, utils.FilterValEnd); hasSelEnd {
fileName = fileName + utils.NestingSep + val[:len(val)-1]
break
}
fileName = fileName + utils.NestingSep + val
}
if !hasSelEnd {
return nil, fmt.Errorf("filter rule <%s> needs to end in )", fldPath)
}
if cP.fileName != fileName {
cP.cache.Set(fldPath, nil)
return
}
case fldPath[0] == utils.MetaReq:
case fldPath[0] == utils.MetaCfg:
data, err = cP.cfg.FieldAsInterface(fldPath[1:])
if err != nil {
return
}
cP.cache.Set(fldPath, data)
return
default:
return nil, fmt.Errorf("invalid prefix for : %s", fldPath)
}
var cfgFieldIdx int
if cfgFieldIdx, err = strconv.Atoi(fldPath[len(fldPath)-1]); err != nil || len(cP.req) <= cfgFieldIdx {
return nil, fmt.Errorf("Ignoring record: %q with error : %+v", cP.req, err)
}
data = cP.req[cfgFieldIdx]
cP.cache.Set(fldPath, data)
return
}
// FieldAsString is part of utils.DataProvider interface
func (cP *csvProvider) FieldAsString(fldPath []string) (data string, err error) {
var valIface any
valIface, err = cP.FieldAsInterface(fldPath)
if err != nil {
return
}
return utils.IfaceAsString(valIface), nil
}

443
loaders/libloader_test.go
View File

@@ -1,443 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"fmt"
"reflect"
"testing"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
)
func TestDataUpdateFromCSVOneFile(t *testing.T) {
attrSFlds := []*config.FCTemplate{
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
{Tag: "Contexts",
Path: "Contexts",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)},
{Tag: "FilterIDs",
Path: "FilterIDs",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)},
{Tag: "ActivationInterval",
Path: "ActivationInterval",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)},
{Tag: "Path",
Path: "Path",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)},
{Tag: "Initial",
Path: "Initial",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)},
{Tag: "Substitute",
Path: "Substitute",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)},
{Tag: "Append",
Path: "Append",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)},
{Tag: "Weight",
Path: "Weight",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)},
}
rows := [][]string{
{"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"},
{"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""},
}
lData := make(LoaderData)
if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds,
"cgrates.org", nil); err != nil {
t.Error(err)
}
eLData := LoaderData{"Tenant": "cgrates.org",
"ID": "ATTR_1",
"Contexts": "*sessions;*cdrs",
"FilterIDs": "*string:Account:1007",
"ActivationInterval": "2014-01-14T00:00:00Z",
"Path": "Account",
"Initial": "*any",
"Substitute": "1001",
"Append": "false",
"Weight": "10",
}
if !reflect.DeepEqual(eLData, lData) {
t.Errorf("expecting: %+v, received: %+v", eLData, lData)
}
lData = make(LoaderData)
if err := lData.UpdateFromCSV("Attributes.csv", rows[1], attrSFlds,
"cgrates.org", nil); err != nil {
t.Error(err)
}
eLData = LoaderData{"Tenant": "cgrates.org",
"ID": "ATTR_1",
"Contexts": "",
"FilterIDs": "",
"ActivationInterval": "",
"Path": "Subject",
"Initial": "*any",
"Substitute": "1001",
"Append": "true",
"Weight": "",
}
if !reflect.DeepEqual(eLData, lData) {
t.Errorf("expecting: %+v, received: %+v", eLData, lData)
}
}
func TestDataUpdateFromCSVOneFile2(t *testing.T) {
attrSFlds := []*config.FCTemplate{
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
{Tag: "Contexts",
Path: "Contexts",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)},
{Tag: "FilterIDs",
Path: "FilterIDs",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)},
{Tag: "ActivationInterval",
Path: "ActivationInterval",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)},
{Tag: "Path",
Path: "Path",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)},
{Tag: "Initial",
Path: "Initial",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)},
{Tag: "Substitute",
Path: "Substitute",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)},
{Tag: "Append",
Path: "Append",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)},
{Tag: "Weight",
Path: "Weight",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)},
}
rows := [][]string{
{"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"},
{"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""},
}
lData := make(LoaderData)
if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds,
"cgrates.org", nil); err != nil {
t.Error(err)
}
eLData := LoaderData{"Tenant": "cgrates.org",
"ID": "ATTR_1",
"Contexts": "*sessions;*cdrs",
"FilterIDs": "*string:Account:1007",
"ActivationInterval": "2014-01-14T00:00:00Z",
"Path": "Account",
"Initial": "*any",
"Substitute": "1001",
"Append": "false",
"Weight": "10",
}
if !reflect.DeepEqual(eLData, lData) {
t.Errorf("expecting: %+v, received: %+v", eLData, lData)
}
lData = make(LoaderData)
if err := lData.UpdateFromCSV("Attributes.csv", rows[1], attrSFlds,
"cgrates.org", nil); err != nil {
t.Error(err)
}
eLData = LoaderData{"Tenant": "cgrates.org",
"ID": "ATTR_1",
"Contexts": "",
"FilterIDs": "",
"ActivationInterval": "",
"Path": "Subject",
"Initial": "*any",
"Substitute": "1001",
"Append": "true",
"Weight": "",
}
if !reflect.DeepEqual(eLData, lData) {
t.Errorf("expecting: %+v, received: %+v", eLData, lData)
}
}
func TestDataUpdateFromCSVMultiFiles(t *testing.T) {
attrSFlds := []*config.FCTemplate{
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep),
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep),
Mandatory: true},
{Tag: "Contexts",
Path: "Contexts",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("*any", utils.InfieldSep)},
{Tag: "Path",
Path: "Path",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File1.csv).5", utils.InfieldSep)},
{Tag: "Initial",
Path: "Initial",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.InfieldSep)},
{Tag: "Substitute",
Path: "Substitute",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.InfieldSep)},
{Tag: "Append",
Path: "Append",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("true", utils.InfieldSep)},
{Tag: "Weight",
Path: "Weight",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("10", utils.InfieldSep)},
}
loadRun1 := map[string][]string{
"File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"},
"File2.csv": {"ignored", "ATTR_1"},
}
lData := make(LoaderData)
for fName, record := range loadRun1 {
if err := lData.UpdateFromCSV(fName, record, attrSFlds,
"cgrates.org", nil); err != nil {
t.Error(err)
}
}
eLData := LoaderData{"Tenant": "cgrates.org",
"ID": "ATTR_1",
"Contexts": "*any",
"Path": "Subject",
"Initial": "*any",
"Substitute": "1001",
"Append": "true",
"Weight": "10",
}
if !reflect.DeepEqual(eLData, lData) {
t.Errorf("expecting: %+v, received: %+v", eLData, lData)
}
}
func TestGetRateIDsLoaderData(t *testing.T) {
ldrData := LoaderData{
"File1.csv": []string{"Subject", "*any", "1001"},
}
expected := "cannot find RateIDs in <map[File1.csv:[Subject *any 1001]]>"
if _, err := ldrData.GetRateIDs(); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
// func TestUpdateFromCsvParseValueError(t *testing.T) {
// ldrData := LoaderData{
// "File1.csv": []string{"Subject", "*any", "1001"},
// }
// tnt := config.NewRSRParsersMustCompile("asd{*duration_seconds}", utils.InfieldSep)
// expected := "time: invalid duration \"asd\""
// if err := ldrData.UpdateFromCSV("File1.csv", nil, nil, tnt, nil); err == nil || err.Error() != expected {
// t.Errorf("Expected %+v, received %+v", expected, err)
// }
// }
func TestUpdateFromCsvWithFiltersError(t *testing.T) {
attrSFlds := []*config.FCTemplate{
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep),
Filters: []string{"*string:~*req.Account:10"},
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep),
Filters: []string{"*string:~*req.Account:10"},
Mandatory: true},
}
loadRunStr := map[string][]string{
"File1.csv": {"cgrates.org", "TEST_1"},
}
lData := make(LoaderData)
cfg := config.NewDefaultCGRConfig()
data, err := engine.NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
filterS := engine.NewFilterS(cfg, nil, dm)
for fName, record := range loadRunStr {
expected := "Ignoring record: [\"cgrates.org\" \"TEST_1\"] with error : strconv.Atoi: parsing \"Account\": invalid syntax"
if err := lData.UpdateFromCSV(fName, record, attrSFlds,
"cgrates.org", filterS); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
}
func TestUpdateFromCsvWithFiltersContinue(t *testing.T) {
attrSFlds := []*config.FCTemplate{
{Tag: "TenantID",
Path: "Tenant",
Type: utils.MetaString,
Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep),
Filters: []string{`*string:~*req.2:10`},
Mandatory: true},
{Tag: "ProfileID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep),
Filters: []string{`*string:~*req.2:10`},
Mandatory: true},
}
loadRunStr := map[string][]string{
"File1.csv": {"Subject", "*any", "1001"},
}
lData := make(LoaderData)
cfg := config.NewDefaultCGRConfig()
data, err := engine.NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
filterS := engine.NewFilterS(cfg, nil, dm)
for fName, record := range loadRunStr {
if err := lData.UpdateFromCSV(fName, record, attrSFlds,
"cgrates.org", filterS); err != nil {
t.Error(err)
}
}
}
func TestLoadersFieldAsInterfaceError(t *testing.T) {
loadRun1 := map[string][]string{
"File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"},
}
csvProv := newCsvProvider(loadRun1["File1.csv"], "File1.csv")
csvString := csvProv.String()
expected := "{}"
if csvString != expected {
t.Errorf("Expected %+v, received %+v", expected, csvString)
}
expected = "invalid prefix for : [File2.csv]"
if _, err := csvProv.FieldAsInterface([]string{"File2.csv"}); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
expected = "filter rule <[*file() ]> needs to end in )"
if _, err := csvProv.FieldAsInterface([]string{"*file()", ""}); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+q", expected, err)
}
expected = "filter rule <[*file() File1.csv]> needs to end in )"
if _, err := csvProv.FieldAsInterface([]string{"*file()", "File1.csv"}); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+q", expected, err)
}
}
func TestLibLoaderGetRateIDs(t *testing.T) {
cases := []struct {
input LoaderData
expectedIDs []string
expectedErr error
}{
{
input: LoaderData{utils.RateIDs: "id1,id2,id3"},
expectedIDs: []string{"id1,id2,id3"},
expectedErr: nil,
},
{
input: LoaderData{utils.RateIDs: ""},
expectedIDs: []string{},
expectedErr: nil,
},
{
input: LoaderData{},
expectedIDs: nil,
expectedErr: fmt.Errorf("cannot find RateIDs in <map[]>"),
},
}
for _, c := range cases {
ids, err := c.input.GetRateIDs()
if c.expectedErr != nil {
if err == nil || err.Error() != c.expectedErr.Error() {
t.Errorf("Expected error: %v, got: %v", c.expectedErr, err)
}
} else {
if err != nil {
t.Errorf("Expected no error, got: %v", err)
}
}
if !reflect.DeepEqual(ids, c.expectedIDs) {
t.Errorf("Expected IDs: %v, got: %v", c.expectedIDs, ids)
}
}
}
func TestLibLoaderFieldAsStringErrorHandling(t *testing.T) {
cP := &csvProvider{}
_, err := cP.FieldAsString([]string{"invalid", "path"})
if err == nil {
t.Error("Expected an error, but got nil")
}
}

1016
loaders/loader.go
View File

File diff suppressed because it is too large Load Diff

857
loaders/loader_it_test.go
View File

@@ -1,857 +0,0 @@
//go:build integration
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"encoding/csv"
"errors"
"io"
"os"
"path"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/cgrates/birpc"
"github.com/cgrates/birpc/context"
"github.com/cgrates/birpc/jsonrpc"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
loaderCfgPath string
loaderCfgDIR string //run tests for specific configuration
loaderCfg *config.CGRConfig
loaderRPC *birpc.Client
customAttributes = "12012000001\t12018209998\n12012000002\t15512580598\n12012000007\t19085199998\n12012000008\t18622784999\n12012000010\t17329440866\n12012000011\t18623689800\n12012000012\t19082050951\n12012000014\t17329440866\n12012000015\t12018209999\n12012000031\t12018209999\n12012000032\t19082050951\n12012000033\t12018209998\n12012000034\t12018209998\n"
sTestsLoader = []func(t *testing.T){
testLoaderMakeFolders,
testLoaderInitCfg,
testLoaderResetDataDB,
testLoaderStartEngine,
testLoaderRPCConn,
testLoaderPopulateData,
testProcessFile,
testProcessFileLockFolder,
testProcessFileUnableToOpen,
testProcessFileAllFilesPresent,
testProcessFileRenameError,
testAllFilesPresentEmptyCSV,
testIsFolderLocked,
testNewLockFolder,
testNewLockFolderNotFound,
testLoaderLoadAttributes,
testLoaderVerifyOutDir,
testLoaderCheckAttributes,
testLoaderResetDataDB,
testLoaderPopulateDataWithoutMoving,
testLoaderLoadAttributesWithoutMoving,
testLoaderVerifyOutDirWithoutMoving,
testLoaderCheckAttributes,
testLoaderResetDataDB,
testLoaderPopulateDataWithSubpath,
testLoaderLoadAttributesWithSubpath,
testLoaderVerifyOutDirWithSubpath,
testLoaderCheckAttributes,
testLoaderResetDataDB,
testLoaderPopulateDataWithSubpathWithMove,
testLoaderLoadAttributesWithoutSubpathWithMove,
testLoaderVerifyOutDirWithSubpathWithMove,
testLoaderCheckAttributes,
testLoaderResetDataDB,
testLoaderPopulateDataForTemplateLoader,
testLoaderLoadAttributesForTemplateLoader,
testLoaderVerifyOutDirForTemplateLoader,
testLoaderCheckAttributes,
testLoaderResetDataDB,
testLoaderPopulateDataForCustomSep,
testLoaderCheckForCustomSep,
testLoaderVerifyOutDirForCustomSep,
testLoaderKillEngine,
}
)
// Test start here
func TestLoaderIT(t *testing.T) {
switch *utils.DBType {
case utils.MetaInternal:
loaderCfgDIR = "tutinternal"
case utils.MetaMySQL:
loaderCfgDIR = "tutmysql"
case utils.MetaMongo:
loaderCfgDIR = "tutmongo"
case utils.MetaPostgres:
t.SkipNow()
default:
t.Fatal("Unknown Database type")
}
for _, stest := range sTestsLoader {
t.Run(loaderCfgDIR, stest)
}
}
func testLoaderInitCfg(t *testing.T) {
var err error
loaderCfgPath = path.Join(*utils.DataDir, "conf", "samples", "loaders", loaderCfgDIR)
loaderCfg, err = config.NewCGRConfigFromPath(loaderCfgPath)
if err != nil {
t.Fatal(err)
}
}
func testLoaderMakeFolders(t *testing.T) {
// active the loaders here
for _, dir := range loaderPaths {
if err := os.RemoveAll(dir); err != nil {
t.Fatal("Error removing folder: ", dir, err)
}
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal("Error creating folder: ", dir, err)
}
}
}
// Wipe out the cdr database
func testLoaderResetDataDB(t *testing.T) {
if err := engine.InitDataDB(loaderCfg); err != nil {
t.Fatal(err)
}
engine.Cache.Clear(nil)
}
// Start CGR Engine
func testLoaderStartEngine(t *testing.T) {
if _, err := engine.StopStartEngine(loaderCfgPath, 100); err != nil {
t.Fatal(err)
}
}
// Connect rpc client to rater
func testLoaderRPCConn(t *testing.T) {
var err error
switch *utils.Encoding {
case utils.MetaJSON:
loaderRPC, err = jsonrpc.Dial(utils.TCP, loaderCfg.ListenCfg().RPCJSONListen)
case utils.MetaGOB:
loaderRPC, err = birpc.Dial(utils.TCP, loaderCfg.ListenCfg().RPCGOBListen)
default:
loaderRPC, err = nil, errors.New("UNSUPPORTED_RPC")
}
if err != nil {
t.Fatal(err)
}
}
func testLoaderPopulateData(t *testing.T) {
fileName := utils.AttributesCsv
tmpFilePath := path.Join("/tmp", fileName)
if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/In", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func testLoaderLoadAttributes(t *testing.T) {
var reply string
if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
&ArgsProcessFolder{LoaderID: "CustomLoader"}, &reply); err != nil {
t.Error(err)
}
}
func testLoaderVerifyOutDir(t *testing.T) {
time.Sleep(100 * time.Millisecond)
if outContent1, err := os.ReadFile(path.Join("/tmp/Out", utils.AttributesCsv)); err != nil {
t.Error(err)
} else if AttributesCSVContent != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1))
}
}
func testLoaderCheckAttributes(t *testing.T) {
eAttrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "ALS1",
Contexts: []string{"con1", "con2", "con3"},
FilterIDs: []string{"*string:~*req.Account:1001"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)},
Attributes: []*engine.Attribute{
{
FilterIDs: []string{"*string:~*req.Field1:Initial"},
Path: utils.MetaReq + utils.NestingSep + "Field1",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep),
},
{
FilterIDs: []string{},
Path: utils.MetaReq + utils.NestingSep + "Field2",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep),
}},
Blocker: true,
Weight: 20,
}
if *utils.Encoding == utils.MetaGOB { // gob threats empty slices as nil values
eAttrPrf.Attributes[1].FilterIDs = nil
}
var reply *engine.AttributeProfile
if err := loaderRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile,
&utils.TenantIDWithAPIOpts{
TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"},
}, &reply); err != nil {
t.Fatal(err)
}
eAttrPrf.Compile()
reply.Compile()
sort.Strings(eAttrPrf.Contexts)
sort.Strings(reply.Contexts)
if !reflect.DeepEqual(eAttrPrf, reply) {
t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply))
}
}
func testLoaderPopulateDataWithoutMoving(t *testing.T) {
fileName := utils.AttributesCsv
tmpFilePath := path.Join("/tmp/", fileName)
if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/LoaderIn", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func testLoaderLoadAttributesWithoutMoving(t *testing.T) {
var reply string
if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
&ArgsProcessFolder{LoaderID: "WithoutMoveToOut"}, &reply); err != nil {
t.Error(err)
}
}
func testLoaderVerifyOutDirWithoutMoving(t *testing.T) {
time.Sleep(100 * time.Millisecond)
// we expect that after the LoaderS process the file leave in in the input folder
if outContent1, err := os.ReadFile(path.Join("/tmp/LoaderIn", utils.AttributesCsv)); err != nil {
t.Error(err)
} else if AttributesCSVContent != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1))
}
}
func testLoaderPopulateDataWithSubpath(t *testing.T) {
fileName := utils.AttributesCsv
tmpFilePath := path.Join("/tmp/", fileName)
if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.MkdirAll("/tmp/SubpathWithoutMove/folder1", 0755); err != nil {
t.Fatal("Error creating folder: /tmp/SubpathWithoutMove/folder1", err)
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathWithoutMove/folder1", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func testLoaderLoadAttributesWithSubpath(t *testing.T) {
var reply string
if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
&ArgsProcessFolder{LoaderID: "SubpathLoaderWithoutMove"}, &reply); err != nil {
t.Error(err)
}
}
func testLoaderVerifyOutDirWithSubpath(t *testing.T) {
time.Sleep(100 * time.Millisecond)
// we expect that after the LoaderS process the file leave in in the input folder
if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathWithoutMove/folder1", utils.AttributesCsv)); err != nil {
t.Error(err)
} else if AttributesCSVContent != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1))
}
}
func testLoaderPopulateDataWithSubpathWithMove(t *testing.T) {
fileName := utils.AttributesCsv
tmpFilePath := path.Join("/tmp/", fileName)
if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.MkdirAll("/tmp/SubpathLoaderWithMove/folder1", 0755); err != nil {
t.Fatal("Error creating folder: /tmp/SubpathLoaderWithMove/folder1", err)
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathLoaderWithMove/folder1", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func testLoaderLoadAttributesWithoutSubpathWithMove(t *testing.T) {
var reply string
if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
&ArgsProcessFolder{LoaderID: "SubpathLoaderWithMove"}, &reply); err != nil {
t.Error(err)
}
}
func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) {
time.Sleep(100 * time.Millisecond)
if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathOut/folder1", utils.AttributesCsv)); err != nil {
t.Error(err)
} else if AttributesCSVContent != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1))
}
}
func testLoaderPopulateDataForTemplateLoader(t *testing.T) {
fileName := utils.AttributesCsv
tmpFilePath := path.Join("/tmp/", fileName)
if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.MkdirAll("/tmp/templateLoaderIn", 0755); err != nil {
t.Fatal("Error creating folder: /tmp/templateLoaderIn", err)
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/templateLoaderIn", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
}
func testLoaderLoadAttributesForTemplateLoader(t *testing.T) {
var reply string
if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load,
&ArgsProcessFolder{LoaderID: "LoaderWithTemplate"}, &reply); err != nil {
t.Error(err)
}
}
func testLoaderVerifyOutDirForTemplateLoader(t *testing.T) {
time.Sleep(100 * time.Millisecond)
if outContent1, err := os.ReadFile(path.Join("/tmp/templateLoaderOut", utils.AttributesCsv)); err != nil {
t.Error(err)
} else if AttributesCSVContent != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1))
}
}
func testLoaderKillEngine(t *testing.T) {
if err := engine.KillEngine(100); err != nil {
t.Error(err)
}
}
func testLoaderPopulateDataForCustomSep(t *testing.T) {
fileName := utils.Attributes
tmpFilePath := path.Join("/tmp/", fileName)
if err := os.WriteFile(tmpFilePath, []byte(customAttributes), os.ModePerm); err != nil {
t.Fatal(err.Error())
}
if err := os.MkdirAll("/tmp/customSepLoaderIn", 0755); err != nil {
t.Fatal("Error creating folder: /tmp/customSepLoaderIn", err)
}
if err := os.Rename(tmpFilePath, path.Join("/tmp/customSepLoaderIn", fileName)); err != nil {
t.Fatal("Error moving file to processing directory: ", err)
}
time.Sleep(100 * time.Millisecond)
}
func testLoaderCheckForCustomSep(t *testing.T) {
eAttrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "ATTR_12012000001",
Contexts: []string{"*any"},
FilterIDs: []string{"*string:~*req.Destination:12012000001"},
Attributes: []*engine.Attribute{
{
FilterIDs: []string{},
Path: "*req.Destination",
Type: utils.MetaConstant,
Value: config.NewRSRParsersMustCompile("12018209998", utils.InfieldSep),
},
},
}
if *utils.Encoding == utils.MetaGOB { // gob threats empty slices as nil values
eAttrPrf.Attributes[0].FilterIDs = nil
}
var reply *engine.AttributeProfile
if err := loaderRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile,
&utils.TenantIDWithAPIOpts{
TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_12012000001"},
}, &reply); err != nil {
t.Fatal(err)
}
eAttrPrf.Compile()
reply.Compile()
if !reflect.DeepEqual(eAttrPrf, reply) {
t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply))
}
}
func testLoaderVerifyOutDirForCustomSep(t *testing.T) {
time.Sleep(100 * time.Millisecond)
if outContent1, err := os.ReadFile(path.Join("/tmp/customSepLoaderOut", utils.Attributes)); err != nil {
t.Error(err)
} else if customAttributes != string(outContent1) {
t.Errorf("Expecting: %q, received: %q", customAttributes, string(outContent1))
}
}
func testProcessFile(t *testing.T) {
flPath := "/tmp/testProcessFile"
if err := os.MkdirAll(flPath, os.ModePerm); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items)
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testProcessFile",
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
fieldSep: utils.FieldsSep,
tpInDir: flPath,
tpOutDir: "/tmp",
lockFilepath: "/tmp/testProcessFile/.lck",
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaResources: {
{Tag: "Tenant",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
//loader file is empty (loaderType will be empty)
if err := ldr.processFile(utils.ResourcesCsv); err != nil {
t.Error(err)
}
// resCsv := `
// #Tenant[0],ID[1]
// cgrates.org,NewRes1
// `
// rdr := io.NopCloser(strings.NewReader(resCsv))
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
utils.ResourcesCsv: &openedCSVFile{
rdr: io.NopCloser(nil),
csvRdr: csv.NewReader(nil),
},
},
}
expRes := &engine.ResourceProfile{
Tenant: "cgrates.org",
ID: "NewRes1",
FilterIDs: []string{},
ThresholdIDs: []string{},
}
//successfully processed the file
if err := ldr.processFile(utils.ResourcesCsv); err != nil {
t.Error(err)
}
//get ResourceProfile and compare
if rcv, err := ldr.dm.GetResourceProfile(expRes.Tenant, expRes.ID, true, true, utils.NonTransactional); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(rcv, expRes) {
t.Errorf("Expected %+v, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv))
}
if err := ldr.dm.RemoveResourceProfile(expRes.Tenant, expRes.ID, true); err != nil {
t.Error(err)
}
file, err = os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
//cannot move file when tpOutDir is empty
ldr.tpOutDir = utils.EmptyString
if err := ldr.processFile(utils.ResourcesCsv); err != nil {
t.Error(err)
}
if err := os.Remove(path.Join("/tmp", utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.RemoveAll(flPath); err != nil {
t.Error(err)
}
}
func testProcessFileAllFilesPresent(t *testing.T) {
flPath := "/tmp/testProcessFile"
if err := os.MkdirAll(flPath, os.ModePerm); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, "inexistent.csv"))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items)
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testProcessFile",
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
fieldSep: utils.FieldsSep,
tpInDir: flPath,
tpOutDir: "/tmp",
lockFilepath: utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaResources: {
{Tag: "Tenant",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.FieldsSep),
Mandatory: true},
{Tag: "ID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.FieldsSep),
Mandatory: true},
},
}
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
"inexistent.csv": nil,
utils.AttributesCsv: nil,
},
}
if err := ldr.processFile("inexistent.csv"); err != nil {
t.Error(err)
}
if err := os.Remove(path.Join(flPath, "inexistent.csv")); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testProcessFileLockFolder(t *testing.T) {
flPath := "/tmp/testProcessFileLockFolder"
if err := os.MkdirAll(flPath, os.ModePerm); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testProcessFileLockFolder",
tpInDir: flPath,
tpOutDir: "/tmp",
lockFilepath: "/tmp/test/.cgr.lck",
fieldSep: utils.InfieldSep,
}
resCsv := `
#Tenant[0],ID[1]
cgrates.org,NewRes1
`
rdr := io.NopCloser(strings.NewReader(resCsv))
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
utils.ResourcesCsv: &openedCSVFile{
fileName: utils.ResourcesCsv,
rdr: rdr,
},
},
}
//unable to lock the folder, because lockFileName is missing
expected := "open /tmp/test/.cgr.lck: no such file or directory"
if err := ldr.processFile(utils.ResourcesCsv); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testProcessFileUnableToOpen(t *testing.T) {
flPath := "/tmp/testProcessFileUnableToOpen"
if err := os.MkdirAll(flPath, os.ModePerm); err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testProcessFile",
tpInDir: flPath,
fieldSep: ",",
lockFilepath: utils.MetaResources,
}
resCsv := `
#Tenant[0],ID[1]
cgrates.org,NewRes1
`
rdr := io.NopCloser(strings.NewReader(resCsv))
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
`resources`: &openedCSVFile{
fileName: utils.ResourcesCsv,
rdr: rdr,
},
},
}
//unable to lock the folder, because lockFileName is missing
expected := "open /tmp/testProcessFileUnableToOpen/resources: no such file or directory"
if err := ldr.processFile(`resources`); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testProcessFileRenameError(t *testing.T) {
flPath1 := "/tmp/testProcessFileLockFolder"
if err := os.MkdirAll(flPath1, os.ModePerm); err != nil {
t.Error(err)
}
data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items)
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testProcessFileRenameError",
dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil),
fieldSep: utils.FieldsSep,
tpInDir: flPath1,
tpOutDir: "INEXISTING_FILE",
lockFilepath: utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
ldr.dataTpls = map[string][]*config.FCTemplate{
utils.MetaResources: {
{Tag: "Tenant",
Path: "Tenant",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true},
{Tag: "ID",
Path: "ID",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true},
},
}
// resCsv := `
// #Tenant[0],ID[1]
// cgrates.org,NewRes1
// `
// rdr := io.NopCloser(strings.NewReader(resCsv))
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
utils.ResourcesCsv: &openedCSVFile{
rdr: io.NopCloser(nil),
csvRdr: csv.NewReader(nil),
},
},
}
file, err := os.Create(path.Join(flPath1, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
expected := "rename /tmp/testProcessFileLockFolder/Resources.csv INEXISTING_FILE/Resources.csv: no such file or directory"
if err := ldr.processFile(utils.ResourcesCsv); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
if err := os.RemoveAll(flPath1); err != nil {
t.Error(err)
}
}
func testAllFilesPresentEmptyCSV(t *testing.T) {
ldr := &Loader{
ldrID: "testProcessFileRenameError",
lockFilepath: utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
ldr.rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
utils.ResourcesCsv: nil,
},
}
if rcv := ldr.allFilesPresent(utils.MetaResources); rcv {
t.Errorf("Expecting false")
}
}
func testIsFolderLocked(t *testing.T) {
flPath := "/tmp/testIsFolderLocked"
ldr := &Loader{
ldrID: "TestLoadAndRemoveResources",
tpInDir: flPath,
lockFilepath: utils.EmptyString,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
expected := "stat /\x00: invalid argument"
if _, err := ldr.isFolderLocked(); err != nil {
t.Errorf("Expected %+v, received %+v", expected, err)
}
}
func testNewLockFolder(t *testing.T) {
pathL := "/tmp/testNewLockFolder/"
if err := os.MkdirAll(pathL, os.ModePerm); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(pathL, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testNewLockFolder",
tpInDir: "",
lockFilepath: pathL + utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
if err := ldr.lockFolder(); err != nil {
t.Error(err)
}
if err := os.RemoveAll(pathL); err != nil {
t.Error(err)
}
}
func testNewLockFolderNotFound(t *testing.T) {
pathL := "/tmp/testNewLockFolder/"
ldr := &Loader{
ldrID: "testNewLockFolder",
tpInDir: "",
lockFilepath: pathL + utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
errExpect := "open /tmp/testNewLockFolder/Resources.csv: no such file or directory"
if err := ldr.lockFolder(); err == nil || err.Error() != errExpect {
t.Error(err)
}
}
func testNewIsFolderLock(t *testing.T) {
pathL := "/tmp/testNewLockFolder/"
if err := os.MkdirAll(pathL, os.ModePerm); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(pathL, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
ldr := &Loader{
ldrID: "testNewLockFolder",
tpInDir: "",
lockFilepath: pathL + utils.ResourcesCsv,
bufLoaderData: make(map[string][]LoaderData),
timezone: "UTC",
}
if err := ldr.lockFolder(); err != nil {
t.Error(err)
}
isLocked, err := ldr.isFolderLocked()
if !isLocked {
t.Error("Expected the file to be locked")
}
if err := os.RemoveAll(pathL); err != nil {
t.Error(err)
}
}

3860
loaders/loader_test.go
View File

File diff suppressed because it is too large Load Diff

150
loaders/loaders.go
View File

@@ -1,150 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"errors"
"fmt"
"sync"
"time"
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
func NewLoaderService(dm *engine.DataManager, ldrsCfg []*config.LoaderSCfg,
timezone string, cachingDlay time.Duration, filterS *engine.FilterS,
connMgr *engine.ConnManager) (ldrS *LoaderService) {
ldrS = &LoaderService{ldrs: make(map[string]*Loader)}
for _, ldrCfg := range ldrsCfg {
if ldrCfg.Enabled {
ldrS.ldrs[ldrCfg.ID] = NewLoader(dm, ldrCfg, timezone, cachingDlay, filterS, connMgr, ldrCfg.CacheSConns)
}
}
return
}
// LoaderService is the Loader service handling independent Loaders
type LoaderService struct {
sync.RWMutex
ldrs map[string]*Loader
}
// Enabled returns true if at least one loader is enabled
func (ldrS *LoaderService) Enabled() bool {
return len(ldrS.ldrs) != 0
}
func (ldrS *LoaderService) ListenAndServe(stopChan chan struct{}) (err error) {
for _, ldr := range ldrS.ldrs {
if err = ldr.ListenAndServe(stopChan); err != nil {
utils.Logger.Err(fmt.Sprintf("<%s-%s> error: <%s>", utils.LoaderS, ldr.ldrID, err.Error()))
return
}
}
return
}
type ArgsProcessFolder struct {
LoaderID string
ForceLock bool
Caching *string
StopOnError bool
}
func (ldrS *LoaderService) V1Load(ctx *context.Context, args *ArgsProcessFolder,
rply *string) (err error) {
ldrS.RLock()
defer ldrS.RUnlock()
if args.LoaderID == "" {
args.LoaderID = utils.MetaDefault
}
ldr, has := ldrS.ldrs[args.LoaderID]
if !has {
return fmt.Errorf("UNKNOWN_LOADER: %s", args.LoaderID)
}
if locked, err := ldr.isFolderLocked(); err != nil {
return utils.NewErrServerError(err)
} else if locked {
if !args.ForceLock {
return errors.New("ANOTHER_LOADER_RUNNING")
}
if err := ldr.unlockFolder(); err != nil {
return utils.NewErrServerError(err)
}
}
//verify If Caching is present in arguments
caching := config.CgrConfig().GeneralCfg().DefaultCaching
if args.Caching != nil {
caching = *args.Caching
}
if err := ldr.ProcessFolder(caching, utils.MetaStore, args.StopOnError); err != nil {
return utils.NewErrServerError(err)
}
*rply = utils.OK
return
}
func (ldrS *LoaderService) V1Remove(ctx *context.Context, args *ArgsProcessFolder,
rply *string) (err error) {
ldrS.RLock()
defer ldrS.RUnlock()
if args.LoaderID == "" {
args.LoaderID = utils.MetaDefault
}
ldr, has := ldrS.ldrs[args.LoaderID]
if !has {
return fmt.Errorf("UNKNOWN_LOADER: %s", args.LoaderID)
}
if locked, err := ldr.isFolderLocked(); err != nil {
return utils.NewErrServerError(err)
} else if locked {
if args.ForceLock {
if err := ldr.unlockFolder(); err != nil {
return utils.NewErrServerError(err)
}
}
return errors.New("ANOTHER_LOADER_RUNNING")
}
//verify If Caching is present in arguments
caching := config.CgrConfig().GeneralCfg().DefaultCaching
if args.Caching != nil {
caching = *args.Caching
}
if err := ldr.ProcessFolder(caching, utils.MetaRemove, args.StopOnError); err != nil {
return utils.NewErrServerError(err)
}
*rply = utils.OK
return
}
// Reload recreates the loaders map thread safe
func (ldrS *LoaderService) Reload(dm *engine.DataManager, ldrsCfg []*config.LoaderSCfg,
timezone string, cachingDlay time.Duration, filterS *engine.FilterS, connMgr *engine.ConnManager) {
ldrS.Lock()
ldrS.ldrs = make(map[string]*Loader)
for _, ldrCfg := range ldrsCfg {
if ldrCfg.Enabled {
ldrS.ldrs[ldrCfg.ID] = NewLoader(dm, ldrCfg, timezone, cachingDlay, filterS, connMgr, ldrCfg.CacheSConns)
}
}
ldrS.Unlock()
}

661
loaders/loaders_it_test.go
View File

@@ -1,661 +0,0 @@
//go:build integration
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"encoding/csv"
"io"
"os"
"path"
"reflect"
"strings"
"testing"
"time"
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
sTestItLoaders = []func(t *testing.T){
testV1LoadResource,
testV1LoadDefaultIDError,
testV1LoadUnableToDeleteFile,
testV1LoadProcessFolderError,
testV1RemoveResource,
testV1RemoveDefaultIDError,
testV1RemoveUnableToDeleteFile,
testV1RemoveProcessFolderError,
testV1LoadAndRemoveProcessRemoveFolderError,
testLoaderServiceReload,
testLoaderServiceListenAndServe,
}
)
func TestITLoaders(t *testing.T) {
for _, test := range sTestItLoaders {
t.Run("Loaders_IT_Tests", test)
}
}
func testV1LoadResource(t *testing.T) {
utils.Logger.SetLogLevel(7)
flPath := "/tmp/testV1LoadResource"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
if _, err := file.Write([]byte(`#Tenant[0],ID[1]
cgrates.org,NewRes1`)); err != nil {
t.Fatal(err)
}
if err := file.Sync(); err != nil {
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
file, err = os.Create(path.Join(flPath, "res.lck"))
if err != nil {
t.Error(err)
}
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfg := config.NewDefaultCGRConfig().LoaderCfg()
cfg[0] = &config.LoaderSCfg{
ID: "testV1LoadResource",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: "res.lck",
Data: []*config.LoaderDataType{
{
Type: utils.MetaResources,
Filename: utils.ResourcesCsv,
Fields: []*config.FCTemplate{
{
Path: "Tenant",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true,
},
{
Path: "ID",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true,
},
},
},
},
}
for _, tmp := range cfg[0].Data[0].Fields {
tmp.ComputePath()
}
ldrs := NewLoaderService(dm, cfg, "UTC", 0, nil, nil)
var reply string
expected := "ANOTHER_LOADER_RUNNING"
//cannot load when there is another loader running
if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{LoaderID: "testV1LoadResource"},
&reply); err == nil || reply != utils.EmptyString || err.Error() != expected {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply)
}
if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{LoaderID: "testV1LoadResource", ForceLock: true},
&reply); err != nil && reply != utils.OK {
t.Error(err)
}
expRes := &engine.ResourceProfile{
Tenant: "cgrates.org",
ID: "NewRes1",
FilterIDs: make([]string, 0),
ThresholdIDs: make([]string, 0),
}
if rcv, err := dm.GetResourceProfile(expRes.Tenant, expRes.ID,
true, true, utils.NonTransactional); err != nil {
t.Error(err)
} else if !reflect.DeepEqual(rcv, expRes) {
t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv))
}
if err := os.RemoveAll(flPath); err != nil {
t.Error(err)
}
}
func testV1LoadDefaultIDError(t *testing.T) {
flPath := "/tmp/testV1LoadResource"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1LoadDefaultIDError",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
var reply string
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{
LoaderID: utils.EmptyString}, &reply); err == nil && reply != utils.EmptyString && err.Error() != utils.EmptyString {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1LoadUnableToDeleteFile(t *testing.T) {
flPath := "testV1LoadUnableToDeleteFile"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1LoadUnableToDeleteFile",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: "/\x00",
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
var reply string
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
expected := "SERVER_ERROR: stat /\x00/Resources.csv: invalid argument"
if err := ldrs.V1Load(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1LoadUnableToDeleteFile",
ForceLock: true}, &reply); err == nil || err.Error() != expected {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1LoadProcessFolderError(t *testing.T) {
flPath := "testV1LoadProcessFolderError"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#PK
NOT_UINT
`))
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1LoadResource",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
ldrs.ldrs["testV1LoadResource"].dataTpls = map[string][]*config.FCTemplate{
utils.MetaFilters: {
{Tag: "PK",
Path: "PK",
Type: utils.MetaComposed,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.FieldsSep)},
},
}
resCsv := `
//PK
NOT_UINT
`
rdr := io.NopCloser(strings.NewReader(resCsv))
csvRdr := csv.NewReader(rdr)
csvRdr.Comment = '#'
ldrs.ldrs["testV1LoadResource"].rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
"not_a_file": &openedCSVFile{
fileName: utils.ResourcesCsv,
rdr: rdr,
csvRdr: csvRdr,
},
},
}
var reply string
expected := "SERVER_ERROR: open testV1LoadProcessFolderError/not_a_file: no such file or directory"
//try to load by changing the caching method
if err := ldrs.V1Load(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1LoadResource",
ForceLock: true,
Caching: utils.StringPointer("not_a_chaching_method"),
StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString {
t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply)
}
if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1RemoveResource(t *testing.T) {
engine.Cache.Clear(nil)
flPath := "/tmp/testV1RemoveResource"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`#Tenant[0],ID[1]
cgrates.org,NewRes1`))
file.Close()
file, err = os.Create(path.Join(flPath, "lock.cgr"))
if err != nil {
t.Error(err)
}
file.Close()
idb, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Fatal(err)
}
dm := engine.NewDataManager(idb, config.CgrConfig().CacheCfg(), nil)
cfg := config.NewDefaultCGRConfig().LoaderCfg()
cfg[0] = &config.LoaderSCfg{
ID: "testV1RemoveResource",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: "lock.cgr",
Data: []*config.LoaderDataType{
{
Type: utils.MetaResources,
Filename: utils.ResourcesCsv,
Fields: []*config.FCTemplate{
{
Path: "Tenant",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep),
Mandatory: true,
},
{
Path: "ID",
Type: utils.MetaVariable,
Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep),
Mandatory: true,
},
},
},
},
}
for _, tmp := range cfg[0].Data[0].Fields {
tmp.ComputePath()
}
ldrs := NewLoaderService(dm, cfg, time.UTC.String(), 0, nil, nil)
//To remove a resource, we need to set it first
if err := dm.SetResourceProfile(&engine.ResourceProfile{
Tenant: "cgrates.org",
ID: "NewRes1",
}, true); err != nil {
t.Error(err)
}
var reply string
expected := "ANOTHER_LOADER_RUNNING"
//cannot load when there is another loader running
if err := ldrs.V1Remove(context.Background(), &ArgsProcessFolder{LoaderID: "testV1RemoveResource"},
&reply); err == nil || reply != utils.EmptyString || err.Error() != expected {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply)
}
os.Remove(path.Join(flPath, "lock.cgr"))
if err := ldrs.V1Remove(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1RemoveResource",
ForceLock: true}, &reply); err != nil && reply != utils.OK {
t.Error(err)
}
//nothing to get from dataBase
if _, err := dm.GetResourceProfile("cgrates.org", "NewRes1",
true, true, utils.NonTransactional); err != utils.ErrNotFound {
t.Error(err)
}
if err := os.RemoveAll(flPath); err != nil {
t.Error(err)
}
}
func testV1RemoveDefaultIDError(t *testing.T) {
flPath := "/tmp/testV1RemoveDefaultIDError"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
file.Write([]byte(`
#Tenant[0],ID[1]
cgrates.org,NewRes1
`))
file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1RemoveDefaultIDError",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
var reply string
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
expected := "UNKNOWN_LOADER: *default"
if err := ldrs.V1Remove(context.Background(), &ArgsProcessFolder{
LoaderID: utils.EmptyString}, &reply); err == nil || reply != utils.EmptyString || err.Error() != expected {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1RemoveUnableToDeleteFile(t *testing.T) {
flPath := "testV1RemoveUnableToDeleteFile"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1RemoveUnableToDeleteFile",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: "/\x00",
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
var reply string
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
expected := "SERVER_ERROR: stat /\x00/Resources.csv: invalid argument"
if err := ldrs.V1Remove(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1RemoveUnableToDeleteFile",
ForceLock: true}, &reply); err == nil || err.Error() != expected {
t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1LoadAndRemoveProcessRemoveFolderError(t *testing.T) {
flPath := "/tmp/testV1RemoveProcessFolderError"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
file, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
defer file.Close()
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1RemoveProcessFolderError",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
Data: nil,
}
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
ldrs.ldrs["testV1RemoveProcessFolderError"].lockFilepath = flPath
ldrs.ldrs["testV1RemoveProcessFolderError"].rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
"not_a_file": nil,
},
}
var reply string
expected := "SERVER_ERROR: remove /tmp/testV1RemoveProcessFolderError: directory not empty"
//try to load by changing the caching method, but there is not a lockFileName
if err := ldrs.V1Load(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1RemoveProcessFolderError",
ForceLock: true,
Caching: utils.StringPointer("not_a_chaching_method"),
StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString {
t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply)
}
//try to remove by changing the caching method
if err := ldrs.V1Remove(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1RemoveProcessFolderError",
ForceLock: true,
Caching: utils.StringPointer("not_a_chaching_method"),
StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString {
t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testV1RemoveProcessFolderError(t *testing.T) {
flPath := "testV1RemoveProcessFolderError"
if err := os.MkdirAll(flPath, 0777); err != nil {
t.Error(err)
}
_, err := os.Create(path.Join(flPath, utils.ResourcesCsv))
if err != nil {
t.Error(err)
}
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1RemoveProcessFolderError",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: "notResource.csv",
Data: nil,
}
ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil)
ldrs.ldrs["testV1RemoveProcessFolderError"].rdrs = map[string]map[string]*openedCSVFile{
utils.MetaResources: {
"not_a_file2": &openedCSVFile{
fileName: utils.ResourcesCsv,
},
},
}
var reply string
expected := "SERVER_ERROR: open testV1RemoveProcessFolderError/not_a_file2: no such file or directory"
//try to load by changing the caching method
if err := ldrs.V1Remove(context.Background(),
&ArgsProcessFolder{
LoaderID: "testV1RemoveProcessFolderError",
ForceLock: true,
Caching: utils.StringPointer("not_a_chaching_method"),
StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString {
t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply)
}
if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil {
t.Error(err)
} else if err := os.Remove(flPath); err != nil {
t.Error(err)
}
}
func testLoaderServiceListenAndServe(t *testing.T) {
ldr := &Loader{
runDelay: -1,
tpInDir: "/tmp/TestLoaderServiceListenAndServe",
}
ldrs := &LoaderService{
ldrs: map[string]*Loader{
"TEST_LOADER": ldr,
},
}
stopChan := make(chan struct{}, 1)
stopChan <- struct{}{}
expected := "no such file or directory"
if err := ldrs.ListenAndServe(stopChan); err == nil || err.Error() != expected {
t.Errorf("Expected %+v, received %+v", expected, err)
}
ldrs.ldrs["TEST_LOADER"].tpInDir = utils.EmptyString
if err := ldrs.ListenAndServe(stopChan); err != nil {
t.Error(err)
}
}
func testLoaderServiceReload(t *testing.T) {
flPath := "/tmp/testLoaderServiceReload"
data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items)
if err != nil {
t.Error(err)
}
dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil)
cfgLdr := config.NewDefaultCGRConfig().LoaderCfg()
cfgLdr[0] = &config.LoaderSCfg{
ID: "testV1LoadResource",
Enabled: true,
FieldSeparator: utils.FieldsSep,
TpInDir: flPath,
TpOutDir: "/tmp",
LockFilePath: utils.ResourcesCsv,
Data: nil,
}
ldrs := &LoaderService{}
ldrs.Reload(dm, cfgLdr, "UTC", 0, nil, nil)
if ldrs.ldrs == nil {
t.Error("Expected to be populated")
}
}

174
loaders/loaders_test.go
View File

@@ -1,174 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package loaders
import (
"testing"
"time"
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
)
func TestLoaderServiceListenAndServe(t *testing.T) {
stopChan := make(chan struct{})
defer close(stopChan)
loaderService := &LoaderService{
ldrs: map[string]*Loader{
"loader1": {ldrID: "loader1"},
"loader2": {ldrID: "loader2"},
"loader3": {ldrID: "error"},
},
}
loaderService.ldrs["loader3"].ldrID = "loader3"
t.Run("All loaders succeed", func(t *testing.T) {
err := loaderService.ListenAndServe(stopChan)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
})
}
func TestLoaderServiceV1Load(t *testing.T) {
ctx := context.Background()
loaderService := &LoaderService{
ldrs: map[string]*Loader{},
}
t.Run("Unknown loader", func(t *testing.T) {
args := &ArgsProcessFolder{LoaderID: "unknown"}
var rply string
err := loaderService.V1Load(ctx, args, &rply)
if err == nil || err.Error() != "UNKNOWN_LOADER: unknown" {
t.Errorf("expected error 'UNKNOWN_LOADER: unknown', got %v", err)
}
})
t.Run("Another loader running without force", func(t *testing.T) {
args := &ArgsProcessFolder{LoaderID: "loader1"}
var rply string
err := loaderService.V1Load(ctx, args, &rply)
if err == nil || err.Error() == "ANOTHER_LOADER_RUNNING" {
t.Errorf("expected error 'ANOTHER_LOADER_RUNNING', got %v", err)
}
})
t.Run("Unlock folder and process", func(t *testing.T) {
args := &ArgsProcessFolder{LoaderID: "loader1", ForceLock: true}
var rply string
err := loaderService.V1Load(ctx, args, &rply)
if err == nil {
t.Errorf("expected no error, got %v", err)
}
if rply == "OK" {
t.Errorf("expected reply 'OK', got %s", rply)
}
})
t.Run("Process with no locking issues", func(t *testing.T) {
args := &ArgsProcessFolder{LoaderID: "loader2"}
var rply string
err := loaderService.V1Load(ctx, args, &rply)
if err == nil {
t.Errorf("expected no error, got %v", err)
}
if rply == "OK" {
t.Errorf("expected reply 'OK', got %s", rply)
}
})
}
func TestLoaderServiceReload(t *testing.T) {
loaderService := &LoaderService{}
dm := &engine.DataManager{}
filterS := &engine.FilterS{}
connMgr := &engine.ConnManager{}
timezone := "UTC"
cachingDelay := time.Duration(5 * time.Second)
ldrsCfg := []*config.LoaderSCfg{
{ID: "loader1", Enabled: true},
{ID: "loader2", Enabled: false},
{ID: "loader3", Enabled: true},
}
loaderService.Reload(dm, ldrsCfg, timezone, cachingDelay, filterS, connMgr)
if len(loaderService.ldrs) != 2 {
t.Errorf("expected 2 enabled loaders, got %d", len(loaderService.ldrs))
}
if _, exists := loaderService.ldrs["loader1"]; !exists {
t.Error("expected loader1 to be in the loaders map")
}
if _, exists := loaderService.ldrs["loader3"]; !exists {
t.Error("expected loader3 to be in the loaders map")
}
if _, exists := loaderService.ldrs["loader2"]; exists {
t.Error("did not expect loader2 to be in the loaders map")
}
}
func TestLoaderServiceV1Remove(t *testing.T) {
ctx := context.Background()
loaderService := &LoaderService{
ldrs: map[string]*Loader{},
}
args := &ArgsProcessFolder{LoaderID: "unknown"}
var rply string
err := loaderService.V1Remove(ctx, args, &rply)
if err == nil || err.Error() != "UNKNOWN_LOADER: unknown" {
t.Errorf("expected error 'UNKNOWN_LOADER: unknown', got %v", err)
}
}
func TestNewLoaderService(t *testing.T) {
dm := &engine.DataManager{}
timezone := "UTC"
cachingDlay := time.Second
filterS := &engine.FilterS{}
connMgr := &engine.ConnManager{}
ldrsCfg := []*config.LoaderSCfg{
{ID: "loader1", Enabled: true},
{ID: "loader2", Enabled: false},
{ID: "loader3", Enabled: true},
}
ldrService := NewLoaderService(dm, ldrsCfg, timezone, cachingDlay, filterS, connMgr)
if len(ldrService.ldrs) != 2 {
t.Errorf("expected 2 loaders, got %d", len(ldrService.ldrs))
}
if _, exists := ldrService.ldrs["loader1"]; !exists {
t.Errorf("expected loader1 to be present")
}
if _, exists := ldrService.ldrs["loader2"]; exists {
t.Errorf("expected loader2 to not be present")
}
if _, exists := ldrService.ldrs["loader3"]; !exists {
t.Errorf("expected loader3 to be present")
}
}

3
services/analyzers_it_test.go
View File

@@ -56,8 +56,7 @@ func TestAnalyzerSReload(t *testing.T) {
anzRPC := make(chan birpc.ClientConnector, 1)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, anzRPC, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(anz,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(anz, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/apiers_it_test.go
View File

@@ -69,8 +69,7 @@ func TestApiersReload(t *testing.T) {
make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
apiSv2 := NewAPIerSv2Service(apiSv1, cfg, server, make(chan birpc.ClientConnector, 1), anz, srvDep)
srvMngr.AddServices(apiSv1, apiSv2, schS, tS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb)
srvMngr.AddServices(apiSv1, apiSv2, schS, tS, db, stordb)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

6
services/asteriskagent_it_test.go
View File

@@ -71,8 +71,7 @@ func TestAsteriskAgentReload(t *testing.T) {
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, cm, anz, srvDep)
astService := NewAsteriskAgent(cfg, shdChan, cm, srvDep)
srvMngr.AddServices(astService, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(astService, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}
@@ -144,8 +143,7 @@ func TestAsteriskAgentReload2(t *testing.T) {
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, cm, anz, srvDep)
astSrv := NewAsteriskAgent(cfg, shdChan, cm, srvDep)
srvMngr.AddServices(astSrv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(astSrv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/attributes_it_test.go
View File

@@ -58,8 +58,7 @@ func TestAttributeSReload(t *testing.T) {
chS, filterSChan, server, attrRPC,
anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(attrS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(attrS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

4
services/cdrs_it_test.go
View File

@@ -77,9 +77,7 @@ func TestCdrsReload(t *testing.T) {
cdrsRPC := make(chan birpc.ClientConnector, 1)
cdrS := NewCDRServer(cfg, db, stordb, filterSChan, server,
cdrsRPC, nil, anz, srvDep)
srvMngr.AddServices(cdrS, ralS, schS, chrS,
NewLoaderService(cfg, db, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb)
srvMngr.AddServices(cdrS, ralS, schS, chrS, db, stordb)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

4
services/chargers_it_test.go
View File

@@ -58,9 +58,7 @@ func TestChargerSReload(t *testing.T) {
attrS := NewAttributeService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep)
chrS := NewChargerService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(attrS, chrS,
NewLoaderService(cfg, db, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(attrS, chrS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/cores_it_test.go
View File

@@ -54,8 +54,7 @@ func TestCoreSReload(t *testing.T) {
caps := engine.NewCaps(1, "test_caps")
coreS := NewCoreService(cfg, caps, server, coreRPC, anz, nil, nil, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(coreS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(coreS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

2
services/datadb.go
View File

@@ -145,7 +145,7 @@ func (db *DataDBService) mandatoryDB() bool {
return db.cfg.RalsCfg().Enabled || db.cfg.SchedulerCfg().Enabled || db.cfg.ChargerSCfg().Enabled ||
db.cfg.AttributeSCfg().Enabled || db.cfg.ResourceSCfg().Enabled || db.cfg.StatSCfg().Enabled ||
db.cfg.ThresholdSCfg().Enabled || db.cfg.RouteSCfg().Enabled || db.cfg.DispatcherSCfg().Enabled ||
db.cfg.LoaderCfg().Enabled() || db.cfg.ApierCfg().Enabled || db.cfg.AnalyzerSCfg().Enabled || db.cfg.ERsCfg().Enabled
db.cfg.ApierCfg().Enabled || db.cfg.AnalyzerSCfg().Enabled || db.cfg.ERsCfg().Enabled
}
// GetDM returns the DataManager

3
services/datadb_it_test.go
View File

@@ -56,8 +56,7 @@ func TestDataDBReload(t *testing.T) {
db := NewDataDBService(cfg, cM, false, srvDep)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
srvMngr.AddServices(NewAttributeService(cfg, db,
chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep),
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep), db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/diameteragent_it_test.go
View File

@@ -61,8 +61,7 @@ func TestDiameterAgentReload1(t *testing.T) {
shdChan, nil, anz, srvDep)
diamSrv := NewDiameterAgent(cfg, filterSChan, shdChan, nil, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(diamSrv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(diamSrv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

4
services/dispatchers_it_test.go
View File

@@ -60,9 +60,7 @@ func TestDispatcherSReload(t *testing.T) {
srv := NewDispatcherService(cfg, db, chS, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(attrS, srv,
NewLoaderService(cfg, db, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(attrS, srv, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

6
services/dnsagent_it_test.go
View File

@@ -67,8 +67,7 @@ func TestDNSAgentStartReloadShut(t *testing.T) {
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, nil, anz, srvDep)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
runtime.Gosched()
time.Sleep(10 * time.Millisecond) //need to switch to gorutine
if err := srv.Shutdown(); err != nil {
@@ -124,8 +123,7 @@ func TestDNSAgentReloadFirst(t *testing.T) {
shdChan, nil, anz, srvDep)
srv := NewDNSAgent(cfg, filterSChan, shdChan, nil, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/ees_it_test.go
View File

@@ -68,8 +68,7 @@ func TestEventExporterSReload(t *testing.T) {
anz, srvDep)
ees := NewEventExporterService(cfg, filterSChan, engine.NewConnManager(cfg, nil),
server, make(chan birpc.ClientConnector, 2), anz, srvDep)
srvMngr.AddServices(ees, attrS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(ees, attrS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/ers_it_test.go
View File

@@ -70,8 +70,7 @@ func TestEventReaderSReload(t *testing.T) {
intERsConn := make(chan birpc.ClientConnector, 1)
erS := NewEventReaderService(cfg, db, filterSChan, shdChan, nil, server, intERsConn, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(erS, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(erS, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/freeswitchagent_it_test.go
View File

@@ -72,8 +72,7 @@ func TestFreeSwitchAgentReload(t *testing.T) {
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, cm, anz, srvDep)
srv := NewFreeswitchAgent(cfg, shdChan, cm, srvDep)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), cm, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/httpagent_it_test.go
View File

@@ -66,8 +66,7 @@ func TestHTTPAgentReload(t *testing.T) {
shdChan, nil, anz, srvDep)
srv := NewHTTPAgent(cfg, filterSChan, server, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/kamailioagent_it_test.go
View File

@@ -76,8 +76,7 @@ func TestKamailioAgentReload(t *testing.T) {
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, cm, anz, srvDep)
srv := NewKamailioAgent(cfg, shdChan, cm, srvDep)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), cm, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

154
services/loaders.go
View File

@@ -1,154 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package services
import (
"sync"
"github.com/cgrates/birpc"
v1 "github.com/cgrates/cgrates/apier/v1"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/cores"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/loaders"
"github.com/cgrates/cgrates/utils"
)
// NewLoaderService returns the Loader Service
func NewLoaderService(cfg *config.CGRConfig, dm *DataDBService,
filterSChan chan *engine.FilterS, server *cores.Server,
internalLoaderSChan chan birpc.ClientConnector,
connMgr *engine.ConnManager, anz *AnalyzerService,
srvDep map[string]*sync.WaitGroup) *LoaderService {
return &LoaderService{
connChan: internalLoaderSChan,
cfg: cfg,
dm: dm,
filterSChan: filterSChan,
server: server,
connMgr: connMgr,
stopChan: make(chan struct{}),
anz: anz,
srvDep: srvDep,
}
}
// LoaderService implements Service interface
type LoaderService struct {
sync.RWMutex
cfg *config.CGRConfig
dm *DataDBService
filterSChan chan *engine.FilterS
server *cores.Server
stopChan chan struct{}
ldrs *loaders.LoaderService
connChan chan birpc.ClientConnector
connMgr *engine.ConnManager
anz *AnalyzerService
srvDep map[string]*sync.WaitGroup
}
// Start should handle the sercive start
func (ldrs *LoaderService) Start() error {
if ldrs.IsRunning() {
return utils.ErrServiceAlreadyRunning
}
filterS := <-ldrs.filterSChan
ldrs.filterSChan <- filterS
dbchan := ldrs.dm.GetDMChan()
datadb := <-dbchan
dbchan <- datadb
ldrs.Lock()
defer ldrs.Unlock()
ldrs.ldrs = loaders.NewLoaderService(datadb, ldrs.cfg.LoaderCfg(),
ldrs.cfg.GeneralCfg().DefaultTimezone, ldrs.cfg.GeneralCfg().CachingDelay, filterS, ldrs.connMgr)
if !ldrs.ldrs.Enabled() {
return nil
}
if err := ldrs.ldrs.ListenAndServe(ldrs.stopChan); err != nil {
return err
}
srv, err := engine.NewService(v1.NewLoaderSv1(ldrs.ldrs))
if err != nil {
return err
}
if !ldrs.cfg.DispatcherSCfg().Enabled {
ldrs.server.RpcRegister(srv)
}
ldrs.connChan <- ldrs.anz.GetInternalCodec(srv, utils.LoaderS)
return nil
}
// Reload handles the change of config
func (ldrs *LoaderService) Reload() (err error) {
filterS := <-ldrs.filterSChan
ldrs.filterSChan <- filterS
dbchan := ldrs.dm.GetDMChan()
datadb := <-dbchan
dbchan <- datadb
close(ldrs.stopChan)
ldrs.stopChan = make(chan struct{})
ldrs.RLock()
ldrs.ldrs.Reload(datadb, ldrs.cfg.LoaderCfg(), ldrs.cfg.GeneralCfg().DefaultTimezone, ldrs.cfg.GeneralCfg().CachingDelay,
filterS, ldrs.connMgr)
if err = ldrs.ldrs.ListenAndServe(ldrs.stopChan); err != nil {
return
}
ldrs.RUnlock()
return
}
// Shutdown stops the service
func (ldrs *LoaderService) Shutdown() (err error) {
ldrs.Lock()
ldrs.ldrs = nil
close(ldrs.stopChan)
<-ldrs.connChan
ldrs.Unlock()
return
}
// IsRunning returns if the service is running
func (ldrs *LoaderService) IsRunning() bool {
ldrs.RLock()
defer ldrs.RUnlock()
return ldrs.ldrs != nil && ldrs.ldrs.Enabled()
}
// ServiceName returns the service name
func (ldrs *LoaderService) ServiceName() string {
return utils.LoaderS
}
// ShouldRun returns if the service should be running
func (ldrs *LoaderService) ShouldRun() bool {
return ldrs.cfg.LoaderCfg().Enabled()
}
// GetLoaderS returns the initialized LoaderService
func (ldrs *LoaderService) GetLoaderS() *loaders.LoaderService {
return ldrs.ldrs
}

200
services/loaders_it_test.go
View File

@@ -1,200 +0,0 @@
//go:build integration
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package services
import (
"os"
"path"
"sync"
"testing"
"time"
"github.com/cgrates/birpc"
"github.com/cgrates/birpc/context"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/cores"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/servmanager"
"github.com/cgrates/cgrates/utils"
)
func testCreateDirs(t *testing.T) {
for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut",
"/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} {
if err := os.RemoveAll(dir); err != nil {
t.Fatal("Error removing folder: ", dir, err)
}
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatal("Error creating folder: ", dir, err)
}
}
if err := os.WriteFile(path.Join("/tmp/In", utils.AttributesCsv), []byte(`
#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight
cgrates.org,ALS1,con1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*string:~*req.Field1:Initial,*req.Field1,*variable,Sub1,true,20
cgrates.org,ALS1,con2;con3,,,,*req.Field2,*variable,Sub2,true,20
`), 0644); err != nil {
t.Fatal(err.Error())
}
}
func TestLoaderSReload(t *testing.T) {
testCreateDirs(t)
cfg := config.NewDefaultCGRConfig()
cfg.TemplatesCfg()["attrTemplateLoader"] = []*config.FCTemplate{
{
Type: utils.MetaVariable,
Path: "*req.Accounts",
Value: config.NewRSRParsersMustCompile("1001", utils.InfieldSep),
},
}
utils.Logger, _ = utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID)
utils.Logger.SetLogLevel(7)
shdChan := utils.NewSyncedChan()
shdWg := new(sync.WaitGroup)
filterSChan := make(chan *engine.FilterS, 1)
filterSChan <- nil
server := cores.NewServer(nil)
srvMngr := servmanager.NewServiceManager(cfg, shdChan, shdWg, nil)
srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)}
db := NewDataDBService(cfg, nil, false, srvDep)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
conMngr := engine.NewConnManager(cfg, nil)
srv := NewLoaderService(cfg, db, filterSChan,
server, make(chan birpc.ClientConnector, 1),
conMngr, anz, srvDep)
srvMngr.AddServices(srv, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}
if db.IsRunning() {
t.Errorf("Expected service to be down")
}
if srv.IsRunning() {
t.Errorf("Expected service to be down")
}
var reply string
if err := cfg.V1ReloadConfig(context.Background(),
&config.ReloadArgs{
Path: path.Join("/usr", "share", "cgrates", "conf", "samples", "loaders", "tutinternal"),
Section: config.LoaderJson,
}, &reply); err != nil {
t.Fatal(err)
} else if reply != utils.OK {
t.Errorf("Expecting OK ,received %s", reply)
}
time.Sleep(10 * time.Millisecond)
if !db.IsRunning() {
t.Fatal("Expected service to be running")
}
time.Sleep(10 * time.Millisecond)
if !srv.IsRunning() {
t.Fatal("Expected service to be running")
}
err := srv.Start()
if err == nil || err != utils.ErrServiceAlreadyRunning {
t.Errorf("\nExpecting <%+v>,\n Received <%+v>", utils.ErrServiceAlreadyRunning, err)
}
time.Sleep(10 * time.Millisecond)
err = srv.Reload()
if err != nil {
t.Errorf("\nExpecting <nil>,\n Received <%+v>", err)
}
time.Sleep(10 * time.Millisecond)
for _, v := range cfg.LoaderCfg() {
v.Enabled = false
}
time.Sleep(10 * time.Millisecond)
cfg.GetReloadChan(config.LoaderJson) <- struct{}{}
time.Sleep(10 * time.Millisecond)
if srv.IsRunning() {
t.Errorf("Expected service to be down")
}
shdChan.CloseOnce()
time.Sleep(10 * time.Millisecond)
testCleanupFiles(t)
}
func testCleanupFiles(t *testing.T) {
for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove",
"/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} {
if err := os.RemoveAll(dir); err != nil {
t.Fatal("Error removing folder: ", dir, err)
}
}
}
func TestLoaderSReload2(t *testing.T) {
cfg := config.NewDefaultCGRConfig()
for _, ld := range cfg.LoaderCfg() {
ld.Enabled = false
}
shdChan := utils.NewSyncedChan()
filterSChan := make(chan *engine.FilterS, 1)
filterSChan <- nil
server := cores.NewServer(nil)
srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)}
db := NewDataDBService(cfg, nil, false, srvDep)
db.dbchan <- new(engine.DataManager)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
srv := NewLoaderService(cfg, db, filterSChan,
server, make(chan birpc.ClientConnector, 1),
nil, anz, srvDep)
err := srv.Start()
if err != nil {
t.Fatal(err)
}
}
func TestLoaderSReload3(t *testing.T) {
cfg := config.NewDefaultCGRConfig()
for _, ld := range cfg.LoaderCfg() {
ld.Enabled = false
}
cfg.LoaderCfg()[0].Enabled = true
cfg.LoaderCfg()[0].TpInDir = "/tmp/TestLoaderSReload3"
cfg.LoaderCfg()[0].RunDelay = -1
shdChan := utils.NewSyncedChan()
filterSChan := make(chan *engine.FilterS, 1)
filterSChan <- nil
server := cores.NewServer(nil)
srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)}
db := NewDataDBService(cfg, nil, false, srvDep)
db.dbchan <- new(engine.DataManager)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
srv := NewLoaderService(cfg, db, filterSChan,
server, make(chan birpc.ClientConnector, 1),
nil, anz, srvDep)
err := srv.Start()
if err == nil || err.Error() != "no such file or directory" {
t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "no such file or directory", err)
}
err = srv.Reload()
if err == nil || err.Error() != "no such file or directory" {
t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "no such file or directory", err)
}
}

97
services/loaders_test.go
View File

@@ -1,97 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
package services
import (
"reflect"
"sync"
"testing"
"github.com/cgrates/birpc"
"github.com/cgrates/cgrates/loaders"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/cores"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
// TestLoaderSCoverage for cover testing
func TestLoaderSCoverage(t *testing.T) {
cfg := config.NewDefaultCGRConfig()
shdChan := utils.NewSyncedChan()
filterSChan := make(chan *engine.FilterS, 1)
filterSChan <- nil
server := cores.NewServer(nil)
srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)}
db := NewDataDBService(cfg, nil, false, srvDep)
internalLoaderSChan := make(chan birpc.ClientConnector, 1)
rpcInternal := map[string]chan birpc.ClientConnector{}
cM := engine.NewConnManager(cfg, rpcInternal)
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
srv := NewLoaderService(cfg, db,
filterSChan, server, internalLoaderSChan,
cM, anz, srvDep)
if srv == nil {
t.Errorf("\nExpecting <nil>,\n Received <%+v>", utils.ToJSON(srv))
}
if srv.IsRunning() {
t.Errorf("Expected service to be down")
}
srv.ldrs = loaders.NewLoaderService(&engine.DataManager{},
[]*config.LoaderSCfg{{
ID: "test_id",
Enabled: true,
Tenant: "",
DryRun: false,
RunDelay: 0,
LockFilePath: "",
CacheSConns: nil,
FieldSeparator: "",
TpInDir: "",
TpOutDir: "",
Data: nil,
}}, "", 0,
&engine.FilterS{}, nil)
if !srv.IsRunning() {
t.Errorf("Expected service to be running")
}
serviceName := srv.ServiceName()
if !reflect.DeepEqual(serviceName, utils.LoaderS) {
t.Errorf("\nExpecting <%+v>,\n Received <%+v>", utils.LoaderS, serviceName)
}
shouldRun := srv.ShouldRun()
if !reflect.DeepEqual(shouldRun, false) {
t.Errorf("\nExpecting <false>,\n Received <%+v>", shouldRun)
}
if !reflect.DeepEqual(srv.GetLoaderS(), srv.ldrs) {
t.Errorf("\nExpecting <%+v>,\n Received <%+v>", srv.ldrs, srv.GetLoaderS())
}
srv.stopChan = make(chan struct{}, 1)
chS := engine.NewCacheS(cfg, nil, nil)
cacheSrv, err := engine.NewService(chS)
if err != nil {
t.Fatal(err)
}
srv.connChan <- cacheSrv
srv.Shutdown()
if srv.IsRunning() {
t.Errorf("Expected service to be down")
}
}

9
services/radiusagent_it_test.go
View File

@@ -69,8 +69,7 @@ func TestRadiusAgentReloadStartShut(t *testing.T) {
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1),
shdChan, nil, anz, srvDep)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
runtime.Gosched()
time.Sleep(10 * time.Millisecond) //need to switch to gorutine
if err := srv.Shutdown(); err != nil {
@@ -126,8 +125,7 @@ func TestRadiusAgentReload1(t *testing.T) {
shdChan, nil, anz, srvDep)
srv := NewRadiusAgent(cfg, filterSChan, shdChan, nil, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}
@@ -197,8 +195,7 @@ func TestRadiusAgentReload2(t *testing.T) {
shdChan, nil, anz, srvDep)
srv := NewRadiusAgent(cfg, filterSChan, shdChan, nil, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/rals_it_test.go
View File

@@ -74,8 +74,7 @@ func TestRalsReload(t *testing.T) {
make(chan birpc.ClientConnector, 1),
make(chan birpc.ClientConnector, 1),
shdChan, nil, anz, srvDep, filterSChan)
srvMngr.AddServices(ralS, schS, tS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb)
srvMngr.AddServices(ralS, schS, tS, db, stordb)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

4
services/registrarc_it_test.go
View File

@@ -60,9 +60,7 @@ func TestDispatcherHReload(t *testing.T) {
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
connMngr := engine.NewConnManager(cfg, nil)
srv := NewRegistrarCService(cfg, server, connMngr, anz, srvDep)
srvMngr.AddServices(srv,
NewLoaderService(cfg, db, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/resources_it_test.go
View File

@@ -61,8 +61,7 @@ func TestResourceSReload(t *testing.T) {
tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
reS := NewResourceService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(tS, reS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(tS, reS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/routes_it_test.go
View File

@@ -55,8 +55,7 @@ func TestRouteSReload(t *testing.T) {
db := NewDataDBService(cfg, nil, false, srvDep)
routeS := NewRouteService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(routeS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(routeS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/schedulers_it_test.go
View File

@@ -54,8 +54,7 @@ func TestSchedulerSReload(t *testing.T) {
anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep)
schS := NewSchedulerService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(schS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(schS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

3
services/sipagent_it_test.go
View File

@@ -64,8 +64,7 @@ func TestSIPAgentReload(t *testing.T) {
shdChan, nil, anz, srvDep)
srv := NewSIPAgent(cfg, filterSChan, shdChan, nil, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(srv, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(srv, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Fatal(err)
}

3
services/stats_it_test.go
View File

@@ -61,8 +61,7 @@ func TestStatSReload(t *testing.T) {
tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
sS := NewStatService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(tS, sS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(tS, sS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

4
services/stordb_it_test.go
View File

@@ -61,9 +61,7 @@ func TestStorDBReload(t *testing.T) {
cdrsRPC := make(chan birpc.ClientConnector, 1)
cdrS := NewCDRServer(cfg, db, stordb, filterSChan, server,
cdrsRPC, nil, anz, srvDep)
srvMngr.AddServices(cdrS, ralS, schS, chrS,
NewLoaderService(cfg, db, filterSChan, server,
make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb)
srvMngr.AddServices(cdrS, ralS, schS, chrS, db, stordb)
if err := engine.InitStorDb(cfg); err != nil {
t.Fatal(err)
}

6
services/thresholds_it_test.go
View File

@@ -57,8 +57,7 @@ func TestThresholdSReload(t *testing.T) {
db := NewDataDBService(cfg, nil, false, srvDep)
tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(tS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(tS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}
@@ -127,8 +126,7 @@ func TestThresholdSReload2(t *testing.T) {
db := NewDataDBService(cfg, nil, false, srvDep)
tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep)
engine.NewConnManager(cfg, nil)
srvMngr.AddServices(tS,
NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db)
srvMngr.AddServices(tS, db)
if err := srvMngr.StartServices(); err != nil {
t.Error(err)
}

2
servmanager/servmanager.go
View File

@@ -204,8 +204,6 @@ func (srvMngr *ServiceManager) handleReload() {
go srvMngr.reloadService(utils.HTTPAgent)
case <-srvMngr.GetConfig().GetReloadChan(config.PrometheusAgentJSON):
go srvMngr.reloadService(utils.PrometheusAgent)
case <-srvMngr.GetConfig().GetReloadChan(config.LoaderJson):
go srvMngr.reloadService(utils.LoaderS)
case <-srvMngr.GetConfig().GetReloadChan(config.AnalyzerCfgJson):
go srvMngr.reloadService(utils.AnalyzerS)
case <-srvMngr.GetConfig().GetReloadChan(config.DispatcherSJson):

11
utils/consts.go
View File

@@ -906,7 +906,6 @@ const (
OrderIDEnd = "OrderIDEnd"
MinCost = "MinCost"
MaxCost = "MaxCost"
MetaLoaders = "*loaders"
TmpSuffix = ".tmp"
MetaDiamreq = "*diamreq"
MetaRadDAReq = "*radDAReq"
@@ -1209,7 +1208,6 @@ const (
ErS = "ErS"
FilterS = "FilterS"
GuardianS = "GuardianS"
LoaderS = "LoaderS"
RALs = "RALs"
RegistrarC = "RegistrarC"
ReplicatorS = "ReplicatorS"
@@ -1236,7 +1234,6 @@ const (
DispatcherSLow = "dispatchers"
AnalyzerSLow = "analyzers"
SchedulerSLow = "schedulers"
LoaderSLow = "loaders"
RALsLow = "rals"
ReplicatorLow = "replicator"
ApierSLow = "apiers"
@@ -2048,14 +2045,6 @@ const (
AnalyzerSv1StringQuery = "AnalyzerSv1.StringQuery"
)
// LoaderS APIs
const (
LoaderSv1 = "LoaderSv1"
LoaderSv1Load = "LoaderSv1.Load"
LoaderSv1Remove = "LoaderSv1.Remove"
LoaderSv1Ping = "LoaderSv1.Ping"
)
// CacheS APIs
const (
CacheSv1 = "CacheSv1"