From 3153bc837841ff0483e67db0c7df7becedd5faaf Mon Sep 17 00:00:00 2001 From: gezimbll Date: Fri, 23 Jan 2026 16:40:36 +0100 Subject: [PATCH] remove loaders subsystem --- apier/v1/loaders.go | 43 - apier/v1/loaders_test.go | 36 - apier/v1/preload_it_test.go | 184 - cmd/cgr-engine/cgr-engine.go | 48 +- cmd/cgr-engine/cgr-engine_flags_test.go | 7 - config/config.go | 41 +- config/config_defaults.go | 190 - config/config_json.go | 15 +- config/config_json_test.go | 563 --- config/config_test.go | 798 +--- config/configsanity.go | 46 - config/configsanity_test.go | 110 - config/libconfig_json.go | 21 - config/loaderscfg.go | 277 -- config/loaderscfg_test.go | 586 --- console/loader_load.go | 65 - console/loader_load_test.go | 54 - console/loader_remove.go | 65 - console/loader_remove_test.go | 54 - console/ping.go | 2 - console/ping_test.go | 39 - .../loaders_console_mongo/cgrates.json | 164 - .../loaders_console_mysql/cgrates.json | 161 - .../samples/loaders/tutinternal/cgrates.json | 256 -- .../samples/loaders/tutmongo/cgrates.json | 304 -- .../samples/loaders/tutmysql/cgrates.json | 304 -- integration_test.sh | 2 +- loaders/lib_test.go | 739 ---- loaders/libloader.go | 166 - loaders/libloader_test.go | 443 -- loaders/loader.go | 1016 ----- loaders/loader_it_test.go | 857 ---- loaders/loader_test.go | 3860 ----------------- loaders/loaders.go | 150 - loaders/loaders_it_test.go | 661 --- loaders/loaders_test.go | 174 - services/analyzers_it_test.go | 3 +- services/apiers_it_test.go | 3 +- services/asteriskagent_it_test.go | 6 +- services/attributes_it_test.go | 3 +- services/cdrs_it_test.go | 4 +- services/chargers_it_test.go | 4 +- services/cores_it_test.go | 3 +- services/datadb.go | 2 +- services/datadb_it_test.go | 3 +- services/diameteragent_it_test.go | 3 +- services/dispatchers_it_test.go | 4 +- services/dnsagent_it_test.go | 6 +- services/ees_it_test.go | 3 +- services/ers_it_test.go | 3 +- services/freeswitchagent_it_test.go | 3 +- services/httpagent_it_test.go | 3 +- services/kamailioagent_it_test.go | 3 +- services/loaders.go | 154 - services/loaders_it_test.go | 200 - services/loaders_test.go | 97 - services/radiusagent_it_test.go | 9 +- services/rals_it_test.go | 3 +- services/registrarc_it_test.go | 4 +- services/resources_it_test.go | 3 +- services/routes_it_test.go | 3 +- services/schedulers_it_test.go | 3 +- services/sipagent_it_test.go | 3 +- services/stats_it_test.go | 3 +- services/stordb_it_test.go | 4 +- services/thresholds_it_test.go | 6 +- servmanager/servmanager.go | 2 - utils/consts.go | 11 - 68 files changed, 44 insertions(+), 13021 deletions(-) delete mode 100644 apier/v1/loaders.go delete mode 100644 apier/v1/loaders_test.go delete mode 100644 apier/v1/preload_it_test.go delete mode 100644 config/loaderscfg.go delete mode 100644 config/loaderscfg_test.go delete mode 100644 console/loader_load.go delete mode 100644 console/loader_load_test.go delete mode 100644 console/loader_remove.go delete mode 100644 console/loader_remove_test.go delete mode 100644 data/conf/samples/loaders/loaders_console_mongo/cgrates.json delete mode 100644 data/conf/samples/loaders/loaders_console_mysql/cgrates.json delete mode 100644 data/conf/samples/loaders/tutinternal/cgrates.json delete mode 100644 data/conf/samples/loaders/tutmongo/cgrates.json delete mode 100644 data/conf/samples/loaders/tutmysql/cgrates.json delete mode 100644 loaders/lib_test.go delete mode 100644 loaders/libloader.go delete mode 100644 loaders/libloader_test.go delete mode 100644 loaders/loader.go delete mode 100644 loaders/loader_it_test.go delete mode 100644 loaders/loader_test.go delete mode 100644 loaders/loaders.go delete mode 100644 loaders/loaders_it_test.go delete mode 100644 loaders/loaders_test.go delete mode 100644 services/loaders.go delete mode 100644 services/loaders_it_test.go delete mode 100644 services/loaders_test.go diff --git a/apier/v1/loaders.go b/apier/v1/loaders.go deleted file mode 100644 index c23d397e1..000000000 --- a/apier/v1/loaders.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/loaders" -) - -func NewLoaderSv1(ldrS *loaders.LoaderService) *LoaderSv1 { - return &LoaderSv1{ldrS: ldrS} -} - -// Exports RPC from LoaderService -type LoaderSv1 struct { - ldrS *loaders.LoaderService -} - -func (ldrSv1 *LoaderSv1) Load(ctx *context.Context, args *loaders.ArgsProcessFolder, - rply *string) error { - return ldrSv1.ldrS.V1Load(ctx, args, rply) -} - -func (ldrSv1 *LoaderSv1) Remove(ctx *context.Context, args *loaders.ArgsProcessFolder, - rply *string) error { - return ldrSv1.ldrS.V1Remove(ctx, args, rply) -} diff --git a/apier/v1/loaders_test.go b/apier/v1/loaders_test.go deleted file mode 100644 index 0d8c94e7d..000000000 --- a/apier/v1/loaders_test.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "testing" - - "github.com/cgrates/cgrates/loaders" -) - -func TestNewLoaderSv1(t *testing.T) { - mockLoaderService := &loaders.LoaderService{} - result := NewLoaderSv1(mockLoaderService) - if result == nil { - t.Errorf("Expected non-nil result, but got nil") - } - if result.ldrS != mockLoaderService { - t.Errorf("Expected ldrS to be %v, but got %v", mockLoaderService, result.ldrS) - } -} diff --git a/apier/v1/preload_it_test.go b/apier/v1/preload_it_test.go deleted file mode 100644 index d37b81e36..000000000 --- a/apier/v1/preload_it_test.go +++ /dev/null @@ -1,184 +0,0 @@ -//go:build integration -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package v1 - -import ( - "os" - "os/exec" - "path" - "reflect" - "sort" - "testing" - "time" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/birpc/jsonrpc" - "github.com/cgrates/cgrates/utils" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" -) - -var ( - preloadCfgPath string - preloadCfgDIR string - preloadCfg *config.CGRConfig - preloadRPC *birpc.Client - - preloadTests = []func(t *testing.T){ - testCreateDirs, - testPreloadITInitConfig, - testPreloadITStartEngine, - testPreloadITRpcConn, - testPreloadITVerifyAttributes, - testCleanupFiles, - testPreloadITKillEngine, - } -) - -func TestPreload(t *testing.T) { - preloadCfgDIR = "tutinternal" - for _, test := range preloadTests { - t.Run(preloadCfgDIR, test) - } -} - -func testCreateDirs(t *testing.T) { - for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut", - "/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - - if err := os.WriteFile(path.Join("/tmp/In", utils.AttributesCsv), []byte(` -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight -cgrates.org,ALS1,con1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*string:~*req.Field1:Initial,*req.Field1,*variable,Sub1,true,20 -cgrates.org,ALS1,con2;con3,,,,*req.Field2,*variable,Sub2,true,20 -`), 0644); err != nil { - t.Fatal(err.Error()) - } -} - -func testPreloadITInitConfig(t *testing.T) { - var err error - preloadCfgPath = path.Join(*utils.DataDir, "conf", "samples", "loaders", preloadCfgDIR) - if preloadCfg, err = config.NewCGRConfigFromPath(preloadCfgPath); err != nil { - t.Fatal("Got config error: ", err.Error()) - } -} - -func testPreloadITStartEngine(t *testing.T) { - enginePath, err := exec.LookPath("cgr-engine") - if err != nil { - t.Error(err) - } - engine := exec.Command(enginePath, "-config_path", preloadCfgPath, "-preload", "CustomLoader") - if err := engine.Start(); err != nil { - t.Error(err) - } - fib := utils.FibDuration(time.Millisecond, 0) - var connected bool - for i := 0; i < 25; i++ { - time.Sleep(fib()) - if _, err := jsonrpc.Dial(utils.TCP, preloadCfg.ListenCfg().RPCJSONListen); err != nil { - t.Logf("Error <%s> when opening test connection to: <%s>", - err.Error(), preloadCfg.ListenCfg().RPCJSONListen) - } else { - connected = true - break - } - } - if !connected { - t.Errorf("engine did not open port <%s>", preloadCfg.ListenCfg().RPCJSONListen) - } - time.Sleep(100 * time.Millisecond) -} - -func testPreloadITRpcConn(t *testing.T) { - var err error - preloadRPC, err = newRPCClient(preloadCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed - if err != nil { - t.Fatal("Could not connect to rater: ", err.Error()) - } -} - -func testPreloadITVerifyAttributes(t *testing.T) { - eAttrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ALS1", - FilterIDs: []string{"*string:~*req.Account:1001"}, - Contexts: []string{"con1", "con2", "con3"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, - Attributes: []*engine.Attribute{ - { - FilterIDs: []string{"*string:~*req.Field1:Initial"}, - Path: "*req.Field1", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep), - }, - { - FilterIDs: []string{}, - Path: "*req.Field2", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep), - }, - }, - Blocker: true, - Weight: 20.0, - } - - var reply *engine.AttributeProfile - if err := preloadRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile, - utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"}}, &reply); err != nil { - t.Fatal(err) - } - reply.Compile() - sort.Strings(reply.Contexts) - if !reflect.DeepEqual(eAttrPrf, reply) { - eAttrPrf.Attributes[1].FilterIDs = nil - if !reflect.DeepEqual(eAttrPrf, reply) { - t.Errorf("Expecting : %+v,\n received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply)) - } - } -} - -func testCleanupFiles(t *testing.T) { - for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - } -} - -func testPreloadITKillEngine(t *testing.T) { - if err := engine.KillEngine(*utils.WaitRater); err != nil { - t.Error(err) - } -} diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index ed0fb7aaa..cd0720dba 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -28,7 +28,6 @@ import ( "runtime" "runtime/pprof" "strconv" - "strings" "sync" "syscall" "time" @@ -36,7 +35,6 @@ import ( "github.com/cgrates/birpc" "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/cores" - "github.com/cgrates/cgrates/loaders" "github.com/cgrates/cgrates/registrarc" v1 "github.com/cgrates/cgrates/apier/v1" @@ -64,7 +62,6 @@ var ( syslogger = cgrEngineFlags.String(utils.LoggerCfg, utils.EmptyString, "Logger type <*syslog|*stdout>") nodeID = cgrEngineFlags.String(utils.NodeIDCfg, utils.EmptyString, "Node ID of the engine") logLevel = cgrEngineFlags.Int(utils.LogLevelCfg, -1, "Log level (0=emergency to 7=debug)") - preload = cgrEngineFlags.String(utils.PreloadCgr, utils.EmptyString, "Loader IDs used to load data before engine starts") setVersions = cgrEngineFlags.Bool(utils.SetVersionsCgr, false, "Overwrite database versions (equivalent to cgr-migrator -exec=*set_versions)") cfg *config.CGRConfig @@ -145,7 +142,7 @@ func startRPC(server *cores.Server, internalRaterChan, internalCdrSChan, internalRsChan, internalIPsChan, internalStatSChan, internalAttrSChan, internalChargerSChan, internalThdSChan, internalTrendSChan, internalSuplSChan, internalSMGChan, internalAnalyzerSChan, internalDispatcherSChan, - internalLoaderSChan, internalRALsv1Chan, internalCacheSChan, + internalRALsv1Chan, internalCacheSChan, internalEEsChan, internalERsChan chan birpc.ClientConnector, shdChan *utils.SyncedChan) { if !cfg.DispatcherSCfg().Enabled { @@ -174,8 +171,6 @@ func startRPC(server *cores.Server, internalRaterChan, internalSuplSChan <- splS case analyzerS := <-internalAnalyzerSChan: internalAnalyzerSChan <- analyzerS - case loaderS := <-internalLoaderSChan: - internalLoaderSChan <- loaderS case ralS := <-internalRALsv1Chan: internalRALsv1Chan <- ralS case chS := <-internalCacheSChan: // added in order to start the RPC before precaching is done @@ -300,32 +295,6 @@ func singnalHandler(shdWg *sync.WaitGroup, shdChan *utils.SyncedChan) { } } -func runPreload(loader *services.LoaderService, internalLoaderSChan chan birpc.ClientConnector, - shdChan *utils.SyncedChan) { - if !cfg.LoaderCfg().Enabled() { - utils.Logger.Err(fmt.Sprintf("<%s> not enabled but required by preload mechanism", utils.LoaderS)) - shdChan.CloseOnce() - return - } - - ldrs := <-internalLoaderSChan - internalLoaderSChan <- ldrs - - var reply string - for _, loaderID := range strings.Split(*preload, utils.FieldsSep) { - if err := loader.GetLoaderS().V1Load(context.TODO(), - &loaders.ArgsProcessFolder{ - ForceLock: true, // force lock will unlock the file in case is locked and return error - LoaderID: loaderID, - StopOnError: true, - }, &reply); err != nil { - utils.Logger.Err(fmt.Sprintf("<%s> preload failed on loadID <%s> , err: <%s>", utils.LoaderS, loaderID, err.Error())) - shdChan.CloseOnce() - return - } - } -} - func main() { cgrEngineFlags.Parse(os.Args[1:]) vers, err := utils.GetCGRVersion() @@ -443,7 +412,6 @@ func main() { internalResponderChan := make(chan birpc.ClientConnector, 1) internalAPIerSv1Chan := make(chan birpc.ClientConnector, 1) internalAPIerSv2Chan := make(chan birpc.ClientConnector, 1) - internalLoaderSChan := make(chan birpc.ClientConnector, 1) internalEEsChan := make(chan birpc.ClientConnector, 1) internalERsChan := make(chan birpc.ClientConnector, 1) @@ -457,7 +425,6 @@ func main() { utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCDRs): internalCDRServerChan, utils.ConcatenatedKey(utils.MetaInternal, utils.MetaChargers): internalChargerSChan, utils.ConcatenatedKey(utils.MetaInternal, utils.MetaGuardian): internalGuardianSChan, - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaLoaders): internalLoaderSChan, utils.ConcatenatedKey(utils.MetaInternal, utils.MetaResources): internalResourceSChan, utils.ConcatenatedKey(utils.MetaInternal, utils.MetaIPs): internalIPsChan, utils.ConcatenatedKey(utils.MetaInternal, utils.MetaResponder): internalResponderChan, @@ -498,7 +465,6 @@ func main() { utils.GlobalVarS: new(sync.WaitGroup), utils.HTTPAgent: new(sync.WaitGroup), utils.KamailioAgent: new(sync.WaitGroup), - utils.LoaderS: new(sync.WaitGroup), utils.RadiusAgent: new(sync.WaitGroup), utils.RALService: new(sync.WaitGroup), utils.ResourceS: new(sync.WaitGroup), @@ -617,9 +583,6 @@ func main() { smg := services.NewSessionService(cfg, dmService, server, internalSessionSChan, shdChan, connManager, anz, srvDep) - ldrs := services.NewLoaderService(cfg, dmService, filterSChan, server, - internalLoaderSChan, connManager, anz, srvDep) - srvManager.AddServices(gvService, attrS, chrS, tS, stS, trS, rnS, reS, ips, routeS, schS, rals, apiSv1, apiSv2, cdrS, smg, coreS, services.NewDNSAgent(cfg, filterSChan, shdChan, connManager, caps, srvDep), @@ -630,7 +593,7 @@ func main() { services.NewDiameterAgent(cfg, filterSChan, shdChan, connManager, caps, srvDep), // partial reload services.NewHTTPAgent(cfg, filterSChan, server, connManager, srvDep), // no reload services.NewPrometheusAgent(cfg, connManager, server, srvDep), - ldrs, anz, dspS, dspH, dmService, storDBService, + anz, dspS, dspH, dmService, storDBService, services.NewEventExporterService(cfg, filterSChan, connManager, server, internalEEsChan, anz, srvDep), services.NewEventReaderService(cfg, dmService, filterSChan, @@ -659,7 +622,6 @@ func main() { engine.IntRPC.AddInternalRPCClient(utils.CDRsV2, internalCDRServerChan) engine.IntRPC.AddInternalRPCClient(utils.ChargerSv1, internalChargerSChan) engine.IntRPC.AddInternalRPCClient(utils.GuardianSv1, internalGuardianSChan) - engine.IntRPC.AddInternalRPCClient(utils.LoaderSv1, internalLoaderSChan) engine.IntRPC.AddInternalRPCClient(utils.ResourceSv1, internalResourceSChan) engine.IntRPC.AddInternalRPCClient(utils.IPsV1, internalIPsChan) engine.IntRPC.AddInternalRPCClient(utils.Responder, internalResponderChan) @@ -683,16 +645,12 @@ func main() { log.Fatal(err) } - if *preload != utils.EmptyString { - runPreload(ldrs, internalLoaderSChan, shdChan) - } - // Serve rpc connections go startRPC(server, internalResponderChan, internalCDRServerChan, internalResourceSChan, internalIPsChan, internalStatSChan, internalAttributeSChan, internalChargerSChan, internalThresholdSChan, internalTrendSChan, internalRouteSChan, internalSessionSChan, internalAnalyzerSChan, - internalDispatcherSChan, internalLoaderSChan, internalRALsChan, + internalDispatcherSChan, internalRALsChan, internalCacheSChan, internalEEsChan, internalERsChan, shdChan) if *memProfDir != utils.EmptyString { diff --git a/cmd/cgr-engine/cgr-engine_flags_test.go b/cmd/cgr-engine/cgr-engine_flags_test.go index c4388ea45..bffa9ba89 100644 --- a/cmd/cgr-engine/cgr-engine_flags_test.go +++ b/cmd/cgr-engine/cgr-engine_flags_test.go @@ -133,13 +133,6 @@ func TestCgrEngineFlags(t *testing.T) { defaultVal: -1, want: 7, }, - { - name: "preload", - flags: []string{"-preload", "TestPreloadID"}, - flagVar: preload, - defaultVal: "", - want: "TestPreloadID", - }, { name: "setVersions", flags: []string{"-set_versions"}, diff --git a/config/config.go b/config/config.go index f5e650734..bc06a4e90 100644 --- a/config/config.go +++ b/config/config.go @@ -46,7 +46,6 @@ var ( dfltKamConnConfig *KamConnCfg // Default Kamailio Connection configuration dfltRemoteHost *RemoteHost dfltAstConnCfg *AsteriskConnCfg - dfltLoaderConfig *LoaderSCfg ) func newDbDefaults() dbDefaults { @@ -171,7 +170,6 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { cfg.migratorCgrCfg.OutDataDBOpts = &DataDBOpts{} cfg.migratorCgrCfg.OutStorDBOpts = &StorDBOpts{} cfg.mailerCfg = new(MailerCfg) - cfg.loaderCfg = make(LoaderSCfgs, 0) cfg.apier = new(ApierCfg) cfg.ersCfg = new(ERsCfg) cfg.eesCfg = &EEsCfg{ @@ -230,7 +228,6 @@ func newCGRConfig(config []byte) (cfg *CGRConfig, err error) { dfltFsConnConfig = cfg.fsAgentCfg.EventSocketConns[0] // We leave it crashing here on purpose if no Connection defaults defined dfltKamConnConfig = cfg.kamAgentCfg.EvapiConns[0] dfltAstConnCfg = cfg.asteriskAgentCfg.AsteriskConns[0] - dfltLoaderConfig = cfg.loaderCfg[0].Clone() dfltRemoteHost = new(RemoteHost) *dfltRemoteHost = *cfg.rpcConns[utils.MetaLocalHost].Conns[0] err = cfg.checkConfigSanity() @@ -288,7 +285,6 @@ type CGRConfig struct { dfltEvRdr *EventReaderCfg // default event reader dfltEvExp *EventExporterCfg // default event exporter - loaderCfg LoaderSCfgs // LoaderS configs httpAgentCfg HTTPAgentCfgs // HttpAgent configs rldChans map[string]chan struct{} // index here the channels used for reloads @@ -374,7 +370,7 @@ func (cfg *CGRConfig) loadFromJSONCfg(jsnCfg *CgrJsonCfg) (err error) { cfg.loadAsteriskAgentCfg, cfg.loadDiameterAgentCfg, cfg.loadRadiusAgentCfg, cfg.loadDNSAgentCfg, cfg.loadHTTPAgentCfg, cfg.loadPrometheusAgentCfg, cfg.loadAttributeSCfg, cfg.loadChargerSCfg, cfg.loadResourceSCfg, cfg.loadStatSCfg, cfg.loadTrendSCfg, - cfg.loadRankingSCfg, cfg.loadThresholdSCfg, cfg.loadRouteSCfg, cfg.loadLoaderSCfg, + cfg.loadRankingSCfg, cfg.loadThresholdSCfg, cfg.loadRouteSCfg, cfg.loadMailerCfg, cfg.loadSureTaxCfg, cfg.loadDispatcherSCfg, cfg.loadLoaderCgrCfg, cfg.loadMigratorCgrCfg, cfg.loadTLSCgrCfg, cfg.loadAnalyzerCgrCfg, cfg.loadApierCfg, cfg.loadErsCfg, cfg.loadEesCfg, @@ -690,23 +686,6 @@ func (cfg *CGRConfig) loadRouteSCfg(jsnCfg *CgrJsonCfg) (err error) { return cfg.routeSCfg.loadFromJSONCfg(jsnRouteSCfg) } -// loadLoaderSCfg loads the LoaderS section of the configuration -func (cfg *CGRConfig) loadLoaderSCfg(jsnCfg *CgrJsonCfg) (err error) { - var jsnLoaderCfg []*LoaderJsonCfg - if jsnLoaderCfg, err = jsnCfg.LoaderJsonCfg(); err != nil { - return - } - // cfg.loaderCfg = make(LoaderSCfgs, len(jsnLoaderCfg)) - for _, profile := range jsnLoaderCfg { - loadSCfgp := NewDfltLoaderSCfg() - if err = loadSCfgp.loadFromJSONCfg(profile, cfg.templates, cfg.generalCfg.RSRSep); err != nil { - return - } - cfg.loaderCfg = append(cfg.loaderCfg, loadSCfgp) // use append so the loaderS profile to be loaded from multiple files - } - return -} - // loadMailerCfg loads the Mailer section of the configuration func (cfg *CGRConfig) loadMailerCfg(jsnCfg *CgrJsonCfg) (err error) { var jsnMailerCfg *MailerJsonCfg @@ -1019,13 +998,6 @@ func (cfg *CGRConfig) CacheCfg() *CacheCfg { return cfg.cacheCfg } -// LoaderCfg returns the Loader Service -func (cfg *CGRConfig) LoaderCfg() LoaderSCfgs { - cfg.lks[LoaderJson].Lock() - defer cfg.lks[LoaderJson].Unlock() - return cfg.loaderCfg -} - // LoaderCgrCfg returns the config for cgr-loader func (cfg *CGRConfig) LoaderCgrCfg() *LoaderCgrCfg { cfg.lks[CgrLoaderCfgJson].Lock() @@ -1311,7 +1283,6 @@ func (cfg *CGRConfig) getLoadFunctions() map[string]func(*CgrJsonCfg) error { RANKINGS_JSON: cfg.loadRankingSCfg, THRESHOLDS_JSON: cfg.loadThresholdSCfg, RouteSJson: cfg.loadRouteSCfg, - LoaderJson: cfg.loadLoaderSCfg, MAILER_JSN: cfg.loadMailerCfg, SURETAX_JSON: cfg.loadSureTaxCfg, CgrLoaderCfgJson: cfg.loadLoaderCgrCfg, @@ -1487,7 +1458,7 @@ func (cfg *CGRConfig) reloadSections(sections ...string) { subsystemsThatNeedDataDB := utils.NewStringSet([]string{DATADB_JSN, SCHEDULER_JSN, RALS_JSN, CDRS_JSN, SessionSJson, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON, - RouteSJson, LoaderJson, DispatcherSJson, ApierS, IPsJSON, + RouteSJson, DispatcherSJson, ApierS, IPsJSON, }) subsystemsThatNeedStorDB := utils.NewStringSet([]string{STORDB_JSN, RALS_JSN, CDRS_JSN, ApierS}) needsDataDB := false @@ -1572,8 +1543,6 @@ func (cfg *CGRConfig) reloadSections(sections ...string) { cfg.rldChans[RouteSJson] <- struct{}{} case JanusAgentJson: cfg.rldChans[JanusAgentJson] <- struct{}{} - case LoaderJson: - cfg.rldChans[LoaderJson] <- struct{}{} case DispatcherSJson: cfg.rldChans[DispatcherSJson] <- struct{}{} case AnalyzerCfgJson: @@ -1595,7 +1564,6 @@ func (cfg *CGRConfig) reloadSections(sections ...string) { // AsMapInterface returns the config as a map[string]any func (cfg *CGRConfig) AsMapInterface(separator string) (mp map[string]any) { return map[string]any{ - LoaderJson: cfg.loaderCfg.AsMapInterface(separator), HttpAgentJson: cfg.httpAgentCfg.AsMapInterface(separator), RPCConnsJsonName: cfg.rpcConns.AsMapInterface(), GENERAL_JSN: cfg.generalCfg.AsMapInterface(), @@ -1778,8 +1746,6 @@ func (cfg *CGRConfig) V1GetConfig(ctx *context.Context, args *SectionWithAPIOpts mp = cfg.DispatcherSCfg().AsMapInterface() case RegistrarCJson: mp = cfg.RegistrarCCfg().AsMapInterface() - case LoaderJson: - mp = cfg.LoaderCfg().AsMapInterface(cfg.GeneralCfg().RSRSep) case CgrLoaderCfgJson: mp = cfg.LoaderCgrCfg().AsMapInterface() case CgrMigratorCfgJson: @@ -1954,8 +1920,6 @@ func (cfg *CGRConfig) V1GetConfigAsJSON(ctx *context.Context, args *SectionWithA mp = cfg.DispatcherSCfg().AsMapInterface() case RegistrarCJson: mp = cfg.RegistrarCCfg().AsMapInterface() - case LoaderJson: - mp = cfg.LoaderCfg().AsMapInterface(cfg.GeneralCfg().RSRSep) case CgrLoaderCfgJson: mp = cfg.LoaderCgrCfg().AsMapInterface() case CgrMigratorCfgJson: @@ -2042,7 +2006,6 @@ func (cfg *CGRConfig) Clone() (cln *CGRConfig) { dfltEvRdr: cfg.dfltEvRdr.Clone(), dfltEvExp: cfg.dfltEvExp.Clone(), - loaderCfg: cfg.loaderCfg.Clone(), httpAgentCfg: cfg.httpAgentCfg.Clone(), rpcConns: cfg.rpcConns.Clone(), templates: cfg.templates.Clone(), diff --git a/config/config_defaults.go b/config/config_defaults.go index 8b0e301bf..6503723eb 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -1003,196 +1003,6 @@ const CGRATES_CFG_JSON = ` }, }, - -"loaders": [ // LoaderS config - { - "id": "*default", // identifier of the Loader - "enabled": false, // starts as service: . - "tenant": "", // tenant used in filterS.Pass - "dry_run": false, // do not send the CDRs to CDRS, just parse them - "run_delay": "0", // sleep interval in seconds between consecutive runs, -1 to use automation via inotify or 0 to disable running all together - "lockfile_path": ".cgr.lck", // Filename containing concurrency lock in case of delayed processing - "caches_conns": ["*internal"], - "field_separator": ",", // separator used in case of csv files - "tp_in_dir": "/var/spool/cgrates/loader/in", // absolute path towards the directory where the TPs are stored - "tp_out_dir": "/var/spool/cgrates/loader/out", // absolute path towards the directory where processed TPs will be moved - "data":[ // data profiles to load - { - "type": "*attributes", // data source type - "file_name": "Attributes.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - { - "type": "*filters", // data source type - "file_name": "Filters.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.2"}, - {"tag": "Element", "path": "Element", "type": "*variable", "value": "~*req.3"}, - {"tag": "Values", "path": "Values", "type": "*variable", "value": "~*req.4"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.5"}, - ], - }, - { - "type": "*resources", // data source type - "file_name": "Resources.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "TTL", "path": "UsageTTL", "type": "*variable", "value": "~*req.4"}, - {"tag": "Limit", "path": "Limit", "type": "*variable", "value": "~*req.5"}, - {"tag": "AllocationMessage", "path": "AllocationMessage", "type": "*variable", "value": "~*req.6"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.7"}, - {"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.8"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.9"}, - {"tag": "ThresholdIDs", "path": "ThresholdIDs", "type": "*variable", "value": "~*req.10"}, - ], - }, - { - "type": "*ips", // data source type - "file_name": "IPs.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "TTL", "path": "TTL", "type": "*variable", "value": "~*req.4"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.5"}, - {"tag": "AddressPool", "path": "AddressPool", "type": "*variable", "value": "~*req.6"}, - {"tag": "Allocation", "path": "Allocation", "type": "*variable", "value": "~*req.7"}, - {"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.8"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.9"} - ] - }, - { - "type": "*stats", // data source type - "file_name": "Stats.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "QueueLength", "path": "QueueLength", "type": "*variable", "value": "~*req.4"}, - {"tag": "TTL", "path": "TTL", "type": "*variable", "value": "~*req.5"}, - {"tag": "MinItems", "path": "MinItems", "type": "*variable", "value": "~*req.6"}, - {"tag": "MetricIDs", "path": "MetricIDs", "type": "*variable", "value": "~*req.7"}, - {"tag": "MetricFilterIDs", "path": "MetricFilterIDs", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Stored", "path": "Stored", "type": "*variable", "value": "~*req.10"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.11"}, - {"tag": "ThresholdIDs", "path": "ThresholdIDs", "type": "*variable", "value": "~*req.12"}, - ], - }, - { - "type": "*thresholds", // data source type - "file_name": "Thresholds.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "MaxHits", "path": "MaxHits", "type": "*variable", "value": "~*req.4"}, - {"tag": "MinHits", "path": "MinHits", "type": "*variable", "value": "~*req.5"}, - {"tag": "MinSleep", "path": "MinSleep", "type": "*variable", "value": "~*req.6"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.7"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.8"}, - {"tag": "ActionIDs", "path": "ActionIDs", "type": "*variable", "value": "~*req.9"}, - {"tag": "Async", "path": "Async", "type": "*variable", "value": "~*req.10"}, - ], - }, - { - "type": "*routes", // data source type - "file_name": "Routes.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "Sorting", "path": "Sorting", "type": "*variable", "value": "~*req.4"}, - {"tag": "SortingParameters", "path": "SortingParameters", "type": "*variable", "value": "~*req.5"}, - {"tag": "RouteID", "path": "RouteID", "type": "*variable", "value": "~*req.6"}, - {"tag": "RouteFilterIDs", "path": "RouteFilterIDs", "type": "*variable", "value": "~*req.7"}, - {"tag": "RouteAccountIDs", "path": "RouteAccountIDs", "type": "*variable", "value": "~*req.8"}, - {"tag": "RouteRatingPlanIDs", "path": "RouteRatingPlanIDs", "type": "*variable", "value": "~*req.9"}, - {"tag": "RouteResourceIDs", "path": "RouteResourceIDs", "type": "*variable", "value": "~*req.10"}, - {"tag": "RouteStatIDs", "path": "RouteStatIDs", "type": "*variable", "value": "~*req.11"}, - {"tag": "RouteWeight", "path": "RouteWeight", "type": "*variable", "value": "~*req.12"}, - {"tag": "RouteBlocker", "path": "RouteBlocker", "type": "*variable", "value": "~*req.13"}, - {"tag": "RouteParameters", "path": "RouteParameters", "type": "*variable", "value": "~*req.14"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.15"}, - ], - }, - { - "type": "*chargers", // data source type - "file_name": "Chargers.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.2"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.3"}, - {"tag": "RunID", "path": "RunID", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeIDs", "path": "AttributeIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.6"}, - ], - }, - { - "type": "*dispatchers", // data source type - "file_name": "DispatcherProfiles.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "Strategy", "path": "Strategy", "type": "*variable", "value": "~*req.5"}, - {"tag": "StrategyParameters", "path": "StrategyParameters", "type": "*variable", "value": "~*req.6"}, - {"tag": "ConnID", "path": "ConnID", "type": "*variable", "value": "~*req.7"}, - {"tag": "ConnFilterIDs", "path": "ConnFilterIDs", "type": "*variable", "value": "~*req.8"}, - {"tag": "ConnWeight", "path": "ConnWeight", "type": "*variable", "value": "~*req.9"}, - {"tag": "ConnBlocker", "path": "ConnBlocker", "type": "*variable", "value": "~*req.10"}, - {"tag": "ConnParameters", "path": "ConnParameters", "type": "*variable", "value": "~*req.11"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.12"}, - ], - }, - { - "type": "*dispatcher_hosts", // data source type - "file_name": "DispatcherHosts.csv", // file name in the tp_in_dir - "fields": [ - {"tag": "Tenant", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Address", "path": "Address", "type": "*variable", "value": "~*req.2"}, - {"tag": "Transport", "path": "Transport", "type": "*variable", "value": "~*req.3"}, - {"tag": "ConnectAttempts", "path": "ConnectAttempts", "type": "*variable", "value":"~*req.4"}, - {"tag": "Reconnects", "path": "Reconnects", "type": "*variable", "value":"~*req.5"}, - {"tag": "MaxReconnectInterval", "path": "MaxReconnectInterval", "type": "*variable", "value":"~*req.6"}, - {"tag": "ConnectTimeout", "path": "ConnectTimeout", "type": "*variable", "value":"~*req.7"}, - {"tag": "ReplyTimeout", "path": "ReplyTimeout", "type": "*variable", "value":"~*req.8"}, - {"tag": "TLS", "path": "TLS", "type": "*variable", "value": "~*req.9"}, - {"tag": "ClientKey", "path": "ClientKey", "type": "*variable", "value":"~*req.10"}, - {"tag": "ClientCertificate", "path": "ClientCertificate", "type": "*variable", "value":"~*req.11"}, - {"tag": "CaCertificate", "path": "CaCertificate", "type": "*variable", "value":"~*req.12"}, - ], - }, - ], - }, -], - - "mailer": { "server": "localhost", // the server to use when sending emails out "auth_user": "cgrates", // authenticate to email server using this user diff --git a/config/config_json.go b/config/config_json.go index 078361b5c..fe52cdbe3 100644 --- a/config/config_json.go +++ b/config/config_json.go @@ -48,7 +48,6 @@ const ( TRENDS_JSON = "trends" RANKINGS_JSON = "rankings" RouteSJson = "routes" - LoaderJson = "loaders" MAILER_JSN = "mailer" SURETAX_JSON = "suretax" DispatcherSJson = "dispatchers" @@ -77,7 +76,7 @@ var ( sortedCfgSections = []string{GENERAL_JSN, RPCConnsJsonName, DATADB_JSN, STORDB_JSN, LISTEN_JSN, TlsCfgJson, HTTP_JSN, SCHEDULER_JSN, CACHE_JSN, FilterSjsn, RALS_JSN, CDRS_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN, DA_JSN, RA_JSN, HttpAgentJson, DNSAgentJson, PrometheusAgentJSON, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, TRENDS_JSON, RANKINGS_JSON, - THRESHOLDS_JSON, RouteSJson, LoaderJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, JanusAgentJson, + THRESHOLDS_JSON, RouteSJson, MAILER_JSN, SURETAX_JSON, CgrLoaderCfgJson, CgrMigratorCfgJson, DispatcherSJson, JanusAgentJson, AnalyzerCfgJson, ApierS, EEsJson, SIPAgentJson, RegistrarCJson, TemplatesJson, ConfigSJson, APIBanCfgJson, SentryPeerCfgJson, CoreSCfgJson, IPsJSON} ) @@ -433,18 +432,6 @@ func (jsnCfg CgrJsonCfg) RouteSJsonCfg() (*RouteSJsonCfg, error) { return cfg, nil } -func (jsnCfg CgrJsonCfg) LoaderJsonCfg() ([]*LoaderJsonCfg, error) { - rawCfg, hasKey := jsnCfg[LoaderJson] - if !hasKey { - return nil, nil - } - cfg := make([]*LoaderJsonCfg, 0) - if err := json.Unmarshal(*rawCfg, &cfg); err != nil { - return nil, err - } - return cfg, nil -} - func (jsnCfg CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) { rawCfg, hasKey := jsnCfg[MAILER_JSN] if !hasKey { diff --git a/config/config_json_test.go b/config/config_json_test.go index b357078e5..704efb729 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -1379,569 +1379,6 @@ func TestDfRouteSJsonCfg(t *testing.T) { } } -func TestDfLoaderJsonCfg(t *testing.T) { - eCfg := []*LoaderJsonCfg{ - { - ID: utils.StringPointer(utils.MetaDefault), - Enabled: utils.BoolPointer(false), - Tenant: utils.StringPointer(""), - Dry_run: utils.BoolPointer(false), - Run_delay: utils.StringPointer("0"), - Lockfile_path: utils.StringPointer(".cgr.lck"), - Caches_conns: &[]string{utils.MetaInternal}, - Field_separator: utils.StringPointer(","), - Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in"), - Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out"), - Data: &[]*LoaderJsonDataType{ - { - Type: utils.StringPointer(utils.MetaAttributes), - File_name: utils.StringPointer(utils.AttributesCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer("TenantID"), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("ProfileID"), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Contexts"), - Path: utils.StringPointer(utils.Contexts), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer(utils.FilterIDs), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("AttributeFilterIDs"), - Path: utils.StringPointer("AttributeFilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("Path"), - Path: utils.StringPointer(utils.Path), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("Type"), - Path: utils.StringPointer("Type"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("Value"), - Path: utils.StringPointer("Value"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("Blocker"), - Path: utils.StringPointer("Blocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer(utils.Weight), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaFilters), - File_name: utils.StringPointer(utils.FiltersCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Type"), - Path: utils.StringPointer("Type"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("Element"), - Path: utils.StringPointer("Element"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("Values"), - Path: utils.StringPointer("Values"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaResources), - File_name: utils.StringPointer(utils.ResourcesCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("TTL"), - Path: utils.StringPointer("UsageTTL"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("Limit"), - Path: utils.StringPointer("Limit"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("AllocationMessage"), - Path: utils.StringPointer("AllocationMessage"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("Blocker"), - Path: utils.StringPointer("Blocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("Stored"), - Path: utils.StringPointer("Stored"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("ThresholdIDs"), - Path: utils.StringPointer("ThresholdIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaIPs), - File_name: utils.StringPointer(utils.IPsCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("TTL"), - Path: utils.StringPointer("TTL"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("Type"), - Path: utils.StringPointer("Type"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer(utils.AddressPool), - Path: utils.StringPointer(utils.AddressPool), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer(utils.Allocation), - Path: utils.StringPointer(utils.Allocation), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("Stored"), - Path: utils.StringPointer("Stored"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaStats), - File_name: utils.StringPointer(utils.StatsCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("QueueLength"), - Path: utils.StringPointer("QueueLength"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("TTL"), - Path: utils.StringPointer("TTL"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("MinItems"), - Path: utils.StringPointer("MinItems"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("MetricIDs"), - Path: utils.StringPointer("MetricIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("MetricFilterIDs"), - Path: utils.StringPointer("MetricFilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("Blocker"), - Path: utils.StringPointer("Blocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("Stored"), - Path: utils.StringPointer("Stored"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.11")}, - - {Tag: utils.StringPointer("ThresholdIDs"), - Path: utils.StringPointer("ThresholdIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.12")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaThresholds), - File_name: utils.StringPointer(utils.ThresholdsCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("MaxHits"), - Path: utils.StringPointer("MaxHits"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("MinHits"), - Path: utils.StringPointer("MinHits"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("MinSleep"), - Path: utils.StringPointer("MinSleep"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("Blocker"), - Path: utils.StringPointer("Blocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("ActionIDs"), - Path: utils.StringPointer("ActionIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("Async"), - Path: utils.StringPointer("Async"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaRoutes), - File_name: utils.StringPointer(utils.RoutesCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("Sorting"), - Path: utils.StringPointer("Sorting"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("SortingParameters"), - Path: utils.StringPointer("SortingParameters"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("RouteID"), - Path: utils.StringPointer("RouteID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("RouteFilterIDs"), - Path: utils.StringPointer("RouteFilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("RouteAccountIDs"), - Path: utils.StringPointer("RouteAccountIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("RouteRatingPlanIDs"), - Path: utils.StringPointer("RouteRatingPlanIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("RouteResourceIDs"), - Path: utils.StringPointer("RouteResourceIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - {Tag: utils.StringPointer("RouteStatIDs"), - Path: utils.StringPointer("RouteStatIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.11")}, - {Tag: utils.StringPointer("RouteWeight"), - Path: utils.StringPointer("RouteWeight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.12")}, - {Tag: utils.StringPointer("RouteBlocker"), - Path: utils.StringPointer("RouteBlocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.13")}, - {Tag: utils.StringPointer("RouteParameters"), - Path: utils.StringPointer("RouteParameters"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.14")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.15")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaChargers), - File_name: utils.StringPointer(utils.ChargersCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("RunID"), - Path: utils.StringPointer("RunID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("AttributeIDs"), - Path: utils.StringPointer("AttributeIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaDispatchers), - File_name: utils.StringPointer(utils.DispatcherProfilesCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Contexts"), - Path: utils.StringPointer("Contexts"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("FilterIDs"), - Path: utils.StringPointer("FilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("ActivationInterval"), - Path: utils.StringPointer("ActivationInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4")}, - {Tag: utils.StringPointer("Strategy"), - Path: utils.StringPointer("Strategy"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5")}, - {Tag: utils.StringPointer("StrategyParameters"), - Path: utils.StringPointer("StrategyParameters"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6")}, - {Tag: utils.StringPointer("ConnID"), - Path: utils.StringPointer("ConnID"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7")}, - {Tag: utils.StringPointer("ConnFilterIDs"), - Path: utils.StringPointer("ConnFilterIDs"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8")}, - {Tag: utils.StringPointer("ConnWeight"), - Path: utils.StringPointer("ConnWeight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9")}, - {Tag: utils.StringPointer("ConnBlocker"), - Path: utils.StringPointer("ConnBlocker"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10")}, - {Tag: utils.StringPointer("ConnParameters"), - Path: utils.StringPointer("ConnParameters"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.11")}, - {Tag: utils.StringPointer("Weight"), - Path: utils.StringPointer("Weight"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.12")}, - }, - }, - { - Type: utils.StringPointer(utils.MetaDispatcherHosts), - File_name: utils.StringPointer(utils.DispatcherHostsCsv), - Fields: &[]*FcTemplateJsonCfg{ - {Tag: utils.StringPointer(utils.Tenant), - Path: utils.StringPointer(utils.Tenant), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.0"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer(utils.ID), - Path: utils.StringPointer(utils.ID), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.1"), - Mandatory: utils.BoolPointer(true)}, - {Tag: utils.StringPointer("Address"), - Path: utils.StringPointer("Address"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.2")}, - {Tag: utils.StringPointer("Transport"), - Path: utils.StringPointer("Transport"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.3")}, - {Tag: utils.StringPointer("ConnectAttempts"), - Path: utils.StringPointer("ConnectAttempts"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.4"), - }, - {Tag: utils.StringPointer("Reconnects"), - Path: utils.StringPointer("Reconnects"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.5"), - }, - {Tag: utils.StringPointer("MaxReconnectInterval"), - Path: utils.StringPointer("MaxReconnectInterval"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.6"), - }, - {Tag: utils.StringPointer("ConnectTimeout"), - Path: utils.StringPointer("ConnectTimeout"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.7"), - }, - {Tag: utils.StringPointer("ReplyTimeout"), - Path: utils.StringPointer("ReplyTimeout"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.8"), - }, - {Tag: utils.StringPointer("TLS"), - Path: utils.StringPointer("TLS"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.9"), - }, - {Tag: utils.StringPointer("ClientKey"), - Path: utils.StringPointer("ClientKey"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.10"), - }, - {Tag: utils.StringPointer("ClientCertificate"), - Path: utils.StringPointer("ClientCertificate"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.11"), - }, - {Tag: utils.StringPointer("CaCertificate"), - Path: utils.StringPointer("CaCertificate"), - Type: utils.StringPointer(utils.MetaVariable), - Value: utils.StringPointer("~*req.12"), - }, - }, - }, - }, - }, - } - dfCgrJSONCfg, err := NewCgrJsonCfgFromBytes([]byte(CGRATES_CFG_JSON)) - if err != nil { - t.Error(err) - } - if cfg, err := dfCgrJSONCfg.LoaderJsonCfg(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCfg, cfg) { - t.Errorf("Expecting: %s \n but received \n Received: %s ", - utils.ToJSON(eCfg), utils.ToJSON(cfg)) - } -} - func TestDfMailerJsonCfg(t *testing.T) { eCfg := &MailerJsonCfg{ Server: utils.StringPointer("localhost"), diff --git a/config/config_test.go b/config/config_test.go index d7316004c..938aae605 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1435,23 +1435,6 @@ func TestLoadThresholdSCfgError(t *testing.T) { } } -func TestLoadLoaderSCfgError(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "run_delay": 0, - }, - ], -}` - expected := "json: cannot unmarshal number into Go struct field LoaderJsonCfg.Run_delay of type string" - cgrConfig := NewDefaultCGRConfig() - if cgrCfgJSON, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { - t.Error(err) - } else if err := cgrConfig.loadLoaderSCfg(cgrCfgJSON); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - func TestLoadRouteSCfgError(t *testing.T) { cfgJSONStr := `{ "routes": { @@ -2117,29 +2100,6 @@ func TestFilterSConfig(t *testing.T) { } } -func TestLoaderConfig(t *testing.T) { - ten := "" - expected := LoaderSCfgs{ - { - Enabled: false, - ID: utils.MetaDefault, - Tenant: ten, - LockFilePath: ".cgr.lck", - CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - FieldSeparator: ",", - TpInDir: "/var/spool/cgrates/loader/in", - TpOutDir: "/var/spool/cgrates/loader/out", - Data: nil, - }, - } - cgrConfig := NewDefaultCGRConfig() - newConfig := cgrConfig.LoaderCfg() - newConfig[0].Data = nil - if !reflect.DeepEqual(expected, newConfig) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected), utils.ToJSON(newConfig)) - } -} - func TestDispatcherSConfig(t *testing.T) { expected := &DispatcherSCfg{ Enabled: false, @@ -2526,708 +2486,6 @@ func TestRLockAndRUnlock(t *testing.T) { cgrCfg.RUnlocks("attributes", "ees", "general") } -func TestCgrLoaderCfgITDefaults(t *testing.T) { - eCfg := LoaderSCfgs{ - { - ID: utils.MetaDefault, - Enabled: false, - DryRun: false, - RunDelay: 0, - LockFilePath: ".cgr.lck", - CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - FieldSeparator: ",", - TpInDir: "/var/spool/cgrates/loader/in", - TpOutDir: "/var/spool/cgrates/loader/out", - Data: []*LoaderDataType{ - { - Type: utils.MetaAttributes, - Filename: utils.AttributesCsv, - Fields: []*FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339, - }, - { - Tag: "ProfileID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339, - }, - { - Tag: "Contexts", - Path: "Contexts", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339, - }, - { - Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339, - }, - { - Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - { - Tag: "AttributeFilterIDs", - Path: "AttributeFilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - { - Tag: "Path", - Path: "Path", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Type", - Path: "Type", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Value", - Path: "Value", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Blocker", - Path: "Blocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339, - }, - }, - }, - { - Type: utils.MetaFilters, - Filename: utils.FiltersCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "Type", - Path: "Type", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Element", - Path: "Element", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Values", - Path: "Values", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaResources, - Filename: utils.ResourcesCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "TTL", - Path: "UsageTTL", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Limit", - Path: "Limit", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "AllocationMessage", - Path: "AllocationMessage", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Blocker", - Path: "Blocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Stored", - Path: "Stored", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ThresholdIDs", - Path: "ThresholdIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaIPs, - Filename: utils.IPsCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "TTL", - Path: "TTL", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Type", - Path: "Type", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: utils.AddressPool, - Path: utils.AddressPool, - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: utils.Allocation, - Path: utils.Allocation, - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Stored", - Path: "Stored", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaStats, - Filename: utils.StatsCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "QueueLength", - Path: "QueueLength", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "TTL", - Path: "TTL", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MinItems", - Path: "MinItems", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MetricIDs", - Path: "MetricIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MetricFilterIDs", - Path: "MetricFilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Blocker", - Path: "Blocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Stored", - Path: "Stored", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - Layout: time.RFC3339}, - - {Tag: "ThresholdIDs", - Path: "ThresholdIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaThresholds, - Filename: utils.ThresholdsCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MaxHits", - Path: "MaxHits", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MinHits", - Path: "MinHits", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "MinSleep", - Path: "MinSleep", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Blocker", - Path: "Blocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActionIDs", - Path: "ActionIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Async", - Path: "Async", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaRoutes, - Filename: utils.RoutesCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Sorting", - Path: "Sorting", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "SortingParameters", - Path: "SortingParameters", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteID", - Path: "RouteID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteFilterIDs", - Path: "RouteFilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteAccountIDs", - Path: "RouteAccountIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteRatingPlanIDs", - Path: "RouteRatingPlanIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteResourceIDs", - Path: "RouteResourceIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteStatIDs", - Path: "RouteStatIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteWeight", - Path: "RouteWeight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteBlocker", - Path: "RouteBlocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.13", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RouteParameters", - Path: "RouteParameters", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.14", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.15", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaChargers, - Filename: utils.ChargersCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "RunID", - Path: "RunID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "AttributeIDs", - Path: "AttributeIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339}, - }, - }, - { - Type: utils.MetaDispatchers, - Filename: utils.DispatcherProfilesCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "Contexts", - Path: "Contexts", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Strategy", - Path: "Strategy", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "StrategyParameters", - Path: "StrategyParameters", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnID", - Path: "ConnID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnFilterIDs", - Path: "ConnFilterIDs", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnWeight", - Path: "ConnWeight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnBlocker", - Path: "ConnBlocker", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnParameters", - Path: "ConnParameters", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - Layout: time.RFC3339, - }, - }, - }, - { - Type: utils.MetaDispatcherHosts, - Filename: utils.DispatcherHostsCsv, - Fields: []*FCTemplate{ - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339}, - {Tag: "Address", - Path: "Address", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Transport", - Path: "Transport", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnectAttempts", - Path: "ConnectAttempts", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "Reconnects", - Path: "Reconnects", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "MaxReconnectInterval", - Path: "MaxReconnectInterval", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ConnectTimeout", - Path: "ConnectTimeout", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ReplyTimeout", - Path: "ReplyTimeout", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "TLS", - Path: "TLS", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ClientKey", - Path: "ClientKey", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "ClientCertificate", - Path: "ClientCertificate", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - Layout: time.RFC3339, - }, - {Tag: "CaCertificate", - Path: "CaCertificate", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - Layout: time.RFC3339, - }, - }, - }, - }, - }, - } - for _, profile := range eCfg { - for _, fields := range profile.Data { - for _, v := range fields.Fields { - v.ComputePath() - } - } - } - if !reflect.DeepEqual(eCfg, cgrCfg.loaderCfg) { - t.Errorf("received: %+v, \n expecting: %+v", - utils.ToJSON(eCfg), utils.ToJSON(cgrCfg.loaderCfg)) - } -} - func TestCgrCfgJSONDefaultDispatcherSCfg(t *testing.T) { eDspSCfg := &DispatcherSCfg{ Enabled: false, @@ -3421,38 +2679,6 @@ func TestCgrCfgV1GetConfigAllConfig(t *testing.T) { } } -func TestCgrCfgV1GetConfigSectionLoader(t *testing.T) { - var reply map[string]any - expected := map[string]any{ - LoaderJson: []map[string]any{ - { - utils.IDCfg: "*default", - utils.EnabledCfg: false, - utils.TenantCfg: utils.EmptyString, - utils.DryRunCfg: false, - utils.RunDelayCfg: "0", - utils.LockFilePathCfg: ".cgr.lck", - utils.CachesConnsCfg: []string{utils.MetaInternal}, - utils.FieldSepCfg: ",", - utils.TpInDirCfg: "/var/spool/cgrates/loader/in", - utils.TpOutDirCfg: "/var/spool/cgrates/loader/out", - utils.DataCfg: []map[string]any{}, - }, - }, - } - cgrCfg := NewDefaultCGRConfig() - if err := cgrCfg.V1GetConfig(context.Background(), &SectionWithAPIOpts{Section: LoaderJson}, &reply); err != nil { - t.Error(err) - } else if mp, can := reply[LoaderJson].([]map[string]any); !can { - t.Errorf("Unexpected type: %t", reply[LoaderJson]) - } else { - mp[0][utils.DataCfg] = []map[string]any{} - if !reflect.DeepEqual(expected[LoaderJson], mp) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected[LoaderJson]), utils.ToJSON(mp)) - } - } -} - func TestCgrCfgV1GetConfigSectionHTTPAgent(t *testing.T) { var reply map[string]any expected := map[string]any{ @@ -5187,17 +4413,6 @@ func TestV1GetConfigAsJSONDispatcherH(t *testing.T) { } } -func TestV1GetConfigAsJSONLoaders(t *testing.T) { - var reply string - expected := `{"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.5"},{"path":"AddressPool","tag":"AddressPool","type":"*variable","value":"~*req.6"},{"path":"Allocation","tag":"Allocation","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"}],"file_name":"IPs.csv","flags":null,"type":"*ips"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"ConnectAttempts","tag":"ConnectAttempts","type":"*variable","value":"~*req.4"},{"path":"Reconnects","tag":"Reconnects","type":"*variable","value":"~*req.5"},{"path":"MaxReconnectInterval","tag":"MaxReconnectInterval","type":"*variable","value":"~*req.6"},{"path":"ConnectTimeout","tag":"ConnectTimeout","type":"*variable","value":"~*req.7"},{"path":"ReplyTimeout","tag":"ReplyTimeout","type":"*variable","value":"~*req.8"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.9"},{"path":"ClientKey","tag":"ClientKey","type":"*variable","value":"~*req.10"},{"path":"ClientCertificate","tag":"ClientCertificate","type":"*variable","value":"~*req.11"},{"path":"CaCertificate","tag":"CaCertificate","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lockfile_path":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}]}` - cgrCfg := NewDefaultCGRConfig() - if err := cgrCfg.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Section: LoaderJson}, &reply); err != nil { - t.Error(err) - } else if expected != reply { - t.Errorf("Expected %+v \n, received %+v", expected, reply) - } -} - func TestV1GetConfigAsJSONCgrLoader(t *testing.T) { var reply string expected := `{"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""}}` @@ -5424,7 +4639,7 @@ func TestV1GetConfigAsJSONAllConfig(t *testing.T) { }` var reply string cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSON) - expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","ari_websocket":false,"connect_attempts":3,"max_reconnect_interval":"0s","password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"route_profile":false,"sessions_conns":["*birpc_internal"]},"attributes":{"any_context":true,"apiers_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*processRuns":1,"*profileIDs":[],"*profileIgnoreFilters":false,"*profileRuns":0},"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*action_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*action_triggers":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*apiban":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*caps_events":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*cdr_ids":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*charger_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*closed_sessions":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*diameter_messages":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_loads":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_routes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatchers":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*event_charges":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_ips":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*event_resources":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_allocations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*radius_packets":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*ranking_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rankings":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rating_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rating_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*replication_hosts":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*reverse_destinations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rpc_connections":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rpc_responses":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*sentrypeer":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":true,"ttl":"24h0m0s"},"*shared_groups":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*stir":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*timings":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*trend_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*trends":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*uch":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"}},"remote_conns":[],"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"compress_stored_cost":false,"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_allocations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ranking_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rankings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*sessions_backup":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*trend_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*trends":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"internalDBBackupPath":"/var/lib/cgrates/internal_db/backup/datadb","internalDBDumpInterval":"0s","internalDBDumpPath":"/var/lib/cgrates/internal_db/datadb","internalDBFileSizeLimit":1073741824,"internalDBRewriteInterval":"0s","internalDBStartTimeout":"5m0s","mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectAttempts":20,"redisConnectTimeout":"0s","redisMaxConns":10,"redisPoolPipelineLimit":0,"redisPoolPipelineWindow":"150µs","redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_failed_dir":"","replication_filtered":false,"replication_interval":"0s"},"diameter_agent":{"asr_template":"","conn_health_check_interval":"0s","conn_status_stat_queue_ids":[],"conn_status_threshold_ids":[],"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listeners":[{"address":"127.0.0.1:3868","network":"tcp"}],"origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"stats_conns":[],"synced_conn_requests":false,"thresholds_conns":[],"vendor_id":0},"dispatchers":{"any_subsystem":true,"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"prevent_loop":false,"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listeners":[{"address":"127.0.0.1:53","network":"udp"}],"request_processors":[],"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*amqp_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*amqpv1_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*els":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*file_csv":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*kafka_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*nats_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*s3_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*sql":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*sqs_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"concurrent_requests":0,"export_path":"/var/spool/cgrates/ees","failed_posts_dir":"/var/spool/cgrates/failed_posts","fields":[],"filters":[],"flags":[],"id":"*default","metrics_reset_schedule":"","opts":{},"synchronous":false,"timezone":"","type":"*none"}],"failed_posts":{"dir":"/var/spool/cgrates/failed_posts","static_ttl":true,"ttl":"5s"}},"ers":{"concurrent_events":1,"ees_conns":[],"enabled":false,"partial_cache_ttl":"1s","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","max_reconnect_interval":"5m0s","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"natsSubject":"cgrates_cdrs","partialCacheAction":"*none","partialOrderField":"~*req.AnswerTime"},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","reconnects":-1,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","start_delay":"0","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[]},"filters":{"apiers_conns":[],"rankings_conns":[],"resources_conns":[],"stats_conns":[],"trends_conns":[]},"freeswitch_agent":{"active_session_delimiter":",","create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","max_reconnect_interval":"0s","password":"ClueCon","reconnects":5,"reply_timeout":"1m0s"}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","route_profile":false,"sched_transfer_extension":"CGRateS","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"caching_delay":"0","connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0s","forceAttemptHttp2":true,"idleConnTimeout":"1m30s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0s","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","pprof_path":"/debug/pprof/","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"ips":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*allocationID":"","*ttl":259200000000000},"prefix_indexed_fields":[],"store_interval":"0s","string_indexed_fields":null,"suffix_indexed_fields":[]},"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","max_reconnect_interval":"0s","reconnects":5}],"route_profile":false,"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"loaders":[{"caches_conns":["*internal"],"data":[{"fields":[{"mandatory":true,"path":"Tenant","tag":"TenantID","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ProfileID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"AttributeFilterIDs","tag":"AttributeFilterIDs","type":"*variable","value":"~*req.5"},{"path":"Path","tag":"Path","type":"*variable","value":"~*req.6"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.7"},{"path":"Value","tag":"Value","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.10"}],"file_name":"Attributes.csv","flags":null,"type":"*attributes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.2"},{"path":"Element","tag":"Element","type":"*variable","value":"~*req.3"},{"path":"Values","tag":"Values","type":"*variable","value":"~*req.4"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.5"}],"file_name":"Filters.csv","flags":null,"type":"*filters"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"UsageTTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Limit","tag":"Limit","type":"*variable","value":"~*req.5"},{"path":"AllocationMessage","tag":"AllocationMessage","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.10"}],"file_name":"Resources.csv","flags":null,"type":"*resources"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.4"},{"path":"Type","tag":"Type","type":"*variable","value":"~*req.5"},{"path":"AddressPool","tag":"AddressPool","type":"*variable","value":"~*req.6"},{"path":"Allocation","tag":"Allocation","type":"*variable","value":"~*req.7"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.8"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.9"}],"file_name":"IPs.csv","flags":null,"type":"*ips"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"QueueLength","tag":"QueueLength","type":"*variable","value":"~*req.4"},{"path":"TTL","tag":"TTL","type":"*variable","value":"~*req.5"},{"path":"MinItems","tag":"MinItems","type":"*variable","value":"~*req.6"},{"path":"MetricIDs","tag":"MetricIDs","type":"*variable","value":"~*req.7"},{"path":"MetricFilterIDs","tag":"MetricFilterIDs","type":"*variable","value":"~*req.8"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.9"},{"path":"Stored","tag":"Stored","type":"*variable","value":"~*req.10"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.11"},{"path":"ThresholdIDs","tag":"ThresholdIDs","type":"*variable","value":"~*req.12"}],"file_name":"Stats.csv","flags":null,"type":"*stats"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"MaxHits","tag":"MaxHits","type":"*variable","value":"~*req.4"},{"path":"MinHits","tag":"MinHits","type":"*variable","value":"~*req.5"},{"path":"MinSleep","tag":"MinSleep","type":"*variable","value":"~*req.6"},{"path":"Blocker","tag":"Blocker","type":"*variable","value":"~*req.7"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.8"},{"path":"ActionIDs","tag":"ActionIDs","type":"*variable","value":"~*req.9"},{"path":"Async","tag":"Async","type":"*variable","value":"~*req.10"}],"file_name":"Thresholds.csv","flags":null,"type":"*thresholds"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"Sorting","tag":"Sorting","type":"*variable","value":"~*req.4"},{"path":"SortingParameters","tag":"SortingParameters","type":"*variable","value":"~*req.5"},{"path":"RouteID","tag":"RouteID","type":"*variable","value":"~*req.6"},{"path":"RouteFilterIDs","tag":"RouteFilterIDs","type":"*variable","value":"~*req.7"},{"path":"RouteAccountIDs","tag":"RouteAccountIDs","type":"*variable","value":"~*req.8"},{"path":"RouteRatingPlanIDs","tag":"RouteRatingPlanIDs","type":"*variable","value":"~*req.9"},{"path":"RouteResourceIDs","tag":"RouteResourceIDs","type":"*variable","value":"~*req.10"},{"path":"RouteStatIDs","tag":"RouteStatIDs","type":"*variable","value":"~*req.11"},{"path":"RouteWeight","tag":"RouteWeight","type":"*variable","value":"~*req.12"},{"path":"RouteBlocker","tag":"RouteBlocker","type":"*variable","value":"~*req.13"},{"path":"RouteParameters","tag":"RouteParameters","type":"*variable","value":"~*req.14"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.15"}],"file_name":"Routes.csv","flags":null,"type":"*routes"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.2"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.3"},{"path":"RunID","tag":"RunID","type":"*variable","value":"~*req.4"},{"path":"AttributeIDs","tag":"AttributeIDs","type":"*variable","value":"~*req.5"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.6"}],"file_name":"Chargers.csv","flags":null,"type":"*chargers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Contexts","tag":"Contexts","type":"*variable","value":"~*req.2"},{"path":"FilterIDs","tag":"FilterIDs","type":"*variable","value":"~*req.3"},{"path":"ActivationInterval","tag":"ActivationInterval","type":"*variable","value":"~*req.4"},{"path":"Strategy","tag":"Strategy","type":"*variable","value":"~*req.5"},{"path":"StrategyParameters","tag":"StrategyParameters","type":"*variable","value":"~*req.6"},{"path":"ConnID","tag":"ConnID","type":"*variable","value":"~*req.7"},{"path":"ConnFilterIDs","tag":"ConnFilterIDs","type":"*variable","value":"~*req.8"},{"path":"ConnWeight","tag":"ConnWeight","type":"*variable","value":"~*req.9"},{"path":"ConnBlocker","tag":"ConnBlocker","type":"*variable","value":"~*req.10"},{"path":"ConnParameters","tag":"ConnParameters","type":"*variable","value":"~*req.11"},{"path":"Weight","tag":"Weight","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherProfiles.csv","flags":null,"type":"*dispatchers"},{"fields":[{"mandatory":true,"path":"Tenant","tag":"Tenant","type":"*variable","value":"~*req.0"},{"mandatory":true,"path":"ID","tag":"ID","type":"*variable","value":"~*req.1"},{"path":"Address","tag":"Address","type":"*variable","value":"~*req.2"},{"path":"Transport","tag":"Transport","type":"*variable","value":"~*req.3"},{"path":"ConnectAttempts","tag":"ConnectAttempts","type":"*variable","value":"~*req.4"},{"path":"Reconnects","tag":"Reconnects","type":"*variable","value":"~*req.5"},{"path":"MaxReconnectInterval","tag":"MaxReconnectInterval","type":"*variable","value":"~*req.6"},{"path":"ConnectTimeout","tag":"ConnectTimeout","type":"*variable","value":"~*req.7"},{"path":"ReplyTimeout","tag":"ReplyTimeout","type":"*variable","value":"~*req.8"},{"path":"TLS","tag":"TLS","type":"*variable","value":"~*req.9"},{"path":"ClientKey","tag":"ClientKey","type":"*variable","value":"~*req.10"},{"path":"ClientCertificate","tag":"ClientCertificate","type":"*variable","value":"~*req.11"},{"path":"CaCertificate","tag":"CaCertificate","type":"*variable","value":"~*req.12"}],"file_name":"DispatcherHosts.csv","flags":null,"type":"*dispatcher_hosts"}],"dry_run":false,"enabled":false,"field_separator":",","id":"*default","lockfile_path":".cgr.lck","run_delay":"0","tenant":"","tp_in_dir":"/var/spool/cgrates/loader/in","tp_out_dir":"/var/spool/cgrates/loader/out"}],"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"0s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectAttempts":20,"redisConnectTimeout":"0s","redisMaxConns":10,"redisPoolPipelineLimit":0,"redisPoolPipelineWindow":"150µs","redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"*redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"0s","mysqlDSNParams":null,"mysqlLocation":"","pgSSLMode":"","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":0,"sqlMaxOpenConns":0},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"*mysql","out_stordb_user":"cgrates","users_filters":null},"prometheus_agent":{"apiers_conns":[],"cache_ids":[],"caches_conns":[],"collect_go_metrics":false,"collect_process_metrics":false,"cores_conns":[],"enabled":false,"path":"/prometheus","stat_queue_ids":[],"stats_conns":[]},"radius_agent":{"client_dictionaries":{"*default":["/usr/share/cgrates/radius/dict/"]},"client_secrets":{"*default":"CGRateS.org"},"coa_template":"*coa","dmr_template":"*dmr","enabled":false,"listeners":[{"acct_address":"127.0.0.1:1813","auth_address":"127.0.0.1:1812","network":"udp"}],"request_processors":[],"requests_cache_key":"","sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"enabled":false,"fallback_depth":3,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"sessions_conns":[],"stats_conns":[],"thresholds_conns":[]},"rankings":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"scheduled_ids":{},"stats_conns":[],"store_interval":"","thresholds_conns":[]},"registrarc":{"dispatchers":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*units":1,"*usageID":""},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*context":"*routes","*ignoreErrors":false,"*maxCost":""},"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"dynaprepaid_actionplans":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sentrypeer":{"Audience":"https://sentrypeer.com/api","ClientID":"","ClientSecret":"","GrantType":"client_credentials","IpUrl":"https://sentrypeer.com/api/ip-addresses","NumberUrl":"https://sentrypeer.com/api/phone-numbers","TokenURL":"https://authz.sentrypeer.com/oauth/token"},"sessions":{"alterable_fields":[],"attributes_conns":[],"backup_interval":"0","cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":2,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"ips_conns":[],"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stale_chan_max_extra_usage":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[],"timezone":""},"stats":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*profileIDs":[],"*profileIgnoreFilters":false},"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"CGRateS.org","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_account_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destination_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_ips":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rankings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_trends":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"internalDBBackupPath":"/var/lib/cgrates/internal_db/backup/stordb","internalDBDumpInterval":"0s","internalDBDumpPath":"/var/lib/cgrates/internal_db/stordb","internalDBFileSizeLimit":1073741824,"internalDBRewriteInterval":"0s","internalDBStartTimeout":"5m0s","mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","mysqlDSNParams":{},"mysqlLocation":"Local","pgSSLMode":"disable","pgSchema":"","sqlConnMaxLifetime":"0s","sqlLogLevel":3,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*coa":[{"path":"*radDAReq.User-Name","tag":"User-Name","type":"*variable","value":"~*oreq.User-Name"},{"path":"*radDAReq.NAS-IP-Address","tag":"NAS-IP-Address","type":"*variable","value":"~*oreq.NAS-IP-Address"},{"path":"*radDAReq.Acct-Session-Id","tag":"Acct-Session-Id","type":"*variable","value":"~*oreq.Acct-Session-Id"},{"path":"*radDAReq.Filter-Id","tag":"Filter-Id","type":"*variable","value":"~*req.CustomFilter"}],"*dmr":[{"path":"*radDAReq.User-Name","tag":"User-Name","type":"*variable","value":"~*oreq.User-Name"},{"path":"*radDAReq.NAS-IP-Address","tag":"NAS-IP-Address","type":"*variable","value":"~*oreq.NAS-IP-Address"},{"path":"*radDAReq.Acct-Session-Id","tag":"Acct-Session-Id","type":"*variable","value":"~*oreq.Acct-Session-Id"},{"path":"*radDAReq.Reply-Message","tag":"Reply-Message","type":"*variable","value":"~*req.DisconnectCause"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*profileIDs":[],"*profileIgnoreFilters":false},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4},"trends":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"scheduled_ids":{},"stats_conns":[],"store_interval":"","store_uncompressed_limit":0,"thresholds_conns":[]}}` + expected := `{"analyzers":{"cleanup_interval":"1h0m0s","db_path":"/var/spool/cgrates/analyzers","enabled":false,"index_type":"*scorch","ttl":"24h0m0s"},"apiban":{"keys":[]},"apiers":{"attributes_conns":[],"caches_conns":["*internal"],"ees_conns":[],"enabled":false,"scheduler_conns":[]},"asterisk_agent":{"asterisk_conns":[{"address":"127.0.0.1:8088","alias":"","ari_websocket":false,"connect_attempts":3,"max_reconnect_interval":"0s","password":"CGRateS.org","reconnects":5,"user":"cgrates"}],"create_cdr":false,"enabled":false,"route_profile":false,"sessions_conns":["*birpc_internal"]},"attributes":{"any_context":true,"apiers_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*processRuns":1,"*profileIDs":[],"*profileIgnoreFilters":false,"*profileRuns":0},"prefix_indexed_fields":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"caches":{"partitions":{"*account_action_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*action_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*action_triggers":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*apiban":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"2m0s"},"*attribute_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*caps_events":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*cdr_ids":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10m0s"},"*charger_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*closed_sessions":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*destinations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*diameter_messages":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*dispatcher_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_loads":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_routes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*dispatchers":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*event_charges":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"10s"},"*event_ips":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*event_resources":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_allocations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*ip_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*radius_packets":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*ranking_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rankings":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rating_plans":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rating_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*replication_hosts":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*reverse_destinations":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rpc_connections":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*rpc_responses":{"limit":0,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"2s"},"*sentrypeer":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":true,"ttl":"24h0m0s"},"*shared_groups":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*stir":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"},"*threshold_filter_indexes":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*timings":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*trend_profiles":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*trends":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*uch":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"3h0m0s"}},"remote_conns":[],"replication_conns":[]},"cdrs":{"attributes_conns":[],"chargers_conns":[],"compress_stored_cost":false,"ees_conns":[],"enabled":false,"extra_fields":[],"online_cdr_exports":[],"rals_conns":[],"scheduler_conns":[],"session_cost_retries":5,"stats_conns":[],"store_cdrs":true,"thresholds_conns":[]},"chargers":{"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"suffix_indexed_fields":[]},"configs":{"enabled":false,"root_dir":"/var/spool/cgrates/configs","url":"/configs/"},"cores":{"caps":0,"caps_stats_interval":"0","caps_strategy":"*busy","shutdown_timeout":"1s"},"data_db":{"db_host":"127.0.0.1","db_name":"10","db_password":"","db_port":6379,"db_type":"*redis","db_user":"cgrates","items":{"*account_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*accounts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*attribute_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*charger_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_allocations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ip_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*load_ids":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*ranking_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rankings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resource_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*reverse_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*route_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*sessions_backup":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*stat_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueue_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*statqueues":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_filter_indexes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*threshold_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*trend_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*trends":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"internalDBBackupPath":"/var/lib/cgrates/internal_db/backup/datadb","internalDBDumpInterval":"0s","internalDBDumpPath":"/var/lib/cgrates/internal_db/datadb","internalDBFileSizeLimit":1073741824,"internalDBRewriteInterval":"0s","internalDBStartTimeout":"5m0s","mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectAttempts":20,"redisConnectTimeout":"0s","redisMaxConns":10,"redisPoolPipelineLimit":0,"redisPoolPipelineWindow":"150µs","redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"remote_conn_id":"","remote_conns":[],"replication_cache":"","replication_conns":[],"replication_failed_dir":"","replication_filtered":false,"replication_interval":"0s"},"diameter_agent":{"asr_template":"","conn_health_check_interval":"0s","conn_status_stat_queue_ids":[],"conn_status_threshold_ids":[],"dictionaries_path":"/usr/share/cgrates/diameter/dict/","enabled":false,"forced_disconnect":"*none","listeners":[{"address":"127.0.0.1:3868","network":"tcp"}],"origin_host":"CGR-DA","origin_realm":"cgrates.org","product_name":"CGRateS","rar_template":"","request_processors":[],"sessions_conns":["*birpc_internal"],"stats_conns":[],"synced_conn_requests":false,"thresholds_conns":[],"vendor_id":0},"dispatchers":{"any_subsystem":true,"attributes_conns":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"prefix_indexed_fields":[],"prevent_loop":false,"suffix_indexed_fields":[]},"dns_agent":{"enabled":false,"listeners":[{"address":"127.0.0.1:53","network":"udp"}],"request_processors":[],"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[],"timezone":""},"ees":{"attributes_conns":[],"cache":{"*amqp_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*amqpv1_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*els":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*file_csv":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false,"ttl":"5s"},"*kafka_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*nats_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*s3_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*sql":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false},"*sqs_json_map":{"limit":-1,"precache":false,"remote":false,"replicate":false,"static_ttl":false}},"enabled":false,"exporters":[{"attempts":1,"attribute_context":"","attribute_ids":[],"concurrent_requests":0,"export_path":"/var/spool/cgrates/ees","failed_posts_dir":"/var/spool/cgrates/failed_posts","fields":[],"filters":[],"flags":[],"id":"*default","metrics_reset_schedule":"","opts":{},"synchronous":false,"timezone":"","type":"*none"}],"failed_posts":{"dir":"/var/spool/cgrates/failed_posts","static_ttl":true,"ttl":"5s"}},"ers":{"concurrent_events":1,"ees_conns":[],"enabled":false,"partial_cache_ttl":"1s","readers":[{"cache_dump_fields":[],"concurrent_requests":1024,"fields":[{"mandatory":true,"path":"*cgreq.ToR","tag":"ToR","type":"*variable","value":"~*req.2"},{"mandatory":true,"path":"*cgreq.OriginID","tag":"OriginID","type":"*variable","value":"~*req.3"},{"mandatory":true,"path":"*cgreq.RequestType","tag":"RequestType","type":"*variable","value":"~*req.4"},{"mandatory":true,"path":"*cgreq.Tenant","tag":"Tenant","type":"*variable","value":"~*req.6"},{"mandatory":true,"path":"*cgreq.Category","tag":"Category","type":"*variable","value":"~*req.7"},{"mandatory":true,"path":"*cgreq.Account","tag":"Account","type":"*variable","value":"~*req.8"},{"mandatory":true,"path":"*cgreq.Subject","tag":"Subject","type":"*variable","value":"~*req.9"},{"mandatory":true,"path":"*cgreq.Destination","tag":"Destination","type":"*variable","value":"~*req.10"},{"mandatory":true,"path":"*cgreq.SetupTime","tag":"SetupTime","type":"*variable","value":"~*req.11"},{"mandatory":true,"path":"*cgreq.AnswerTime","tag":"AnswerTime","type":"*variable","value":"~*req.12"},{"mandatory":true,"path":"*cgreq.Usage","tag":"Usage","type":"*variable","value":"~*req.13"}],"filters":[],"flags":[],"id":"*default","max_reconnect_interval":"5m0s","opts":{"csvFieldSeparator":",","csvHeaderDefineChar":":","csvRowLength":0,"natsSubject":"cgrates_cdrs","partialCacheAction":"*none","partialOrderField":"~*req.AnswerTime"},"partial_commit_fields":[],"processed_path":"/var/spool/cgrates/ers/out","reconnects":-1,"run_delay":"0","source_path":"/var/spool/cgrates/ers/in","start_delay":"0","tenant":"","timezone":"","type":"*none"}],"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[]},"filters":{"apiers_conns":[],"rankings_conns":[],"resources_conns":[],"stats_conns":[],"trends_conns":[]},"freeswitch_agent":{"active_session_delimiter":",","create_cdr":false,"empty_balance_ann_file":"","empty_balance_context":"","enabled":false,"event_socket_conns":[{"address":"127.0.0.1:8021","alias":"127.0.0.1:8021","max_reconnect_interval":"0s","password":"ClueCon","reconnects":5,"reply_timeout":"1m0s"}],"extra_fields":"","low_balance_ann_file":"","max_wait_connection":"2s","route_profile":false,"sched_transfer_extension":"CGRateS","sessions_conns":["*birpc_internal"],"subscribe_park":true},"general":{"caching_delay":"0","connect_attempts":5,"connect_timeout":"1s","dbdata_encoding":"*msgpack","default_caching":"*reload","default_category":"call","default_request_type":"*rated","default_tenant":"cgrates.org","default_timezone":"Local","digest_equal":":","digest_separator":",","locking_timeout":"0","log_level":6,"logger":"*syslog","max_parallel_conns":100,"max_reconnect_interval":"0","node_id":"ENGINE1","poster_attempts":3,"reconnects":-1,"reply_timeout":"2s","rounding_decimals":5,"rsr_separator":";","tpexport_dir":"/var/spool/cgrates/tpe"},"http":{"auth_users":{},"client_opts":{"dialFallbackDelay":"300ms","dialKeepAlive":"30s","dialTimeout":"30s","disableCompression":false,"disableKeepAlives":false,"expectContinueTimeout":"0s","forceAttemptHttp2":true,"idleConnTimeout":"1m30s","maxConnsPerHost":0,"maxIdleConns":100,"maxIdleConnsPerHost":2,"responseHeaderTimeout":"0s","skipTlsVerify":false,"tlsHandshakeTimeout":"10s"},"freeswitch_cdrs_url":"/freeswitch_json","http_cdrs":"/cdr_http","json_rpc_url":"/jsonrpc","pprof_path":"/debug/pprof/","registrars_url":"/registrar","use_basic_auth":false,"ws_url":"/ws"},"http_agent":[],"ips":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*allocationID":"","*ttl":259200000000000},"prefix_indexed_fields":[],"store_interval":"0s","string_indexed_fields":null,"suffix_indexed_fields":[]},"kamailio_agent":{"create_cdr":false,"enabled":false,"evapi_conns":[{"address":"127.0.0.1:8448","alias":"","max_reconnect_interval":"0s","reconnects":5}],"route_profile":false,"sessions_conns":["*birpc_internal"],"timezone":""},"listen":{"http":"127.0.0.1:2080","http_tls":"127.0.0.1:2280","rpc_gob":"127.0.0.1:2013","rpc_gob_tls":"127.0.0.1:2023","rpc_json":"127.0.0.1:2012","rpc_json_tls":"127.0.0.1:2022"},"loader":{"caches_conns":["*localhost"],"data_path":"./","disable_reverse":false,"field_separator":",","gapi_credentials":".gapi/credentials.json","gapi_token":".gapi/token.json","scheduler_conns":["*localhost"],"tpid":""},"mailer":{"auth_password":"CGRateS.org","auth_user":"cgrates","from_address":"cgr-mailer@localhost.localdomain","server":"localhost"},"migrator":{"out_datadb_encoding":"msgpack","out_datadb_host":"127.0.0.1","out_datadb_name":"10","out_datadb_opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"0s","redisCACertificate":"","redisClientCertificate":"","redisClientKey":"","redisCluster":false,"redisClusterOndownDelay":"0s","redisClusterSync":"5s","redisConnectAttempts":20,"redisConnectTimeout":"0s","redisMaxConns":10,"redisPoolPipelineLimit":0,"redisPoolPipelineWindow":"150µs","redisReadTimeout":"0s","redisSentinel":"","redisTLS":false,"redisWriteTimeout":"0s"},"out_datadb_password":"","out_datadb_port":"6379","out_datadb_type":"*redis","out_datadb_user":"cgrates","out_stordb_host":"127.0.0.1","out_stordb_name":"cgrates","out_stordb_opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"0s","mysqlDSNParams":null,"mysqlLocation":"","pgSSLMode":"","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":0,"sqlMaxOpenConns":0},"out_stordb_password":"","out_stordb_port":"3306","out_stordb_type":"*mysql","out_stordb_user":"cgrates","users_filters":null},"prometheus_agent":{"apiers_conns":[],"cache_ids":[],"caches_conns":[],"collect_go_metrics":false,"collect_process_metrics":false,"cores_conns":[],"enabled":false,"path":"/prometheus","stat_queue_ids":[],"stats_conns":[]},"radius_agent":{"client_dictionaries":{"*default":["/usr/share/cgrates/radius/dict/"]},"client_secrets":{"*default":"CGRateS.org"},"coa_template":"*coa","dmr_template":"*dmr","enabled":false,"listeners":[{"acct_address":"127.0.0.1:1813","auth_address":"127.0.0.1:1812","network":"udp"}],"request_processors":[],"requests_cache_key":"","sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[]},"rals":{"balance_rating_subject":{"*any":"*zero1ns","*voice":"*zero1s"},"enabled":false,"fallback_depth":3,"max_computed_usage":{"*any":"189h0m0s","*data":"107374182400","*mms":"10000","*sms":"10000","*voice":"72h0m0s"},"max_increments":1000000,"remove_expired":true,"rp_subject_prefix_matching":false,"sessions_conns":[],"stats_conns":[],"thresholds_conns":[]},"rankings":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"scheduled_ids":{},"stats_conns":[],"store_interval":"","thresholds_conns":[]},"registrarc":{"dispatchers":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]},"rpc":{"hosts":[],"refresh_interval":"5m0s","registrars_conns":[]}},"resources":{"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*units":1,"*usageID":""},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[],"thresholds_conns":[]},"routes":{"attributes_conns":[],"default_ratio":1,"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*context":"*routes","*ignoreErrors":false,"*maxCost":""},"prefix_indexed_fields":[],"rals_conns":[],"resources_conns":[],"stats_conns":[],"suffix_indexed_fields":[]},"rpc_conns":{"*bijson_localhost":{"conns":[{"address":"127.0.0.1:2014","transport":"*birpc_json"}],"poolSize":0,"strategy":"*first"},"*birpc_internal":{"conns":[{"address":"*birpc_internal","transport":""}],"poolSize":0,"strategy":"*first"},"*internal":{"conns":[{"address":"*internal","transport":""}],"poolSize":0,"strategy":"*first"},"*localhost":{"conns":[{"address":"127.0.0.1:2012","transport":"*json"}],"poolSize":0,"strategy":"*first"}},"schedulers":{"cdrs_conns":[],"dynaprepaid_actionplans":[],"enabled":false,"filters":[],"stats_conns":[],"thresholds_conns":[]},"sentrypeer":{"Audience":"https://sentrypeer.com/api","ClientID":"","ClientSecret":"","GrantType":"client_credentials","IpUrl":"https://sentrypeer.com/api/ip-addresses","NumberUrl":"https://sentrypeer.com/api/phone-numbers","TokenURL":"https://authz.sentrypeer.com/oauth/token"},"sessions":{"alterable_fields":[],"attributes_conns":[],"backup_interval":"0","cdrs_conns":[],"channel_sync_interval":"0","chargers_conns":[],"client_protocol":2,"debit_interval":"0","default_usage":{"*any":"3h0m0s","*data":"1048576","*sms":"1","*voice":"3h0m0s"},"enabled":false,"ips_conns":[],"listen_bigob":"","listen_bijson":"127.0.0.1:2014","min_dur_low_balance":"0","rals_conns":[],"replication_conns":[],"resources_conns":[],"routes_conns":[],"scheduler_conns":[],"session_indexes":[],"session_ttl":"0","stale_chan_max_extra_usage":"0","stats_conns":[],"stir":{"allowed_attest":["*any"],"default_attest":"A","payload_maxduration":"-1","privatekey_path":"","publickey_path":""},"store_session_costs":false,"terminate_attempts":5,"thresholds_conns":[]},"sip_agent":{"enabled":false,"listen":"127.0.0.1:5060","listen_net":"udp","request_processors":[],"retransmission_timer":1000000000,"sessions_conns":["*internal"],"stats_conns":[],"thresholds_conns":[],"timezone":""},"stats":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*profileIDs":[],"*profileIgnoreFilters":false},"prefix_indexed_fields":[],"store_interval":"","store_uncompressed_limit":0,"suffix_indexed_fields":[],"thresholds_conns":[]},"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"CGRateS.org","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_account_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destination_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_ips":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rankings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_trends":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"internalDBBackupPath":"/var/lib/cgrates/internal_db/backup/stordb","internalDBDumpInterval":"0s","internalDBDumpPath":"/var/lib/cgrates/internal_db/stordb","internalDBFileSizeLimit":1073741824,"internalDBRewriteInterval":"0s","internalDBStartTimeout":"5m0s","mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","mysqlDSNParams":{},"mysqlLocation":"Local","pgSSLMode":"disable","pgSchema":"","sqlConnMaxLifetime":"0s","sqlLogLevel":3,"sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]},"suretax":{"bill_to_number":"","business_unit":"","client_number":"","client_tracking":"~*req.CGRID","customer_number":"~*req.Subject","include_local_cost":false,"orig_number":"~*req.Subject","p2pplus4":"","p2pzipcode":"","plus4":"","regulatory_code":"03","response_group":"03","response_type":"D4","return_file_code":"0","sales_type_code":"R","tax_exemption_code_list":"","tax_included":"0","tax_situs_rule":"04","term_number":"~*req.Destination","timezone":"UTC","trans_type_code":"010101","unit_type":"00","units":"1","url":"","validation_key":"","zipcode":""},"templates":{"*asr":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"}],"*cca":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"path":"*rep.Result-Code","tag":"ResultCode","type":"*constant","value":"2001"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"},{"mandatory":true,"path":"*rep.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"mandatory":true,"path":"*rep.CC-Request-Type","tag":"CCRequestType","type":"*variable","value":"~*req.CC-Request-Type"},{"mandatory":true,"path":"*rep.CC-Request-Number","tag":"CCRequestNumber","type":"*variable","value":"~*req.CC-Request-Number"}],"*cdrLog":[{"mandatory":true,"path":"*cdr.ToR","tag":"ToR","type":"*variable","value":"~*req.BalanceType"},{"mandatory":true,"path":"*cdr.OriginHost","tag":"OriginHost","type":"*constant","value":"127.0.0.1"},{"mandatory":true,"path":"*cdr.RequestType","tag":"RequestType","type":"*constant","value":"*none"},{"mandatory":true,"path":"*cdr.Tenant","tag":"Tenant","type":"*variable","value":"~*req.Tenant"},{"mandatory":true,"path":"*cdr.Account","tag":"Account","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Subject","tag":"Subject","type":"*variable","value":"~*req.Account"},{"mandatory":true,"path":"*cdr.Cost","tag":"Cost","type":"*variable","value":"~*req.Cost"},{"mandatory":true,"path":"*cdr.Source","tag":"Source","type":"*constant","value":"*cdrLog"},{"mandatory":true,"path":"*cdr.Usage","tag":"Usage","type":"*constant","value":"1"},{"mandatory":true,"path":"*cdr.RunID","tag":"RunID","type":"*variable","value":"~*req.ActionType"},{"mandatory":true,"path":"*cdr.SetupTime","tag":"SetupTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.AnswerTime","tag":"AnswerTime","type":"*constant","value":"*now"},{"mandatory":true,"path":"*cdr.PreRated","tag":"PreRated","type":"*constant","value":"true"}],"*coa":[{"path":"*radDAReq.User-Name","tag":"User-Name","type":"*variable","value":"~*oreq.User-Name"},{"path":"*radDAReq.NAS-IP-Address","tag":"NAS-IP-Address","type":"*variable","value":"~*oreq.NAS-IP-Address"},{"path":"*radDAReq.Acct-Session-Id","tag":"Acct-Session-Id","type":"*variable","value":"~*oreq.Acct-Session-Id"},{"path":"*radDAReq.Filter-Id","tag":"Filter-Id","type":"*variable","value":"~*req.CustomFilter"}],"*dmr":[{"path":"*radDAReq.User-Name","tag":"User-Name","type":"*variable","value":"~*oreq.User-Name"},{"path":"*radDAReq.NAS-IP-Address","tag":"NAS-IP-Address","type":"*variable","value":"~*oreq.NAS-IP-Address"},{"path":"*radDAReq.Acct-Session-Id","tag":"Acct-Session-Id","type":"*variable","value":"~*oreq.Acct-Session-Id"},{"path":"*radDAReq.Reply-Message","tag":"Reply-Message","type":"*variable","value":"~*req.DisconnectCause"}],"*err":[{"mandatory":true,"path":"*rep.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*rep.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*vars.OriginHost"},{"mandatory":true,"path":"*rep.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*vars.OriginRealm"}],"*errSip":[{"mandatory":true,"path":"*rep.Request","tag":"Request","type":"*constant","value":"SIP/2.0 500 Internal Server Error"}],"*rar":[{"mandatory":true,"path":"*diamreq.Session-Id","tag":"SessionId","type":"*variable","value":"~*req.Session-Id"},{"mandatory":true,"path":"*diamreq.Origin-Host","tag":"OriginHost","type":"*variable","value":"~*req.Destination-Host"},{"mandatory":true,"path":"*diamreq.Origin-Realm","tag":"OriginRealm","type":"*variable","value":"~*req.Destination-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Realm","tag":"DestinationRealm","type":"*variable","value":"~*req.Origin-Realm"},{"mandatory":true,"path":"*diamreq.Destination-Host","tag":"DestinationHost","type":"*variable","value":"~*req.Origin-Host"},{"mandatory":true,"path":"*diamreq.Auth-Application-Id","tag":"AuthApplicationId","type":"*variable","value":"~*vars.*appid"},{"path":"*diamreq.Re-Auth-Request-Type","tag":"ReAuthRequestType","type":"*constant","value":"0"}]},"thresholds":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"exists_indexed_fields":[],"indexed_selects":true,"nested_fields":false,"opts":{"*profileIDs":[],"*profileIgnoreFilters":false},"prefix_indexed_fields":[],"store_interval":"","suffix_indexed_fields":[]},"tls":{"ca_certificate":"","client_certificate":"","client_key":"","server_certificate":"","server_key":"","server_name":"","server_policy":4},"trends":{"ees_conns":[],"ees_exporter_ids":[],"enabled":false,"scheduled_ids":{},"stats_conns":[],"store_interval":"","store_uncompressed_limit":0,"thresholds_conns":[]}}` if err != nil { t.Fatal(err) } @@ -5785,14 +5000,14 @@ func TestReloadSections(t *testing.T) { subsystemsThatNeedDataDB := utils.NewStringSet([]string{SCHEDULER_JSN, RALS_JSN, CDRS_JSN, SessionSJson, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON, - RouteSJson, LoaderJson, DispatcherSJson, ApierS, IPsJSON}) + RouteSJson, DispatcherSJson, ApierS, IPsJSON}) subsystemsThatNeedStorDB := utils.NewStringSet([]string{RALS_JSN, CDRS_JSN, ApierS}) cfgCgr := NewDefaultCGRConfig() for _, section := range []string{RPCConnsJsonName, HTTP_JSN, SCHEDULER_JSN, RALS_JSN, CDRS_JSN, ERsJson, SessionSJson, AsteriskAgentJSN, FreeSWITCHAgentJSN, KamailioAgentJSN, DA_JSN, RA_JSN, HttpAgentJson, DNSAgentJson, ATTRIBUTE_JSN, ChargerSCfgJson, RESOURCES_JSON, STATS_JSON, THRESHOLDS_JSON, RouteSJson, - LoaderJson, DispatcherSJson, ApierS, EEsJson, SIPAgentJson, RegistrarCJson, AnalyzerCfgJson, IPsJSON} { + DispatcherSJson, ApierS, EEsJson, SIPAgentJson, RegistrarCJson, AnalyzerCfgJson, IPsJSON} { for _, section := range sortedCfgSections { cfgCgr.rldChans[section] = make(chan struct{}, 1) } @@ -5902,9 +5117,6 @@ func TestCGRConfigClone(t *testing.T) { if !reflect.DeepEqual(cfg.dfltEvExp, rcv.dfltEvExp) { t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(cfg.dfltEvExp), utils.ToJSON(rcv.dfltEvExp)) } - if !reflect.DeepEqual(cfg.loaderCfg, rcv.loaderCfg) { - t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(cfg.loaderCfg), utils.ToJSON(rcv.loaderCfg)) - } if !reflect.DeepEqual(cfg.httpAgentCfg, rcv.httpAgentCfg) { t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(cfg.httpAgentCfg), utils.ToJSON(rcv.httpAgentCfg)) } @@ -6039,8 +5251,8 @@ func TestCGRConfigClone(t *testing.T) { func TestCGRConfigGetDP(t *testing.T) { cfg := NewDefaultCGRConfig() - cfg.LockSections(HttpAgentJson, LoaderJson, ChargerSCfgJson) - cfg.UnlockSections(HttpAgentJson, LoaderJson, ChargerSCfgJson) + cfg.LockSections(HttpAgentJson, ChargerSCfgJson) + cfg.UnlockSections(HttpAgentJson, ChargerSCfgJson) exp := utils.MapStorage(cfg.AsMapInterface(cfg.generalCfg.RSRSep)) dp := cfg.GetDataProvider() if !reflect.DeepEqual(dp, exp) { diff --git a/config/configsanity.go b/config/configsanity.go index 1128db864..af0c085ad 100644 --- a/config/configsanity.go +++ b/config/configsanity.go @@ -22,7 +22,6 @@ import ( "fmt" "math" "os" - "path" "slices" "strings" @@ -119,51 +118,6 @@ func (cfg *CGRConfig) checkConfigSanity() error { } } } - // Loaders sanity checks - for _, ldrSCfg := range cfg.loaderCfg { - if !ldrSCfg.Enabled { - continue - } - if _, err := os.Stat(ldrSCfg.TpInDir); err != nil && os.IsNotExist(err) { // if loader is enabled tpInDir must exist - return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpInDir) - } - if ldrSCfg.TpOutDir != utils.EmptyString { // tpOutDir support empty string for no moving files after process - if _, err := os.Stat(ldrSCfg.TpOutDir); err != nil && os.IsNotExist(err) { - return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, ldrSCfg.TpOutDir) - } - } - if ldrSCfg.LockFilePath != utils.EmptyString { // tpOutDir support empty string for no moving files after process - pathL := ldrSCfg.GetLockFilePath() - if _, err := os.Stat(path.Dir(pathL)); err != nil && os.IsNotExist(err) { - return fmt.Errorf("<%s> nonexistent folder: %s", utils.LoaderS, pathL) - } - } - for _, data := range ldrSCfg.Data { - if !possibleLoaderTypes.Has(data.Type) { - return fmt.Errorf("<%s> unsupported data type %s", utils.LoaderS, data.Type) - } - - for _, field := range data.Fields { - if field.Type != utils.MetaComposed && field.Type != utils.MetaString && field.Type != utils.MetaVariable { - return fmt.Errorf("<%s> invalid field type %s for %s at %s", utils.LoaderS, field.Type, data.Type, field.Tag) - } - if field.Path == utils.EmptyString { - return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, utils.NewErrMandatoryIeMissing(utils.Path), data.Type, field.Tag) - } - if err := utils.IsPathValidForExporters(field.Path); err != nil { - return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, field.Path, utils.Path) - } - for _, val := range field.Value { - if err := utils.IsPathValidForExporters(val.path); err != nil { - return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, val.path, utils.Values) - } - } - if err := utils.CheckInLineFilter(field.Filters); err != nil { - return fmt.Errorf("<%s> %s for %s at %s", utils.LoaderS, err, field.Filters, utils.Filters) - } - } - } - } // SessionS checks if cfg.sessionSCfg.Enabled { if cfg.sessionSCfg.TerminateAttempts < 1 { diff --git a/config/configsanity_test.go b/config/configsanity_test.go index aa7ca4c80..309c63754 100644 --- a/config/configsanity_test.go +++ b/config/configsanity_test.go @@ -138,116 +138,6 @@ func TestConfigSanityCDRServer(t *testing.T) { } } -func TestConfigSanityLoaders(t *testing.T) { - cfg := NewDefaultCGRConfig() - cfg.loaderCfg = LoaderSCfgs{ - &LoaderSCfg{ - Enabled: true, - TpInDir: "/not/exist", - Data: []*LoaderDataType{{ - Type: "strsdfing", - }}, - }, - } - expected := " nonexistent folder: /not/exist" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.loaderCfg = LoaderSCfgs{ - &LoaderSCfg{ - Enabled: true, - TpInDir: "/", - TpOutDir: "/not/exist", - Data: []*LoaderDataType{{ - Type: "strsdfing", - }}, - }, - } - expected = " nonexistent folder: /not/exist" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.loaderCfg = LoaderSCfgs{ - &LoaderSCfg{ - Enabled: true, - TpInDir: "/", - TpOutDir: "/", - Data: []*LoaderDataType{{ - Type: "wrongtype", - }}, - }, - } - expected = " unsupported data type wrongtype" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.loaderCfg = LoaderSCfgs{ - &LoaderSCfg{ - Enabled: true, - TpInDir: "/", - TpOutDir: "/", - Data: []*LoaderDataType{{ - Type: utils.MetaStats, - Fields: []*FCTemplate{{ - Type: utils.MetaStats, - Tag: "test1", - }}, - }}, - }, - } - expected = " invalid field type *stats for *stats at test1" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.loaderCfg = LoaderSCfgs{ - &LoaderSCfg{ - Enabled: true, - TpInDir: "/", - TpOutDir: "/", - Data: []*LoaderDataType{{ - Type: utils.MetaStats, - Fields: []*FCTemplate{{ - Type: utils.MetaComposed, - Tag: "test1", - Path: utils.EmptyString, - }}, - }}, - }, - } - expected = " MANDATORY_IE_MISSING: [Path] for *stats at test1" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - - cfg.loaderCfg[0].Data[0].Fields[0].Path = "~req." - expected = " Empty field path for ~req. at Path" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.loaderCfg[0].Data[0].Fields[0].Path = "~*req.Destination" - - cfg.loaderCfg[0].Data[0].Fields[0].Value = RSRParsers{{}} - cfg.loaderCfg[0].Data[0].Fields[0].Value[0].path = "~req." - expected = " Empty field path for ~req. at Values" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.loaderCfg[0].Data[0].Fields[0].Value[0].path = "*req.Destination" - - cfg.loaderCfg[0].Data[0].Fields[0].Filters = make([]string, 1) - cfg.loaderCfg[0].Data[0].Fields[0].Filters = []string{"*string:~*req..Field"} - expected = " inline parse error for string: <*string:~*req..Field> for [*string:~*req..Field] at Filters" - if err := cfg.checkConfigSanity(); err == nil || err.Error() != expected { - t.Errorf("Expecting: %+q received: %+q", expected, err) - } - cfg.loaderCfg[0].Data[0].Fields[0].Filters = []string{"~req.Valid.Field"} - -} - func TestConfigSanitySessionS(t *testing.T) { cfg := NewDefaultCGRConfig() cfg.sessionSCfg = &SessionSCfg{ diff --git a/config/libconfig_json.go b/config/libconfig_json.go index 7308c889c..a6fb4366e 100644 --- a/config/libconfig_json.go +++ b/config/libconfig_json.go @@ -800,27 +800,6 @@ type RouteSJsonCfg struct { Opts *RoutesOptsJson } -type LoaderJsonDataType struct { - Type *string - File_name *string - Flags *[]string - Fields *[]*FcTemplateJsonCfg -} - -type LoaderJsonCfg struct { - ID *string - Enabled *bool - Tenant *string - Dry_run *bool - Run_delay *string - Lockfile_path *string - Caches_conns *[]string - Field_separator *string - Tp_in_dir *string - Tp_out_dir *string - Data *[]*LoaderJsonDataType -} - // Mailer config section type MailerJsonCfg struct { Server *string diff --git a/config/loaderscfg.go b/config/loaderscfg.go deleted file mode 100644 index 996190934..000000000 --- a/config/loaderscfg.go +++ /dev/null @@ -1,277 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package config - -import ( - "os" - "path" - "path/filepath" - "time" - - "github.com/cgrates/cgrates/utils" -) - -// NewDfltLoaderSCfg returns the first cached default value for a LoaderSCfg connection -func NewDfltLoaderSCfg() *LoaderSCfg { - if dfltLoaderConfig == nil { - return new(LoaderSCfg) - } - dfltVal := *dfltLoaderConfig - return &dfltVal -} - -// LoaderSCfgs to export some methods for LoaderS profiles -type LoaderSCfgs []*LoaderSCfg - -// AsMapInterface returns the config as a map[string]any -func (ldrs LoaderSCfgs) AsMapInterface(separator string) (loaderCfg []map[string]any) { - loaderCfg = make([]map[string]any, len(ldrs)) - for i, item := range ldrs { - loaderCfg[i] = item.AsMapInterface(separator) - } - return -} - -// Enabled returns true if Loader Service is enabled -func (ldrs LoaderSCfgs) Enabled() bool { - for _, ldr := range ldrs { - if ldr.Enabled { - return true - } - } - return false -} - -// Clone itself into a new LoaderSCfgs -func (ldrs LoaderSCfgs) Clone() (cln LoaderSCfgs) { - cln = make(LoaderSCfgs, len(ldrs)) - for i, ldr := range ldrs { - cln[i] = ldr.Clone() - } - return -} - -// LoaderSCfg the config for a loader -type LoaderSCfg struct { - ID string - Enabled bool - Tenant string - DryRun bool - RunDelay time.Duration - LockFilePath string - CacheSConns []string - FieldSeparator string - TpInDir string - TpOutDir string - Data []*LoaderDataType -} - -// LoaderDataType the template for profile loading -type LoaderDataType struct { - Type string - Filename string - Flags utils.FlagsWithParams - Fields []*FCTemplate -} - -func (lData *LoaderDataType) loadFromJSONCfg(jsnCfg *LoaderJsonDataType, msgTemplates map[string][]*FCTemplate, separator string) (err error) { - if jsnCfg == nil { - return nil - } - if jsnCfg.Type != nil { - lData.Type = *jsnCfg.Type - } - if jsnCfg.File_name != nil { - lData.Filename = *jsnCfg.File_name - } - if jsnCfg.Flags != nil { - lData.Flags = utils.FlagsWithParamsFromSlice(*jsnCfg.Flags) - } - if jsnCfg.Fields != nil { - if lData.Fields, err = FCTemplatesFromFCTemplatesJSONCfg(*jsnCfg.Fields, separator); err != nil { - return - } - if tpls, err := InflateTemplates(lData.Fields, msgTemplates); err != nil { - return err - } else if tpls != nil { - lData.Fields = tpls - } - } - return nil -} - -func (l *LoaderSCfg) loadFromJSONCfg(jsnCfg *LoaderJsonCfg, msgTemplates map[string][]*FCTemplate, separator string) (err error) { - if jsnCfg == nil { - return nil - } - if jsnCfg.ID != nil { - l.ID = *jsnCfg.ID - } - if jsnCfg.Enabled != nil { - l.Enabled = *jsnCfg.Enabled - } - if jsnCfg.Tenant != nil { - l.Tenant = *jsnCfg.Tenant - } - if jsnCfg.Dry_run != nil { - l.DryRun = *jsnCfg.Dry_run - } - if jsnCfg.Run_delay != nil { - if l.RunDelay, err = utils.ParseDurationWithNanosecs(*jsnCfg.Run_delay); err != nil { - return - } - } - if jsnCfg.Caches_conns != nil { - l.CacheSConns = make([]string, len(*jsnCfg.Caches_conns)) - for idx, connID := range *jsnCfg.Caches_conns { - // if we have the connection internal we change the name so we can have internal rpc for each subsystem - if connID == utils.MetaInternal { - l.CacheSConns[idx] = utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches) - } else { - l.CacheSConns[idx] = connID - } - } - } - if jsnCfg.Field_separator != nil { - l.FieldSeparator = *jsnCfg.Field_separator - } - if jsnCfg.Tp_in_dir != nil { - l.TpInDir = *jsnCfg.Tp_in_dir - } - if jsnCfg.Tp_out_dir != nil { - l.TpOutDir = *jsnCfg.Tp_out_dir - } - if jsnCfg.Lockfile_path != nil { - l.LockFilePath = *jsnCfg.Lockfile_path - } - if jsnCfg.Data != nil { - data := make([]*LoaderDataType, len(*jsnCfg.Data)) - for idx, jsnLoCfg := range *jsnCfg.Data { - data[idx] = new(LoaderDataType) - if err := data[idx].loadFromJSONCfg(jsnLoCfg, msgTemplates, separator); err != nil { - return err - } - } - l.Data = data - } - - return nil -} - -func (l LoaderSCfg) GetLockFilePath() (pathL string) { - pathL = l.LockFilePath - if !filepath.IsAbs(pathL) { - pathL = path.Join(l.TpInDir, pathL) - } - - if file, err := os.Stat(pathL); err == nil && file.IsDir() { - pathL = path.Join(pathL, l.ID+".lck") - } - return -} - -// Clone itself into a new LoaderDataType -func (lData LoaderDataType) Clone() (cln *LoaderDataType) { - cln = &LoaderDataType{ - Type: lData.Type, - Filename: lData.Filename, - Flags: lData.Flags.Clone(), - Fields: make([]*FCTemplate, len(lData.Fields)), - } - for idx, val := range lData.Fields { - cln.Fields[idx] = val.Clone() - } - return -} - -// Clone itself into a new LoadersConfig -func (l LoaderSCfg) Clone() (cln *LoaderSCfg) { - cln = &LoaderSCfg{ - ID: l.ID, - Enabled: l.Enabled, - Tenant: l.Tenant, - DryRun: l.DryRun, - RunDelay: l.RunDelay, - LockFilePath: l.LockFilePath, - CacheSConns: make([]string, len(l.CacheSConns)), - FieldSeparator: l.FieldSeparator, - TpInDir: l.TpInDir, - TpOutDir: l.TpOutDir, - Data: make([]*LoaderDataType, len(l.Data)), - } - - copy(cln.CacheSConns, l.CacheSConns) - - for idx, fld := range l.Data { - cln.Data[idx] = fld.Clone() - } - return -} - -// AsMapInterface returns the config as a map[string]any -func (lData *LoaderDataType) AsMapInterface(separator string) (initialMP map[string]any) { - initialMP = map[string]any{ - utils.TypeCf: lData.Type, - utils.FilenameCfg: lData.Filename, - utils.FlagsCfg: lData.Flags.SliceFlags(), - } - - fields := make([]map[string]any, len(lData.Fields)) - for i, item := range lData.Fields { - fields[i] = item.AsMapInterface(separator) - } - initialMP[utils.FieldsCfg] = fields - return -} - -// AsMapInterface returns the config as a map[string]any -func (l *LoaderSCfg) AsMapInterface(separator string) (initialMP map[string]any) { - initialMP = map[string]any{ - utils.IDCfg: l.ID, - utils.TenantCfg: l.Tenant, - utils.EnabledCfg: l.Enabled, - utils.DryRunCfg: l.DryRun, - utils.LockFilePathCfg: l.LockFilePath, - utils.FieldSepCfg: l.FieldSeparator, - utils.TpInDirCfg: l.TpInDir, - utils.TpOutDirCfg: l.TpOutDir, - utils.RunDelayCfg: "0", - } - if l.Data != nil { - data := make([]map[string]any, len(l.Data)) - for i, item := range l.Data { - data[i] = item.AsMapInterface(separator) - } - initialMP[utils.DataCfg] = data - } - if l.RunDelay != 0 { - initialMP[utils.RunDelayCfg] = l.RunDelay.String() - } - if l.CacheSConns != nil { - cacheSConns := make([]string, len(l.CacheSConns)) - for i, item := range l.CacheSConns { - cacheSConns[i] = item - if item == utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches) { - cacheSConns[i] = utils.MetaInternal - } - } - initialMP[utils.CachesConnsCfg] = cacheSConns - } - return -} diff --git a/config/loaderscfg_test.go b/config/loaderscfg_test.go deleted file mode 100644 index 3bc0d6bb7..000000000 --- a/config/loaderscfg_test.go +++ /dev/null @@ -1,586 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ -package config - -import ( - "os" - "path" - "reflect" - "testing" - "time" - - "github.com/cgrates/cgrates/utils" -) - -func TestLoaderSCfgloadFromJsonCfgCase1(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "id": "*default", - "enabled": true, - "tenant": "cgrates.org", - "lockfile_path": ".cgr.lck", - "caches_conns": ["*internal","*conn1"], - "field_separator": ",", - "tp_in_dir": "/var/spool/cgrates/loader/in", - "tp_out_dir": "/var/spool/cgrates/loader/out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "flags": [], - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*composed", "value": "~req.0", "mandatory": true,"layout": "2006-01-02T15:04:05Z07:00"}, - ], - }, - ], - }, - ], -}` - val, err := NewRSRParsers("~req.0", utils.InfieldSep) - if err != nil { - t.Error(err) - } - ten := "cgrates.org" - - expected := LoaderSCfgs{ - { - Enabled: true, - ID: utils.MetaDefault, - Tenant: ten, - LockFilePath: ".cgr.lck", - CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "*conn1"}, - FieldSeparator: ",", - TpInDir: "/var/spool/cgrates/loader/in", - TpOutDir: "/var/spool/cgrates/loader/out", - Data: []*LoaderDataType{ - { - Type: "*attributes", - Filename: "Attributes.csv", - Flags: utils.FlagsWithParams{}, - Fields: []*FCTemplate{ - { - Tag: "TenantID", - Path: "Tenant", - pathSlice: []string{"Tenant"}, - Type: utils.MetaComposed, - Value: val, - Mandatory: true, - Layout: time.RFC3339, - }, - }, - }, - }, - }, - } - newCfg := new(CGRConfig) - newCfg.generalCfg = new(GeneralCfg) - newCfg.generalCfg.RSRSep = ";" - if jsonCfg, err := NewCgrJsonCfgFromBytes([]byte(cfgJSONStr)); err != nil { - t.Error(err) - } else if err = newCfg.loadLoaderSCfg(jsonCfg); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(expected, newCfg.loaderCfg) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expected), utils.ToJSON(newCfg.loaderCfg)) - } -} - -// func TestLoaderSCfgloadFromJsonCfgCase2(t *testing.T) { -// cfgJSON := &LoaderJsonCfg{ -// Tenant: utils.StringPointer("a{*"), -// } -// expected := "invalid converter terminator in rule: " -// jsonCfg := NewDefaultCGRConfig() -// if err = jsonCfg.loaderCfg[0].loadFromJSONCfg(nil, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err != nil { -// t.Error(err) -// } else if err = jsonCfg.loaderCfg[0].loadFromJSONCfg(cfgJSON, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { -// t.Errorf("Expected %+v, received %+v", expected, err) -// } -// } - -func TestLoaderSCfgloadFromJsonCfgCase3(t *testing.T) { - cfg := &LoaderJsonCfg{ - Data: &[]*LoaderJsonDataType{ - { - Fields: &[]*FcTemplateJsonCfg{ - { - Value: utils.StringPointer("a{*"), - }, - }, - }, - }, - } - expected := "invalid converter terminator in rule: " - jsonCfg := NewDefaultCGRConfig() - if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestLoaderSCfgloadFromJsonCfgCase4(t *testing.T) { - cfg := &LoaderJsonCfg{ - Data: &[]*LoaderJsonDataType{ - { - Fields: &[]*FcTemplateJsonCfg{ - { - Type: utils.StringPointer(utils.MetaTemplate), - }, - }, - }, - }, - } - expected := "no template with id: <>" - jsonCfg := NewDefaultCGRConfig() - if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestLoaderSCfgloadFromJsonCfgCase5(t *testing.T) { - cfg := &LoaderJsonCfg{ - Data: &[]*LoaderJsonDataType{ - { - Fields: &[]*FcTemplateJsonCfg{ - { - Tag: utils.StringPointer("randomTag"), - Path: utils.StringPointer("randomPath"), - Type: utils.StringPointer(utils.MetaTemplate), - Value: utils.StringPointer("randomTemplate"), - }, - }, - }, - }, - } - expectedFields := LoaderSCfgs{ - { - Data: []*LoaderDataType{ - { - Fields: []*FCTemplate{ - { - Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - }, - }, - }, - }, - } - msgTemplates := map[string][]*FCTemplate{ - "randomTemplate": { - { - Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaVariable, - Value: NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - }, - } - jsonCfg := NewDefaultCGRConfig() - if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, msgTemplates, jsonCfg.generalCfg.RSRSep); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(jsonCfg.loaderCfg[0].Data[0].Fields[0], expectedFields[0].Data[0].Fields[0]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expectedFields[0].Data[0].Fields[0]), utils.ToJSON(jsonCfg.loaderCfg[0].Data[0].Fields[0])) - } - - if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(nil, msgTemplates, jsonCfg.generalCfg.RSRSep); err != nil { - t.Error(err) - } -} - -func TestLoaderSCfgloadFromJsonCfgCase6(t *testing.T) { - cfg := &LoaderJsonCfg{ - Data: &[]*LoaderJsonDataType{nil}, - } - jsonCfg := NewDefaultCGRConfig() - if err := jsonCfg.loaderCfg[0].loadFromJSONCfg(cfg, jsonCfg.templates, jsonCfg.generalCfg.RSRSep); err != nil { - t.Error(err) - } -} - -func TestEnabledCase1(t *testing.T) { - jsonCfg := NewDefaultCGRConfig() - - if enabled := jsonCfg.loaderCfg.Enabled(); enabled { - t.Errorf("Expected %+v", enabled) - } -} -func TestEnabledCase2(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "enabled": true, - }, - ], -}` - if jsonCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { - t.Error(err) - } else if enabled := jsonCfg.loaderCfg.Enabled(); !enabled { - t.Errorf("Expected %+v", enabled) - } -} - -func TestLoaderCfgloadFromJsonCfg(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "enabled": true, - "run_delay": "1sa", - }, - ], -}` - expected := "time: unknown unit \"sa\" in duration \"1sa\"" - if _, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err == nil || err.Error() != expected { - t.Errorf("Expected error: %s ,received: %v", expected, err) - } -} - -func TestLoaderCfgAsMapInterfaceCase1(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "id": "*default", - "enabled": false, - "tenant": "~*req.Destination1", - "dry_run": false, - "run_delay": "0", - "lockfile_path": ".cgr.lck", - "caches_conns": ["*internal:*caches"], - "field_separator": ",", - "tp_in_dir": "/var/spool/cgrates/loader/in", - "tp_out_dir": "/var/spool/cgrates/loader/out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - ], - }, - ], - }, - ], -}` - eMap := []map[string]any{ - { - utils.IDCfg: "*default", - utils.EnabledCfg: false, - utils.TenantCfg: "~*req.Destination1", - utils.DryRunCfg: false, - utils.RunDelayCfg: "0", - utils.LockFilePathCfg: ".cgr.lck", - utils.CachesConnsCfg: []string{utils.MetaInternal}, - utils.FieldSepCfg: ",", - utils.TpInDirCfg: "/var/spool/cgrates/loader/in", - utils.TpOutDirCfg: "/var/spool/cgrates/loader/out", - utils.DataCfg: []map[string]any{ - { - utils.TypeCfg: "*attributes", - utils.FilenameCfg: "Attributes.csv", - utils.FieldsCfg: []map[string]any{ - { - utils.TagCfg: "TenantID", - utils.PathCfg: "Tenant", - utils.TypeCfg: "*variable", - utils.ValueCfg: "~*req.0", - utils.MandatoryCfg: true, - }, { - utils.TagCfg: "ProfileID", - utils.PathCfg: "ID", - utils.TypeCfg: "*variable", - utils.ValueCfg: "~*req.1", - utils.MandatoryCfg: true, - }, - }, - }, - }, - }, - } - if cfgCgr, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { - t.Error(err) - } else { - rcv := cgrCfg.loaderCfg.AsMapInterface(cfgCgr.generalCfg.RSRSep) - if !reflect.DeepEqual(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0], - rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0]), - utils.ToJSON(rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[0])) - } else if !reflect.DeepEqual(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1], - rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1]) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(eMap[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1]), - utils.ToJSON(rcv[0][utils.DataCfg].([]map[string]any)[0][utils.FieldsCfg].([]map[string]any)[1])) - } else if !reflect.DeepEqual(eMap[0][utils.CachesConnsCfg], rcv[0][utils.CachesConnsCfg]) { - t.Errorf("Expected %+v, received %+v", eMap[0][utils.CachesConnsCfg], rcv[0][utils.CachesConnsCfg]) - } else if !reflect.DeepEqual(eMap[0][utils.TpInDirCfg], rcv[0][utils.TpInDirCfg]) { - t.Errorf("Expected %+v, received %+v", eMap[0][utils.TpInDirCfg], rcv[0][utils.TpInDirCfg]) - } else if !reflect.DeepEqual(eMap[0][utils.LockFilePathCfg], rcv[0][utils.LockFilePathCfg]) { - t.Errorf("Expected %+v, received %+v", eMap[0][utils.LockFilePathCfg], rcv[0][utils.LockFilePathCfg]) - } - } -} - -func TestLoaderCfgAsMapInterfaceCase2(t *testing.T) { - cfgJSONStr := `{ - "loaders": [ - { - "id": "*default", - "enabled": false, - "tenant": "~*req.Destination1", - "dry_run": false, - "run_delay": "1", - "lockfile_path": ".cgr.lck", - "caches_conns": ["*conn1"], - "field_separator": ",", - "tp_in_dir": "/var/spool/cgrates/loader/in", - "tp_out_dir": "/var/spool/cgrates/loader/out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - ], - }, - ], - }, - ], -}` - eMap := []map[string]any{ - { - utils.IDCfg: "*default", - utils.EnabledCfg: false, - utils.TenantCfg: "~*req.Destination1", - utils.DryRunCfg: false, - utils.RunDelayCfg: "0", - utils.LockFilePathCfg: ".cgr.lck", - utils.CachesConnsCfg: []string{"*conn1"}, - utils.FieldSepCfg: ",", - utils.TpInDirCfg: "/var/spool/cgrates/loader/in", - utils.TpOutDirCfg: "/var/spool/cgrates/loader/out", - utils.DataCfg: []map[string]any{ - { - utils.TypeCfg: "*attributes", - utils.FilenameCfg: "Attributes.csv", - utils.FieldsCfg: []map[string]any{ - { - utils.TagCfg: "TenantID", - utils.PathCfg: "Tenant", - utils.TypeCfg: "*variable", - utils.ValueCfg: "~*req.0", - utils.MandatoryCfg: true, - }, { - utils.TagCfg: "ProfileID", - utils.PathCfg: "ID", - utils.TypeCfg: "*variable", - utils.ValueCfg: "~*req.1", - utils.MandatoryCfg: true, - }, - }, - }, - }, - }, - } - if jsonCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { - t.Error(err) - } else if rcv := jsonCfg.loaderCfg.AsMapInterface(jsonCfg.generalCfg.RSRSep); !reflect.DeepEqual(rcv[0][utils.Tenant], eMap[0][utils.Tenant]) { - t.Errorf("Expected %+v, received %+v", rcv[0][utils.Tenant], eMap[0][utils.Tenant]) - } -} - -func TestLoaderSCfgsClone(t *testing.T) { - ban := LoaderSCfgs{{ - Enabled: true, - ID: utils.MetaDefault, - Tenant: "cgrate.org", - LockFilePath: ".cgr.lck", - CacheSConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches), "*conn1"}, - FieldSeparator: ",", - TpInDir: "/var/spool/cgrates/loader/in", - TpOutDir: "/var/spool/cgrates/loader/out", - Data: []*LoaderDataType{{ - Type: "*attributes", - Filename: "Attributes.csv", - Flags: utils.FlagsWithParams{}, - Fields: []*FCTemplate{ - { - Tag: "TenantID", - Path: "Tenant", - pathSlice: []string{"Tenant"}, - Type: utils.MetaComposed, - Value: NewRSRParsersMustCompile("cgrate.org", utils.InfieldSep), - Mandatory: true, - Layout: time.RFC3339, - }, - }}, - }, - }} - rcv := ban.Clone() - if !reflect.DeepEqual(ban, rcv) { - t.Errorf("Expected: %+v\nReceived: %+v", utils.ToJSON(ban), utils.ToJSON(rcv)) - } - if rcv[0].CacheSConns[1] = ""; ban[0].CacheSConns[1] != "*conn1" { - t.Errorf("Expected clone to not modify the cloned") - } - if rcv[0].Data[0].Type = ""; ban[0].Data[0].Type != "*attributes" { - t.Errorf("Expected clone to not modify the cloned") - } -} - -func TestLockFolderRelativePath(t *testing.T) { - ldr := &LoaderSCfg{ - TpInDir: "/var/spool/cgrates/loader/in/", - TpOutDir: "/var/spool/cgrates/loader/out/", - LockFilePath: utils.ResourcesCsv, - } - - jsonCfg := &LoaderJsonCfg{ - ID: utils.StringPointer("loaderid"), - Enabled: utils.BoolPointer(true), - Tenant: utils.StringPointer("cgrates.org"), - Dry_run: utils.BoolPointer(false), - Lockfile_path: utils.StringPointer(utils.ResourcesCsv), - Field_separator: utils.StringPointer(utils.InfieldSep), - Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"), - Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"), - } - expPath := path.Join(ldr.LockFilePath) - if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil { - t.Error(err) - } else if ldr.LockFilePath != expPath { - t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath) - } -} -func TestLockFolderNonRelativePath(t *testing.T) { - ldr := &LoaderSCfg{ - TpInDir: "/var/spool/cgrates/loader/in/", - TpOutDir: "/var/spool/cgrates/loader/out/", - LockFilePath: utils.ResourcesCsv, - } - - jsonCfg := &LoaderJsonCfg{ - ID: utils.StringPointer("loaderid"), - Enabled: utils.BoolPointer(true), - Tenant: utils.StringPointer("cgrates.org"), - Dry_run: utils.BoolPointer(false), - Lockfile_path: utils.StringPointer(path.Join("/tmp/", utils.ResourcesCsv)), - Field_separator: utils.StringPointer(utils.InfieldSep), - Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"), - Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"), - } - expPath := path.Join("/tmp/", utils.ResourcesCsv) - if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil { - t.Error(err) - } else if ldr.LockFilePath != expPath { - t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath) - } -} - -func TestLockFolderIsDir(t *testing.T) { - ldr := &LoaderSCfg{ - LockFilePath: "test", - } - - jsonCfg := &LoaderJsonCfg{ - ID: utils.StringPointer("loaderid"), - Enabled: utils.BoolPointer(true), - Tenant: utils.StringPointer("cgrates.org"), - Dry_run: utils.BoolPointer(false), - Lockfile_path: utils.StringPointer("/tmp"), - Field_separator: utils.StringPointer(utils.InfieldSep), - Tp_in_dir: utils.StringPointer("/var/spool/cgrates/loader/in/"), - Tp_out_dir: utils.StringPointer("/var/spool/cgrates/loader/out/"), - } - expPath := path.Join("/tmp") - - if err := ldr.loadFromJSONCfg(jsonCfg, map[string][]*FCTemplate{}, utils.InfieldSep); err != nil { - t.Error(err) - } else if ldr.LockFilePath != expPath { - t.Errorf("Expected %v \n but received \n %v", expPath, ldr.LockFilePath) - } -} - -func TestLockGetLockFilePath(t *testing.T) { - - l := LoaderSCfg{ - LockFilePath: "folder", - TpInDir: "/dir", - ID: "4"} - expected := "/dir/folder" - if val := l.GetLockFilePath(); val != expected { - t.Error(val) - } - -} - -func TestLoadersCfgGetLockFilePath(t *testing.T) { - str := "test" - bl := true - l := LoaderSCfg{ - ID: str, - Enabled: bl, - Tenant: str, - DryRun: bl, - RunDelay: 1 * time.Second, - LockFilePath: str, - CacheSConns: []string{str}, - FieldSeparator: str, - TpInDir: str, - TpOutDir: str, - Data: []*LoaderDataType{}, - } - - rcv := l.GetLockFilePath() - - if rcv != "test/test" { - t.Error(rcv) - } -} - -func TestGetLockFilePath(t *testing.T) { - loader := LoaderSCfg{ - LockFilePath: "/cgrates/cgrates/lockfile.lck", - } - if loader.GetLockFilePath() != "/cgrates/cgrates/lockfile.lck" { - t.Error("Expected /cgrates/cgrates/lockfile.lck") - } - - loader = LoaderSCfg{ - LockFilePath: "relative.lck", - TpInDir: "/base/dir", - } - if loader.GetLockFilePath() != "/base/dir/relative.lck" { - t.Error("Expected /base/dir/relative.lck") - } - - tmpDir := "/tmp/cgrates_cgrates" - _ = os.Mkdir(tmpDir, 0755) - defer os.Remove(tmpDir) - - loader = LoaderSCfg{ - LockFilePath: tmpDir, - ID: "loader123", - } - expected := "/tmp/cgrates_cgrates/loader123.lck" - if loader.GetLockFilePath() != expected { - t.Errorf("Expected %v, but got %v", expected, loader.GetLockFilePath()) - } -} diff --git a/console/loader_load.go b/console/loader_load.go deleted file mode 100644 index 7457da178..000000000 --- a/console/loader_load.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/loaders" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdLoaderLoad{ - name: "loader_load", - rpcMethod: utils.LoaderSv1Load, - rpcParams: &loaders.ArgsProcessFolder{}, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -type CmdLoaderLoad struct { - name string - rpcMethod string - rpcParams *loaders.ArgsProcessFolder - *CommandExecuter -} - -func (self *CmdLoaderLoad) Name() string { - return self.name -} - -func (self *CmdLoaderLoad) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdLoaderLoad) RpcParams(reset bool) any { - if reset || self.rpcParams == nil { - self.rpcParams = &loaders.ArgsProcessFolder{} - } - return self.rpcParams -} - -func (self *CmdLoaderLoad) PostprocessRpcParams() error { - return nil -} - -func (self *CmdLoaderLoad) RpcResult() any { - var s string - return &s -} diff --git a/console/loader_load_test.go b/console/loader_load_test.go deleted file mode 100644 index f116fc74d..000000000 --- a/console/loader_load_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package console - -import ( - "reflect" - "strings" - "testing" - - v1 "github.com/cgrates/cgrates/apier/v1" - - "github.com/cgrates/cgrates/utils" -) - -func TestCmdLoaderLoad(t *testing.T) { - // commands map is initiated in init function - command := commands["loader_load"] - // verify if ApierSv1 object has method on it - m, ok := reflect.TypeOf(new(v1.LoaderSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1]) - if !ok { - t.Fatal("method not found") - } - if m.Type.NumIn() != 4 { // expecting 4 inputs - t.Fatalf("invalid number of input parameters ") - } - // verify the type of input parameter - if ok := m.Type.In(2).AssignableTo(reflect.TypeOf(command.RpcParams(true))); !ok { - t.Fatalf("cannot assign input parameter") - } - // verify the type of output parameter - if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok { - t.Fatalf("cannot assign output parameter") - } - // for coverage purpose - if err := command.PostprocessRpcParams(); err != nil { - t.Fatal(err) - } -} diff --git a/console/loader_remove.go b/console/loader_remove.go deleted file mode 100644 index 0c20da872..000000000 --- a/console/loader_remove.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package console - -import ( - "github.com/cgrates/cgrates/loaders" - "github.com/cgrates/cgrates/utils" -) - -func init() { - c := &CmdLoaderRemove{ - name: "loader_remove", - rpcMethod: utils.LoaderSv1Remove, - rpcParams: &loaders.ArgsProcessFolder{}, - } - commands[c.Name()] = c - c.CommandExecuter = &CommandExecuter{c} -} - -type CmdLoaderRemove struct { - name string - rpcMethod string - rpcParams *loaders.ArgsProcessFolder - *CommandExecuter -} - -func (self *CmdLoaderRemove) Name() string { - return self.name -} - -func (self *CmdLoaderRemove) RpcMethod() string { - return self.rpcMethod -} - -func (self *CmdLoaderRemove) RpcParams(reset bool) any { - if reset || self.rpcParams == nil { - self.rpcParams = &loaders.ArgsProcessFolder{} - } - return self.rpcParams -} - -func (self *CmdLoaderRemove) PostprocessRpcParams() error { - return nil -} - -func (self *CmdLoaderRemove) RpcResult() any { - var s string - return &s -} diff --git a/console/loader_remove_test.go b/console/loader_remove_test.go deleted file mode 100644 index 21815e91b..000000000 --- a/console/loader_remove_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package console - -import ( - "reflect" - "strings" - "testing" - - v1 "github.com/cgrates/cgrates/apier/v1" - - "github.com/cgrates/cgrates/utils" -) - -func TestCmdLoaderRemove(t *testing.T) { - // commands map is initiated in init function - command := commands["loader_remove"] - // verify if ApierSv1 object has method on it - m, ok := reflect.TypeOf(new(v1.LoaderSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1]) - if !ok { - t.Fatal("method not found") - } - if m.Type.NumIn() != 4 { // expecting 4 inputs - t.Fatalf("invalid number of input parameters ") - } - // verify the type of input parameter - if ok := m.Type.In(2).AssignableTo(reflect.TypeOf(command.RpcParams(true))); !ok { - t.Fatalf("cannot assign input parameter") - } - // verify the type of output parameter - if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok { - t.Fatalf("cannot assign output parameter") - } - // for coverage purpose - if err := command.PostprocessRpcParams(); err != nil { - t.Fatal(err) - } -} diff --git a/console/ping.go b/console/ping.go index ef245a363..6563f981c 100644 --- a/console/ping.go +++ b/console/ping.go @@ -65,8 +65,6 @@ func (self *CmdApierPing) RpcMethod() string { return utils.ThresholdSv1Ping case utils.SessionsLow: return utils.SessionSv1Ping - case utils.LoaderSLow: - return utils.LoaderSv1Ping case utils.DispatcherSLow: return utils.DispatcherSv1Ping case utils.AnalyzerSLow: diff --git a/console/ping_test.go b/console/ping_test.go index 86a7cd683..dfbdea48c 100644 --- a/console/ping_test.go +++ b/console/ping_test.go @@ -303,45 +303,6 @@ func TestCmdPingSessionsLow(t *testing.T) { } } -func TestCmdPingLoaderSLow(t *testing.T) { - // commands map is initiated in init function - command := commands["ping"] - castCommand, canCast := command.(*CmdApierPing) - if !canCast { - t.Fatalf("cannot cast") - } - castCommand.item = utils.LoaderSLow - result2 := command.RpcMethod() - if !reflect.DeepEqual(result2, utils.LoaderSv1Ping) { - t.Errorf("Expected <%+v>, Received <%+v>", utils.LoaderSv1Ping, result2) - } - srv, err := engine.NewService(&v1.LoaderSv1{}) - if err != nil { - t.Fatal(err) - } - mType, ok := srv.Methods["Ping"] - if !ok { - t.Fatal("method not found") - } - m := mType.Method - if m.Type.NumIn() != 4 { // expecting 4 inputs - t.Fatalf("invalid number of input parameters ") - } - // for coverage purpose - result := command.RpcParams(true) - if !reflect.DeepEqual(result, new(StringWrapper)) { - t.Errorf("Expected <%T>, Received <%T>", new(StringWrapper), result) - } - // verify the type of output parameter - if ok := m.Type.In(3).AssignableTo(reflect.TypeOf(command.RpcResult())); !ok { - t.Fatalf("cannot assign output parameter") - } - // for coverage purpose - if err := command.PostprocessRpcParams(); err != nil { - t.Fatal(err) - } -} - func TestCmdPingDispatcherSLow(t *testing.T) { // commands map is initiated in init function command := commands["ping"] diff --git a/data/conf/samples/loaders/loaders_console_mongo/cgrates.json b/data/conf/samples/loaders/loaders_console_mongo/cgrates.json deleted file mode 100644 index 71762787a..000000000 --- a/data/conf/samples/loaders/loaders_console_mongo/cgrates.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - // CGRateS Configuration file - - -"general": { - "log_level": 7, - "reply_timeout": "30s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - - -"data_db": { - "db_type": "mongo", - "db_name": "10", - "db_port": 27017, -}, - - -"stor_db": { - "db_type": "mongo", - "db_name": "cgrates", - "db_port": 27017, - "db_password": "", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*localhost"], -}, - - -"cdrs": { - "enabled": true, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "stats_conns": ["*localhost"], - "resources_conns": ["*localhost"], -}, - - -"attributes": { // Attribute service - "enabled": true, // starts Attribute service: . -}, - - -"loaders": [ - { - "id": "CustomLoader", - "enabled": true, - "dry_run": false, - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/In", - "tp_out_dir": "/tmp/Out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, -], - - -"sessions": { - "enabled": true, - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator": { - "out_datadb_type": "mongo", - "out_datadb_port": "27017", - "out_datadb_name": "10", - "out_stordb_type": "mongo", - "out_stordb_port": "27017", - "out_stordb_name": "cgrates", - "users_filters":["Account"], -}, - - -"templates": { - "attrTemplateLoader": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], -}, - - - - "apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, -} - \ No newline at end of file diff --git a/data/conf/samples/loaders/loaders_console_mysql/cgrates.json b/data/conf/samples/loaders/loaders_console_mysql/cgrates.json deleted file mode 100644 index 68c193925..000000000 --- a/data/conf/samples/loaders/loaders_console_mysql/cgrates.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - // CGRateS Configuration file - // - - -"general": { - "log_level": 7, - "reply_timeout": "50s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: - "db_port": 6379, // data_db port to reach the database - "db_name": "10", // data_db database name to connect to -}, - -"stor_db": { - "db_password": "CGRateS.org", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*internal"], -}, - - -"cdrs": { - "enabled": true, - "chargers_conns":["*internal"], -}, - - -"loaders": [ - { - "id": "CustomLoader", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/In", - "tp_out_dir": "/tmp/Out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, -], - - - -"attributes": { - "enabled": true, - "prefix_indexed_fields":["*req.Destination"], -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "prefix_indexed_fields":["*req.Destination"], - "stats_conns": ["*internal"], - "resources_conns": ["*internal"], -}, - - -"sessions": { - "enabled": true, - "routes_conns": ["*internal"], - "resources_conns": ["*internal"], - "attributes_conns": ["*internal"], - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator":{ - "out_stordb_password": "CGRateS.org", - "users_filters":["Account"], -}, - - -"templates": { - "attrTemplateLoader": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], -}, - - - - "apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, -} \ No newline at end of file diff --git a/data/conf/samples/loaders/tutinternal/cgrates.json b/data/conf/samples/loaders/tutinternal/cgrates.json deleted file mode 100644 index 2c2c2b74b..000000000 --- a/data/conf/samples/loaders/tutinternal/cgrates.json +++ /dev/null @@ -1,256 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, - "reply_timeout": "50s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - - -"data_db": { - "db_type": "*internal", -}, - - -"stor_db": { - "db_type": "*internal" -}, - - -"rals": { - "enabled": true, -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*internal"], -}, - - -"cdrs": { - "enabled": true, - "chargers_conns":["*internal"], -}, - - -"loaders": [ - { - "id": "CustomLoader", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/In", - "tp_out_dir": "/tmp/Out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "WithoutMoveToOut", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/LoaderIn", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithoutMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathWithoutMove", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathLoaderWithMove", - "tp_out_dir": "/tmp/SubpathOut", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "LoaderWithTemplate", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/templateLoaderIn", - "tp_out_dir": "/tmp/templateLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} - ], - }, - ], - }, - { - "id": "CustomSep", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "-1", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": "\t", - "tp_in_dir": "/tmp/customSepLoaderIn", - "tp_out_dir": "/tmp/customSepLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"}, - ], - }, - ], - } -], - - - -"attributes": { - "enabled": true, - "prefix_indexed_fields":["*req.Destination"], -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - -"templates": { - "attrTemplateLoader": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], -}, - -"apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/data/conf/samples/loaders/tutmongo/cgrates.json b/data/conf/samples/loaders/tutmongo/cgrates.json deleted file mode 100644 index eba1182b1..000000000 --- a/data/conf/samples/loaders/tutmongo/cgrates.json +++ /dev/null @@ -1,304 +0,0 @@ -{ -// CGRateS Configuration file - - -"general": { - "log_level": 7, - "reply_timeout": "30s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - - -"data_db": { - "db_type": "mongo", - "db_name": "10", - "db_port": 27017, -}, - - -"stor_db": { - "db_type": "mongo", - "db_name": "cgrates", - "db_port": 27017, - "db_password": "", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*localhost"], -}, - - -"cdrs": { - "enabled": true, -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "stats_conns": ["*localhost"], - "resources_conns": ["*localhost"], -}, - - -"attributes": { // Attribute service - "enabled": true, // starts Attribute service: . -}, - - -"loaders": [ - { - "id": "CustomLoader", - "enabled": true, - "dry_run": false, - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/In", - "tp_out_dir": "/tmp/Out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "WithoutMoveToOut", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/LoaderIn", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithoutMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathWithoutMove", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathLoaderWithMove", - "tp_out_dir": "/tmp/SubpathOut", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "LoaderWithTemplate", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/templateLoaderIn", - "tp_out_dir": "/tmp/templateLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} - ], - }, - ], - }, - { - "id": "CustomSep", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "-1", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": "\t", - "tp_in_dir": "/tmp/customSepLoaderIn", - "tp_out_dir": "/tmp/customSepLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"}, - ], - }, - ], - } -], - - -"sessions": { - "enabled": true, - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator": { - "out_datadb_type": "mongo", - "out_datadb_port": "27017", - "out_datadb_name": "10", - "out_stordb_type": "mongo", - "out_stordb_port": "27017", - "out_stordb_name": "cgrates", - "users_filters":["Account"], -}, - - -"templates": { - "attrTemplateLoader": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], -}, - - - - "apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, -} diff --git a/data/conf/samples/loaders/tutmysql/cgrates.json b/data/conf/samples/loaders/tutmysql/cgrates.json deleted file mode 100644 index 0f0ea2a49..000000000 --- a/data/conf/samples/loaders/tutmysql/cgrates.json +++ /dev/null @@ -1,304 +0,0 @@ -{ -// CGRateS Configuration file -// - - -"general": { - "log_level": 7, - "reply_timeout": "50s", -}, - - -"listen": { - "rpc_json": ":2012", - "rpc_gob": ":2013", - "http": ":2080", -}, - -"data_db": { // database used to store runtime data (eg: accounts, cdr stats) - "db_type": "redis", // data_db type: - "db_port": 6379, // data_db port to reach the database - "db_name": "10", // data_db database name to connect to -}, - -"stor_db": { - "db_password": "CGRateS.org", -}, - - -"rals": { - "enabled": true, - "thresholds_conns": ["*internal"], -}, - - -"schedulers": { - "enabled": true, - "cdrs_conns": ["*internal"], -}, - - -"cdrs": { - "enabled": true, - "chargers_conns":["*internal"], -}, - - -"loaders": [ - { - "id": "CustomLoader", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/In", - "tp_out_dir": "/tmp/Out", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "WithoutMoveToOut", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/LoaderIn", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithoutMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathWithoutMove", - "tp_out_dir": "", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "SubpathLoaderWithMove", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/SubpathLoaderWithMove", - "tp_out_dir": "/tmp/SubpathOut", - "data":[ - { - "type": "*attributes", - "file_name": "folder1/Attributes.csv", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], - }, - ], - }, - { - "id": "LoaderWithTemplate", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "0", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": ",", - "tp_in_dir": "/tmp/templateLoaderIn", - "tp_out_dir": "/tmp/templateLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes.csv", - "fields": [ - {"tag": "AttributesFields","type": "*template", "value": "attrTemplateLoader"} - ], - }, - ], - }, - { - "id": "CustomSep", - "enabled": true, - "dry_run": false, - "tenant": "cgrates.org", - "run_delay": "-1", - "lockfile_path": ".cgr.lock", - "caches_conns": ["*internal"], - "field_separator": "\t", - "tp_in_dir": "/tmp/customSepLoaderIn", - "tp_out_dir": "/tmp/customSepLoaderOut", - "data":[ - { - "type": "*attributes", - "file_name": "Attributes", - "fields": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "cgrates.org", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "ATTR_;~*req.0", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "*any"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "`*string:~*req.Destination:`;~*req.0"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "*req.Destination"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "*constant"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.1"}, - ], - }, - ], - } -], - - - -"attributes": { - "enabled": true, - "prefix_indexed_fields":["*req.Destination"], -}, - - -"chargers": { - "enabled": true, - "attributes_conns": ["*internal"], -}, - - -"resources": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"] -}, - - -"stats": { - "enabled": true, - "store_interval": "1s", - "thresholds_conns": ["*internal"], -}, - -"thresholds": { - "enabled": true, - "store_interval": "1s", -}, - - -"routes": { - "enabled": true, - "prefix_indexed_fields":["*req.Destination"], - "stats_conns": ["*internal"], - "resources_conns": ["*internal"], -}, - - -"sessions": { - "enabled": true, - "routes_conns": ["*internal"], - "resources_conns": ["*internal"], - "attributes_conns": ["*internal"], - "rals_conns": ["*internal"], - "cdrs_conns": ["*internal"], - "chargers_conns": ["*internal"], -}, - - -"migrator":{ - "out_stordb_password": "CGRateS.org", - "users_filters":["Account"], -}, - - -"templates": { - "attrTemplateLoader": [ - {"tag": "TenantID", "path": "Tenant", "type": "*variable", "value": "~*req.0", "mandatory": true}, - {"tag": "ProfileID", "path": "ID", "type": "*variable", "value": "~*req.1", "mandatory": true}, - {"tag": "Contexts", "path": "Contexts", "type": "*variable", "value": "~*req.2"}, - {"tag": "FilterIDs", "path": "FilterIDs", "type": "*variable", "value": "~*req.3"}, - {"tag": "ActivationInterval", "path": "ActivationInterval", "type": "*variable", "value": "~*req.4"}, - {"tag": "AttributeFilterIDs", "path": "AttributeFilterIDs", "type": "*variable", "value": "~*req.5"}, - {"tag": "Path", "path": "Path", "type": "*variable", "value": "~*req.6"}, - {"tag": "Type", "path": "Type", "type": "*variable", "value": "~*req.7"}, - {"tag": "Value", "path": "Value", "type": "*variable", "value": "~*req.8"}, - {"tag": "Blocker", "path": "Blocker", "type": "*variable", "value": "~*req.9"}, - {"tag": "Weight", "path": "Weight", "type": "*variable", "value": "~*req.10"}, - ], -}, - - - - "apiers": { - "enabled": true, - "scheduler_conns": ["*internal"], -}, - - -} diff --git a/integration_test.sh b/integration_test.sh index 0c89839d9..ab989c4df 100755 --- a/integration_test.sh +++ b/integration_test.sh @@ -8,7 +8,7 @@ # Example: # ./integration_test.sh -dbtype=*mysql -rpc=*gob -packages=("agents" "apier/v1" "apier/v2" "cmd/cgr-loader" "dispatchers" "engine" "ers" "general_tests" "loaders" "registrarc" "sessions") +packages=("agents" "apier/v1" "apier/v2" "cmd/cgr-loader" "dispatchers" "engine" "ers" "general_tests" "registrarc" "sessions") dbtypes=("*internal" "*mysql" "*mongo" "*postgres") # Tests that are independent of the dbtype flag and run only once diff --git a/loaders/lib_test.go b/loaders/lib_test.go deleted file mode 100644 index 482a4bc27..000000000 --- a/loaders/lib_test.go +++ /dev/null @@ -1,739 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "encoding/csv" - "io" - "strings" - "testing" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/rpcclient" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var loaderPaths = []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut", - "/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} - -type testMockCacheConn struct { - calls map[string]func(arg any, rply any) error -} - -func (s *testMockCacheConn) Call(ctx *context.Context, method string, arg any, rply any) error { - if call, has := s.calls[method]; !has { - return rpcclient.ErrUnsupporteServiceMethod - } else { - return call(arg, rply) - } -} - -func TestProcessContentCallsRemoveItems(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1RemoveItems: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestProcessContentCallsRemoveItems", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_ID -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(utils.MetaAttributes, utils.MetaRemove); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - // Calling the method again while caching method is invalid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected = "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestProcessContentCallsClear(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestProcessContentCallsClear", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_ID -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(utils.MetaAttributes, utils.MetaClear); err != nil { - t.Error(err) - } - - //inexisting method(*none) of cache and reinitialized the reader will do nothing - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.processContent(utils.MetaAttributes, utils.MetaNone); err != nil { - t.Error(err) - } - - // Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.processContent(utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsReload(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1ReloadCache: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_2 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_2", - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaReload); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaReload); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsLoad(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1LoadCache: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_3 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_3", - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaLoad); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaLoad); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveContentCallsRemove(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1RemoveItems: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_4 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_4", - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaRemove); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaRemove); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - //inexisting method(*none) of cache and reinitialized the reader will do nothing - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaNone); err != nil { - t.Error(err) - } -} - -func TestRemoveContentCallsClear(t *testing.T) { - // Clear cache because connManager sets the internal connection in cache - engine.Cache.Clear([]string{utils.CacheRPCConnections}) - - sMock := &testMockCacheConn{ - calls: map[string]func(arg any, rply any) error{ - utils.CacheSv1Clear: func(arg any, rply any) error { - prply, can := rply.(*string) - if !can { - t.Errorf("Wrong argument type : %T", rply) - return nil - } - *prply = utils.OK - return nil - }, - }, - } - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - - internalCacheSChan := make(chan birpc.ClientConnector, 1) - internalCacheSChan <- sMock - ldr := &Loader{ - ldrID: "TestRemoveContentCallsReload", - bufLoaderData: make(map[string][]LoaderData), - connMgr: engine.NewConnManager(config.CgrConfig(), map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): internalCacheSChan, - }), - cacheConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - attributeCsv := ` -#Tenant[0],ID[1] -cgrates.org,MOCK_RELOAD_3 -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - attrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "MOCK_RELOAD_3", - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaClear); err != nil { - t.Error(err) - } - - //Calling the method again while cacheConnsID is not valid - ldr.cacheConns = []string{utils.MetaInternal} - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - - //set and remove again from database - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - expected := "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(utils.MetaAttributes, utils.MetaClear); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - // Calling the method again while caching method is invalid - rdr = io.NopCloser(strings.NewReader(attributeCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{ - fileName: utils.AttributesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.dm.SetAttributeProfile(attrPrf, true); err != nil { - t.Error(err) - } - expected = "UNSUPPORTED_SERVICE_METHOD" - if err := ldr.removeContent(utils.MetaAttributes, "invalid_caching_api"); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} diff --git a/loaders/libloader.go b/loaders/libloader.go deleted file mode 100644 index dc51ed084..000000000 --- a/loaders/libloader.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "fmt" - "strconv" - "strings" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type LoaderData map[string]any - -func (ld LoaderData) TenantID() string { - return utils.ConcatenatedKey(utils.IfaceAsString(ld[utils.Tenant]), - utils.IfaceAsString(ld[utils.ID])) -} - -func (ld LoaderData) GetRateIDs() ([]string, error) { - if _, has := ld[utils.RateIDs]; !has { - return nil, fmt.Errorf("cannot find RateIDs in <%+v>", ld) - } - if rateIDs := ld[utils.RateIDs].(string); len(rateIDs) != 0 { - return strings.Split(rateIDs, utils.InfieldSep), nil - } - return []string{}, nil -} - -// UpdateFromCSV will update LoaderData with data received from fileName, -// contained in record and processed with cfgTpl -func (ld LoaderData) UpdateFromCSV(fileName string, record []string, - cfgTpl []*config.FCTemplate, tnt string, filterS *engine.FilterS) (err error) { - csvProvider := newCsvProvider(record, fileName) - tenant := tnt - for _, cfgFld := range cfgTpl { - // Make sure filters are matching - if len(cfgFld.Filters) != 0 { - if pass, err := filterS.Pass(tenant, - cfgFld.Filters, csvProvider); err != nil { - return err - } else if !pass { - continue // Not passes filters, ignore this CDR - } - } - out, err := cfgFld.Value.ParseDataProvider(csvProvider) - if err != nil { - return err - } - switch cfgFld.Type { - case utils.MetaComposed: - if _, has := ld[cfgFld.Path]; !has { - ld[cfgFld.Path] = out - } else if valOrig, canCast := ld[cfgFld.Path].(string); canCast { - valOrig += out - ld[cfgFld.Path] = valOrig - } - case utils.MetaVariable: - ld[cfgFld.Path] = out - case utils.MetaString: - if _, has := ld[cfgFld.Path]; !has { - ld[cfgFld.Path] = out - } - } - } - return -} - -// newCsvProvider constructs a DataProvider -func newCsvProvider(record []string, fileName string) (dP utils.DataProvider) { - return &csvProvider{ - req: record, - fileName: fileName, - cache: utils.MapStorage{}, - cfg: config.CgrConfig().GetDataProvider(), - } -} - -// csvProvider implements utils.DataProvider so we can pass it to filters -type csvProvider struct { - req []string - fileName string - cache utils.MapStorage - cfg utils.DataProvider -} - -// String is part of utils.DataProvider interface -// when called, it will display the already parsed values out of cache -func (cP *csvProvider) String() string { - return utils.ToJSON(cP) -} - -// FieldAsInterface is part of utils.DataProvider interface -func (cP *csvProvider) FieldAsInterface(fldPath []string) (data any, err error) { - if data, err = cP.cache.FieldAsInterface(fldPath); err == nil || - err != utils.ErrNotFound { // item found in cache - return - } - err = nil // cancel previous err - - switch { - case strings.HasPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart): - fileName := strings.TrimPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart) - hasSelEnd := false - for _, val := range fldPath[1:] { - if hasSelEnd = strings.HasSuffix(val, utils.FilterValEnd); hasSelEnd { - fileName = fileName + utils.NestingSep + val[:len(val)-1] - break - } - fileName = fileName + utils.NestingSep + val - } - if !hasSelEnd { - return nil, fmt.Errorf("filter rule <%s> needs to end in )", fldPath) - } - if cP.fileName != fileName { - cP.cache.Set(fldPath, nil) - return - } - case fldPath[0] == utils.MetaReq: - case fldPath[0] == utils.MetaCfg: - data, err = cP.cfg.FieldAsInterface(fldPath[1:]) - if err != nil { - return - } - cP.cache.Set(fldPath, data) - return - default: - return nil, fmt.Errorf("invalid prefix for : %s", fldPath) - } - var cfgFieldIdx int - if cfgFieldIdx, err = strconv.Atoi(fldPath[len(fldPath)-1]); err != nil || len(cP.req) <= cfgFieldIdx { - return nil, fmt.Errorf("Ignoring record: %q with error : %+v", cP.req, err) - } - data = cP.req[cfgFieldIdx] - - cP.cache.Set(fldPath, data) - return -} - -// FieldAsString is part of utils.DataProvider interface -func (cP *csvProvider) FieldAsString(fldPath []string) (data string, err error) { - var valIface any - valIface, err = cP.FieldAsInterface(fldPath) - if err != nil { - return - } - return utils.IfaceAsString(valIface), nil -} diff --git a/loaders/libloader_test.go b/loaders/libloader_test.go deleted file mode 100644 index fc3916b71..000000000 --- a/loaders/libloader_test.go +++ /dev/null @@ -1,443 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "fmt" - "reflect" - "testing" - - "github.com/cgrates/cgrates/engine" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -func TestDataUpdateFromCSVOneFile(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Tag: "Contexts", - Path: "Contexts", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Tag: "Path", - Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Tag: "Initial", - Path: "Initial", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Tag: "Substitute", - Path: "Substitute", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Tag: "Append", - Path: "Append", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - } - - rows := [][]string{ - {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, - {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, - } - lData := make(LoaderData) - if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*sessions;*cdrs", - "FilterIDs": "*string:Account:1007", - "ActivationInterval": "2014-01-14T00:00:00Z", - "Path": "Account", - "Initial": "*any", - "Substitute": "1001", - "Append": "false", - "Weight": "10", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } - lData = make(LoaderData) - if err := lData.UpdateFromCSV("Attributes.csv", rows[1], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData = LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "", - "FilterIDs": "", - "ActivationInterval": "", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - "Weight": "", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } -} - -func TestDataUpdateFromCSVOneFile2(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Tag: "Contexts", - Path: "Contexts", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Tag: "Path", - Path: "Path", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Tag: "Initial", - Path: "Initial", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Tag: "Substitute", - Path: "Substitute", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Tag: "Append", - Path: "Append", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - } - - rows := [][]string{ - {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, - {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, - } - lData := make(LoaderData) - if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*sessions;*cdrs", - "FilterIDs": "*string:Account:1007", - "ActivationInterval": "2014-01-14T00:00:00Z", - "Path": "Account", - "Initial": "*any", - "Substitute": "1001", - "Append": "false", - "Weight": "10", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } - lData = make(LoaderData) - if err := lData.UpdateFromCSV("Attributes.csv", rows[1], attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - eLData = LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "", - "FilterIDs": "", - "ActivationInterval": "", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - "Weight": "", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } -} - -func TestDataUpdateFromCSVMultiFiles(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Mandatory: true}, - {Tag: "Contexts", - Path: "Contexts", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("*any", utils.InfieldSep)}, - {Tag: "Path", - Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).5", utils.InfieldSep)}, - {Tag: "Initial", - Path: "Initial", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.InfieldSep)}, - {Tag: "Substitute", - Path: "Substitute", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.InfieldSep)}, - {Tag: "Append", - Path: "Append", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("true", utils.InfieldSep)}, - {Tag: "Weight", - Path: "Weight", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("10", utils.InfieldSep)}, - } - - loadRun1 := map[string][]string{ - "File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, - "File2.csv": {"ignored", "ATTR_1"}, - } - lData := make(LoaderData) - for fName, record := range loadRun1 { - if err := lData.UpdateFromCSV(fName, record, attrSFlds, - "cgrates.org", nil); err != nil { - t.Error(err) - } - } - - eLData := LoaderData{"Tenant": "cgrates.org", - "ID": "ATTR_1", - "Contexts": "*any", - "Path": "Subject", - "Initial": "*any", - "Substitute": "1001", - "Append": "true", - "Weight": "10", - } - if !reflect.DeepEqual(eLData, lData) { - t.Errorf("expecting: %+v, received: %+v", eLData, lData) - } -} - -func TestGetRateIDsLoaderData(t *testing.T) { - ldrData := LoaderData{ - "File1.csv": []string{"Subject", "*any", "1001"}, - } - expected := "cannot find RateIDs in " - if _, err := ldrData.GetRateIDs(); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -// func TestUpdateFromCsvParseValueError(t *testing.T) { -// ldrData := LoaderData{ -// "File1.csv": []string{"Subject", "*any", "1001"}, -// } -// tnt := config.NewRSRParsersMustCompile("asd{*duration_seconds}", utils.InfieldSep) -// expected := "time: invalid duration \"asd\"" -// if err := ldrData.UpdateFromCSV("File1.csv", nil, nil, tnt, nil); err == nil || err.Error() != expected { -// t.Errorf("Expected %+v, received %+v", expected, err) -// } -// } - -func TestUpdateFromCsvWithFiltersError(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Filters: []string{"*string:~*req.Account:10"}, - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Filters: []string{"*string:~*req.Account:10"}, - Mandatory: true}, - } - loadRunStr := map[string][]string{ - "File1.csv": {"cgrates.org", "TEST_1"}, - } - lData := make(LoaderData) - - cfg := config.NewDefaultCGRConfig() - data, err := engine.NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - filterS := engine.NewFilterS(cfg, nil, dm) - - for fName, record := range loadRunStr { - expected := "Ignoring record: [\"cgrates.org\" \"TEST_1\"] with error : strconv.Atoi: parsing \"Account\": invalid syntax" - if err := lData.UpdateFromCSV(fName, record, attrSFlds, - "cgrates.org", filterS); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - } -} - -func TestUpdateFromCsvWithFiltersContinue(t *testing.T) { - attrSFlds := []*config.FCTemplate{ - {Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Filters: []string{`*string:~*req.2:10`}, - Mandatory: true}, - {Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Filters: []string{`*string:~*req.2:10`}, - Mandatory: true}, - } - loadRunStr := map[string][]string{ - "File1.csv": {"Subject", "*any", "1001"}, - } - lData := make(LoaderData) - - cfg := config.NewDefaultCGRConfig() - data, err := engine.NewInternalDB(nil, nil, true, nil, cfg.DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - filterS := engine.NewFilterS(cfg, nil, dm) - - for fName, record := range loadRunStr { - if err := lData.UpdateFromCSV(fName, record, attrSFlds, - "cgrates.org", filterS); err != nil { - t.Error(err) - } - } -} - -func TestLoadersFieldAsInterfaceError(t *testing.T) { - loadRun1 := map[string][]string{ - "File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, - } - csvProv := newCsvProvider(loadRun1["File1.csv"], "File1.csv") - - csvString := csvProv.String() - expected := "{}" - if csvString != expected { - t.Errorf("Expected %+v, received %+v", expected, csvString) - } - - expected = "invalid prefix for : [File2.csv]" - if _, err := csvProv.FieldAsInterface([]string{"File2.csv"}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - expected = "filter rule <[*file() ]> needs to end in )" - if _, err := csvProv.FieldAsInterface([]string{"*file()", ""}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+q", expected, err) - } - - expected = "filter rule <[*file() File1.csv]> needs to end in )" - if _, err := csvProv.FieldAsInterface([]string{"*file()", "File1.csv"}); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+q", expected, err) - } -} - -func TestLibLoaderGetRateIDs(t *testing.T) { - cases := []struct { - input LoaderData - expectedIDs []string - expectedErr error - }{ - { - input: LoaderData{utils.RateIDs: "id1,id2,id3"}, - expectedIDs: []string{"id1,id2,id3"}, - expectedErr: nil, - }, - { - input: LoaderData{utils.RateIDs: ""}, - expectedIDs: []string{}, - expectedErr: nil, - }, - { - input: LoaderData{}, - expectedIDs: nil, - expectedErr: fmt.Errorf("cannot find RateIDs in "), - }, - } - for _, c := range cases { - ids, err := c.input.GetRateIDs() - if c.expectedErr != nil { - if err == nil || err.Error() != c.expectedErr.Error() { - t.Errorf("Expected error: %v, got: %v", c.expectedErr, err) - } - } else { - if err != nil { - t.Errorf("Expected no error, got: %v", err) - } - } - if !reflect.DeepEqual(ids, c.expectedIDs) { - t.Errorf("Expected IDs: %v, got: %v", c.expectedIDs, ids) - } - } -} - -func TestLibLoaderFieldAsStringErrorHandling(t *testing.T) { - cP := &csvProvider{} - _, err := cP.FieldAsString([]string{"invalid", "path"}) - if err == nil { - t.Error("Expected an error, but got nil") - } -} diff --git a/loaders/loader.go b/loaders/loader.go deleted file mode 100644 index 465062ab4..000000000 --- a/loaders/loader.go +++ /dev/null @@ -1,1016 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "encoding/csv" - "fmt" - "io" - "os" - "path" - "strings" - "time" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type openedCSVFile struct { - fileName string - rdr io.ReadCloser // keep reference so we can close it when done - csvRdr *csv.Reader -} - -func NewLoader(dm *engine.DataManager, cfg *config.LoaderSCfg, - timezone string, cachingDlay time.Duration, filterS *engine.FilterS, - connMgr *engine.ConnManager, cacheConns []string) (ldr *Loader) { - ldr = &Loader{ - enabled: cfg.Enabled, - tenant: cfg.Tenant, - dryRun: cfg.DryRun, - ldrID: cfg.ID, - tpInDir: cfg.TpInDir, - tpOutDir: cfg.TpOutDir, - lockFilepath: cfg.GetLockFilePath(), - fieldSep: cfg.FieldSeparator, - runDelay: cfg.RunDelay, - cachingDelay: cachingDlay, - dataTpls: make(map[string][]*config.FCTemplate), - flagsTpls: make(map[string]utils.FlagsWithParams), - rdrs: make(map[string]map[string]*openedCSVFile), - bufLoaderData: make(map[string][]LoaderData), - dm: dm, - timezone: timezone, - filterS: filterS, - connMgr: connMgr, - cacheConns: cacheConns, - } - for _, ldrData := range cfg.Data { - ldr.dataTpls[ldrData.Type] = ldrData.Fields - ldr.flagsTpls[ldrData.Type] = ldrData.Flags - ldr.rdrs[ldrData.Type] = make(map[string]*openedCSVFile) - if ldrData.Filename != "" { - ldr.rdrs[ldrData.Type][ldrData.Filename] = nil - } - for _, cfgFld := range ldrData.Fields { // add all possible files to be opened - for _, cfgFldVal := range cfgFld.Value { - rule := cfgFldVal.Rules - if !strings.HasPrefix(rule, utils.DynamicDataPrefix+utils.MetaFile+utils.FilterValStart) { - continue - } - if idxEnd := strings.Index(rule, utils.FilterValEnd); idxEnd != -1 { - ldr.rdrs[ldrData.Type][rule[7:idxEnd]] = nil - } - } - } - } - return -} - -// Loader is one instance loading from a folder -type Loader struct { - enabled bool - tenant string - dryRun bool - ldrID string - tpInDir string - tpOutDir string - lockFilepath string - fieldSep string - runDelay time.Duration - cachingDelay time.Duration - dataTpls map[string][]*config.FCTemplate // map[loaderType]*config.FCTemplate - flagsTpls map[string]utils.FlagsWithParams //map[loaderType]utils.FlagsWithParams - rdrs map[string]map[string]*openedCSVFile // map[loaderType]map[fileName]*openedCSVFile for common incremental read - bufLoaderData map[string][]LoaderData // cache of data read, indexed on tenantID - dm *engine.DataManager - timezone string - filterS *engine.FilterS - connMgr *engine.ConnManager - cacheConns []string -} - -func (ldr *Loader) ListenAndServe(stopChan chan struct{}) (err error) { - utils.Logger.Info(fmt.Sprintf("Starting <%s-%s>", utils.LoaderS, ldr.ldrID)) - return ldr.serve(stopChan) -} - -// ProcessFolder will process the content in the folder with locking -func (ldr *Loader) ProcessFolder(caching, loadOption string, stopOnError bool) (err error) { - if err = ldr.lockFolder(); err != nil { - return - } - defer ldr.unlockFolder() - for ldrType := range ldr.rdrs { - if err = ldr.processFiles(ldrType, caching, loadOption); err != nil { - if stopOnError { - return - } - utils.Logger.Warning(fmt.Sprintf("<%s-%s> loaderType: <%s> cannot open files, err: %s", - utils.LoaderS, ldr.ldrID, ldrType, err.Error())) - continue - } - } - return ldr.moveFiles() -} - -// lockFolder will attempt to lock the folder by creating the lock file -func (ldr *Loader) lockFolder() (err error) { - // If the path is an empty string, we should not be locking - if ldr.lockFilepath == utils.EmptyString { - return - } - _, err = os.OpenFile(ldr.lockFilepath, - os.O_RDONLY|os.O_CREATE, 0644) - return -} - -func (ldr *Loader) unlockFolder() (err error) { - // If the path is an empty string, we should not be locking - if ldr.lockFilepath == utils.EmptyString { - return - } - if _, err = os.Stat(ldr.lockFilepath); err == nil { - return os.Remove(ldr.lockFilepath) - } - return -} - -func (ldr *Loader) isFolderLocked() (locked bool, err error) { - // If the path is an empty string, we should not be locking - if ldr.lockFilepath == utils.EmptyString { - return - } - if _, err = os.Stat(ldr.lockFilepath); err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return -} - -// unreferenceFile will cleanup an used file by closing and removing from referece map -func (ldr *Loader) unreferenceFile(loaderType, fileName string) (err error) { - openedCSVFile := ldr.rdrs[loaderType][fileName] - ldr.rdrs[loaderType][fileName] = nil - return openedCSVFile.rdr.Close() -} - -func (ldr *Loader) moveFiles() (err error) { - if ldr.tpOutDir == utils.EmptyString { - return - } - filesInDir, _ := os.ReadDir(ldr.tpInDir) - for _, file := range filesInDir { - fName := file.Name() - if fName == ldr.lockFilepath { - continue - } - oldPath := path.Join(ldr.tpInDir, fName) - newPath := path.Join(ldr.tpOutDir, fName) - if err = os.Rename(oldPath, newPath); err != nil { - return - } - } - return -} - -func (ldr *Loader) processFiles(loaderType, caching, loadOption string) (err error) { - for fName := range ldr.rdrs[loaderType] { - var rdr *os.File - if rdr, err = os.Open(path.Join(ldr.tpInDir, fName)); err != nil { - return err - } - csvReader := csv.NewReader(rdr) - csvReader.Comma = rune(ldr.fieldSep[0]) - csvReader.Comment = '#' - ldr.rdrs[loaderType][fName] = &openedCSVFile{ - fileName: fName, rdr: rdr, csvRdr: csvReader} - defer ldr.unreferenceFile(loaderType, fName) - } - // based on load option will store or remove the content - switch loadOption { - case utils.MetaStore: - return ldr.processContent(loaderType, caching) - case utils.MetaRemove: - return ldr.removeContent(loaderType, caching) - } - return -} - -// processContent will process the contect and will store it into database -func (ldr *Loader) processContent(loaderType, caching string) (err error) { - // start processing lines - keepLooping := true // controls looping - lineNr := 0 - for keepLooping { - lineNr++ - var hasErrors bool - lData := make(LoaderData) // one row - for fName, rdr := range ldr.rdrs[loaderType] { - var record []string - if record, err = rdr.csvRdr.Read(); err != nil { - if err == io.EOF { - keepLooping = false - break - } - hasErrors = true - utils.Logger.Warning( - fmt.Sprintf("<%s> <%s> reading line: %d, error: %s", - utils.LoaderS, ldr.ldrID, lineNr, err.Error())) - } - if hasErrors { // if any of the readers will give errors, we ignore the line - continue - } - - if err := lData.UpdateFromCSV(fName, record, - ldr.dataTpls[loaderType], ldr.tenant, ldr.filterS); err != nil { - utils.Logger.Warning( - fmt.Sprintf("<%s> <%s> line: %d, error: %s", - utils.LoaderS, ldr.ldrID, lineNr, err.Error())) - hasErrors = true - continue - } - // Record from map - // update dataDB - } - if len(lData) == 0 { // no data, could be the last line in file - continue - } - tntID := lData.TenantID() - if _, has := ldr.bufLoaderData[tntID]; !has && - len(ldr.bufLoaderData) == 1 { // process previous records before going futher - var prevTntID string - for prevTntID = range ldr.bufLoaderData { - break // have stolen the existing key in buffer - } - if err = ldr.storeLoadedData(loaderType, - map[string][]LoaderData{prevTntID: ldr.bufLoaderData[prevTntID]}, caching); err != nil { - return - } - delete(ldr.bufLoaderData, prevTntID) - } - ldr.bufLoaderData[tntID] = append(ldr.bufLoaderData[tntID], lData) - } - // proceed with last element in bufLoaderData - var tntID string - for tntID = range ldr.bufLoaderData { - break // get the first tenantID - } - if err = ldr.storeLoadedData(loaderType, - map[string][]LoaderData{tntID: ldr.bufLoaderData[tntID]}, caching); err != nil { - return - } - delete(ldr.bufLoaderData, tntID) - return -} - -func (ldr *Loader) storeLoadedData(loaderType string, - lds map[string][]LoaderData, caching string) (err error) { - var ids []string - cacheArgs := make(map[string][]string) - var cacheIDs []string // verify if we need to clear indexe - switch loaderType { - case utils.MetaAttributes: - cacheIDs = []string{utils.CacheAttributeFilterIndexes} - for _, lDataSet := range lds { - attrModels := make(engine.AttributeMdls, len(lDataSet)) - for i, ld := range lDataSet { - attrModels[i] = new(engine.AttributeMdl) - if err = utils.UpdateStructWithIfaceMap(attrModels[i], ld); err != nil { - return - } - } - for _, tpApf := range attrModels.AsTPAttributes() { - apf, err := engine.APItoAttributeProfile(tpApf, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: AttributeProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(apf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, apf.TenantID()) - if err := ldr.dm.SetAttributeProfile(apf, true); err != nil { - return err - } - } - cacheArgs[utils.CacheAttributeProfiles] = ids - } - case utils.MetaResources: - cacheIDs = []string{utils.CacheResourceFilterIndexes} - for _, lDataSet := range lds { - resModels := make(engine.ResourceMdls, len(lDataSet)) - for i, ld := range lDataSet { - resModels[i] = new(engine.ResourceMdl) - if err = utils.UpdateStructWithIfaceMap(resModels[i], ld); err != nil { - return - } - } - - for _, tpRes := range resModels.AsTPResources() { - res, err := engine.APItoResource(tpRes, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ResourceProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(res))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, res.TenantID()) - if err := ldr.dm.SetResourceProfile(res, true); err != nil { - return err - } - cacheArgs[utils.CacheResourceProfiles] = ids - cacheArgs[utils.CacheResources] = ids - } - } - case utils.MetaIPs: - cacheIDs = []string{utils.CacheIPFilterIndexes} - for _, lDataSet := range lds { - ipMdls := make(engine.IPMdls, len(lDataSet)) - for i, ld := range lDataSet { - ipMdls[i] = new(engine.IPMdl) - if err = utils.UpdateStructWithIfaceMap(ipMdls[i], ld); err != nil { - return - } - } - - for _, tpIP := range ipMdls.AsTPIPs() { - res, err := engine.APItoIP(tpIP, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: IPProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(res))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, res.TenantID()) - if err := ldr.dm.SetIPProfile(res, true); err != nil { - return err - } - cacheArgs[utils.CacheIPProfiles] = ids - cacheArgs[utils.CacheIPAllocations] = ids - } - } - case utils.MetaFilters: - for _, lDataSet := range lds { - fltrModels := make(engine.FilterMdls, len(lDataSet)) - for i, ld := range lDataSet { - fltrModels[i] = new(engine.FilterMdl) - if err = utils.UpdateStructWithIfaceMap(fltrModels[i], ld); err != nil { - return - } - } - - for _, tpFltr := range fltrModels.AsTPFilter() { - fltrPrf, err := engine.APItoFilter(tpFltr, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: Filter: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(fltrPrf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, fltrPrf.TenantID()) - if err := ldr.dm.SetFilter(fltrPrf, true); err != nil { - return err - } - cacheArgs[utils.CacheFilters] = ids - } - } - case utils.MetaStats: - cacheIDs = []string{utils.CacheStatFilterIndexes} - for _, lDataSet := range lds { - stsModels := make(engine.StatMdls, len(lDataSet)) - for i, ld := range lDataSet { - stsModels[i] = new(engine.StatMdl) - if err = utils.UpdateStructWithIfaceMap(stsModels[i], ld); err != nil { - return - } - } - for _, tpSts := range stsModels.AsTPStats() { - stsPrf, err := engine.APItoStats(tpSts, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: StatsQueueProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(stsPrf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, stsPrf.TenantID()) - if err := ldr.dm.SetStatQueueProfile(stsPrf, true); err != nil { - return err - } - cacheArgs[utils.CacheStatQueueProfiles] = ids - cacheArgs[utils.CacheStatQueues] = ids - } - } - case utils.MetaTrends: - for _, lDataSet := range lds { - trsModels := make(engine.TrendsMdls, len(lDataSet)) - for i, ld := range lDataSet { - trsModels[i] = new(engine.TrendsMdl) - if err = utils.UpdateStructWithIfaceMap(trsModels[i], ld); err != nil { - return - } - } - for _, tpTrs := range trsModels.AsTPTrends() { - trsPrf, err := engine.APItoTrends(tpTrs) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: TrendProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(trsPrf))) - continue - } - - ids = append(ids, trsPrf.TenantID()) - if err := ldr.dm.SetTrendProfile(trsPrf); err != nil { - return err - } - } - } - case utils.MetaRankings: - cacheIDs = []string{utils.CacheRankingFilterIndexes} - for _, lDataSet := range lds { - rnkModels := make(engine.RankingsMdls, len(lDataSet)) - for i, ld := range lDataSet { - rnkModels[i] = new(engine.RankingsMdl) - if err = utils.UpdateStructWithIfaceMap(rnkModels[i], ld); err != nil { - return - } - } - for _, tpRgs := range rnkModels.AsTPRanking() { - rgsPrf, err := engine.APItoRanking(tpRgs) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: RankingProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(rgsPrf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, rgsPrf.TenantID()) - if err := ldr.dm.SetRankingProfile(rgsPrf); err != nil { - return err - } - cacheArgs[utils.CacheRankingFilterIndexes] = ids - } - } - case utils.MetaThresholds: - cacheIDs = []string{utils.CacheThresholdFilterIndexes} - for _, lDataSet := range lds { - thModels := make(engine.ThresholdMdls, len(lDataSet)) - for i, ld := range lDataSet { - thModels[i] = new(engine.ThresholdMdl) - if err = utils.UpdateStructWithIfaceMap(thModels[i], ld); err != nil { - return - } - } - for _, tpTh := range thModels.AsTPThreshold() { - thPrf, err := engine.APItoThresholdProfile(tpTh, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ThresholdProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(thPrf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, thPrf.TenantID()) - if err := ldr.dm.SetThresholdProfile(thPrf, true); err != nil { - return err - } - cacheArgs[utils.CacheThresholdProfiles] = ids - cacheArgs[utils.CacheThresholds] = ids - } - } - case utils.MetaRoutes: - cacheIDs = []string{utils.CacheRouteFilterIndexes} - for _, lDataSet := range lds { - sppModels := make(engine.RouteMdls, len(lDataSet)) - for i, ld := range lDataSet { - sppModels[i] = new(engine.RouteMdl) - if err = utils.UpdateStructWithIfaceMap(sppModels[i], ld); err != nil { - return - } - } - - for _, tpSpp := range sppModels.AsTPRouteProfile() { - spPrf, err := engine.APItoRouteProfile(tpSpp, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: RouteProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(spPrf))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, spPrf.TenantID()) - if err := ldr.dm.SetRouteProfile(spPrf, true); err != nil { - return err - } - cacheArgs[utils.CacheRouteProfiles] = ids - } - } - case utils.MetaChargers: - cacheIDs = []string{utils.CacheChargerFilterIndexes} - for _, lDataSet := range lds { - cppModels := make(engine.ChargerMdls, len(lDataSet)) - for i, ld := range lDataSet { - cppModels[i] = new(engine.ChargerMdl) - if err = utils.UpdateStructWithIfaceMap(cppModels[i], ld); err != nil { - return - } - } - - for _, tpCPP := range cppModels.AsTPChargers() { - cpp, err := engine.APItoChargerProfile(tpCPP, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ChargerProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(cpp))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, cpp.TenantID()) - if err := ldr.dm.SetChargerProfile(cpp, true); err != nil { - return err - } - cacheArgs[utils.CacheChargerProfiles] = ids - } - } - case utils.MetaDispatchers: - cacheIDs = []string{utils.CacheDispatcherFilterIndexes} - for _, lDataSet := range lds { - dispModels := make(engine.DispatcherProfileMdls, len(lDataSet)) - for i, ld := range lDataSet { - dispModels[i] = new(engine.DispatcherProfileMdl) - if err = utils.UpdateStructWithIfaceMap(dispModels[i], ld); err != nil { - return - } - } - for _, tpDsp := range dispModels.AsTPDispatcherProfiles() { - dsp, err := engine.APItoDispatcherProfile(tpDsp, ldr.timezone) - if err != nil { - return err - } - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherProfile: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(dsp))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, dsp.TenantID()) - if err := ldr.dm.SetDispatcherProfile(dsp, true); err != nil { - return err - } - cacheArgs[utils.CacheDispatcherProfiles] = ids - } - } - case utils.MetaDispatcherHosts: - for _, lDataSet := range lds { - dispModels := make(engine.DispatcherHostMdls, len(lDataSet)) - for i, ld := range lDataSet { - dispModels[i] = new(engine.DispatcherHostMdl) - if err = utils.UpdateStructWithIfaceMap(dispModels[i], ld); err != nil { - return - } - } - tpDsps, err := dispModels.AsTPDispatcherHosts() - if err != nil { - return err - } - for _, tpDsp := range tpDsps { - dsp := engine.APItoDispatcherHost(tpDsp) - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherHost: %s", - utils.LoaderS, ldr.ldrID, utils.ToJSON(dsp))) - continue - } - // get IDs so we can reload in cache - ids = append(ids, dsp.TenantID()) - if err := ldr.dm.SetDispatcherHost(dsp); err != nil { - return err - } - cacheArgs[utils.CacheDispatcherHosts] = ids - } - } - } - // delay if needed before cache reload - if ldr.cachingDelay != 0 { - utils.Logger.Info(fmt.Sprintf("<%v> Delaying cache reload for %v", utils.LoaderS, ldr.cachingDelay)) - time.Sleep(ldr.cachingDelay) - } - if len(ldr.cacheConns) != 0 { - return engine.CallCache(ldr.connMgr, ldr.cacheConns, caching, cacheArgs, cacheIDs, nil, false, ldr.tenant) - } - return -} - -// removeContent will process the content and will remove it from database -func (ldr *Loader) removeContent(loaderType, caching string) (err error) { - // start processing lines - keepLooping := true // controls looping - lineNr := 0 - for keepLooping { - lineNr++ - var hasErrors bool - lData := make(LoaderData) // one row - for fName, rdr := range ldr.rdrs[loaderType] { - var record []string - if record, err = rdr.csvRdr.Read(); err != nil { - if err == io.EOF { - keepLooping = false - break - } - hasErrors = true - utils.Logger.Warning( - fmt.Sprintf("<%s> <%s> reading line: %d, error: %s", - utils.LoaderS, ldr.ldrID, lineNr, err.Error())) - } - if hasErrors { // if any of the readers will give errors, we ignore the line - continue - } - - if err := lData.UpdateFromCSV(fName, record, - ldr.dataTpls[loaderType], ldr.tenant, ldr.filterS); err != nil { - utils.Logger.Warning( - fmt.Sprintf("<%s> <%s> line: %d, error: %s", - utils.LoaderS, ldr.ldrID, lineNr, err.Error())) - hasErrors = true - continue - } - // Record from map - // update dataDB - } - if len(lData) == 0 { // no data, could be the last line in file - continue - } - tntID := lData.TenantID() - if _, has := ldr.bufLoaderData[tntID]; !has && - len(ldr.bufLoaderData) == 1 { // process previous records before going futher - var prevTntID string - for prevTntID = range ldr.bufLoaderData { - break // have stolen the existing key in buffer - } - if err = ldr.removeLoadedData(loaderType, - map[string][]LoaderData{prevTntID: ldr.bufLoaderData[prevTntID]}, caching); err != nil { - return - } - delete(ldr.bufLoaderData, prevTntID) - } - ldr.bufLoaderData[tntID] = append(ldr.bufLoaderData[tntID], lData) - } - // proceed with last element in bufLoaderData - var tntID string - for tntID = range ldr.bufLoaderData { - break // get the first tenantID - } - if err = ldr.removeLoadedData(loaderType, - map[string][]LoaderData{tntID: ldr.bufLoaderData[tntID]}, caching); err != nil { - return - } - delete(ldr.bufLoaderData, tntID) - return -} - -// removeLoadedData will remove the data from database -// since we remove we don't need to compose the struct we only need the Tenant and the ID of the profile -func (ldr *Loader) removeLoadedData(loaderType string, lds map[string][]LoaderData, caching string) (err error) { - var ids []string - cacheArgs := make(map[string][]string) - var cacheIDs []string // verify if we need to clear indexe - switch loaderType { - case utils.MetaAttributes: - cacheIDs = []string{utils.CacheAttributeFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: AttributeProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveAttributeProfile(tntIDStruct.Tenant, tntIDStruct.ID, - true); err != nil { - return err - } - cacheArgs[utils.CacheAttributeProfiles] = ids - } - } - - case utils.MetaResources: - cacheIDs = []string{utils.CacheResourceFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ResourceProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveResourceProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheResourceProfiles] = ids - cacheArgs[utils.CacheResources] = ids - } - } - case utils.MetaFilters: - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: Filter: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveFilter(tntIDStruct.Tenant, tntIDStruct.ID, - true); err != nil { - return err - } - cacheArgs[utils.CacheFilters] = ids - } - } - case utils.MetaStats: - cacheIDs = []string{utils.CacheStatFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: StatsQueueProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveStatQueueProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheStatQueueProfiles] = ids - cacheArgs[utils.CacheStatQueues] = ids - } - } - case utils.MetaRankings: - cacheIDs = []string{utils.CacheRankingFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: RankingProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveRankingProfile(tntIDStruct.Tenant, - tntIDStruct.ID); err != nil { - return err - } - cacheArgs[utils.CacheRankingProfiles] = ids - } - } - case utils.MetaThresholds: - cacheIDs = []string{utils.CacheThresholdFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ThresholdProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveThresholdProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheThresholdProfiles] = ids - cacheArgs[utils.CacheThresholds] = ids - } - } - case utils.MetaRoutes: - cacheIDs = []string{utils.CacheRouteFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: RouteProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveRouteProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheRouteProfiles] = ids - } - } - case utils.MetaChargers: - cacheIDs = []string{utils.CacheChargerFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: ChargerProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveChargerProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheChargerProfiles] = ids - } - } - case utils.MetaDispatchers: - cacheIDs = []string{utils.CacheDispatcherFilterIndexes} - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherProfileID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveDispatcherProfile(tntIDStruct.Tenant, - tntIDStruct.ID, true); err != nil { - return err - } - cacheArgs[utils.CacheDispatcherProfiles] = ids - } - } - case utils.MetaDispatcherHosts: - for tntID := range lds { - if ldr.dryRun { - utils.Logger.Info( - fmt.Sprintf("<%s-%s> DRY_RUN: DispatcherHostID: %s", - utils.LoaderS, ldr.ldrID, tntID)) - } else { - tntIDStruct := utils.NewTenantID(tntID) - // get IDs so we can reload in cache - ids = append(ids, tntID) - if err := ldr.dm.RemoveDispatcherHost(tntIDStruct.Tenant, - tntIDStruct.ID); err != nil { - return err - } - cacheArgs[utils.CacheDispatcherHosts] = ids - } - } - } - // delay if needed before cache reload - if ldr.cachingDelay != 0 { - utils.Logger.Info(fmt.Sprintf("<%v> Delaying cache reload for %v", utils.LoaderS, ldr.cachingDelay)) - time.Sleep(ldr.cachingDelay) - } - if len(ldr.cacheConns) != 0 { - return engine.CallCache(ldr.connMgr, ldr.cacheConns, caching, cacheArgs, cacheIDs, nil, false, ldr.tenant) - } - return -} - -func (ldr *Loader) serve(stopChan chan struct{}) (err error) { - switch ldr.runDelay { - case time.Duration(0): // 0 disables the automatic read, maybe done per API - return - case time.Duration(-1): - return utils.WatchDir(ldr.tpInDir, ldr.processFile, - utils.LoaderS+"-"+ldr.ldrID, stopChan) - default: - go ldr.handleFolder(stopChan) - } - return -} - -func (ldr *Loader) handleFolder(stopChan chan struct{}) { - for { - go ldr.ProcessFolder(config.CgrConfig().GeneralCfg().DefaultCaching, utils.MetaStore, false) - timer := time.NewTimer(ldr.runDelay) - select { - case <-stopChan: - utils.Logger.Info( - fmt.Sprintf("<%s-%s> stop monitoring path <%s>", - utils.LoaderS, ldr.ldrID, ldr.tpInDir)) - timer.Stop() - return - case <-timer.C: - } - } -} - -func (ldr *Loader) processFile(itmID string) (err error) { - loaderType := ldr.getLdrType(itmID) - if len(loaderType) == 0 { - return - } - if err = ldr.lockFolder(); err != nil { - return - } - defer ldr.unlockFolder() - if ldr.rdrs[loaderType][itmID] != nil { - ldr.unreferenceFile(loaderType, itmID) - } - var rdr *os.File - if rdr, err = os.Open(path.Join(ldr.tpInDir, itmID)); err != nil { - return - } - csvReader := csv.NewReader(rdr) - csvReader.Comma = rune(ldr.fieldSep[0]) - csvReader.Comment = '#' - ldr.rdrs[loaderType][itmID] = &openedCSVFile{ - fileName: itmID, rdr: rdr, csvRdr: csvReader} - if !ldr.allFilesPresent(loaderType) { - return - } - for fName := range ldr.rdrs[loaderType] { - defer ldr.unreferenceFile(loaderType, fName) - } - - err = ldr.processContent(loaderType, config.CgrConfig().GeneralCfg().DefaultCaching) - - if ldr.tpOutDir == utils.EmptyString { - return - } - for fName := range ldr.rdrs[loaderType] { - oldPath := path.Join(ldr.tpInDir, fName) - newPath := path.Join(ldr.tpOutDir, fName) - if nerr := os.Rename(oldPath, newPath); nerr != nil { - return nerr - } - } - return -} - -func (ldr *Loader) allFilesPresent(ldrType string) bool { - for _, rdr := range ldr.rdrs[ldrType] { - if rdr == nil { - return false - } - } - return true -} - -// getLdrType returns loaderType for the given fileName -func (ldr *Loader) getLdrType(fName string) (ldrType string) { - for ldr, rdrs := range ldr.rdrs { - if _, has := rdrs[fName]; has { - return ldr - } - } - return -} diff --git a/loaders/loader_it_test.go b/loaders/loader_it_test.go deleted file mode 100644 index bd42502bd..000000000 --- a/loaders/loader_it_test.go +++ /dev/null @@ -1,857 +0,0 @@ -//go:build integration -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ -package loaders - -import ( - "encoding/csv" - "errors" - "io" - "os" - "path" - "reflect" - "sort" - "strings" - "testing" - "time" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/birpc/jsonrpc" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - loaderCfgPath string - loaderCfgDIR string //run tests for specific configuration - loaderCfg *config.CGRConfig - loaderRPC *birpc.Client - customAttributes = "12012000001\t12018209998\n12012000002\t15512580598\n12012000007\t19085199998\n12012000008\t18622784999\n12012000010\t17329440866\n12012000011\t18623689800\n12012000012\t19082050951\n12012000014\t17329440866\n12012000015\t12018209999\n12012000031\t12018209999\n12012000032\t19082050951\n12012000033\t12018209998\n12012000034\t12018209998\n" - - sTestsLoader = []func(t *testing.T){ - testLoaderMakeFolders, - testLoaderInitCfg, - testLoaderResetDataDB, - testLoaderStartEngine, - testLoaderRPCConn, - testLoaderPopulateData, - testProcessFile, - testProcessFileLockFolder, - testProcessFileUnableToOpen, - testProcessFileAllFilesPresent, - testProcessFileRenameError, - testAllFilesPresentEmptyCSV, - testIsFolderLocked, - testNewLockFolder, - testNewLockFolderNotFound, - testLoaderLoadAttributes, - testLoaderVerifyOutDir, - testLoaderCheckAttributes, - testLoaderResetDataDB, - testLoaderPopulateDataWithoutMoving, - testLoaderLoadAttributesWithoutMoving, - testLoaderVerifyOutDirWithoutMoving, - testLoaderCheckAttributes, - testLoaderResetDataDB, - testLoaderPopulateDataWithSubpath, - testLoaderLoadAttributesWithSubpath, - testLoaderVerifyOutDirWithSubpath, - testLoaderCheckAttributes, - testLoaderResetDataDB, - testLoaderPopulateDataWithSubpathWithMove, - testLoaderLoadAttributesWithoutSubpathWithMove, - testLoaderVerifyOutDirWithSubpathWithMove, - testLoaderCheckAttributes, - testLoaderResetDataDB, - testLoaderPopulateDataForTemplateLoader, - testLoaderLoadAttributesForTemplateLoader, - testLoaderVerifyOutDirForTemplateLoader, - testLoaderCheckAttributes, - testLoaderResetDataDB, - testLoaderPopulateDataForCustomSep, - testLoaderCheckForCustomSep, - testLoaderVerifyOutDirForCustomSep, - testLoaderKillEngine, - } -) - -// Test start here -func TestLoaderIT(t *testing.T) { - switch *utils.DBType { - case utils.MetaInternal: - loaderCfgDIR = "tutinternal" - case utils.MetaMySQL: - loaderCfgDIR = "tutmysql" - case utils.MetaMongo: - loaderCfgDIR = "tutmongo" - case utils.MetaPostgres: - t.SkipNow() - default: - t.Fatal("Unknown Database type") - } - - for _, stest := range sTestsLoader { - t.Run(loaderCfgDIR, stest) - } -} - -func testLoaderInitCfg(t *testing.T) { - var err error - loaderCfgPath = path.Join(*utils.DataDir, "conf", "samples", "loaders", loaderCfgDIR) - loaderCfg, err = config.NewCGRConfigFromPath(loaderCfgPath) - if err != nil { - t.Fatal(err) - } -} - -func testLoaderMakeFolders(t *testing.T) { - // active the loaders here - for _, dir := range loaderPaths { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } -} - -// Wipe out the cdr database -func testLoaderResetDataDB(t *testing.T) { - if err := engine.InitDataDB(loaderCfg); err != nil { - t.Fatal(err) - } - engine.Cache.Clear(nil) -} - -// Start CGR Engine -func testLoaderStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(loaderCfgPath, 100); err != nil { - t.Fatal(err) - } -} - -// Connect rpc client to rater -func testLoaderRPCConn(t *testing.T) { - var err error - switch *utils.Encoding { - case utils.MetaJSON: - loaderRPC, err = jsonrpc.Dial(utils.TCP, loaderCfg.ListenCfg().RPCJSONListen) - case utils.MetaGOB: - loaderRPC, err = birpc.Dial(utils.TCP, loaderCfg.ListenCfg().RPCGOBListen) - default: - loaderRPC, err = nil, errors.New("UNSUPPORTED_RPC") - } - if err != nil { - t.Fatal(err) - } -} - -func testLoaderPopulateData(t *testing.T) { - fileName := utils.AttributesCsv - tmpFilePath := path.Join("/tmp", fileName) - if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/In", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testLoaderLoadAttributes(t *testing.T) { - var reply string - if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load, - &ArgsProcessFolder{LoaderID: "CustomLoader"}, &reply); err != nil { - t.Error(err) - } -} - -func testLoaderVerifyOutDir(t *testing.T) { - time.Sleep(100 * time.Millisecond) - if outContent1, err := os.ReadFile(path.Join("/tmp/Out", utils.AttributesCsv)); err != nil { - t.Error(err) - } else if AttributesCSVContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1)) - } -} - -func testLoaderCheckAttributes(t *testing.T) { - eAttrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ALS1", - Contexts: []string{"con1", "con2", "con3"}, - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, - Attributes: []*engine.Attribute{ - { - FilterIDs: []string{"*string:~*req.Field1:Initial"}, - Path: utils.MetaReq + utils.NestingSep + "Field1", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep), - }, - { - FilterIDs: []string{}, - Path: utils.MetaReq + utils.NestingSep + "Field2", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep), - }}, - Blocker: true, - Weight: 20, - } - if *utils.Encoding == utils.MetaGOB { // gob threats empty slices as nil values - eAttrPrf.Attributes[1].FilterIDs = nil - } - var reply *engine.AttributeProfile - if err := loaderRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile, - &utils.TenantIDWithAPIOpts{ - TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ALS1"}, - }, &reply); err != nil { - t.Fatal(err) - } - eAttrPrf.Compile() - reply.Compile() - sort.Strings(eAttrPrf.Contexts) - sort.Strings(reply.Contexts) - if !reflect.DeepEqual(eAttrPrf, reply) { - t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply)) - } -} - -func testLoaderPopulateDataWithoutMoving(t *testing.T) { - fileName := utils.AttributesCsv - tmpFilePath := path.Join("/tmp/", fileName) - if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/LoaderIn", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testLoaderLoadAttributesWithoutMoving(t *testing.T) { - var reply string - if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load, - &ArgsProcessFolder{LoaderID: "WithoutMoveToOut"}, &reply); err != nil { - t.Error(err) - } -} - -func testLoaderVerifyOutDirWithoutMoving(t *testing.T) { - time.Sleep(100 * time.Millisecond) - // we expect that after the LoaderS process the file leave in in the input folder - if outContent1, err := os.ReadFile(path.Join("/tmp/LoaderIn", utils.AttributesCsv)); err != nil { - t.Error(err) - } else if AttributesCSVContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1)) - } -} - -func testLoaderPopulateDataWithSubpath(t *testing.T) { - fileName := utils.AttributesCsv - tmpFilePath := path.Join("/tmp/", fileName) - if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.MkdirAll("/tmp/SubpathWithoutMove/folder1", 0755); err != nil { - t.Fatal("Error creating folder: /tmp/SubpathWithoutMove/folder1", err) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathWithoutMove/folder1", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testLoaderLoadAttributesWithSubpath(t *testing.T) { - var reply string - if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load, - &ArgsProcessFolder{LoaderID: "SubpathLoaderWithoutMove"}, &reply); err != nil { - t.Error(err) - } -} - -func testLoaderVerifyOutDirWithSubpath(t *testing.T) { - time.Sleep(100 * time.Millisecond) - // we expect that after the LoaderS process the file leave in in the input folder - if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathWithoutMove/folder1", utils.AttributesCsv)); err != nil { - t.Error(err) - } else if AttributesCSVContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1)) - } -} - -func testLoaderPopulateDataWithSubpathWithMove(t *testing.T) { - fileName := utils.AttributesCsv - tmpFilePath := path.Join("/tmp/", fileName) - if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.MkdirAll("/tmp/SubpathLoaderWithMove/folder1", 0755); err != nil { - t.Fatal("Error creating folder: /tmp/SubpathLoaderWithMove/folder1", err) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/SubpathLoaderWithMove/folder1", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testLoaderLoadAttributesWithoutSubpathWithMove(t *testing.T) { - var reply string - if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load, - &ArgsProcessFolder{LoaderID: "SubpathLoaderWithMove"}, &reply); err != nil { - t.Error(err) - } -} - -func testLoaderVerifyOutDirWithSubpathWithMove(t *testing.T) { - time.Sleep(100 * time.Millisecond) - if outContent1, err := os.ReadFile(path.Join("/tmp/SubpathOut/folder1", utils.AttributesCsv)); err != nil { - t.Error(err) - } else if AttributesCSVContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1)) - } -} - -func testLoaderPopulateDataForTemplateLoader(t *testing.T) { - fileName := utils.AttributesCsv - tmpFilePath := path.Join("/tmp/", fileName) - if err := os.WriteFile(tmpFilePath, []byte(AttributesCSVContent), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.MkdirAll("/tmp/templateLoaderIn", 0755); err != nil { - t.Fatal("Error creating folder: /tmp/templateLoaderIn", err) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/templateLoaderIn", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } -} - -func testLoaderLoadAttributesForTemplateLoader(t *testing.T) { - var reply string - if err := loaderRPC.Call(context.Background(), utils.LoaderSv1Load, - &ArgsProcessFolder{LoaderID: "LoaderWithTemplate"}, &reply); err != nil { - t.Error(err) - } -} - -func testLoaderVerifyOutDirForTemplateLoader(t *testing.T) { - time.Sleep(100 * time.Millisecond) - if outContent1, err := os.ReadFile(path.Join("/tmp/templateLoaderOut", utils.AttributesCsv)); err != nil { - t.Error(err) - } else if AttributesCSVContent != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", AttributesCSVContent, string(outContent1)) - } -} - -func testLoaderKillEngine(t *testing.T) { - if err := engine.KillEngine(100); err != nil { - t.Error(err) - } -} - -func testLoaderPopulateDataForCustomSep(t *testing.T) { - fileName := utils.Attributes - tmpFilePath := path.Join("/tmp/", fileName) - if err := os.WriteFile(tmpFilePath, []byte(customAttributes), os.ModePerm); err != nil { - t.Fatal(err.Error()) - } - if err := os.MkdirAll("/tmp/customSepLoaderIn", 0755); err != nil { - t.Fatal("Error creating folder: /tmp/customSepLoaderIn", err) - } - if err := os.Rename(tmpFilePath, path.Join("/tmp/customSepLoaderIn", fileName)); err != nil { - t.Fatal("Error moving file to processing directory: ", err) - } - time.Sleep(100 * time.Millisecond) -} - -func testLoaderCheckForCustomSep(t *testing.T) { - eAttrPrf := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ATTR_12012000001", - Contexts: []string{"*any"}, - FilterIDs: []string{"*string:~*req.Destination:12012000001"}, - Attributes: []*engine.Attribute{ - { - FilterIDs: []string{}, - Path: "*req.Destination", - Type: utils.MetaConstant, - Value: config.NewRSRParsersMustCompile("12018209998", utils.InfieldSep), - }, - }, - } - if *utils.Encoding == utils.MetaGOB { // gob threats empty slices as nil values - eAttrPrf.Attributes[0].FilterIDs = nil - } - var reply *engine.AttributeProfile - if err := loaderRPC.Call(context.Background(), utils.APIerSv1GetAttributeProfile, - &utils.TenantIDWithAPIOpts{ - TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_12012000001"}, - }, &reply); err != nil { - t.Fatal(err) - } - eAttrPrf.Compile() - reply.Compile() - if !reflect.DeepEqual(eAttrPrf, reply) { - t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(eAttrPrf), utils.ToJSON(reply)) - } -} - -func testLoaderVerifyOutDirForCustomSep(t *testing.T) { - time.Sleep(100 * time.Millisecond) - if outContent1, err := os.ReadFile(path.Join("/tmp/customSepLoaderOut", utils.Attributes)); err != nil { - t.Error(err) - } else if customAttributes != string(outContent1) { - t.Errorf("Expecting: %q, received: %q", customAttributes, string(outContent1)) - } -} - -func testProcessFile(t *testing.T) { - flPath := "/tmp/testProcessFile" - if err := os.MkdirAll(flPath, os.ModePerm); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items) - if err != nil { - t.Error(err) - } - ldr := &Loader{ - ldrID: "testProcessFile", - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - fieldSep: utils.FieldsSep, - tpInDir: flPath, - tpOutDir: "/tmp", - lockFilepath: "/tmp/testProcessFile/.lck", - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - - //loader file is empty (loaderType will be empty) - if err := ldr.processFile(utils.ResourcesCsv); err != nil { - t.Error(err) - } - - // resCsv := ` - // #Tenant[0],ID[1] - // cgrates.org,NewRes1 - // ` - // rdr := io.NopCloser(strings.NewReader(resCsv)) - - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: &openedCSVFile{ - rdr: io.NopCloser(nil), - csvRdr: csv.NewReader(nil), - }, - }, - } - - expRes := &engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "NewRes1", - FilterIDs: []string{}, - ThresholdIDs: []string{}, - } - - //successfully processed the file - if err := ldr.processFile(utils.ResourcesCsv); err != nil { - t.Error(err) - } - - //get ResourceProfile and compare - if rcv, err := ldr.dm.GetResourceProfile(expRes.Tenant, expRes.ID, true, true, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rcv, expRes) { - t.Errorf("Expected %+v, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv)) - } - - if err := ldr.dm.RemoveResourceProfile(expRes.Tenant, expRes.ID, true); err != nil { - t.Error(err) - } - - file, err = os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - //cannot move file when tpOutDir is empty - ldr.tpOutDir = utils.EmptyString - if err := ldr.processFile(utils.ResourcesCsv); err != nil { - t.Error(err) - } - - if err := os.Remove(path.Join("/tmp", utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.RemoveAll(flPath); err != nil { - t.Error(err) - } -} - -func testProcessFileAllFilesPresent(t *testing.T) { - flPath := "/tmp/testProcessFile" - if err := os.MkdirAll(flPath, os.ModePerm); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, "inexistent.csv")) - if err != nil { - t.Error(err) - } - - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items) - if err != nil { - t.Error(err) - } - ldr := &Loader{ - ldrID: "testProcessFile", - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - fieldSep: utils.FieldsSep, - tpInDir: flPath, - tpOutDir: "/tmp", - lockFilepath: utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.FieldsSep), - Mandatory: true}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.FieldsSep), - Mandatory: true}, - }, - } - - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "inexistent.csv": nil, - utils.AttributesCsv: nil, - }, - } - - if err := ldr.processFile("inexistent.csv"); err != nil { - t.Error(err) - } - - if err := os.Remove(path.Join(flPath, "inexistent.csv")); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testProcessFileLockFolder(t *testing.T) { - flPath := "/tmp/testProcessFileLockFolder" - if err := os.MkdirAll(flPath, os.ModePerm); err != nil { - t.Error(err) - } - _, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - ldr := &Loader{ - ldrID: "testProcessFileLockFolder", - tpInDir: flPath, - tpOutDir: "/tmp", - lockFilepath: "/tmp/test/.cgr.lck", - fieldSep: utils.InfieldSep, - } - - resCsv := ` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -` - rdr := io.NopCloser(strings.NewReader(resCsv)) - - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: &openedCSVFile{ - fileName: utils.ResourcesCsv, - rdr: rdr, - }, - }, - } - - //unable to lock the folder, because lockFileName is missing - expected := "open /tmp/test/.cgr.lck: no such file or directory" - if err := ldr.processFile(utils.ResourcesCsv); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testProcessFileUnableToOpen(t *testing.T) { - flPath := "/tmp/testProcessFileUnableToOpen" - if err := os.MkdirAll(flPath, os.ModePerm); err != nil { - t.Error(err) - } - - ldr := &Loader{ - ldrID: "testProcessFile", - tpInDir: flPath, - fieldSep: ",", - lockFilepath: utils.MetaResources, - } - resCsv := ` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -` - rdr := io.NopCloser(strings.NewReader(resCsv)) - - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - `resources`: &openedCSVFile{ - fileName: utils.ResourcesCsv, - rdr: rdr, - }, - }, - } - - //unable to lock the folder, because lockFileName is missing - expected := "open /tmp/testProcessFileUnableToOpen/resources: no such file or directory" - if err := ldr.processFile(`resources`); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testProcessFileRenameError(t *testing.T) { - flPath1 := "/tmp/testProcessFileLockFolder" - if err := os.MkdirAll(flPath1, os.ModePerm); err != nil { - t.Error(err) - } - data, err := engine.NewInternalDB(nil, nil, true, nil, loaderCfg.DataDbCfg().Items) - if err != nil { - t.Error(err) - } - ldr := &Loader{ - ldrID: "testProcessFileRenameError", - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - fieldSep: utils.FieldsSep, - tpInDir: flPath1, - tpOutDir: "INEXISTING_FILE", - lockFilepath: utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Tag: "ID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - - // resCsv := ` - // #Tenant[0],ID[1] - // cgrates.org,NewRes1 - // ` - // rdr := io.NopCloser(strings.NewReader(resCsv)) - - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: &openedCSVFile{ - rdr: io.NopCloser(nil), - csvRdr: csv.NewReader(nil), - }, - }, - } - - file, err := os.Create(path.Join(flPath1, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - expected := "rename /tmp/testProcessFileLockFolder/Resources.csv INEXISTING_FILE/Resources.csv: no such file or directory" - if err := ldr.processFile(utils.ResourcesCsv); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - if err := os.RemoveAll(flPath1); err != nil { - t.Error(err) - } -} - -func testAllFilesPresentEmptyCSV(t *testing.T) { - ldr := &Loader{ - ldrID: "testProcessFileRenameError", - lockFilepath: utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: nil, - }, - } - if rcv := ldr.allFilesPresent(utils.MetaResources); rcv { - t.Errorf("Expecting false") - } -} - -func testIsFolderLocked(t *testing.T) { - flPath := "/tmp/testIsFolderLocked" - ldr := &Loader{ - ldrID: "TestLoadAndRemoveResources", - tpInDir: flPath, - lockFilepath: utils.EmptyString, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - expected := "stat /\x00: invalid argument" - if _, err := ldr.isFolderLocked(); err != nil { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func testNewLockFolder(t *testing.T) { - pathL := "/tmp/testNewLockFolder/" - if err := os.MkdirAll(pathL, os.ModePerm); err != nil { - t.Error(err) - } - - _, err := os.Create(path.Join(pathL, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - ldr := &Loader{ - ldrID: "testNewLockFolder", - tpInDir: "", - lockFilepath: pathL + utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - - if err := ldr.lockFolder(); err != nil { - t.Error(err) - } - if err := os.RemoveAll(pathL); err != nil { - t.Error(err) - } -} - -func testNewLockFolderNotFound(t *testing.T) { - pathL := "/tmp/testNewLockFolder/" - ldr := &Loader{ - ldrID: "testNewLockFolder", - tpInDir: "", - lockFilepath: pathL + utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - - errExpect := "open /tmp/testNewLockFolder/Resources.csv: no such file or directory" - if err := ldr.lockFolder(); err == nil || err.Error() != errExpect { - t.Error(err) - } -} - -func testNewIsFolderLock(t *testing.T) { - pathL := "/tmp/testNewLockFolder/" - if err := os.MkdirAll(pathL, os.ModePerm); err != nil { - t.Error(err) - } - - _, err := os.Create(path.Join(pathL, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - ldr := &Loader{ - ldrID: "testNewLockFolder", - tpInDir: "", - lockFilepath: pathL + utils.ResourcesCsv, - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - - if err := ldr.lockFolder(); err != nil { - t.Error(err) - } - - isLocked, err := ldr.isFolderLocked() - if !isLocked { - t.Error("Expected the file to be locked") - } - - if err := os.RemoveAll(pathL); err != nil { - t.Error(err) - } -} diff --git a/loaders/loader_test.go b/loaders/loader_test.go deleted file mode 100644 index d74c8c3cc..000000000 --- a/loaders/loader_test.go +++ /dev/null @@ -1,3860 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "encoding/csv" - "io" - "os" - "path" - "reflect" - "sort" - "strings" - "testing" - "time" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/rpcclient" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/utils" -) - -var data, _ = engine.NewInternalDB(nil, nil, false, nil, config.CgrConfig().DataDbCfg().Items) - -const ( - ResourcesCSVContent = ` -#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],TTL[4],Limit[5],AllocationMessage[6],Blocker[7],Stored[8],Weight[9],Thresholds[10] -cgrates.org,ResGroup21,*string:~*req.Account:1001,2014-07-29T15:00:00Z,1s,2,call,true,true,10, -cgrates.org,ResGroup22,*string:~*req.Account:dan,2014-07-29T15:00:00Z,3600s,2,premium_call,true,true,10, -` - StatsCSVContent = ` -#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],QueueLength[4],TTL[5],MinItems[6],Metrics[7],MetricFilterIDs[8],Stored[9],Blocker[10],Weight[11],ThresholdIDs[12] -cgrates.org,TestStats,*string:~*req.Account:1001,2014-07-29T15:00:00Z,100,1s,2,*sum#~*req.Value;*average#~*req.Value,,true,true,20,Th1;Th2 -cgrates.org,TestStats,,,,,2,*sum#~*req.Usage,,true,true,20, -cgrates.org,TestStats2,FLTR_1,2014-07-29T15:00:00Z,100,1s,2,*sum#~*req.Value;*sum#~*req.Usage;*average#~*req.Value;*average#~*req.Usage,,true,true,20,Th -cgrates.org,TestStats2,,,,,2,*sum#~*req.Cost;*average#~*req.Cost,,true,true,20, -` - ThresholdsCSVContent = ` -#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,Threshold1,*string:~*req.Account:1001;*string:~*req.RunID:*default,2014-07-29T15:00:00Z,12,10,1s,true,10,THRESH1,true -` - FiltersCSVContent = ` -#Tenant[0],ID[1],Type[2],Element[3],Values[4],ActivationInterval[5] -cgrates.org,FLTR_1,*string,~*req.Account,1001;1002,2014-07-29T15:00:00Z -cgrates.org,FLTR_1,*prefix,~*req.Destination,10;20,2014-07-29T15:00:00Z -cgrates.org,FLTR_1,*rsr,~*req.Subject,~^1.*1$, -cgrates.org,FLTR_1,*rsr,~*req.Destination,1002, -cgrates.org,FLTR_ACNT_dan,*string,~*req.Account,dan,2014-07-29T15:00:00Z -cgrates.org,FLTR_DST_DE,*destinations,~*req.Destination,DST_DE,2014-07-29T15:00:00Z -cgrates.org,FLTR_DST_NL,*destinations,~*req.Destination,DST_NL,2014-07-29T15:00:00Z -` - RoutesCSVContent = ` -#Tenant[0],ID[1],FilterIDs[2],ActivationInterval[3],Sorting[4],SortingParameters[5],RouteID[6],RouteFilterIDs[7],RouteAccountIDs[8],RouteRatingPlanIDs[9],RouteResourceIDs[10],RouteStatIDs[11],RouteWeight[12],RouteBlocker[13],RouteParameters[14],Weight[15] -cgrates.org,RoutePrf1,*string:~*req.Account:dan,2014-07-29T15:00:00Z,*lc,,route1,FLTR_ACNT_dan,Account1;Account1_1,RPL_1,ResGroup1,Stat1,10,true,param1,20 -cgrates.org,RoutePrf1,,,,,route1,,,RPL_2,ResGroup2,,10,,, -cgrates.org,RoutePrf1,,,,,route1,FLTR_DST_DE,Account2,RPL_3,ResGroup3,Stat2,10,,, -cgrates.org,RoutePrf1,,,,,route1,,,,ResGroup4,Stat3,10,,, -` - AttributesCSVContent = ` -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight -cgrates.org,ALS1,con1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*string:~*req.Field1:Initial,*req.Field1,*variable,Sub1,true,20 -cgrates.org,ALS1,con2;con3,,,,*req.Field2,*variable,Sub2,true,20 -` - ChargersCSVContent = ` -#Tenant,ID,FilterIDs,ActivationInterval,RunID,AttributeIDs,Weight -cgrates.org,Charger1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*rated,ATTR_1001_SIMPLEAUTH,20 -` - DispatcherCSVContent = ` -#Tenant,ID,FilterIDs,ActivationInterval,Strategy,Hosts,Weight -cgrates.org,D1,*any,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*first,,C1,*gt:~*req.Usage:10,10,false,192.168.56.203,20 -cgrates.org,D1,,,,*first,,C2,*lt:~*req.Usage:10,10,false,192.168.56.204, -` - DispatcherHostCSVContent = ` -#Tenant[0],ID[1],Address[2],Transport[3],ConnectAttempts[4],Reconnects[5],MaxReconnectInterval[6],ConnectTimeout[7],ReplyTimeout[8],Tls[9],ClientKey[10],ClientCertificate[11],CaCertificate[12] -cgrates.org,ALL,127.0.0.1:6012,*json,1,3,5m,1m,2m,false,,, -` -) - -func TestLoaderProcessContentSingleFile(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - bufLoaderData: make(map[string][]LoaderData), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "Contexts", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "AttributeFilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Path: "Type", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Path: "Value", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Path: "Blocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(AttributesCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - "Attributes.csv": &openedCSVFile{fileName: "Attributes.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - - //cannot set AttributeProfile when dryrun is true - ldr.dryRun = true - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err != nil { - t.Error(err) - } - - //processContent successfully when dryrun is false - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(AttributesCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - "Attributes.csv": &openedCSVFile{fileName: "Attributes.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err != nil { - t.Error(err) - } - - eAP := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ALS1", - Contexts: []string{"con1", "con2", "con3"}, - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, - Attributes: []*engine.Attribute{ - { - FilterIDs: []string{"*string:~*req.Field1:Initial"}, - Path: utils.MetaReq + utils.NestingSep + "Field1", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep), - }, - { - FilterIDs: []string{}, - Path: utils.MetaReq + utils.NestingSep + "Field2", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep), - }}, - Blocker: true, - Weight: 20, - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - if ap, err := ldr.dm.GetAttributeProfile("cgrates.org", "ALS1", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eAP.Attributes, ap.Attributes) { - t.Errorf("expecting: %s, \n received: %s", - utils.ToJSON(eAP), utils.ToJSON(ap)) - } - - //cannot set AttributeProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(AttributesCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{fileName: utils.AttributesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expectedErr := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err == nil || err != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoaderProcessContentMultiFiles(t *testing.T) { - file1CSV := `ignored,ignored,ignored,ignored,ignored,,*req.Subject,1001,ignored,ignored` - file2CSV := `ignored,TestLoader2` - ldr := &Loader{ - ldrID: "TestLoaderProcessContentMultiFiles", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Mandatory: true}, - {Path: "Contexts", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("*any", utils.InfieldSep)}, - {Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.InfieldSep)}, - {Path: "Value", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("10", utils.InfieldSep)}, - }, - } - rdr1 := io.NopCloser(strings.NewReader(file1CSV)) - csvRdr1 := csv.NewReader(rdr1) - csvRdr1.Comment = '#' - rdr2 := io.NopCloser(strings.NewReader(file2CSV)) - csvRdr2 := csv.NewReader(rdr2) - csvRdr2.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - "File1.csv": &openedCSVFile{fileName: "File1.csv", - rdr: rdr1, csvRdr: csvRdr1}, - "File2.csv": &openedCSVFile{fileName: "File2.csv", - rdr: rdr2, csvRdr: csvRdr2}}, - } - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err != nil { - t.Error(err) - } - eAP := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "TestLoader2", - FilterIDs: []string{}, - Contexts: []string{utils.MetaAny}, - Attributes: []*engine.Attribute{ - { - Path: utils.MetaReq + utils.NestingSep + "Subject", - FilterIDs: []string{}, - Value: config.NewRSRParsersMustCompile("1001", utils.InfieldSep), - }}, - Weight: 10.0, - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - if ap, err := ldr.dm.GetAttributeProfile("cgrates.org", "TestLoader2", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eAP, ap) { - t.Errorf("expecting: %s, \n received: %s", - utils.ToJSON(eAP), utils.ToJSON(ap)) - } -} - -func TestLoaderProcessResource(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessResources", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "UsageTTL", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "Limit", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "AllocationMessage", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Path: "Blocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Path: "Stored", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - {Path: "Thresholds", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(ResourcesCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err != nil { - t.Error(err) - } - eResPrf1 := &engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "ResGroup21", - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC), - }, - UsageTTL: time.Second, - AllocationMessage: "call", - Weight: 10, - Limit: 2, - Blocker: true, - Stored: true, - ThresholdIDs: []string{}, - } - eResPrf2 := &engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "ResGroup22", - FilterIDs: []string{"*string:~*req.Account:dan"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC), - }, - UsageTTL: 3600 * time.Second, - AllocationMessage: "premium_call", - Weight: 10, - Limit: 2, - Blocker: true, - Stored: true, - ThresholdIDs: []string{}, - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - if resPrf, err := ldr.dm.GetResourceProfile("cgrates.org", "ResGroup21", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eResPrf1, resPrf) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eResPrf1), utils.ToJSON(resPrf)) - } - if resPrf, err := ldr.dm.GetResourceProfile("cgrates.org", "ResGroup22", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eResPrf2, resPrf) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eResPrf2), utils.ToJSON(resPrf)) - } -} - -func TestLoaderProcessFilters(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessFilters", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaFilters: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "Type", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "Element", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "Values", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(FiltersCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - "Filters.csv": &openedCSVFile{fileName: "Filters.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - - //Cannot set filterProfile when dryrun is true - ldr.dryRun = true - if err := ldr.processContent(utils.MetaFilters, utils.EmptyString); err != nil { - t.Error(err) - } - - //processContent when dryrun is false - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(FiltersCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - "Filters.csv": &openedCSVFile{fileName: "Filters.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaFilters, utils.EmptyString); err != nil { - t.Error(err) - } - - eFltr1 := &engine.Filter{ - Tenant: "cgrates.org", - ID: "FLTR_1", - Rules: []*engine.FilterRule{ - { - Type: utils.MetaString, - Element: utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.AccountField, - Values: []string{"1001", "1002"}, - }, - { - Type: utils.MetaPrefix, - Element: utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Destination, - Values: []string{"10", "20"}, - }, - { - Type: utils.MetaRSR, - Element: "~*req.Subject", - Values: []string{"~^1.*1$"}, - }, - { - Type: utils.MetaRSR, - Element: "~*req.Destination", - Values: []string{"1002"}, - }, - }, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC), - }, - } - if err := eFltr1.Compile(); err != nil { - t.Error(err) - } - // Compile Value for rsr fields - if err := eFltr1.Rules[2].CompileValues(); err != nil { - t.Error(err) - } - // Compile Value for rsr fields - if err := eFltr1.Rules[3].CompileValues(); err != nil { - t.Error(err) - } - eFltr2 := &engine.Filter{ - Tenant: "cgrates.org", - ID: "FLTR_DST_DE", - Rules: []*engine.FilterRule{ - { - Type: "*destinations", - Element: utils.DynamicDataPrefix + utils.MetaReq + utils.NestingSep + utils.Destination, - Values: []string{"DST_DE"}, - }, - }, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC), - }, - } - if err := eFltr2.Compile(); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - if fltr, err := ldr.dm.GetFilter("cgrates.org", "FLTR_1", - true, true, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eFltr1, fltr) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eFltr1), utils.ToJSON(fltr)) - } - if fltr, err := ldr.dm.GetFilter("cgrates.org", "FLTR_DST_DE", - true, true, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eFltr2, fltr) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eFltr2), utils.ToJSON(fltr)) - } -} - -func TestLoaderProcessThresholds(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaThresholds: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "MaxHits", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "MinHits", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "MinSleep", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Path: "Blocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Path: "ActionIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - {Path: "Async", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(ThresholdsCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{fileName: utils.ThresholdsCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - eTh1 := &engine.ThresholdProfile{ - Tenant: "cgrates.org", - ID: "Threshold1", - FilterIDs: []string{"*string:~*req.Account:1001", "*string:~*req.RunID:*default"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, - MaxHits: 12, - MinHits: 10, - MinSleep: time.Second, - Blocker: true, - Weight: 10, - ActionIDs: []string{"THRESH1"}, - Async: true, - } - aps, err := ldr.dm.GetThresholdProfile("cgrates.org", "Threshold1", - true, false, utils.NonTransactional) - sort.Strings(eTh1.FilterIDs) - sort.Strings(aps.FilterIDs) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eTh1, aps) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eTh1), utils.ToJSON(aps)) - } - - //cannot set thresholdProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(ThresholdsCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - "Thresholds.csv": &openedCSVFile{fileName: "Thresholds.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessStats(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "QueueLength", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "TTL", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "MinItems", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Path: "MetricIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Path: "MetricFilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Path: "Blocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - {Path: "Stored", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.11", utils.InfieldSep)}, - - {Path: "ThresholdIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.12", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(StatsCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: {utils.StatsCsv: &openedCSVFile{rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - eSt1 := &engine.StatQueueProfile{ - Tenant: "cgrates.org", - ID: "TestStats", - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 00, 0, 0, time.UTC), - }, - QueueLength: 100, - TTL: time.Second, - Metrics: []*engine.MetricWithFilters{ - {MetricID: "*sum#~*req.Value"}, - {MetricID: "*average#~*req.Value"}, - {MetricID: "*sum#~*req.Usage"}, - }, - ThresholdIDs: []string{"Th1", "Th2"}, - Blocker: true, - Stored: true, - Weight: 20, - MinItems: 2, - } - - aps, err := ldr.dm.GetStatQueueProfile("cgrates.org", "TestStats", - true, false, utils.NonTransactional) - //sort the slices of Metrics - sort.Slice(eSt1.Metrics, func(i, j int) bool { return eSt1.Metrics[i].MetricID < eSt1.Metrics[j].MetricID }) - sort.Slice(aps.Metrics, func(i, j int) bool { return aps.Metrics[i].MetricID < aps.Metrics[j].MetricID }) - sort.Strings(eSt1.ThresholdIDs) - sort.Strings(aps.ThresholdIDs) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eSt1, aps) { - t.Errorf("expecting: %+v, received: %+v", utils.ToJSON(eSt1), utils.ToJSON(aps)) - } - - //cannot set statsProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(StatsCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: {utils.StatsCsv: &openedCSVFile{rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessStatsWrongMetrics(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessStatsWrongMetrics", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - dataTpls: map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "MetricIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - {Path: "Stored", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep)}, - }, - }, - } - rdr := io.NopCloser(strings.NewReader(`#Metrics[0],Stored[1] -not_a_valid_metric_type,true,`)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{fileName: utils.StatsCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expected := "unsupported metric type " - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } - - //initialize again but with a valid metric and false stored field - rdr = io.NopCloser(strings.NewReader(`#Metrics[0],Stored[1] -*sum#~*req.Value,false`)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{fileName: utils.StatsCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessRoutes(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRoutes: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "Sorting", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "SortingParameters", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "RouteID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - {Path: "RouteFilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep)}, - {Path: "RouteAccountIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep)}, - {Path: "RouteRatingplanIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep)}, - {Path: "RouteResourceIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep)}, - {Path: "RouteStatIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.11", utils.InfieldSep)}, - {Path: "RouteWeight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.12", utils.InfieldSep)}, - {Path: "RouteBlocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.13", utils.InfieldSep)}, - {Path: "RouteParameters", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.14", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.15", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(RoutesCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{fileName: utils.RoutesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaRoutes, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - - eSp := &engine.RouteProfile{ - Tenant: "cgrates.org", - ID: "RoutePrf1", - FilterIDs: []string{"*string:~*req.Account:dan"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC), - }, - Sorting: utils.MetaLC, - SortingParameters: []string{}, - Routes: []*engine.Route{ - { - ID: "route1", - FilterIDs: []string{"FLTR_ACNT_dan"}, - AccountIDs: []string{"Account1", "Account1_1"}, - RatingPlanIDs: []string{"RPL_1"}, - ResourceIDs: []string{"ResGroup1"}, - StatIDs: []string{"Stat1"}, - Weight: 10, - Blocker: true, - RouteParameters: "param1", - }, - { - ID: "route1", - FilterIDs: []string{"FLTR_DST_DE"}, - AccountIDs: []string{"Account2"}, - RatingPlanIDs: []string{"RPL_3"}, - ResourceIDs: []string{"ResGroup3"}, - StatIDs: []string{"Stat2"}, - Weight: 10, - Blocker: false, - RouteParameters: utils.EmptyString, - }, - { - ID: "route1", - RatingPlanIDs: []string{"RPL_2"}, - ResourceIDs: []string{"ResGroup2", "ResGroup4"}, - StatIDs: []string{"Stat3"}, - Weight: 10, - Blocker: false, - RouteParameters: utils.EmptyString, - }, - }, - Weight: 20, - } - sort.Slice(eSp.Routes, func(i, j int) bool { - return strings.Compare(eSp.Routes[i].ID+strings.Join(eSp.Routes[i].FilterIDs, utils.ConcatenatedKeySep), - eSp.Routes[j].ID+strings.Join(eSp.Routes[j].FilterIDs, utils.ConcatenatedKeySep)) < 0 - }) - - aps, err := ldr.dm.GetRouteProfile("cgrates.org", "RoutePrf1", - true, false, utils.NonTransactional) - if err != nil { - t.Fatal(err) - } - sort.Slice(aps.Routes, func(i, j int) bool { - return strings.Compare(aps.Routes[i].ID+strings.Join(aps.Routes[i].FilterIDs, utils.ConcatenatedKeySep), - aps.Routes[j].ID+strings.Join(aps.Routes[j].FilterIDs, utils.ConcatenatedKeySep)) < 0 - }) - if !reflect.DeepEqual(eSp, aps) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eSp), utils.ToJSON(aps)) - } - - //cannot set RoutesProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(RoutesCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{fileName: utils.RoutesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaRoutes, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessChargers(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaChargers: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - {Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep)}, - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep)}, - {Path: "RunID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep)}, - {Path: "AttributeIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep)}, - }, - } - rdr := io.NopCloser(strings.NewReader(ChargersCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{fileName: utils.ChargersCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaChargers, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - eCharger1 := &engine.ChargerProfile{ - Tenant: "cgrates.org", - ID: "Charger1", - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 00, 0, 0, time.UTC), - }, - RunID: "*rated", - AttributeIDs: []string{"ATTR_1001_SIMPLEAUTH"}, - Weight: 20, - } - - if rcv, err := ldr.dm.GetChargerProfile("cgrates.org", "Charger1", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eCharger1, rcv) { - t.Errorf("expecting: %s, received: %s", - utils.ToJSON(eCharger1), utils.ToJSON(rcv)) - } - - //cannot set chargerProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(ChargersCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{fileName: utils.ChargersCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.processContent(utils.MetaChargers, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessDispatches(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatchers: { - { - Tag: "TenantID", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ProfileID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "Subsystems", - Path: "Subsystems", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - }, - { - Tag: "FilterIDs", - Path: "FilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - }, - { - Tag: "ActivationInterval", - Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - }, - { - Tag: "Strategy", - Path: "Strategy", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - }, - { - Tag: "StrategyParameters", - Path: "StrategyParameters", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - }, - { - Tag: "ConnID", - Path: "ConnID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - }, - { - Tag: "ConnFilterIDs", - Path: "ConnFilterIDs", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - }, - { - Tag: "ConnWeight", - Path: "ConnWeight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - }, - { - Tag: "ConnBlocker", - Path: "ConnBlocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - }, - { - Tag: "ConnParameters", - Path: "ConnParameters", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - }, - { - Tag: "Weight", - Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - }, - }, - } - rdr := io.NopCloser(strings.NewReader(DispatcherCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.processContent(utils.MetaDispatchers, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - eDisp := &engine.DispatcherProfile{ - Tenant: "cgrates.org", - ID: "D1", - Subsystems: []string{"*any"}, - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 00, 0, 0, time.UTC), - }, - StrategyParams: map[string]any{}, - Strategy: "*first", - Weight: 20, - Hosts: engine.DispatcherHostProfiles{ - &engine.DispatcherHostProfile{ - ID: "C1", - FilterIDs: []string{"*gt:~*req.Usage:10"}, - Weight: 10, - Params: map[string]any{"0": "192.168.56.203"}, - Blocker: false, - }, - &engine.DispatcherHostProfile{ - ID: "C2", - FilterIDs: []string{"*lt:~*req.Usage:10"}, - Weight: 10, - Params: map[string]any{"0": "192.168.56.204"}, - Blocker: false, - }, - }, - } - - rcv, err := ldr.dm.GetDispatcherProfile("cgrates.org", "D1", - true, false, utils.NonTransactional) - if err != nil { - t.Fatal(err) - } - sort.Slice(eDisp.Hosts, func(i, j int) bool { return eDisp.Hosts[i].ID < eDisp.Hosts[j].ID }) - sort.Slice(rcv.Hosts, func(i, j int) bool { return rcv.Hosts[i].ID < rcv.Hosts[j].ID }) - if !reflect.DeepEqual(eDisp, rcv) { - t.Errorf("expecting: %+v, received: %+v", utils.ToJSON(eDisp), utils.ToJSON(rcv)) - } - - //cannot set DispatchersProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(DispatcherCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.processContent(utils.MetaDispatchers, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderProcessDispatcheHosts(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatcherHosts: { - { - Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "Address", - Path: "Address", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - }, - { - Tag: "Transport", - Path: "Transport", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - }, - { - Tag: "ConnectAttempts", - Path: "ConnectAttempts", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "Reconnects", - Path: "Reconnects", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "MaxReconnectInterval", - Path: "MaxReconnectInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ConnectTimeout", - Path: "ConnectTimeout", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ReplyTimeout", - Path: "ReplyTimeout", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.8", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "TLS", - Path: "TLS", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.9", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ClientKey", - Path: "ClientKey", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.10", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ClientCertificate", - Path: "ClientCertificate", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.11", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "CaCertificate", - Path: "CaCertificate", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.12", utils.InfieldSep), - Mandatory: true, - }, - }, - } - rdr := io.NopCloser(strings.NewReader(DispatcherHostCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.processContent(utils.MetaDispatcherHosts, utils.EmptyString); err != nil { - t.Error(err) - } - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - eDispHost := &engine.DispatcherHost{ - Tenant: "cgrates.org", - RemoteHost: &config.RemoteHost{ - ID: "ALL", - Address: "127.0.0.1:6012", - Transport: utils.MetaJSON, - ConnectAttempts: 1, - Reconnects: 3, - MaxReconnectInterval: 5 * time.Minute, - ConnectTimeout: 1 * time.Minute, - ReplyTimeout: 2 * time.Minute, - TLS: false, - }, - } - - rcv, err := ldr.dm.GetDispatcherHost("cgrates.org", "ALL", - true, false, utils.NonTransactional) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(eDispHost, rcv) { - t.Errorf("expecting: %+v, received: %+v", utils.ToJSON(eDispHost), utils.ToJSON(rcv)) - } - - //cannot set DispatcherHostProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(DispatcherHostCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.processContent(utils.MetaDispatcherHosts, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestLoaderRemoveContentSingleFile(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - rdr := io.NopCloser(strings.NewReader(AttributesCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - "Attributes.csv": &openedCSVFile{fileName: "Attributes.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - // Add two attributeProfiles - ap := &engine.AttributeProfile{ - Tenant: "cgrates.org", - ID: "ALS1", - Contexts: []string{"con1", "con2", "con3"}, - FilterIDs: []string{"*string:~*req.Account:1001"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC)}, - Attributes: []*engine.Attribute{ - { - FilterIDs: []string{"*string:~*req.Field1:Initial"}, - Path: utils.MetaReq + utils.NestingSep + "Field1", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub1", utils.InfieldSep), - }, - { - FilterIDs: []string{}, - Path: utils.MetaReq + utils.NestingSep + "Field2", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("Sub2", utils.InfieldSep), - }}, - Blocker: true, - Weight: 20, - } - if err := ldr.dm.SetAttributeProfile(ap, true); err != nil { - t.Error(err) - } - ap.ID = "Attr2" - if err := ldr.dm.SetAttributeProfile(ap, true); err != nil { - t.Error(err) - } - - if err := ldr.removeContent(utils.MetaAttributes, utils.EmptyString); err != nil { - t.Error(err) - } - - if len(ldr.bufLoaderData) != 0 { - t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) - } - // make sure the first attribute is deleted - if _, err := ldr.dm.GetAttributeProfile("cgrates.org", "ALS1", - true, false, utils.NonTransactional); err == nil || err.Error() != utils.ErrNotFound.Error() { - t.Error(err) - } - // the second should be there - if rcv, err := ldr.dm.GetAttributeProfile("cgrates.org", "Attr2", - true, false, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(ap, rcv) { - t.Errorf("expecting: %s, \n received: %s", - utils.ToJSON(ap), utils.ToJSON(rcv)) - } - - //now should be empty, nothing to remove - if err := ldr.removeContent(utils.MetaAttributes, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(AttributesCSVContent)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - "Attributes.csv": &openedCSVFile{fileName: "Attributes.csv", - rdr: rdr, csvRdr: csvRdr}}, - } - if err := ldr.removeContent(utils.MetaAttributes, utils.EmptyString); err != nil { - t.Error(err) - } -} -func TestNewLoaderWithMultiFiles(t *testing.T) { - - ldrCfg := config.CgrConfig().LoaderCfg()[0].Clone() - ldrCfg.Data[0].Fields = []*config.FCTemplate{ - {Path: "Tenant", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("cgrates.org", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.InfieldSep), - Mandatory: true}, - {Path: "Contexts", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("*any", utils.InfieldSep)}, - {Path: "Path", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.InfieldSep)}, - {Path: "Value", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.InfieldSep)}, - {Path: "Weight", - Type: utils.MetaString, - Value: config.NewRSRParsersMustCompile("10", utils.InfieldSep)}, - } - ldr := NewLoader(engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), ldrCfg, "", 0, nil, nil, nil) - - openRdrs := make(utils.StringSet) - for _, rdr := range ldr.rdrs { - for fileName := range rdr { - openRdrs.Add(fileName) - } - } - expected := utils.StringSet{ - utils.AttributesCsv: {}, - utils.ChargersCsv: {}, - utils.DispatcherHostsCsv: {}, - utils.DispatcherProfilesCsv: {}, - "File1.csv": {}, - "File2.csv": {}, - utils.FiltersCsv: {}, - utils.IPsCsv: {}, - utils.ResourcesCsv: {}, - utils.RoutesCsv: {}, - utils.StatsCsv: {}, - utils.ThresholdsCsv: {}, - } - if !reflect.DeepEqual(expected, openRdrs) { - t.Errorf("Expected %s,received %s", utils.ToJSON(expected), utils.ToJSON(openRdrs)) - } -} - -func TestLoaderAttributesAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderAttributesAsStructErrType", - bufLoaderData: map[string][]LoaderData{}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Path: "Weight", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - attributeCsv := ` -#Weight -true -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{fileName: utils.AttributesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expectedErr := "strconv.ParseFloat: parsing \"true\": invalid syntax" - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Error(err) - } -} - -func TestLoaderAttributesAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderAttributesAsStructErrConversion", - bufLoaderData: map[string][]LoaderData{}, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaAttributes: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - attributeCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(attributeCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaAttributes: { - utils.AttributesCsv: &openedCSVFile{fileName: utils.AttributesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaAttributes, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadResourcesAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadResourcesAsStructErr", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Path: "Blocker", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - resourcesCsv := ` -#Blocker -NOT_A_BOOLEAN -` - rdr := io.NopCloser(strings.NewReader(resourcesCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: &openedCSVFile{fileName: utils.ResourcesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expectedErr := "strconv.ParseBool: parsing \"NOT_A_BOOLEAN\": invalid syntax" - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadResourcesAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadResourcesAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Path: "UsageTTL", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - resourcesCsv := ` -#UsageTTL -12ss -` - rdr := io.NopCloser(strings.NewReader(resourcesCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - utils.ResourcesCsv: &openedCSVFile{fileName: utils.ResourcesCsv, - rdr: rdr, csvRdr: csvRdr}}, - } - expectedErr := "time: unknown unit \"ss\" in duration \"12ss\"" - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadFiltersAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadFiltersAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaFilters: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - filtersCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(filtersCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - utils.FiltersCsv: &openedCSVFile{ - fileName: utils.FiltersCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaFilters, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadFiltersAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadFiltersAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaFilters: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - filtersCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(filtersCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - utils.FiltersCsv: &openedCSVFile{ - fileName: utils.FiltersCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaFilters, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadStatsAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadStatsAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - statsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(statsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadStatsAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadStatsAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - statsCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(statsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadThresholdsAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadThresholdsAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaThresholds: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadThresholdsAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadThresholdsAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaThresholds: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadRoutesAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadRoutesAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRoutes: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{ - fileName: utils.RoutesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaRoutes, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadRoutesAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadRoutesAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRoutes: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{ - fileName: utils.RoutesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaRoutes, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadChargersAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadChargersAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaChargers: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{ - fileName: utils.ChargersCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaChargers, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadChargersAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadChargersAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaChargers: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{ - fileName: utils.ChargersCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaChargers, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadDispatchersAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadDispatchersAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatchers: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaDispatchers, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadDispatcherAsStructErrConversion(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadDispatcherHostsAsStructErrConversion", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatchers: { - {Path: "ActivationInterval", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#ActivationInterval -* * * * * * -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "Unsupported time format" - if err := ldr.processContent(utils.MetaDispatchers, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadDispatcherHostsAsStructErrType(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadDispatcherHostsAsStructErrType", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatcherHosts: { - {Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep)}, - }, - } - thresholdsCsv := ` -#PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherHostsCsv: &openedCSVFile{ - fileName: utils.DispatcherHostsCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expectedErr := "cannot update unsupported struct field: 0" - if err := ldr.processContent(utils.MetaDispatcherHosts, utils.EmptyString); err == nil || err.Error() != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoadAndRemoveResources(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadAndRemoveResources", - bufLoaderData: make(map[string][]LoaderData), - dryRun: true, - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - resourcesCSV := ` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -` - rdr := io.NopCloser(strings.NewReader(resourcesCSV)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - //empty database - if _, err := ldr.dm.GetResourceProfile("cgrates.org", "NewRes1", false, false, utils.NonTransactional); err != utils.ErrNotFound { - t.Error(err) - } - - //because of dryrun, database will be empty again - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err != nil { - t.Error(err) - } - - ldr.dryRun = false - //reinitialized reader because after first process the reader is at the end of the file - rdr = io.NopCloser(strings.NewReader(resourcesCSV)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - - resPrf := &engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "NewRes1", - FilterIDs: []string{}, - ThresholdIDs: []string{}, - } - //NOT_FOUND because is resourceProfile is not set - if _, err := ldr.dm.GetResourceProfile("cgrates.org", "NewRes1", false, false, utils.NonTransactional); err != utils.ErrNotFound { - t.Error(err) - } - - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err != nil { - t.Error(err) - } - - rcv, err := ldr.dm.GetResourceProfile("cgrates.org", "NewRes1", false, false, utils.NonTransactional) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(resPrf, rcv) { - t.Errorf("Expected %+v, received %+v", utils.ToJSON(resPrf), utils.ToJSON(rcv)) - } - - //reinitialized reader because seeker it s at the end of the file - rdr = io.NopCloser(strings.NewReader(resourcesCSV)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - - //cannot remove when dryrun is on true - ldr.dryRun = true - if err := ldr.removeContent(utils.MetaResources, utils.EmptyString); err != nil { - t.Error(err) - } - - //remove successfully when dryrun is false - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(resourcesCSV)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - if err := ldr.removeContent(utils.MetaResources, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing in database - if _, err := ldr.dm.GetResourceProfile("cgrates.org", "NewRes1", false, false, utils.NonTransactional); err != utils.ErrNotFound { - t.Error(err) - } - - //nothing to remove - if err := ldr.removeContent(utils.MetaResources, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot set again ResourceProfile when dataManager is nil - ldr.dm = nil - rdr = io.NopCloser(strings.NewReader(resourcesCSV)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveFilterContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveFilterContents", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaFilters: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - filtersCsv := ` -#Tenant[0],ID[0] -cgrates.org,FILTERS_REM_1 -` - rdr := io.NopCloser(strings.NewReader(filtersCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - utils.FiltersCsv: &openedCSVFile{ - fileName: utils.FiltersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - eFltr1 := &engine.Filter{ - Tenant: "cgrates.org", - ID: "FILTERS_REM_1", - } - if err := ldr.dm.SetFilter(eFltr1, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaFilters, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remove from database - if err := ldr.removeContent(utils.MetaFilters, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove Filter when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(filtersCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - utils.FiltersCsv: &openedCSVFile{ - fileName: utils.FiltersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaFilters, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again FiltersProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(filtersCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaFilters: { - utils.FiltersCsv: &openedCSVFile{ - fileName: utils.FiltersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaFilters, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveStatsContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - statsCsv := ` -#Tenant[0],ProfileID[1] -cgrates.org,REM_STATS_1 -` - rdr := io.NopCloser(strings.NewReader(statsCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expStats := &engine.StatQueueProfile{ - Tenant: "cgrates.org", - ID: "REM_STATS_1", - } - if err := ldr.dm.SetStatQueueProfile(expStats, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remove from database - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove statsQueueProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(statsCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again StatsProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(statsCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveThresholdsContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveThresholdsContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaThresholds: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - thresholdsCsv := ` -#Tenant[0],ID[1] -cgrates.org,REM_THRESHOLDS_1, -` - rdr := io.NopCloser(strings.NewReader(thresholdsCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expThresholdPrf := &engine.ThresholdProfile{ - Tenant: "cgrates.org", - ID: "REM_THRESHOLDS_1", - } - if err := ldr.dm.SetThresholdProfile(expThresholdPrf, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remove from database - if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove statsQueueProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(thresholdsCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again ThresholdsProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(thresholdsCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveRoutesContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveRoutesContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaRoutes: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - routesCsv := ` -#Tenant[0],ID[1] -cgrates.org,ROUTES_REM_1 -` - rdr := io.NopCloser(strings.NewReader(routesCsv)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{ - fileName: routesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expRoutes := &engine.RouteProfile{ - Tenant: "cgrates.org", - ID: "ROUTES_REM_1", - } - if err := ldr.dm.SetRouteProfile(expRoutes, true); err != nil { - t.Error(err) - } - if err := ldr.removeContent(utils.MetaRoutes, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remove from database - if err := ldr.removeContent(utils.MetaRoutes, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove routeProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(routesCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{ - fileName: routesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - if err := ldr.removeContent(utils.MetaRoutes, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again RoutesProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(routesCsv)) - rdrCsv = csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaRoutes: { - utils.RoutesCsv: &openedCSVFile{ - fileName: routesCsv, - rdr: rdr, - csvRdr: rdrCsv, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaRoutes, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveChargersContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveChargersContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaChargers: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - routesCsv := ` -#Tenant[0],ID[1] -cgrates.org,REM_ROUTES_1 -` - rdr := io.NopCloser(strings.NewReader(routesCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{ - fileName: utils.ChargersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expChargers := &engine.ChargerProfile{ - Tenant: "cgrates.org", - ID: "REM_ROUTES_1", - } - if err := ldr.dm.SetChargerProfile(expChargers, true); err != nil { - t.Error(err) - } else if err := ldr.removeContent(utils.MetaChargers, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remvoe from database - if err := ldr.removeContent(utils.MetaChargers, utils.EmptyString); err != utils.ErrNotFound { - t.Error(err) - } - - //cannot remove ChargersProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(routesCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{ - fileName: utils.ChargersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaChargers, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again ChargersProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(routesCsv)) - rdr = io.NopCloser(strings.NewReader(routesCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaChargers: { - utils.ChargersCsv: &openedCSVFile{ - fileName: utils.ChargersCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaChargers, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveDispatchersContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveDispatchersContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatchers: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - dispatchersCsv := ` -#Tenant[0],ID[1] -cgrates.org,REM_DISPATCHERS_1 -` - rdr := io.NopCloser(strings.NewReader(dispatchersCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expDispatchers := &engine.DispatcherProfile{ - Tenant: "cgrates.org", - ID: "REM_DISPATCHERS_1", - } - if err := ldr.dm.SetDispatcherProfile(expDispatchers, true); err != nil { - t.Error(err) - } else if err := ldr.removeContent(utils.MetaDispatchers, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remvoe from database - if err := ldr.removeContent(utils.MetaDispatchers, utils.EmptyString); err != utils.ErrDSPProfileNotFound { - t.Error(err) - } - - //cannot remove DispatchersProfile when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(dispatchersCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaDispatchers, utils.EmptyString); err != nil { - t.Error(err) - } - - //cannot set again DispatchersProfile when dataManager is nil - ldr.dm = nil - ldr.dryRun = false - rdr = io.NopCloser(strings.NewReader(dispatchersCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatchers: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expected := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaDispatchers, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveDispatcherHostsContent(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveDispatcherHostsContent", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatcherHosts: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - } - dispatchersHostsCsv := ` -#Tenant[0],ID[1] -cgrates.org,REM_DISPATCHERH_1 -` - rdr := io.NopCloser(strings.NewReader(dispatchersHostsCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherHostsCsv: &openedCSVFile{ - fileName: utils.DispatcherHostsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expDispatchers := &engine.DispatcherHost{ - Tenant: "cgrates.org", - RemoteHost: &config.RemoteHost{ - ID: "REM_DISPATCHERH_1", - }, - } - if err := ldr.dm.SetDispatcherHost(expDispatchers); err != nil { - t.Error(err) - } else if err := ldr.removeContent(utils.MetaDispatcherHosts, utils.EmptyString); err != nil { - t.Error(err) - } - - //nothing to remove from database - if err := ldr.removeContent(utils.MetaDispatcherHosts, utils.EmptyString); err != utils.ErrDSPHostNotFound { - t.Error(err) - } - - //cannot remove DispatcherHosts when dryrun is true - ldr.dryRun = true - rdr = io.NopCloser(strings.NewReader(dispatchersHostsCsv)) - csvRdr = csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherHostsCsv: &openedCSVFile{ - fileName: utils.DispatcherHostsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - if err := ldr.removeContent(utils.MetaDispatcherHosts, utils.EmptyString); err != nil { - t.Error(err) - } -} - -func TestProcessContentEmptyDataBase(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoaderProcessContent", - bufLoaderData: make(map[string][]LoaderData), - dm: nil, - timezone: "UTC", - } - ldr.dataTpls = map[string][]*config.FCTemplate{ - utils.MetaDispatcherHosts: { - { - Tag: "Tenant", - Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "ID", - Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - }, - { - Tag: "Address", - Path: "Address", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.2", utils.InfieldSep), - }, - { - Tag: "Transport", - Path: "Transport", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.3", utils.InfieldSep), - }, - { - Tag: "TLS", - Path: "TLS", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.4", utils.InfieldSep), - }, - }, - } - rdr := io.NopCloser(strings.NewReader(DispatcherHostCSVContent)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaDispatcherHosts: { - utils.DispatcherProfilesCsv: &openedCSVFile{ - fileName: utils.DispatcherProfilesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - expectedErr := utils.ErrNoDatabaseConn - if err := ldr.processContent(utils.MetaDispatcherHosts, utils.EmptyString); err == nil || err != expectedErr { - t.Errorf("Expected %+v, received %+v", expectedErr, err) - } -} - -func TestLoaderListenAndServe(t *testing.T) { - ldr := &Loader{} - stopChan := make(chan struct{}, 1) - go func() { - stopChan <- struct{}{} - }() - - if err := ldr.ListenAndServe(stopChan); err != nil { - t.Error(err) - } - - ldr.runDelay = -1 - if err := ldr.ListenAndServe(stopChan); err != nil { - t.Error(err) - } - - ldr.runDelay = 1 - if err := ldr.ListenAndServe(stopChan); err != nil { - t.Error(err) - } -} - -func TestRemoveThresholdsMockError(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveThresholdsMockError", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - dataTpls: map[string][]*config.FCTemplate{ - utils.MetaThresholds: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - }, - } - rdr := io.NopCloser(strings.NewReader(`#Tenant[0],ID[1] - cgrates.org,REM_THRESHOLDS_1,`)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaThresholds: { - utils.ThresholdsCsv: &openedCSVFile{ - fileName: utils.ThresholdsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - - expected := utils.ErrNoDatabaseConn - ldr.dm = engine.NewDataManager(&engine.DataDBMock{ - GetThresholdProfileDrvF: func(tenant, id string) (tp *engine.ThresholdProfile, err error) { - return &engine.ThresholdProfile{ - Tenant: "cgrates.org", - ID: "REM_THRESHOLDS_1", - }, nil - }, - SetThresholdProfileDrvF: func(tp *engine.ThresholdProfile) (err error) { return expected }, - RemThresholdProfileDrvF: func(tenant, id string) (err error) { return expected }, - }, config.CgrConfig().CacheCfg(), nil) - if err := ldr.processContent(utils.MetaThresholds, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } else if err := ldr.removeContent(utils.MetaThresholds, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveStatQueueMockError(t *testing.T) { - ldr := &Loader{ - ldrID: "TestRemoveStatQueueError", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - dataTpls: map[string][]*config.FCTemplate{ - utils.MetaStats: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - }, - } - rdr := io.NopCloser(strings.NewReader(`#Tenant[0],ProfileID[1] -cgrates.org,REM_STATS_1`)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaStats: { - utils.StatsCsv: &openedCSVFile{ - fileName: utils.StatsCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - - expected := utils.ErrNoDatabaseConn - ldr.dm = engine.NewDataManager(&engine.DataDBMock{ - GetStatQueueProfileDrvF: func(tenant, id string) (sq *engine.StatQueueProfile, err error) { return nil, nil }, - SetStatQueueProfileDrvF: func(sq *engine.StatQueueProfile) (err error) { return expected }, - RemStatQueueProfileDrvF: func(tenant, id string) (err error) { return expected }, - }, config.CgrConfig().CacheCfg(), nil) - - if err := ldr.removeContent(utils.MetaStats, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } else if err := ldr.processContent(utils.MetaStats, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestRemoveResourcesMockError(t *testing.T) { - ldr := &Loader{ - ldrID: "TestLoadAndRemoveResources", - bufLoaderData: make(map[string][]LoaderData), - dm: engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), - timezone: "UTC", - dataTpls: map[string][]*config.FCTemplate{ - utils.MetaResources: { - {Path: "Tenant", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true}, - {Path: "ID", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true}, - }, - }, - } - rdr := io.NopCloser(strings.NewReader(` #Tenant[0],ID[1] - cgrates.org,NewRes1`)) - rdrCsv := csv.NewReader(rdr) - rdrCsv.Comment = '#' - ldr.rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "Resources.csv": &openedCSVFile{fileName: "Resources.csv", - rdr: rdr, csvRdr: rdrCsv}}, - } - - expected := utils.ErrNoDatabaseConn - ldr.dm = engine.NewDataManager(&engine.DataDBMock{ - GetResourceProfileDrvF: func(tnt, id string) (*engine.ResourceProfile, error) { return nil, nil }, - SetResourceProfileDrvF: func(rp *engine.ResourceProfile) error { return expected }, - RemoveResourceProfileDrvF: func(tnt, id string) error { return expected }, - }, config.CgrConfig().CacheCfg(), nil) - - if err := ldr.removeContent(utils.MetaResources, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } else if err := ldr.processContent(utils.MetaResources, utils.EmptyString); err == nil || err != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } -} - -func TestLoaderHandleFolder(t *testing.T) { - stopChan := make(chan struct{}, 1) - stopChan <- struct{}{} - ldr := &Loader{ - ldrID: "TestLoaderHandleFolder", - runDelay: 1, - } - ldr.handleFolder(stopChan) -} - -func TestLoaderServiceEnabled(t *testing.T) { - //THis is an empty loader, so there is not an active loader - ldrs := &LoaderService{} - if rcv := ldrs.Enabled(); rcv { - t.Errorf("Expected false, received %+v", rcv) - } -} - -type ccMock struct { - calls map[string]func(args any, reply any) error -} - -func (ccM *ccMock) Call(_ *context.Context, serviceMethod string, args any, reply any) (err error) { - if call, has := ccM.calls[serviceMethod]; !has { - return rpcclient.ErrUnsupporteServiceMethod - } else { - return call(args, reply) - } -} - -func TestStoreLoadedDataAttributes(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - AttributeProfileIDs: []string{"cgrates.org:attributesID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Attributes": { - { - "Tenant": "cgrates.org", - "ID": "attributesID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaAttributes, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataResources(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - ResourceIDs: []string{"cgrates.org:resourcesID"}, - ResourceProfileIDs: []string{"cgrates.org:resourcesID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Resources": { - { - "Tenant": "cgrates.org", - "ID": "resourcesID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaResources, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataFilters(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - FilterIDs: []string{"cgrates.org:filtersID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Filters": { - { - "Tenant": "cgrates.org", - "ID": "filtersID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaFilters, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataStats(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - StatsQueueIDs: []string{"cgrates.org:statsID"}, - StatsQueueProfileIDs: []string{"cgrates.org:statsID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "StatsQueue": { - { - "Tenant": "cgrates.org", - "ID": "statsID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaStats, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataThresholds(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - ThresholdIDs: []string{"cgrates.org:thresholdsID"}, - ThresholdProfileIDs: []string{"cgrates.org:thresholdsID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Thresholds": { - { - "Tenant": "cgrates.org", - "ID": "thresholdsID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaThresholds, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataRoutes(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - RouteProfileIDs: []string{"cgrates.org:routesID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Routes": { - { - "Tenant": "cgrates.org", - "ID": "routesID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaRoutes, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataChargers(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - ChargerProfileIDs: []string{"cgrates.org:chargersID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Chargers": { - { - "Tenant": "cgrates.org", - "ID": "chargersID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaChargers, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataDispatchers(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - DispatcherProfileIDs: []string{"cgrates.org:dispatchersID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "Dispatchers": { - { - "Tenant": "cgrates.org", - "ID": "dispatchersID", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaDispatchers, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataDispatcherHosts(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - DispatcherHostIDs: []string{"cgrates.org:dispatcherHostsID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - // ldr := &Loader{ - - // } - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", 0, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "DispatcherHosts": { - { - "Tenant": "cgrates.org", - "ID": "dispatcherHostsID", - "Address": "192.168.100.1", - }, - }, - } - if err := ldr.storeLoadedData(utils.MetaDispatcherHosts, lds, utils.MetaReload); err != nil { - t.Error(err) - } -} - -func TestStoreLoadedDataWithDelay(t *testing.T) { - engine.Cache.Clear(nil) - cfg := config.NewDefaultCGRConfig() - cfg.GeneralCfg().CachingDelay = 5 * time.Millisecond - argExpect := &utils.AttrReloadCacheWithAPIOpts{ - APIOpts: nil, - Tenant: "", - DispatcherHostIDs: []string{"cgrates.org:dispatcherHostsID"}, - } - cM := &ccMock{ - calls: map[string]func(args any, reply any) error{ - utils.CacheSv1ReloadCache: func(args any, reply any) error { - if !reflect.DeepEqual(args, argExpect) { - t.Errorf("Expected %v \nbut received %v", utils.ToJSON(argExpect), utils.ToJSON(args)) - } - return nil - }, - utils.CacheSv1Clear: func(args any, reply any) error { - return nil - }, - }, - } - - rpcInternal := make(chan birpc.ClientConnector, 1) - rpcInternal <- cM - connMgr := engine.NewConnManager(cfg, map[string]chan birpc.ClientConnector{ - utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches): rpcInternal, - }) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) - cacheConns := []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaCaches)} - loaderCfg := config.CgrConfig().LoaderCfg() - fltrS := engine.NewFilterS(cfg, connMgr, dm) - ldr := NewLoader(dm, loaderCfg[0], "", cfg.GeneralCfg().CachingDelay, fltrS, connMgr, cacheConns) - lds := map[string][]LoaderData{ - "DispatcherHosts": { - { - "Tenant": "cgrates.org", - "ID": "dispatcherHostsID", - "Address": "192.168.100.1", - }, - }, - } - tStart := time.Now() - if err := ldr.storeLoadedData(utils.MetaDispatcherHosts, lds, utils.MetaReload); err != nil { - t.Error(err) - } - got := time.Since(tStart) - want := cfg.GeneralCfg().CachingDelay - if diff := got - want; diff < 0 || diff > 15*time.Millisecond { - t.Errorf("storeLoadedData duration = %v, want at least %v (diff %v, margin 15ms)", got, want, diff) - } -} - -func TestLoaderLoaderallFilesPresent(t *testing.T) { - tests := []struct { - name string - ldrType string - rdrs map[string]map[string]*openedCSVFile - expected bool - }{ - { - name: "All files present", - ldrType: "type1", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": &openedCSVFile{}, "file3": &openedCSVFile{}}, - }, - expected: true, - }, - { - name: "Some files nil", - ldrType: "type1", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": nil, "file3": &openedCSVFile{}}, - }, - expected: false, - }, - { - name: "No files for type", - ldrType: "type1", - rdrs: map[string]map[string]*openedCSVFile{}, - expected: true, - }, - { - name: "Type not present", - ldrType: "type2", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": &openedCSVFile{}, "file3": &openedCSVFile{}}, - }, - expected: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ldr := &Loader{rdrs: tt.rdrs} - result := ldr.allFilesPresent(tt.ldrType) - reflect.DeepEqual(tt.expected, result) - }) - } -} - -func TestLoadergetLdrType(t *testing.T) { - tests := []struct { - name string - fName string - rdrs map[string]map[string]*openedCSVFile - expected string - }{ - { - name: "File name exists in one loader type", - fName: "file1", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": &openedCSVFile{}}, - "type2": {"file3": &openedCSVFile{}, "file4": &openedCSVFile{}}, - }, - expected: "type1", - }, - { - name: "File name does not exist in any loader type", - fName: "file5", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": &openedCSVFile{}}, - "type2": {"file3": &openedCSVFile{}, "file4": &openedCSVFile{}}, - }, - expected: "", - }, - { - name: "Multiple loader types, file name present in one", - fName: "file4", - rdrs: map[string]map[string]*openedCSVFile{ - "type1": {"file1": &openedCSVFile{}, "file2": &openedCSVFile{}}, - "type2": {"file3": &openedCSVFile{}, "file4": &openedCSVFile{}}, - "type3": {"file5": &openedCSVFile{}}, - }, - expected: "type2", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ldr := &Loader{rdrs: tt.rdrs} - result := ldr.getLdrType(tt.fName) - reflect.DeepEqual(tt.expected, result) - }) - } -} - -func TestLoaderIsFolderLocked(t *testing.T) { - tests := []struct { - name string - lockFilepath string - expectedLocked bool - expectedErr error - }{ - { - name: "No lock file path", - lockFilepath: "", - expectedLocked: false, - expectedErr: nil, - }, - { - name: "Lock file does not exist", - lockFilepath: "nonexistent/path/to/lockfile", - expectedLocked: false, - expectedErr: nil, - }, - { - name: "Error checking lock file", - lockFilepath: "invalid/file/path", - expectedLocked: false, - expectedErr: os.ErrInvalid, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ldr := &Loader{lockFilepath: tt.lockFilepath} - locked, _ := ldr.isFolderLocked() - if locked != tt.expectedLocked { - t.Errorf("expected locked: %v, got: %v", tt.expectedLocked, locked) - } - - }) - } -} - -func TestProcessFiles(t *testing.T) { - tempDir := t.TempDir() - tempFileName := path.Join(tempDir, "testfile.csv") - fileContent := "header1,header2\nvalue1,value2\n" - if err := os.WriteFile(tempFileName, []byte(fileContent), 0644); err != nil { - t.Fatalf("Failed to create temp CSV file: %v", err) - } - ldr := &Loader{ - tpInDir: tempDir, - fieldSep: ",", - rdrs: map[string]map[string]*openedCSVFile{ - "testLoader": { - "testfile.csv": &openedCSVFile{}, - }, - }, - } - if err := ldr.processFiles("testLoader", "cacheOption", "storeOption"); err != nil { - t.Errorf("processFiles returned an error: %v", err) - } - if _, ok := ldr.rdrs["testLoader"]["testfile.csv"]; !ok { - t.Error("File was not added to ldr.rdrs") - } -} - -func TestProcessFile(t *testing.T) { - tempDir := t.TempDir() - tempFileName := path.Join(tempDir, "testfile.csv") - fileContent := "header1,header2\nvalue1,value2\n" - err := os.WriteFile(tempFileName, []byte(fileContent), 0644) - if err != nil { - t.Fatalf("Failed to create temp CSV file: %v", err) - } - ldr := &Loader{} - err = ldr.processFile("testfile.csv") - if err != nil { - t.Errorf("processFile returned an error: %v", err) - } - if _, ok := ldr.rdrs["testLoader"]["testfile.csv"]; ok { - t.Errorf("File was not added to ldr.rdrs") - } -} - -func TestLoaderMoveFiles(t *testing.T) { - inDir := t.TempDir() - outDir := t.TempDir() - - testFileNames := []string{"file1.csv", "file2.csv", "lockfile"} - for _, name := range testFileNames { - filePath := path.Join(inDir, name) - if _, err := os.Create(filePath); err != nil { - t.Fatalf("Failed to create test file %s: %v", name, err) - } - } - - loader := &Loader{ - tpInDir: inDir, - tpOutDir: outDir, - lockFilepath: "lockfile", - } - - if err := loader.moveFiles(); err != nil { - t.Errorf("moveFiles returned an error: %v", err) - } - - for _, name := range testFileNames { - filePath := path.Join(outDir, name) - if name != loader.lockFilepath { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - t.Errorf("Expected file %s to be in output directory, but it was not found", name) - } - } else { - if _, err := os.Stat(path.Join(inDir, name)); os.IsNotExist(err) { - t.Errorf("Expected lock file %s to remain in input directory, but it was moved", name) - } - } - } -} - -func TestLoaderUnlockFolder(t *testing.T) { - tempDir := t.TempDir() - lockFilePath := path.Join(tempDir, "lockfile") - - if _, err := os.Create(lockFilePath); err != nil { - t.Fatalf("Failed to create lock file: %v", err) - } - - loader := &Loader{ - lockFilepath: lockFilePath, - } - - if err := loader.unlockFolder(); err != nil { - t.Errorf("unlockFolder returned an error: %v", err) - } - - if _, err := os.Stat(lockFilePath); !os.IsNotExist(err) { - t.Errorf("Expected lock file to be removed, but it still exists") - } - - loader.lockFilepath = "" - if err := loader.unlockFolder(); err != nil { - t.Errorf("unlockFolder returned an error with empty lockFilepath: %v", err) - } -} - -func TestLoaderLockFolder(t *testing.T) { - tempDir := t.TempDir() - lockFilePath := path.Join(tempDir, "lockfile") - - loader := &Loader{ - lockFilepath: lockFilePath, - } - - if err := loader.lockFolder(); err != nil { - t.Errorf("lockFolder returned an error: %v", err) - } - - if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { - t.Errorf("Expected lock file to be created, but it was not found") - } - - loader.lockFilepath = "" - if err := loader.lockFolder(); err != nil { - t.Errorf("lockFolder returned an error with empty lockFilepath: %v", err) - } - - emptyLockFilePath := path.Join(tempDir, "nonexistent_lockfile") - if _, err := os.Stat(emptyLockFilePath); err == nil { - t.Errorf("Expected no lock file to be created for empty lockFilepath, but it exists") - } -} diff --git a/loaders/loaders.go b/loaders/loaders.go deleted file mode 100644 index cfd0af693..000000000 --- a/loaders/loaders.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "errors" - "fmt" - "sync" - "time" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -func NewLoaderService(dm *engine.DataManager, ldrsCfg []*config.LoaderSCfg, - timezone string, cachingDlay time.Duration, filterS *engine.FilterS, - connMgr *engine.ConnManager) (ldrS *LoaderService) { - ldrS = &LoaderService{ldrs: make(map[string]*Loader)} - for _, ldrCfg := range ldrsCfg { - if ldrCfg.Enabled { - ldrS.ldrs[ldrCfg.ID] = NewLoader(dm, ldrCfg, timezone, cachingDlay, filterS, connMgr, ldrCfg.CacheSConns) - } - } - return -} - -// LoaderService is the Loader service handling independent Loaders -type LoaderService struct { - sync.RWMutex - ldrs map[string]*Loader -} - -// Enabled returns true if at least one loader is enabled -func (ldrS *LoaderService) Enabled() bool { - return len(ldrS.ldrs) != 0 -} - -func (ldrS *LoaderService) ListenAndServe(stopChan chan struct{}) (err error) { - for _, ldr := range ldrS.ldrs { - if err = ldr.ListenAndServe(stopChan); err != nil { - utils.Logger.Err(fmt.Sprintf("<%s-%s> error: <%s>", utils.LoaderS, ldr.ldrID, err.Error())) - return - } - } - return -} - -type ArgsProcessFolder struct { - LoaderID string - ForceLock bool - Caching *string - StopOnError bool -} - -func (ldrS *LoaderService) V1Load(ctx *context.Context, args *ArgsProcessFolder, - rply *string) (err error) { - ldrS.RLock() - defer ldrS.RUnlock() - if args.LoaderID == "" { - args.LoaderID = utils.MetaDefault - } - ldr, has := ldrS.ldrs[args.LoaderID] - if !has { - return fmt.Errorf("UNKNOWN_LOADER: %s", args.LoaderID) - } - if locked, err := ldr.isFolderLocked(); err != nil { - return utils.NewErrServerError(err) - } else if locked { - if !args.ForceLock { - return errors.New("ANOTHER_LOADER_RUNNING") - } - if err := ldr.unlockFolder(); err != nil { - return utils.NewErrServerError(err) - } - } - //verify If Caching is present in arguments - caching := config.CgrConfig().GeneralCfg().DefaultCaching - if args.Caching != nil { - caching = *args.Caching - } - if err := ldr.ProcessFolder(caching, utils.MetaStore, args.StopOnError); err != nil { - return utils.NewErrServerError(err) - } - *rply = utils.OK - return -} - -func (ldrS *LoaderService) V1Remove(ctx *context.Context, args *ArgsProcessFolder, - rply *string) (err error) { - ldrS.RLock() - defer ldrS.RUnlock() - if args.LoaderID == "" { - args.LoaderID = utils.MetaDefault - } - ldr, has := ldrS.ldrs[args.LoaderID] - if !has { - return fmt.Errorf("UNKNOWN_LOADER: %s", args.LoaderID) - } - if locked, err := ldr.isFolderLocked(); err != nil { - return utils.NewErrServerError(err) - } else if locked { - if args.ForceLock { - if err := ldr.unlockFolder(); err != nil { - return utils.NewErrServerError(err) - } - } - return errors.New("ANOTHER_LOADER_RUNNING") - } - //verify If Caching is present in arguments - caching := config.CgrConfig().GeneralCfg().DefaultCaching - if args.Caching != nil { - caching = *args.Caching - } - if err := ldr.ProcessFolder(caching, utils.MetaRemove, args.StopOnError); err != nil { - return utils.NewErrServerError(err) - } - *rply = utils.OK - return -} - -// Reload recreates the loaders map thread safe -func (ldrS *LoaderService) Reload(dm *engine.DataManager, ldrsCfg []*config.LoaderSCfg, - timezone string, cachingDlay time.Duration, filterS *engine.FilterS, connMgr *engine.ConnManager) { - ldrS.Lock() - ldrS.ldrs = make(map[string]*Loader) - for _, ldrCfg := range ldrsCfg { - if ldrCfg.Enabled { - ldrS.ldrs[ldrCfg.ID] = NewLoader(dm, ldrCfg, timezone, cachingDlay, filterS, connMgr, ldrCfg.CacheSConns) - } - } - ldrS.Unlock() -} diff --git a/loaders/loaders_it_test.go b/loaders/loaders_it_test.go deleted file mode 100644 index 62c8512cf..000000000 --- a/loaders/loaders_it_test.go +++ /dev/null @@ -1,661 +0,0 @@ -//go:build integration -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "encoding/csv" - "io" - "os" - "path" - "reflect" - "strings" - "testing" - "time" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -var ( - sTestItLoaders = []func(t *testing.T){ - testV1LoadResource, - testV1LoadDefaultIDError, - testV1LoadUnableToDeleteFile, - testV1LoadProcessFolderError, - testV1RemoveResource, - testV1RemoveDefaultIDError, - testV1RemoveUnableToDeleteFile, - testV1RemoveProcessFolderError, - testV1LoadAndRemoveProcessRemoveFolderError, - testLoaderServiceReload, - testLoaderServiceListenAndServe, - } -) - -func TestITLoaders(t *testing.T) { - for _, test := range sTestItLoaders { - t.Run("Loaders_IT_Tests", test) - } -} - -func testV1LoadResource(t *testing.T) { - utils.Logger.SetLogLevel(7) - flPath := "/tmp/testV1LoadResource" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - if _, err := file.Write([]byte(`#Tenant[0],ID[1] -cgrates.org,NewRes1`)); err != nil { - t.Fatal(err) - } - if err := file.Sync(); err != nil { - t.Fatal(err) - } - if err := file.Close(); err != nil { - t.Fatal(err) - } - file, err = os.Create(path.Join(flPath, "res.lck")) - if err != nil { - t.Error(err) - } - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfg := config.NewDefaultCGRConfig().LoaderCfg() - cfg[0] = &config.LoaderSCfg{ - ID: "testV1LoadResource", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: "res.lck", - Data: []*config.LoaderDataType{ - { - Type: utils.MetaResources, - Filename: utils.ResourcesCsv, - Fields: []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - { - Path: "ID", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - }, - }, - }, - }, - } - for _, tmp := range cfg[0].Data[0].Fields { - tmp.ComputePath() - } - ldrs := NewLoaderService(dm, cfg, "UTC", 0, nil, nil) - - var reply string - expected := "ANOTHER_LOADER_RUNNING" - //cannot load when there is another loader running - if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{LoaderID: "testV1LoadResource"}, - &reply); err == nil || reply != utils.EmptyString || err.Error() != expected { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply) - } - - if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{LoaderID: "testV1LoadResource", ForceLock: true}, - &reply); err != nil && reply != utils.OK { - t.Error(err) - } - - expRes := &engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "NewRes1", - FilterIDs: make([]string, 0), - ThresholdIDs: make([]string, 0), - } - - if rcv, err := dm.GetResourceProfile(expRes.Tenant, expRes.ID, - true, true, utils.NonTransactional); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rcv, expRes) { - t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expRes), utils.ToJSON(rcv)) - } - - if err := os.RemoveAll(flPath); err != nil { - t.Error(err) - } -} - -func testV1LoadDefaultIDError(t *testing.T) { - flPath := "/tmp/testV1LoadResource" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1LoadDefaultIDError", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - - var reply string - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - if err := ldrs.V1Load(context.Background(), &ArgsProcessFolder{ - LoaderID: utils.EmptyString}, &reply); err == nil && reply != utils.EmptyString && err.Error() != utils.EmptyString { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1LoadUnableToDeleteFile(t *testing.T) { - flPath := "testV1LoadUnableToDeleteFile" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - _, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1LoadUnableToDeleteFile", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: "/\x00", - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - var reply string - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - expected := "SERVER_ERROR: stat /\x00/Resources.csv: invalid argument" - if err := ldrs.V1Load(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1LoadUnableToDeleteFile", - ForceLock: true}, &reply); err == nil || err.Error() != expected { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1LoadProcessFolderError(t *testing.T) { - flPath := "testV1LoadProcessFolderError" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#PK -NOT_UINT -`)) - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1LoadResource", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - ldrs.ldrs["testV1LoadResource"].dataTpls = map[string][]*config.FCTemplate{ - utils.MetaFilters: { - {Tag: "PK", - Path: "PK", - Type: utils.MetaComposed, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.FieldsSep)}, - }, - } - resCsv := ` -//PK -NOT_UINT -` - rdr := io.NopCloser(strings.NewReader(resCsv)) - csvRdr := csv.NewReader(rdr) - csvRdr.Comment = '#' - ldrs.ldrs["testV1LoadResource"].rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "not_a_file": &openedCSVFile{ - fileName: utils.ResourcesCsv, - rdr: rdr, - csvRdr: csvRdr, - }, - }, - } - - var reply string - expected := "SERVER_ERROR: open testV1LoadProcessFolderError/not_a_file: no such file or directory" - //try to load by changing the caching method - if err := ldrs.V1Load(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1LoadResource", - ForceLock: true, - Caching: utils.StringPointer("not_a_chaching_method"), - StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString { - t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply) - } - - if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1RemoveResource(t *testing.T) { - engine.Cache.Clear(nil) - flPath := "/tmp/testV1RemoveResource" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(`#Tenant[0],ID[1] -cgrates.org,NewRes1`)) - file.Close() - - file, err = os.Create(path.Join(flPath, "lock.cgr")) - if err != nil { - t.Error(err) - } - file.Close() - - idb, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Fatal(err) - } - dm := engine.NewDataManager(idb, config.CgrConfig().CacheCfg(), nil) - cfg := config.NewDefaultCGRConfig().LoaderCfg() - cfg[0] = &config.LoaderSCfg{ - ID: "testV1RemoveResource", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: "lock.cgr", - Data: []*config.LoaderDataType{ - { - Type: utils.MetaResources, - Filename: utils.ResourcesCsv, - Fields: []*config.FCTemplate{ - { - Path: "Tenant", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.0", utils.InfieldSep), - Mandatory: true, - }, - { - Path: "ID", - Type: utils.MetaVariable, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.InfieldSep), - Mandatory: true, - }, - }, - }, - }, - } - for _, tmp := range cfg[0].Data[0].Fields { - tmp.ComputePath() - } - ldrs := NewLoaderService(dm, cfg, time.UTC.String(), 0, nil, nil) - //To remove a resource, we need to set it first - if err := dm.SetResourceProfile(&engine.ResourceProfile{ - Tenant: "cgrates.org", - ID: "NewRes1", - }, true); err != nil { - t.Error(err) - } - - var reply string - expected := "ANOTHER_LOADER_RUNNING" - //cannot load when there is another loader running - if err := ldrs.V1Remove(context.Background(), &ArgsProcessFolder{LoaderID: "testV1RemoveResource"}, - &reply); err == nil || reply != utils.EmptyString || err.Error() != expected { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply) - } - - os.Remove(path.Join(flPath, "lock.cgr")) - if err := ldrs.V1Remove(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1RemoveResource", - ForceLock: true}, &reply); err != nil && reply != utils.OK { - t.Error(err) - } - - //nothing to get from dataBase - if _, err := dm.GetResourceProfile("cgrates.org", "NewRes1", - true, true, utils.NonTransactional); err != utils.ErrNotFound { - t.Error(err) - } - if err := os.RemoveAll(flPath); err != nil { - t.Error(err) - } -} - -func testV1RemoveDefaultIDError(t *testing.T) { - flPath := "/tmp/testV1RemoveDefaultIDError" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - file.Write([]byte(` -#Tenant[0],ID[1] -cgrates.org,NewRes1 -`)) - file.Close() - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1RemoveDefaultIDError", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - - var reply string - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - expected := "UNKNOWN_LOADER: *default" - if err := ldrs.V1Remove(context.Background(), &ArgsProcessFolder{ - LoaderID: utils.EmptyString}, &reply); err == nil || reply != utils.EmptyString || err.Error() != expected { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", expected, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1RemoveUnableToDeleteFile(t *testing.T) { - flPath := "testV1RemoveUnableToDeleteFile" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - _, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1RemoveUnableToDeleteFile", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: "/\x00", - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - var reply string - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - expected := "SERVER_ERROR: stat /\x00/Resources.csv: invalid argument" - if err := ldrs.V1Remove(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1RemoveUnableToDeleteFile", - ForceLock: true}, &reply); err == nil || err.Error() != expected { - t.Errorf("Expected %+v and %+v \n, received %+v and %+v", utils.EmptyString, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1LoadAndRemoveProcessRemoveFolderError(t *testing.T) { - flPath := "/tmp/testV1RemoveProcessFolderError" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - file, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - defer file.Close() - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1RemoveProcessFolderError", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - Data: nil, - } - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - - ldrs.ldrs["testV1RemoveProcessFolderError"].lockFilepath = flPath - - ldrs.ldrs["testV1RemoveProcessFolderError"].rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "not_a_file": nil, - }, - } - - var reply string - expected := "SERVER_ERROR: remove /tmp/testV1RemoveProcessFolderError: directory not empty" - //try to load by changing the caching method, but there is not a lockFileName - if err := ldrs.V1Load(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1RemoveProcessFolderError", - ForceLock: true, - Caching: utils.StringPointer("not_a_chaching_method"), - StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString { - t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply) - } - - //try to remove by changing the caching method - if err := ldrs.V1Remove(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1RemoveProcessFolderError", - ForceLock: true, - Caching: utils.StringPointer("not_a_chaching_method"), - StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString { - t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testV1RemoveProcessFolderError(t *testing.T) { - flPath := "testV1RemoveProcessFolderError" - if err := os.MkdirAll(flPath, 0777); err != nil { - t.Error(err) - } - _, err := os.Create(path.Join(flPath, utils.ResourcesCsv)) - if err != nil { - t.Error(err) - } - - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1RemoveProcessFolderError", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: "notResource.csv", - Data: nil, - } - ldrs := NewLoaderService(dm, cfgLdr, "UTC", 0, nil, nil) - ldrs.ldrs["testV1RemoveProcessFolderError"].rdrs = map[string]map[string]*openedCSVFile{ - utils.MetaResources: { - "not_a_file2": &openedCSVFile{ - fileName: utils.ResourcesCsv, - }, - }, - } - - var reply string - expected := "SERVER_ERROR: open testV1RemoveProcessFolderError/not_a_file2: no such file or directory" - //try to load by changing the caching method - if err := ldrs.V1Remove(context.Background(), - &ArgsProcessFolder{ - LoaderID: "testV1RemoveProcessFolderError", - ForceLock: true, - Caching: utils.StringPointer("not_a_chaching_method"), - StopOnError: true}, &reply); err == nil || err.Error() != expected || reply != utils.EmptyString { - t.Errorf("Expected %+q and %+q \n, received %+q and %+q", expected, utils.EmptyString, err, reply) - } - - if err := os.Remove(path.Join(flPath, utils.ResourcesCsv)); err != nil { - t.Error(err) - } else if err := os.Remove(flPath); err != nil { - t.Error(err) - } -} - -func testLoaderServiceListenAndServe(t *testing.T) { - ldr := &Loader{ - runDelay: -1, - tpInDir: "/tmp/TestLoaderServiceListenAndServe", - } - ldrs := &LoaderService{ - ldrs: map[string]*Loader{ - "TEST_LOADER": ldr, - }, - } - stopChan := make(chan struct{}, 1) - stopChan <- struct{}{} - expected := "no such file or directory" - if err := ldrs.ListenAndServe(stopChan); err == nil || err.Error() != expected { - t.Errorf("Expected %+v, received %+v", expected, err) - } - - ldrs.ldrs["TEST_LOADER"].tpInDir = utils.EmptyString - if err := ldrs.ListenAndServe(stopChan); err != nil { - t.Error(err) - } -} - -func testLoaderServiceReload(t *testing.T) { - flPath := "/tmp/testLoaderServiceReload" - data, err := engine.NewInternalDB(nil, nil, true, nil, config.CgrConfig().DataDbCfg().Items) - if err != nil { - t.Error(err) - } - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - cfgLdr := config.NewDefaultCGRConfig().LoaderCfg() - cfgLdr[0] = &config.LoaderSCfg{ - ID: "testV1LoadResource", - Enabled: true, - FieldSeparator: utils.FieldsSep, - TpInDir: flPath, - TpOutDir: "/tmp", - LockFilePath: utils.ResourcesCsv, - Data: nil, - } - ldrs := &LoaderService{} - ldrs.Reload(dm, cfgLdr, "UTC", 0, nil, nil) - if ldrs.ldrs == nil { - t.Error("Expected to be populated") - } -} diff --git a/loaders/loaders_test.go b/loaders/loaders_test.go deleted file mode 100644 index 73ea1f61a..000000000 --- a/loaders/loaders_test.go +++ /dev/null @@ -1,174 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package loaders - -import ( - "testing" - "time" - - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" -) - -func TestLoaderServiceListenAndServe(t *testing.T) { - stopChan := make(chan struct{}) - defer close(stopChan) - loaderService := &LoaderService{ - ldrs: map[string]*Loader{ - "loader1": {ldrID: "loader1"}, - "loader2": {ldrID: "loader2"}, - "loader3": {ldrID: "error"}, - }, - } - - loaderService.ldrs["loader3"].ldrID = "loader3" - t.Run("All loaders succeed", func(t *testing.T) { - err := loaderService.ListenAndServe(stopChan) - if err != nil { - t.Errorf("expected no error, got %v", err) - } - }) -} - -func TestLoaderServiceV1Load(t *testing.T) { - ctx := context.Background() - loaderService := &LoaderService{ - ldrs: map[string]*Loader{}, - } - - t.Run("Unknown loader", func(t *testing.T) { - args := &ArgsProcessFolder{LoaderID: "unknown"} - var rply string - err := loaderService.V1Load(ctx, args, &rply) - if err == nil || err.Error() != "UNKNOWN_LOADER: unknown" { - t.Errorf("expected error 'UNKNOWN_LOADER: unknown', got %v", err) - } - }) - - t.Run("Another loader running without force", func(t *testing.T) { - args := &ArgsProcessFolder{LoaderID: "loader1"} - var rply string - err := loaderService.V1Load(ctx, args, &rply) - if err == nil || err.Error() == "ANOTHER_LOADER_RUNNING" { - t.Errorf("expected error 'ANOTHER_LOADER_RUNNING', got %v", err) - } - }) - - t.Run("Unlock folder and process", func(t *testing.T) { - args := &ArgsProcessFolder{LoaderID: "loader1", ForceLock: true} - var rply string - err := loaderService.V1Load(ctx, args, &rply) - if err == nil { - t.Errorf("expected no error, got %v", err) - } - if rply == "OK" { - t.Errorf("expected reply 'OK', got %s", rply) - } - - }) - - t.Run("Process with no locking issues", func(t *testing.T) { - args := &ArgsProcessFolder{LoaderID: "loader2"} - var rply string - err := loaderService.V1Load(ctx, args, &rply) - if err == nil { - t.Errorf("expected no error, got %v", err) - } - if rply == "OK" { - t.Errorf("expected reply 'OK', got %s", rply) - } - }) -} - -func TestLoaderServiceReload(t *testing.T) { - loaderService := &LoaderService{} - dm := &engine.DataManager{} - filterS := &engine.FilterS{} - connMgr := &engine.ConnManager{} - timezone := "UTC" - cachingDelay := time.Duration(5 * time.Second) - ldrsCfg := []*config.LoaderSCfg{ - {ID: "loader1", Enabled: true}, - {ID: "loader2", Enabled: false}, - {ID: "loader3", Enabled: true}, - } - loaderService.Reload(dm, ldrsCfg, timezone, cachingDelay, filterS, connMgr) - if len(loaderService.ldrs) != 2 { - t.Errorf("expected 2 enabled loaders, got %d", len(loaderService.ldrs)) - } - - if _, exists := loaderService.ldrs["loader1"]; !exists { - t.Error("expected loader1 to be in the loaders map") - } - - if _, exists := loaderService.ldrs["loader3"]; !exists { - t.Error("expected loader3 to be in the loaders map") - } - - if _, exists := loaderService.ldrs["loader2"]; exists { - t.Error("did not expect loader2 to be in the loaders map") - } -} - -func TestLoaderServiceV1Remove(t *testing.T) { - ctx := context.Background() - loaderService := &LoaderService{ - ldrs: map[string]*Loader{}, - } - - args := &ArgsProcessFolder{LoaderID: "unknown"} - var rply string - err := loaderService.V1Remove(ctx, args, &rply) - if err == nil || err.Error() != "UNKNOWN_LOADER: unknown" { - t.Errorf("expected error 'UNKNOWN_LOADER: unknown', got %v", err) - } -} - -func TestNewLoaderService(t *testing.T) { - dm := &engine.DataManager{} - timezone := "UTC" - cachingDlay := time.Second - filterS := &engine.FilterS{} - connMgr := &engine.ConnManager{} - - ldrsCfg := []*config.LoaderSCfg{ - {ID: "loader1", Enabled: true}, - {ID: "loader2", Enabled: false}, - {ID: "loader3", Enabled: true}, - } - - ldrService := NewLoaderService(dm, ldrsCfg, timezone, cachingDlay, filterS, connMgr) - - if len(ldrService.ldrs) != 2 { - t.Errorf("expected 2 loaders, got %d", len(ldrService.ldrs)) - } - - if _, exists := ldrService.ldrs["loader1"]; !exists { - t.Errorf("expected loader1 to be present") - } - - if _, exists := ldrService.ldrs["loader2"]; exists { - t.Errorf("expected loader2 to not be present") - } - - if _, exists := ldrService.ldrs["loader3"]; !exists { - t.Errorf("expected loader3 to be present") - } -} diff --git a/services/analyzers_it_test.go b/services/analyzers_it_test.go index 3ee18a9b4..ce35a6a0d 100644 --- a/services/analyzers_it_test.go +++ b/services/analyzers_it_test.go @@ -56,8 +56,7 @@ func TestAnalyzerSReload(t *testing.T) { anzRPC := make(chan birpc.ClientConnector, 1) anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, anzRPC, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(anz, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(anz, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/apiers_it_test.go b/services/apiers_it_test.go index 7bc2ca434..aedae4f84 100644 --- a/services/apiers_it_test.go +++ b/services/apiers_it_test.go @@ -69,8 +69,7 @@ func TestApiersReload(t *testing.T) { make(chan birpc.ClientConnector, 1), nil, anz, srvDep) apiSv2 := NewAPIerSv2Service(apiSv1, cfg, server, make(chan birpc.ClientConnector, 1), anz, srvDep) - srvMngr.AddServices(apiSv1, apiSv2, schS, tS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb) + srvMngr.AddServices(apiSv1, apiSv2, schS, tS, db, stordb) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/asteriskagent_it_test.go b/services/asteriskagent_it_test.go index 1640e89cd..c95b8d4ee 100644 --- a/services/asteriskagent_it_test.go +++ b/services/asteriskagent_it_test.go @@ -71,8 +71,7 @@ func TestAsteriskAgentReload(t *testing.T) { sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, cm, anz, srvDep) astService := NewAsteriskAgent(cfg, shdChan, cm, srvDep) - srvMngr.AddServices(astService, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(astService, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } @@ -144,8 +143,7 @@ func TestAsteriskAgentReload2(t *testing.T) { sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, cm, anz, srvDep) astSrv := NewAsteriskAgent(cfg, shdChan, cm, srvDep) - srvMngr.AddServices(astSrv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(astSrv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/attributes_it_test.go b/services/attributes_it_test.go index c5a8193c2..44db57bfd 100644 --- a/services/attributes_it_test.go +++ b/services/attributes_it_test.go @@ -58,8 +58,7 @@ func TestAttributeSReload(t *testing.T) { chS, filterSChan, server, attrRPC, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(attrS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(attrS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/cdrs_it_test.go b/services/cdrs_it_test.go index 556b8ec7e..d7886aebd 100644 --- a/services/cdrs_it_test.go +++ b/services/cdrs_it_test.go @@ -77,9 +77,7 @@ func TestCdrsReload(t *testing.T) { cdrsRPC := make(chan birpc.ClientConnector, 1) cdrS := NewCDRServer(cfg, db, stordb, filterSChan, server, cdrsRPC, nil, anz, srvDep) - srvMngr.AddServices(cdrS, ralS, schS, chrS, - NewLoaderService(cfg, db, filterSChan, server, - make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb) + srvMngr.AddServices(cdrS, ralS, schS, chrS, db, stordb) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/chargers_it_test.go b/services/chargers_it_test.go index fbadb4f0d..0bd7565ea 100644 --- a/services/chargers_it_test.go +++ b/services/chargers_it_test.go @@ -58,9 +58,7 @@ func TestChargerSReload(t *testing.T) { attrS := NewAttributeService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep) chrS := NewChargerService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(attrS, chrS, - NewLoaderService(cfg, db, filterSChan, server, - make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(attrS, chrS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/cores_it_test.go b/services/cores_it_test.go index 64914dea5..d7f1772fd 100644 --- a/services/cores_it_test.go +++ b/services/cores_it_test.go @@ -54,8 +54,7 @@ func TestCoreSReload(t *testing.T) { caps := engine.NewCaps(1, "test_caps") coreS := NewCoreService(cfg, caps, server, coreRPC, anz, nil, nil, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(coreS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(coreS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/datadb.go b/services/datadb.go index e78d822ec..b2d75b9f9 100644 --- a/services/datadb.go +++ b/services/datadb.go @@ -145,7 +145,7 @@ func (db *DataDBService) mandatoryDB() bool { return db.cfg.RalsCfg().Enabled || db.cfg.SchedulerCfg().Enabled || db.cfg.ChargerSCfg().Enabled || db.cfg.AttributeSCfg().Enabled || db.cfg.ResourceSCfg().Enabled || db.cfg.StatSCfg().Enabled || db.cfg.ThresholdSCfg().Enabled || db.cfg.RouteSCfg().Enabled || db.cfg.DispatcherSCfg().Enabled || - db.cfg.LoaderCfg().Enabled() || db.cfg.ApierCfg().Enabled || db.cfg.AnalyzerSCfg().Enabled || db.cfg.ERsCfg().Enabled + db.cfg.ApierCfg().Enabled || db.cfg.AnalyzerSCfg().Enabled || db.cfg.ERsCfg().Enabled } // GetDM returns the DataManager diff --git a/services/datadb_it_test.go b/services/datadb_it_test.go index c147109e4..b69f1a6c4 100644 --- a/services/datadb_it_test.go +++ b/services/datadb_it_test.go @@ -56,8 +56,7 @@ func TestDataDBReload(t *testing.T) { db := NewDataDBService(cfg, cM, false, srvDep) anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) srvMngr.AddServices(NewAttributeService(cfg, db, - chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep), - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + chS, filterSChan, server, make(chan birpc.ClientConnector, 1), anz, srvDep), db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/diameteragent_it_test.go b/services/diameteragent_it_test.go index 6054bd31d..d3f1a80a1 100644 --- a/services/diameteragent_it_test.go +++ b/services/diameteragent_it_test.go @@ -61,8 +61,7 @@ func TestDiameterAgentReload1(t *testing.T) { shdChan, nil, anz, srvDep) diamSrv := NewDiameterAgent(cfg, filterSChan, shdChan, nil, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(diamSrv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(diamSrv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/dispatchers_it_test.go b/services/dispatchers_it_test.go index 6b350cb2d..72203fd41 100644 --- a/services/dispatchers_it_test.go +++ b/services/dispatchers_it_test.go @@ -60,9 +60,7 @@ func TestDispatcherSReload(t *testing.T) { srv := NewDispatcherService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(attrS, srv, - NewLoaderService(cfg, db, filterSChan, server, - make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(attrS, srv, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/dnsagent_it_test.go b/services/dnsagent_it_test.go index a08a5bcbb..4dc0dfbd3 100644 --- a/services/dnsagent_it_test.go +++ b/services/dnsagent_it_test.go @@ -67,8 +67,7 @@ func TestDNSAgentStartReloadShut(t *testing.T) { anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, nil, anz, srvDep) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) runtime.Gosched() time.Sleep(10 * time.Millisecond) //need to switch to gorutine if err := srv.Shutdown(); err != nil { @@ -124,8 +123,7 @@ func TestDNSAgentReloadFirst(t *testing.T) { shdChan, nil, anz, srvDep) srv := NewDNSAgent(cfg, filterSChan, shdChan, nil, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/ees_it_test.go b/services/ees_it_test.go index 113cbff24..9ddd90d52 100644 --- a/services/ees_it_test.go +++ b/services/ees_it_test.go @@ -68,8 +68,7 @@ func TestEventExporterSReload(t *testing.T) { anz, srvDep) ees := NewEventExporterService(cfg, filterSChan, engine.NewConnManager(cfg, nil), server, make(chan birpc.ClientConnector, 2), anz, srvDep) - srvMngr.AddServices(ees, attrS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(ees, attrS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/ers_it_test.go b/services/ers_it_test.go index 19106a873..26be1b2de 100644 --- a/services/ers_it_test.go +++ b/services/ers_it_test.go @@ -70,8 +70,7 @@ func TestEventReaderSReload(t *testing.T) { intERsConn := make(chan birpc.ClientConnector, 1) erS := NewEventReaderService(cfg, db, filterSChan, shdChan, nil, server, intERsConn, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(erS, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(erS, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/freeswitchagent_it_test.go b/services/freeswitchagent_it_test.go index f80b7d5ee..02b7e60f6 100644 --- a/services/freeswitchagent_it_test.go +++ b/services/freeswitchagent_it_test.go @@ -72,8 +72,7 @@ func TestFreeSwitchAgentReload(t *testing.T) { sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, cm, anz, srvDep) srv := NewFreeswitchAgent(cfg, shdChan, cm, srvDep) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), cm, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/httpagent_it_test.go b/services/httpagent_it_test.go index cb527756b..521dabe8a 100644 --- a/services/httpagent_it_test.go +++ b/services/httpagent_it_test.go @@ -66,8 +66,7 @@ func TestHTTPAgentReload(t *testing.T) { shdChan, nil, anz, srvDep) srv := NewHTTPAgent(cfg, filterSChan, server, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/kamailioagent_it_test.go b/services/kamailioagent_it_test.go index f96465f9d..e46752c12 100644 --- a/services/kamailioagent_it_test.go +++ b/services/kamailioagent_it_test.go @@ -76,8 +76,7 @@ func TestKamailioAgentReload(t *testing.T) { sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, cm, anz, srvDep) srv := NewKamailioAgent(cfg, shdChan, cm, srvDep) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), cm, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/loaders.go b/services/loaders.go deleted file mode 100644 index b7f5cfd69..000000000 --- a/services/loaders.go +++ /dev/null @@ -1,154 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ - -package services - -import ( - "sync" - - "github.com/cgrates/birpc" - v1 "github.com/cgrates/cgrates/apier/v1" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/cores" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/loaders" - "github.com/cgrates/cgrates/utils" -) - -// NewLoaderService returns the Loader Service -func NewLoaderService(cfg *config.CGRConfig, dm *DataDBService, - filterSChan chan *engine.FilterS, server *cores.Server, - internalLoaderSChan chan birpc.ClientConnector, - connMgr *engine.ConnManager, anz *AnalyzerService, - srvDep map[string]*sync.WaitGroup) *LoaderService { - return &LoaderService{ - connChan: internalLoaderSChan, - cfg: cfg, - dm: dm, - filterSChan: filterSChan, - server: server, - connMgr: connMgr, - stopChan: make(chan struct{}), - anz: anz, - srvDep: srvDep, - } -} - -// LoaderService implements Service interface -type LoaderService struct { - sync.RWMutex - cfg *config.CGRConfig - dm *DataDBService - filterSChan chan *engine.FilterS - server *cores.Server - stopChan chan struct{} - - ldrs *loaders.LoaderService - connChan chan birpc.ClientConnector - connMgr *engine.ConnManager - anz *AnalyzerService - srvDep map[string]*sync.WaitGroup -} - -// Start should handle the sercive start -func (ldrs *LoaderService) Start() error { - if ldrs.IsRunning() { - return utils.ErrServiceAlreadyRunning - } - - filterS := <-ldrs.filterSChan - ldrs.filterSChan <- filterS - dbchan := ldrs.dm.GetDMChan() - datadb := <-dbchan - dbchan <- datadb - - ldrs.Lock() - defer ldrs.Unlock() - - ldrs.ldrs = loaders.NewLoaderService(datadb, ldrs.cfg.LoaderCfg(), - ldrs.cfg.GeneralCfg().DefaultTimezone, ldrs.cfg.GeneralCfg().CachingDelay, filterS, ldrs.connMgr) - - if !ldrs.ldrs.Enabled() { - return nil - } - if err := ldrs.ldrs.ListenAndServe(ldrs.stopChan); err != nil { - return err - } - srv, err := engine.NewService(v1.NewLoaderSv1(ldrs.ldrs)) - if err != nil { - return err - } - if !ldrs.cfg.DispatcherSCfg().Enabled { - ldrs.server.RpcRegister(srv) - } - ldrs.connChan <- ldrs.anz.GetInternalCodec(srv, utils.LoaderS) - return nil -} - -// Reload handles the change of config -func (ldrs *LoaderService) Reload() (err error) { - filterS := <-ldrs.filterSChan - ldrs.filterSChan <- filterS - dbchan := ldrs.dm.GetDMChan() - datadb := <-dbchan - dbchan <- datadb - close(ldrs.stopChan) - ldrs.stopChan = make(chan struct{}) - - ldrs.RLock() - - ldrs.ldrs.Reload(datadb, ldrs.cfg.LoaderCfg(), ldrs.cfg.GeneralCfg().DefaultTimezone, ldrs.cfg.GeneralCfg().CachingDelay, - filterS, ldrs.connMgr) - if err = ldrs.ldrs.ListenAndServe(ldrs.stopChan); err != nil { - return - } - ldrs.RUnlock() - return -} - -// Shutdown stops the service -func (ldrs *LoaderService) Shutdown() (err error) { - ldrs.Lock() - ldrs.ldrs = nil - close(ldrs.stopChan) - <-ldrs.connChan - ldrs.Unlock() - return -} - -// IsRunning returns if the service is running -func (ldrs *LoaderService) IsRunning() bool { - ldrs.RLock() - defer ldrs.RUnlock() - return ldrs.ldrs != nil && ldrs.ldrs.Enabled() -} - -// ServiceName returns the service name -func (ldrs *LoaderService) ServiceName() string { - return utils.LoaderS -} - -// ShouldRun returns if the service should be running -func (ldrs *LoaderService) ShouldRun() bool { - return ldrs.cfg.LoaderCfg().Enabled() -} - -// GetLoaderS returns the initialized LoaderService -func (ldrs *LoaderService) GetLoaderS() *loaders.LoaderService { - return ldrs.ldrs -} diff --git a/services/loaders_it_test.go b/services/loaders_it_test.go deleted file mode 100644 index b32a3a6cd..000000000 --- a/services/loaders_it_test.go +++ /dev/null @@ -1,200 +0,0 @@ -//go:build integration -// +build integration - -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ -package services - -import ( - "os" - "path" - "sync" - "testing" - "time" - - "github.com/cgrates/birpc" - "github.com/cgrates/birpc/context" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/cores" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/servmanager" - "github.com/cgrates/cgrates/utils" -) - -func testCreateDirs(t *testing.T) { - for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut", "/tmp/templateLoaderIn", "/tmp/templateLoaderOut", - "/tmp/customSepLoaderIn", "/tmp/customSepLoaderOut"} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal("Error creating folder: ", dir, err) - } - } - if err := os.WriteFile(path.Join("/tmp/In", utils.AttributesCsv), []byte(` -#Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight -cgrates.org,ALS1,con1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,*string:~*req.Field1:Initial,*req.Field1,*variable,Sub1,true,20 -cgrates.org,ALS1,con2;con3,,,,*req.Field2,*variable,Sub2,true,20 -`), 0644); err != nil { - t.Fatal(err.Error()) - } -} - -func TestLoaderSReload(t *testing.T) { - testCreateDirs(t) - cfg := config.NewDefaultCGRConfig() - cfg.TemplatesCfg()["attrTemplateLoader"] = []*config.FCTemplate{ - { - Type: utils.MetaVariable, - Path: "*req.Accounts", - Value: config.NewRSRParsersMustCompile("1001", utils.InfieldSep), - }, - } - utils.Logger, _ = utils.Newlogger(utils.MetaSysLog, cfg.GeneralCfg().NodeID) - utils.Logger.SetLogLevel(7) - - shdChan := utils.NewSyncedChan() - shdWg := new(sync.WaitGroup) - filterSChan := make(chan *engine.FilterS, 1) - filterSChan <- nil - server := cores.NewServer(nil) - srvMngr := servmanager.NewServiceManager(cfg, shdChan, shdWg, nil) - srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)} - db := NewDataDBService(cfg, nil, false, srvDep) - anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) - conMngr := engine.NewConnManager(cfg, nil) - srv := NewLoaderService(cfg, db, filterSChan, - server, make(chan birpc.ClientConnector, 1), - conMngr, anz, srvDep) - srvMngr.AddServices(srv, db) - if err := srvMngr.StartServices(); err != nil { - t.Fatal(err) - } - - if db.IsRunning() { - t.Errorf("Expected service to be down") - } - - if srv.IsRunning() { - t.Errorf("Expected service to be down") - } - - var reply string - if err := cfg.V1ReloadConfig(context.Background(), - &config.ReloadArgs{ - Path: path.Join("/usr", "share", "cgrates", "conf", "samples", "loaders", "tutinternal"), - Section: config.LoaderJson, - }, &reply); err != nil { - t.Fatal(err) - } else if reply != utils.OK { - t.Errorf("Expecting OK ,received %s", reply) - } - time.Sleep(10 * time.Millisecond) - if !db.IsRunning() { - t.Fatal("Expected service to be running") - } - time.Sleep(10 * time.Millisecond) - if !srv.IsRunning() { - t.Fatal("Expected service to be running") - } - - err := srv.Start() - if err == nil || err != utils.ErrServiceAlreadyRunning { - t.Errorf("\nExpecting <%+v>,\n Received <%+v>", utils.ErrServiceAlreadyRunning, err) - } - time.Sleep(10 * time.Millisecond) - err = srv.Reload() - if err != nil { - t.Errorf("\nExpecting ,\n Received <%+v>", err) - } - time.Sleep(10 * time.Millisecond) - for _, v := range cfg.LoaderCfg() { - v.Enabled = false - } - time.Sleep(10 * time.Millisecond) - cfg.GetReloadChan(config.LoaderJson) <- struct{}{} - time.Sleep(10 * time.Millisecond) - - if srv.IsRunning() { - t.Errorf("Expected service to be down") - } - - shdChan.CloseOnce() - time.Sleep(10 * time.Millisecond) - testCleanupFiles(t) -} -func testCleanupFiles(t *testing.T) { - for _, dir := range []string{"/tmp/In", "/tmp/Out", "/tmp/LoaderIn", "/tmp/SubpathWithoutMove", - "/tmp/SubpathLoaderWithMove", "/tmp/SubpathOut"} { - if err := os.RemoveAll(dir); err != nil { - t.Fatal("Error removing folder: ", dir, err) - } - } -} - -func TestLoaderSReload2(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - for _, ld := range cfg.LoaderCfg() { - ld.Enabled = false - } - shdChan := utils.NewSyncedChan() - filterSChan := make(chan *engine.FilterS, 1) - filterSChan <- nil - server := cores.NewServer(nil) - srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)} - db := NewDataDBService(cfg, nil, false, srvDep) - db.dbchan <- new(engine.DataManager) - anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) - srv := NewLoaderService(cfg, db, filterSChan, - server, make(chan birpc.ClientConnector, 1), - nil, anz, srvDep) - err := srv.Start() - if err != nil { - t.Fatal(err) - } -} - -func TestLoaderSReload3(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - for _, ld := range cfg.LoaderCfg() { - ld.Enabled = false - } - cfg.LoaderCfg()[0].Enabled = true - cfg.LoaderCfg()[0].TpInDir = "/tmp/TestLoaderSReload3" - cfg.LoaderCfg()[0].RunDelay = -1 - shdChan := utils.NewSyncedChan() - filterSChan := make(chan *engine.FilterS, 1) - filterSChan <- nil - server := cores.NewServer(nil) - srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)} - db := NewDataDBService(cfg, nil, false, srvDep) - db.dbchan <- new(engine.DataManager) - anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) - srv := NewLoaderService(cfg, db, filterSChan, - server, make(chan birpc.ClientConnector, 1), - nil, anz, srvDep) - err := srv.Start() - if err == nil || err.Error() != "no such file or directory" { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "no such file or directory", err) - } - err = srv.Reload() - if err == nil || err.Error() != "no such file or directory" { - t.Errorf("\nExpected <%+v>, \nReceived <%+v>", "no such file or directory", err) - } -} diff --git a/services/loaders_test.go b/services/loaders_test.go deleted file mode 100644 index 3c8b27ec0..000000000 --- a/services/loaders_test.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see -*/ -package services - -import ( - "reflect" - "sync" - "testing" - - "github.com/cgrates/birpc" - "github.com/cgrates/cgrates/loaders" - - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/cores" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -// TestLoaderSCoverage for cover testing -func TestLoaderSCoverage(t *testing.T) { - cfg := config.NewDefaultCGRConfig() - shdChan := utils.NewSyncedChan() - filterSChan := make(chan *engine.FilterS, 1) - filterSChan <- nil - server := cores.NewServer(nil) - srvDep := map[string]*sync.WaitGroup{utils.DataDB: new(sync.WaitGroup)} - db := NewDataDBService(cfg, nil, false, srvDep) - internalLoaderSChan := make(chan birpc.ClientConnector, 1) - rpcInternal := map[string]chan birpc.ClientConnector{} - cM := engine.NewConnManager(cfg, rpcInternal) - anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) - srv := NewLoaderService(cfg, db, - filterSChan, server, internalLoaderSChan, - cM, anz, srvDep) - if srv == nil { - t.Errorf("\nExpecting ,\n Received <%+v>", utils.ToJSON(srv)) - } - if srv.IsRunning() { - t.Errorf("Expected service to be down") - } - srv.ldrs = loaders.NewLoaderService(&engine.DataManager{}, - []*config.LoaderSCfg{{ - ID: "test_id", - Enabled: true, - Tenant: "", - DryRun: false, - RunDelay: 0, - LockFilePath: "", - CacheSConns: nil, - FieldSeparator: "", - TpInDir: "", - TpOutDir: "", - Data: nil, - }}, "", 0, - &engine.FilterS{}, nil) - if !srv.IsRunning() { - t.Errorf("Expected service to be running") - } - serviceName := srv.ServiceName() - if !reflect.DeepEqual(serviceName, utils.LoaderS) { - t.Errorf("\nExpecting <%+v>,\n Received <%+v>", utils.LoaderS, serviceName) - } - shouldRun := srv.ShouldRun() - if !reflect.DeepEqual(shouldRun, false) { - t.Errorf("\nExpecting ,\n Received <%+v>", shouldRun) - } - if !reflect.DeepEqual(srv.GetLoaderS(), srv.ldrs) { - t.Errorf("\nExpecting <%+v>,\n Received <%+v>", srv.ldrs, srv.GetLoaderS()) - } - srv.stopChan = make(chan struct{}, 1) - chS := engine.NewCacheS(cfg, nil, nil) - cacheSrv, err := engine.NewService(chS) - if err != nil { - t.Fatal(err) - } - srv.connChan <- cacheSrv - srv.Shutdown() - if srv.IsRunning() { - t.Errorf("Expected service to be down") - } - -} diff --git a/services/radiusagent_it_test.go b/services/radiusagent_it_test.go index 5096772a1..de760928c 100644 --- a/services/radiusagent_it_test.go +++ b/services/radiusagent_it_test.go @@ -69,8 +69,7 @@ func TestRadiusAgentReloadStartShut(t *testing.T) { anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) sS := NewSessionService(cfg, db, server, make(chan birpc.ClientConnector, 1), shdChan, nil, anz, srvDep) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) runtime.Gosched() time.Sleep(10 * time.Millisecond) //need to switch to gorutine if err := srv.Shutdown(); err != nil { @@ -126,8 +125,7 @@ func TestRadiusAgentReload1(t *testing.T) { shdChan, nil, anz, srvDep) srv := NewRadiusAgent(cfg, filterSChan, shdChan, nil, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } @@ -197,8 +195,7 @@ func TestRadiusAgentReload2(t *testing.T) { shdChan, nil, anz, srvDep) srv := NewRadiusAgent(cfg, filterSChan, shdChan, nil, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/rals_it_test.go b/services/rals_it_test.go index 26f8dd257..771222562 100644 --- a/services/rals_it_test.go +++ b/services/rals_it_test.go @@ -74,8 +74,7 @@ func TestRalsReload(t *testing.T) { make(chan birpc.ClientConnector, 1), make(chan birpc.ClientConnector, 1), shdChan, nil, anz, srvDep, filterSChan) - srvMngr.AddServices(ralS, schS, tS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb) + srvMngr.AddServices(ralS, schS, tS, db, stordb) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/registrarc_it_test.go b/services/registrarc_it_test.go index 16a782101..03d4f21c1 100644 --- a/services/registrarc_it_test.go +++ b/services/registrarc_it_test.go @@ -60,9 +60,7 @@ func TestDispatcherHReload(t *testing.T) { anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) connMngr := engine.NewConnManager(cfg, nil) srv := NewRegistrarCService(cfg, server, connMngr, anz, srvDep) - srvMngr.AddServices(srv, - NewLoaderService(cfg, db, filterSChan, server, - make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/resources_it_test.go b/services/resources_it_test.go index 2204a1ad1..fc861c06e 100644 --- a/services/resources_it_test.go +++ b/services/resources_it_test.go @@ -61,8 +61,7 @@ func TestResourceSReload(t *testing.T) { tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) reS := NewResourceService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(tS, reS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(tS, reS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/routes_it_test.go b/services/routes_it_test.go index 6c139217e..6f9f7ed1e 100644 --- a/services/routes_it_test.go +++ b/services/routes_it_test.go @@ -55,8 +55,7 @@ func TestRouteSReload(t *testing.T) { db := NewDataDBService(cfg, nil, false, srvDep) routeS := NewRouteService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(routeS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(routeS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/schedulers_it_test.go b/services/schedulers_it_test.go index df8352d16..fa4b770a4 100644 --- a/services/schedulers_it_test.go +++ b/services/schedulers_it_test.go @@ -54,8 +54,7 @@ func TestSchedulerSReload(t *testing.T) { anz := NewAnalyzerService(cfg, server, filterSChan, shdChan, make(chan birpc.ClientConnector, 1), srvDep) schS := NewSchedulerService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(schS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(schS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/sipagent_it_test.go b/services/sipagent_it_test.go index 506f2e660..86b505ef3 100644 --- a/services/sipagent_it_test.go +++ b/services/sipagent_it_test.go @@ -64,8 +64,7 @@ func TestSIPAgentReload(t *testing.T) { shdChan, nil, anz, srvDep) srv := NewSIPAgent(cfg, filterSChan, shdChan, nil, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(srv, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(srv, sS, db) if err := srvMngr.StartServices(); err != nil { t.Fatal(err) } diff --git a/services/stats_it_test.go b/services/stats_it_test.go index ca1979c0e..2ec9ef58e 100644 --- a/services/stats_it_test.go +++ b/services/stats_it_test.go @@ -61,8 +61,7 @@ func TestStatSReload(t *testing.T) { tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) sS := NewStatService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(tS, sS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(tS, sS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/services/stordb_it_test.go b/services/stordb_it_test.go index 0ba49417b..1e2548837 100644 --- a/services/stordb_it_test.go +++ b/services/stordb_it_test.go @@ -61,9 +61,7 @@ func TestStorDBReload(t *testing.T) { cdrsRPC := make(chan birpc.ClientConnector, 1) cdrS := NewCDRServer(cfg, db, stordb, filterSChan, server, cdrsRPC, nil, anz, srvDep) - srvMngr.AddServices(cdrS, ralS, schS, chrS, - NewLoaderService(cfg, db, filterSChan, server, - make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db, stordb) + srvMngr.AddServices(cdrS, ralS, schS, chrS, db, stordb) if err := engine.InitStorDb(cfg); err != nil { t.Fatal(err) } diff --git a/services/thresholds_it_test.go b/services/thresholds_it_test.go index 442829474..c11422534 100644 --- a/services/thresholds_it_test.go +++ b/services/thresholds_it_test.go @@ -57,8 +57,7 @@ func TestThresholdSReload(t *testing.T) { db := NewDataDBService(cfg, nil, false, srvDep) tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(tS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(tS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } @@ -127,8 +126,7 @@ func TestThresholdSReload2(t *testing.T) { db := NewDataDBService(cfg, nil, false, srvDep) tS := NewThresholdService(cfg, db, chS, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep) engine.NewConnManager(cfg, nil) - srvMngr.AddServices(tS, - NewLoaderService(cfg, db, filterSChan, server, make(chan birpc.ClientConnector, 1), nil, anz, srvDep), db) + srvMngr.AddServices(tS, db) if err := srvMngr.StartServices(); err != nil { t.Error(err) } diff --git a/servmanager/servmanager.go b/servmanager/servmanager.go index b28f0e3a3..38497a850 100644 --- a/servmanager/servmanager.go +++ b/servmanager/servmanager.go @@ -204,8 +204,6 @@ func (srvMngr *ServiceManager) handleReload() { go srvMngr.reloadService(utils.HTTPAgent) case <-srvMngr.GetConfig().GetReloadChan(config.PrometheusAgentJSON): go srvMngr.reloadService(utils.PrometheusAgent) - case <-srvMngr.GetConfig().GetReloadChan(config.LoaderJson): - go srvMngr.reloadService(utils.LoaderS) case <-srvMngr.GetConfig().GetReloadChan(config.AnalyzerCfgJson): go srvMngr.reloadService(utils.AnalyzerS) case <-srvMngr.GetConfig().GetReloadChan(config.DispatcherSJson): diff --git a/utils/consts.go b/utils/consts.go index f3824b608..3aedd544d 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -906,7 +906,6 @@ const ( OrderIDEnd = "OrderIDEnd" MinCost = "MinCost" MaxCost = "MaxCost" - MetaLoaders = "*loaders" TmpSuffix = ".tmp" MetaDiamreq = "*diamreq" MetaRadDAReq = "*radDAReq" @@ -1209,7 +1208,6 @@ const ( ErS = "ErS" FilterS = "FilterS" GuardianS = "GuardianS" - LoaderS = "LoaderS" RALs = "RALs" RegistrarC = "RegistrarC" ReplicatorS = "ReplicatorS" @@ -1236,7 +1234,6 @@ const ( DispatcherSLow = "dispatchers" AnalyzerSLow = "analyzers" SchedulerSLow = "schedulers" - LoaderSLow = "loaders" RALsLow = "rals" ReplicatorLow = "replicator" ApierSLow = "apiers" @@ -2048,14 +2045,6 @@ const ( AnalyzerSv1StringQuery = "AnalyzerSv1.StringQuery" ) -// LoaderS APIs -const ( - LoaderSv1 = "LoaderSv1" - LoaderSv1Load = "LoaderSv1.Load" - LoaderSv1Remove = "LoaderSv1.Remove" - LoaderSv1Ping = "LoaderSv1.Ping" -) - // CacheS APIs const ( CacheSv1 = "CacheSv1"