From f43c5aad8e6d886844181cbd72aa6ec24b420754 Mon Sep 17 00:00:00 2001 From: porosnicuadrian Date: Mon, 7 Jun 2021 10:39:04 +0300 Subject: [PATCH] Tested remove flag from cgr-loader for all subsystems --- cmd/cgr-loader/cgr-loader_it_test.go | 1 + cmd/cgr-loader/cgr-loader_remove_it_test.go | 482 ++++++++++++++++++++ data/tariffplans/testit/Filters.csv | 1 + 3 files changed, 484 insertions(+) create mode 100644 cmd/cgr-loader/cgr-loader_remove_it_test.go diff --git a/cmd/cgr-loader/cgr-loader_it_test.go b/cmd/cgr-loader/cgr-loader_it_test.go index 4adcb4800..74201d35b 100644 --- a/cmd/cgr-loader/cgr-loader_it_test.go +++ b/cmd/cgr-loader/cgr-loader_it_test.go @@ -38,6 +38,7 @@ var ( dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here") dbType = flag.String("dbtype", utils.MetaInternal, "The type of DataBase (Internal/Mongo/mySql)") encoding = flag.String("rpc", utils.MetaJSON, "what encoding whould be used for rpc comunication") + waitRater = flag.Int("wait_rater", 100, "Number of miliseconds to wait for rater to start and cache") ) func TestLoadConfig(t *testing.T) { diff --git a/cmd/cgr-loader/cgr-loader_remove_it_test.go b/cmd/cgr-loader/cgr-loader_remove_it_test.go new file mode 100644 index 000000000..68a1fc9c2 --- /dev/null +++ b/cmd/cgr-loader/cgr-loader_remove_it_test.go @@ -0,0 +1,482 @@ + // +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package main + + import ( + "bytes" + "errors" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" + "net/rpc" + "net/rpc/jsonrpc" + "os/exec" + "path" + "reflect" + "sort" + "testing" + "time" + ) + +var ( + cgrLdrCfgPath string + cgrLdrCfgDir string + cgrLdrCfg *config.CGRConfig + cgrLdrBIRPC *rpc.Client + cgrLdrTests = []func(t *testing.T) { + testCgrLdrInitCfg, + testCgrLdrInitDataDB, + testCgrLdrInitStorDB, + testCgrLdrStartEngine, + testCgrLdrRPCConn, + testCgrLdrGetSubsystemsNotLoadedLoad, + testCgrLdrLoadData, + testCgrLdrGetAttributeProfileAfterLoad, + testCgrLdrGetFilterAfterLoad, + testCgrLdrGetResourceProfileAfterLoad, + testCgrLdrGetResourceAfterLoad, + testCgrLdrGetRouteProfileAfterLoad, + testCgrLdrGetStatsProfileAfterLoad, + testCgrLdrGetStatQueueAfterLoad, + testCgrLdrGetThresholdProfileAfterLoad, + testCgrLdrGetThresholdAfterLoad, + + //remove all data with cgr-loader and remove flag + testCgrLdrRemoveData, + testCgrLdrGetSubsystemsNotLoadedLoad, + testCgrLdrKillEngine, + } +) + +func TestCGRLoaderRemove(t *testing.T) { + switch *dbType{ + case utils.MetaInternal: + cgrLdrCfgDir = "tutinternal" + case utils.MetaMongo: + cgrLdrCfgDir = "tutmongo" + case utils.MetaMySQL: + cgrLdrCfgDir = "tutmysql" + case utils.MetaPostgres: + t.SkipNow() + default: + t.Fatal("Unknown Database type") + } + for _, test := range cgrLdrTests { + t.Run("cgr-loader remove tests", test) + } +} + +func testCgrLdrInitCfg(t *testing.T) { + var err error + cgrLdrCfgPath = path.Join(*dataDir, "conf", "samples", cgrLdrCfgDir) + cgrLdrCfg, err = config.NewCGRConfigFromPath(cgrLdrCfgPath) + if err != nil { + t.Error(err) + } +} + +func testCgrLdrInitDataDB(t *testing.T) { + if err := engine.InitDataDb(cgrLdrCfg); err != nil { + t.Fatal(err) + } +} + +func testCgrLdrInitStorDB(t *testing.T) { + if err := engine.InitStorDb(cgrLdrCfg); err != nil { + t.Fatal(err) + } +} + +func testCgrLdrStartEngine(t *testing.T){ + if _, err := engine.StartEngine(cgrLdrCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +func testCgrLdrRPCConn(t *testing.T) { + var err error + cgrLdrBIRPC, err = newRPCClient(cgrLdrCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +func testCgrLdrGetSubsystemsNotLoadedLoad(t *testing.T) { + //attributesPrf + var replyAttr *engine.AttributeProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetAttributeProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_ACNT_1001"}}, + &replyAttr); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + //filtersPrf + var replyFltr *engine.Filter + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetFilter, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "FLTR_1"}}, + &replyFltr); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // resourcesPrf + var replyResPrf *engine.ResourceProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetResourceProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "RES_ACNT_1001"}}, + &replyResPrf); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // resource + var replyRes *engine.Resource + if err := cgrLdrBIRPC.Call(utils.ResourceSv1GetResource, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "RES_ACNT_1001"}}, + &replyRes); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // routesPrf + var replyRts *engine.RouteProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetRouteProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ROUTE_ACNT_1001"}}, + &replyRts); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // statsPrf + var replySts *engine.StatQueueProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetStatQueueProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "Stat_1"}}, + &replySts); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // statQueue + var replyStQue *engine.StatQueue + if err := cgrLdrBIRPC.Call(utils.StatSv1GetStatQueue, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "Stat_1"}}, + &replyStQue); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // thresholdPrf + var replyThdPrf *engine.ThresholdProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetThresholdProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "THD_ACNT_1"}}, + &replyThdPrf); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } + + // threshold + var rplyThd *engine.Threshold + if err := cgrLdrBIRPC.Call(utils.ThresholdSv1GetThreshold, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "THD_ACNT_1"}}, + &rplyThd); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Errorf("Expected %+q, received %+q", utils.ErrNotFound.Error(), err.Error()) + } +} + +func testCgrLdrLoadData(t *testing.T) { + // *cacheSAddress = "127.0.0.1:2012" + cmd := exec.Command("cgr-loader", "-config_path="+cgrLdrCfgPath, "-path="+path.Join(*dataDir, "tariffplans", "testit")) + output := bytes.NewBuffer(nil) + outerr := bytes.NewBuffer(nil) + cmd.Stdout = output + cmd.Stderr = outerr + if err := cmd.Run(); err != nil { + t.Log(cmd.Args) + t.Log(output.String()) + t.Log(outerr.String()) + t.Fatal(err) + } +} + +func testCgrLdrGetAttributeProfileAfterLoad(t *testing.T) { + extAttrPrf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTR_ACNT_1001", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"FLTR_ACCOUNT_1001"}, + Weight: 10, + Attributes: []*engine.Attribute{ + { + FilterIDs: []string{}, + Path: "*req.OfficeGroup", + Type: utils.MetaConstant, + Value: config.NewRSRParsersMustCompile("Marketing", utils.InfieldSep), + }, + }, + } + var replyAttr *engine.AttributeProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetAttributeProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ATTR_ACNT_1001"}}, + &replyAttr); err != nil { + t.Error(err) + } else { + sort.Strings(extAttrPrf.FilterIDs) + sort.Strings(replyAttr.FilterIDs) + replyAttr.Attributes[0].Value.Compile() + extAttrPrf.Attributes[0].Value.Compile() + if !reflect.DeepEqual(extAttrPrf.Attributes[0].Value[0], replyAttr.Attributes[0].Value[0]) { + t.Errorf("Expected %T \n, received %T", extAttrPrf.Attributes[0].Value, replyAttr.Attributes[0].Value) + } + } +} + +func testCgrLdrGetFilterAfterLoad(t *testing.T) { + expFilter := &engine.Filter{ + Tenant: "cgrates.org", + ID: "FLTR_1", + Rules: []*engine.FilterRule{ + { + Type: utils.MetaString, + Element: "~*req.Account", + Values: []string{"1003","1002"}, + }, + { + Type: utils.MetaPrefix, + Element: "~*req.Destination", + Values: []string{"10","20"}, + }, + { + Type: utils.MetaRSR, + Element: "~*req.Destination", + Values: []string{"1002"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, time.July, 29, 15, 0, 0, 0, time.UTC), + }, + } + var replyFltr *engine.Filter + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetFilter, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "FLTR_1"}}, + &replyFltr); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expFilter, replyFltr) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expFilter), utils.ToJSON(replyFltr)) + } +} + +func testCgrLdrGetResourceProfileAfterLoad(t *testing.T) { + expREsPrf := &engine.ResourceProfile{ + Tenant: "cgrates.org", + ID: "RES_ACNT_1001", + FilterIDs: []string{"FLTR_ACCOUNT_1001"}, + Weight: 10, + UsageTTL: time.Hour, + Limit: 1, + ThresholdIDs: []string{}, + } + var replyRes *engine.ResourceProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetResourceProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "RES_ACNT_1001"}}, + &replyRes); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expREsPrf, replyRes) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expREsPrf), utils.ToJSON(replyRes)) + } +} + +func testCgrLdrGetResourceAfterLoad(t *testing.T) { + expREsPrf := &engine.Resource{ + Tenant: "cgrates.org", + ID: "RES_ACNT_1001", + Usages: map[string]*engine.ResourceUsage{}, + } + var replyRes *engine.Resource + if err := cgrLdrBIRPC.Call(utils.ResourceSv1GetResource, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "RES_ACNT_1001"}}, + &replyRes); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expREsPrf, replyRes) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expREsPrf), utils.ToJSON(replyRes)) + } +} + +func testCgrLdrGetRouteProfileAfterLoad(t *testing.T) { + expRoutePrf := &engine.RouteProfile{ + ID: "ROUTE_ACNT_1001", + Tenant: "cgrates.org", + FilterIDs: []string{"FLTR_ACCOUNT_1001"}, + Weight: 10, + Sorting: utils.MetaWeight, + SortingParameters: []string{}, + Routes: []*engine.Route{ + { + ID: "route1", + Weight: 20, + }, + { + ID: "route2", + Weight: 10, + }, + }, + + } + var replyRts *engine.RouteProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetRouteProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "ROUTE_ACNT_1001"}}, + &replyRts); err != nil { + t.Error(err) + } else { + sort.Slice(expRoutePrf.Routes, func(i, j int) bool { + return expRoutePrf.Routes[i].ID < expRoutePrf.Routes[j].ID + }) + sort.Slice(replyRts.Routes, func(i, j int) bool { + return replyRts.Routes[i].ID < replyRts.Routes[j].ID + }) + if !reflect.DeepEqual(expRoutePrf, replyRts) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expRoutePrf), utils.ToJSON(replyRts)) + } + } +} + +func testCgrLdrGetStatsProfileAfterLoad(t *testing.T) { + expStatsprf := &engine.StatQueueProfile{ + Tenant: "cgrates.org", + ID: "Stat_1", + FilterIDs: []string{"FLTR_STAT_1"}, + Weight: 30, + QueueLength: 100, + TTL: 10 * time.Second, + MinItems: 0, + Metrics: []*engine.MetricWithFilters{ + { + MetricID: "*tcd", + }, + { + MetricID: "*asr", + }, + { + MetricID: "*acd", + }, + }, + Blocker: true, + ThresholdIDs: []string{utils.MetaNone}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, time.July, 29, 15, 0, 0, 0, time.UTC), + }, + } + var replySts *engine.StatQueueProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetStatQueueProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "Stat_1"}}, + &replySts); err != nil { + t.Error(err) + } else { + sort.Slice(expStatsprf.Metrics, func(i, j int) bool { + return expStatsprf.Metrics[i].MetricID < expStatsprf.Metrics[j].MetricID + }) + sort.Slice(replySts.Metrics, func(i, j int) bool { + return replySts.Metrics[i].MetricID < replySts.Metrics[j].MetricID + }) + if !reflect.DeepEqual(expStatsprf, replySts) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expStatsprf), utils.ToJSON(replySts)) + } + } +} + +func testCgrLdrGetStatQueueAfterLoad(t *testing.T) { + expStatQueue := map[string]string{ + "*acd": "N/A", + "*tcd": "N/A", + "*asr": "N/A", + } + replyStQue := make(map[string]string) + if err := cgrLdrBIRPC.Call(utils.StatSv1GetQueueStringMetrics, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "Stat_1"}}, + &replyStQue);err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expStatQueue, replyStQue) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expStatQueue), utils.ToJSON(replyStQue)) + } +} + +func testCgrLdrGetThresholdProfileAfterLoad(t *testing.T) { + expThPrf := &engine.ThresholdProfile{ + Tenant: "cgrates.org", + ID: "THD_ACNT_1001", + FilterIDs: []string{"FLTR_ACCOUNT_1001"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, time.July, 29, 15, 0, 0, 0, time.UTC), + }, + Weight: 10, + MaxHits: -1, + MinHits: 0, + ActionIDs: []string{"TOPUP_MONETARY_10"}, + } + var replyThdPrf *engine.ThresholdProfile + if err := cgrLdrBIRPC.Call(utils.APIerSv1GetThresholdProfile, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "THD_ACNT_1001"}}, + &replyThdPrf); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expThPrf, replyThdPrf) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expThPrf), utils.ToJSON(replyThdPrf)) + } +} + +func testCgrLdrGetThresholdAfterLoad(t *testing.T) { + expThPrf := &engine.Threshold{ + Tenant: "cgrates.org", + ID: "THD_ACNT_1001", + Hits: 0, + } + var replyThdPrf *engine.Threshold + if err := cgrLdrBIRPC.Call(utils.ThresholdSv1GetThreshold, + &utils.TenantIDWithAPIOpts{TenantID: &utils.TenantID{Tenant: "cgrates.org", ID: "THD_ACNT_1001"}}, + &replyThdPrf); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expThPrf, replyThdPrf) { + t.Errorf("Expected %+v \n, received %+v", utils.ToJSON(expThPrf), utils.ToJSON(replyThdPrf)) + } +} + +func testCgrLdrRemoveData(t *testing.T) { + // *cacheSAddress = "127.0.0.1:2012" + cmd := exec.Command("cgr-loader", "-config_path="+cgrLdrCfgPath, "-path="+path.Join(*dataDir, "tariffplans", "testit"), "-remove") + output := bytes.NewBuffer(nil) + outerr := bytes.NewBuffer(nil) + cmd.Stdout = output + cmd.Stderr = outerr + if err := cmd.Run(); err != nil { + t.Log(cmd.Args) + t.Log(output.String()) + t.Log(outerr.String()) + t.Fatal(err) + } +} + +//Kill the engine when it is about to be finished +func testCgrLdrKillEngine(t *testing.T) { + if err := engine.KillEngine(100); err != nil { + t.Error(err) + } +} + + func newRPCClient(cfg *config.ListenCfg) (c *rpc.Client, err error) { + switch *encoding { + case utils.MetaJSON: + return jsonrpc.Dial(utils.TCP, cfg.RPCJSONListen) + case utils.MetaGOB: + return rpc.Dial(utils.TCP, cfg.RPCGOBListen) + default: + return nil, errors.New("UNSUPPORTED_RPC") + } + } diff --git a/data/tariffplans/testit/Filters.csv b/data/tariffplans/testit/Filters.csv index 71445f5ca..77ab0590d 100644 --- a/data/tariffplans/testit/Filters.csv +++ b/data/tariffplans/testit/Filters.csv @@ -26,4 +26,5 @@ cgrates.org,FLTR_QOS_SP1_2,*gte,~*vars.Cost,0.1,2014-07-29T15:00:00Z cgrates.org,FLTR_QOS_SP2_2,*gte,~*vars.Cost,0.2,2014-07-29T15:00:00Z cgrates.org,FLTR_TEST,*string,~*req.Subject,TEST,2014-07-29T15:00:00Z cgrates.org,FLTR_SPP_LOAD_DIST,*string,~*req.DistinctMatch,LoadDistStrategy,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_DE,*string,~*req.Destination,1003,2014-07-29T15:00:00Z cgrates.new,FLTR_1,*prefix,~*req.Destination,207;1207;+1207,