From 6c2fbbec280f12068a5176896b2e068aa09b470d Mon Sep 17 00:00:00 2001 From: gezimbll Date: Mon, 24 Jun 2024 11:06:57 -0400 Subject: [PATCH] added sars model,tp,api --- apier/v1/apier.go | 36 ++- apier/v1/replicator.go | 30 +++ apier/v1/sags_it_test.go | 2 +- apier/v1/sars.go | 67 ++++++ apier/v1/sars_it_test.go | 209 ++++++++++++++++++ apier/v1/tpsars.go | 101 +++++++++ apier/v1/tpsars_it_test.go | 196 ++++++++++++++++ config/config_defaults.go | 6 +- config/config_json_test.go | 14 ++ config/config_test.go | 2 +- .../mysql/create_tariffplan_tables.sql | 24 ++ .../postgres/create_tariffplan_tables.sql | 22 ++ engine/datadbmock.go | 23 ++ engine/datamanager.go | 66 +++++- engine/{sags.go => libsags.go} | 0 engine/libsars.go | 46 ++++ engine/libtest.go | 4 + engine/loader_csv_test.go | 2 +- engine/model_helpers.go | 161 +++++++++++++- engine/models.go | 19 ++ engine/storage_csv.go | 30 ++- engine/storage_interface.go | 5 + engine/storage_internal_datadb.go | 19 ++ engine/storage_internal_stordb.go | 31 +++ engine/storage_mongo_datadb.go | 42 +++- engine/storage_mongo_stordb.go | 49 ++++ engine/storage_redis.go | 23 ++ engine/storage_sql.go | 40 ++++ engine/tpexporter.go | 14 ++ engine/tpimporter_csv.go | 12 + engine/tpreader.go | 36 +++ general_tests/acntacts_test.go | 3 +- general_tests/auth_test.go | 6 +- general_tests/costs1_test.go | 2 +- general_tests/datachrg1_test.go | 2 +- general_tests/ddazmbl1_test.go | 3 +- general_tests/ddazmbl2_test.go | 3 +- general_tests/ddazmbl3_test.go | 3 +- general_tests/smschrg1_test.go | 2 +- loaders/loader.go | 29 +++ services/datadb_it_test.go | 1 + utils/apitpdata.go | 14 ++ utils/consts.go | 26 +++ 43 files changed, 1401 insertions(+), 24 deletions(-) create mode 100644 apier/v1/sars_it_test.go create mode 100644 apier/v1/tpsars.go create mode 100644 apier/v1/tpsars_it_test.go rename engine/{sags.go => libsags.go} (100%) create mode 100644 engine/libsars.go diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 259b30fb3..8466bcc1b 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -1585,7 +1585,7 @@ func (apierSv1 *APIerSv1) ExportToFolder(ctx *context.Context, arg *utils.ArgExp if len(arg.Items) == 0 { arg.Items = []string{utils.MetaAttributes, utils.MetaChargers, utils.MetaDispatchers, utils.MetaDispatcherHosts, utils.MetaFilters, utils.MetaResources, utils.MetaStats, - utils.MetaRoutes, utils.MetaThresholds, utils.MetaSags} + utils.MetaRoutes, utils.MetaThresholds, utils.MetaSags, utils.MetaSars} } if _, err := os.Stat(arg.Path); os.IsNotExist(err) { os.Mkdir(arg.Path, os.ModeDir) @@ -1890,6 +1890,40 @@ func (apierSv1 *APIerSv1) ExportToFolder(ctx *context.Context, arg *utils.ArgExp csvWriter.Write(record) } csvWriter.Flush() + case utils.MetaSars: + prfx := utils.SarsProfilePrefix + keys, err := apierSv1.DataManager.DataDB().GetKeysForPrefix(prfx) + if err != nil { + return err + } + if len(keys) == 0 { + continue + } + f, err := os.Create(path.Join(arg.Path, utils.SarsCsv)) + if err != nil { + return err + } + defer f.Close() + + csvWriter := csv.NewWriter(f) + csvWriter.Comma = utils.CSVSep + if err := csvWriter.Write(engine.SarsMdls{}.CSVHeader()); err != nil { + return err + } + for _, key := range keys { + tntID := strings.SplitN(key[len(prfx):], utils.InInFieldSep, 2) + srsPrf, err := apierSv1.DataManager.GetSarProfile(tntID[0], tntID[1]) + if err != nil { + return err + } + mdl := engine.APItoModelSars(engine.SarProfileToAPI(srsPrf)) + record, err := engine.CsvDump(mdl) + if err != nil { + return err + } + csvWriter.Write(record) + } + csvWriter.Flush() case utils.MetaRoutes: prfx := utils.RouteProfilePrefix keys, err := apierSv1.DataManager.DataDB().GetKeysForPrefix(prfx) diff --git a/apier/v1/replicator.go b/apier/v1/replicator.go index f00e7828f..e8aba3e00 100644 --- a/apier/v1/replicator.go +++ b/apier/v1/replicator.go @@ -147,6 +147,17 @@ func (rplSv1 *ReplicatorSv1) GetSagProfile(ctx *context.Context, tntID *utils.Te return nil } +// GetSarProfile is the remote method coresponding to the dataDb driver method +func (rplSv1 *ReplicatorSv1) GetSarProfile(ctx *context.Context, tntID *utils.TenantIDWithAPIOpts, reply *engine.SarProfile) error { + engine.UpdateReplicationFilters(utils.SarsProfilePrefix, tntID.TenantID.TenantID(), utils.IfaceAsString(tntID.APIOpts[utils.RemoteHostOpt])) + rcv, err := rplSv1.dm.DataDB().GetSarProfileDrv(tntID.Tenant, tntID.ID) + if err != nil { + return err + } + *reply = *rcv + return nil +} + // GetTiming is the remote method coresponding to the dataDb driver method func (rplSv1 *ReplicatorSv1) GetTiming(ctx *context.Context, id *utils.StringWithAPIOpts, reply *utils.TPTiming) error { engine.UpdateReplicationFilters(utils.TimingsPrefix, id.Arg, utils.IfaceAsString(id.APIOpts[utils.RemoteHostOpt])) @@ -445,6 +456,15 @@ func (rplSv1 *ReplicatorSv1) SetSagProfile(ctx *context.Context, sg *engine.SagP return } +// SetSarProfile is the replication method coresponding to the dataDb driver method +func (rplSv1 *ReplicatorSv1) SetSarProfile(ctx *context.Context, sg *engine.SarProfileWithAPIOpts, reply *string) (err error) { + if err = rplSv1.dm.DataDB().SetSarProfileDrv(sg.SarProfile); err != nil { + return + } + *reply = utils.OK + return +} + // SetStatQueue is the replication method coresponding to the dataDb driver method func (rplSv1 *ReplicatorSv1) SetStatQueue(ctx *context.Context, sq *engine.StatQueueWithAPIOpts, reply *string) (err error) { if err = rplSv1.dm.DataDB().SetStatQueueDrv(nil, sq.StatQueue); err != nil { @@ -860,6 +880,16 @@ func (rplSv1 *ReplicatorSv1) RemoveSagProfile(ctx *context.Context, args *utils. return } +// RemoveSarProfile is the replication method coresponding to the dataDb driver method +func (rplSv1 *ReplicatorSv1) RemoveSarProfile(ctx *context.Context, args *utils.TenantIDWithAPIOpts, reply *string) (err error) { + if err = rplSv1.dm.DataDB().RemSarProfileDrv(args.Tenant, args.ID); err != nil { + return + } + + *reply = utils.OK + return +} + // RemoveTiming is the replication method coresponding to the dataDb driver method func (rplSv1 *ReplicatorSv1) RemoveTiming(ctx *context.Context, id *utils.StringWithAPIOpts, reply *string) (err error) { if err = rplSv1.dm.DataDB().RemoveTimingDrv(id.Arg); err != nil { diff --git a/apier/v1/sags_it_test.go b/apier/v1/sags_it_test.go index 517b09ee0..07415d448 100644 --- a/apier/v1/sags_it_test.go +++ b/apier/v1/sags_it_test.go @@ -158,7 +158,7 @@ func testSagSSetSagProfile(t *testing.T) { Sorting: "*asc", ThresholdIDs: []string{"THD1", "THD2"}}, } - if err := sagRPC.Call(context.Background(), utils.APIerSv1GetStatQueueProfile, + if err := sagRPC.Call(context.Background(), utils.APIerSv1GetSagProfile, &utils.TenantID{Tenant: "cgrates.org", ID: "Sag1"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Fatal(err) diff --git a/apier/v1/sars.go b/apier/v1/sars.go index 73e6d99fc..0193bf509 100644 --- a/apier/v1/sars.go +++ b/apier/v1/sars.go @@ -20,6 +20,7 @@ package v1 import ( "github.com/cgrates/birpc/context" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -33,3 +34,69 @@ func (sa *SarSv1) Ping(ctx *context.Context, ign *utils.CGREvent, reply *string) *reply = utils.Pong return nil } + +func (apierSv1 *APIerSv1) GetSarProfile(ctx *context.Context, arg *utils.TenantID, reply *engine.SarProfile) (err error) { + if missing := utils.MissingStructFields(arg, []string{utils.ID}); len(missing) != 0 { + return utils.NewErrMandatoryIeMissing(missing...) + } + tnt := arg.Tenant + if tnt == utils.EmptyString { + tnt = apierSv1.Config.GeneralCfg().DefaultTenant + } + sg, err := apierSv1.DataManager.GetSarProfile(tnt, arg.ID) + if err != nil { + return utils.APIErrorHandler(err) + } + *reply = *sg + return +} + +func (apierSv1 *APIerSv1) GetSarProfileIDs(ctx *context.Context, args *utils.PaginatorWithTenant, sgPrfIDs *[]string) (err error) { + tnt := args.Tenant + if tnt == utils.EmptyString { + tnt = apierSv1.Config.GeneralCfg().DefaultTenant + } + prfx := utils.SarsProfilePrefix + tnt + utils.ConcatenatedKeySep + keys, err := apierSv1.DataManager.DataDB().GetKeysForPrefix(prfx) + if err != nil { + return err + } + if len(keys) == 0 { + return utils.ErrNotFound + } + sgIDs := make([]string, len(keys)) + for i, key := range keys { + sgIDs[i] = key[len(prfx):] + } + *sgPrfIDs = args.PaginateStringSlice(sgIDs) + return +} + +func (apierSv1 *APIerSv1) SetSarProfile(ctx *context.Context, arg *engine.SarProfileWithAPIOpts, reply *string) error { + if missing := utils.MissingStructFields(arg.SarProfile, []string{utils.ID}); len(missing) != 0 { + return utils.NewErrMandatoryIeMissing(missing...) + } + if arg.Tenant == utils.EmptyString { + arg.Tenant = apierSv1.Config.GeneralCfg().DefaultTenant + } + if err := apierSv1.DataManager.SetSarProfile(arg.SarProfile); err != nil { + return utils.APIErrorHandler(err) + } + *reply = utils.OK + return nil +} + +func (apierSv1 *APIerSv1) RemoveSarProfile(ctx *context.Context, args *utils.TenantIDWithAPIOpts, reply *string) error { + if missing := utils.MissingStructFields(args, []string{utils.ID}); len(missing) != 0 { //Params missing + return utils.NewErrMandatoryIeMissing(missing...) + } + tnt := args.Tenant + if tnt == utils.EmptyString { + tnt = apierSv1.Config.GeneralCfg().DefaultTenant + } + if err := apierSv1.DataManager.RemoveSagProfile(tnt, args.ID); err != nil { + return utils.APIErrorHandler(err) + } + *reply = utils.OK + return nil +} diff --git a/apier/v1/sars_it_test.go b/apier/v1/sars_it_test.go new file mode 100644 index 000000000..26a5aef32 --- /dev/null +++ b/apier/v1/sars_it_test.go @@ -0,0 +1,209 @@ +//go:build integration +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package v1 + +import ( + "path" + "testing" + "time" + + "github.com/cgrates/birpc" + "github.com/cgrates/birpc/context" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var ( + sarCfgPath string + sarCfg *config.CGRConfig + sarRPC *birpc.Client + sarProfile *engine.SarProfileWithAPIOpts + sarConfigDIR string + + sTestsSar = []func(t *testing.T){ + testSarSInitCfg, + testSarSInitDataDb, + testSarSResetStorDb, + testSarSStartEngine, + testSarSRPCConn, + testSarSLoadAdd, + testSarSetSarProfile, + testSarSGetSarProfileIDs, + testSarSUpdateSarProfile, + testSarSRemSarProfile, + testSarSKillEngine, + } +) + +func TestSarSIT(t *testing.T) { + switch *utils.DBType { + case utils.MetaInternal: + sarConfigDIR = "tutinternal" + case utils.MetaMySQL: + sarConfigDIR = "tutmysql" + case utils.MetaMongo: + sarConfigDIR = "tutmongo" + case utils.MetaPostgres: + t.SkipNow() + default: + t.Fatal("Unknown Database type") + } + for _, stest := range sTestsSar { + t.Run(sarConfigDIR, stest) + } +} + +func testSarSInitCfg(t *testing.T) { + var err error + sarCfgPath = path.Join(*utils.DataDir, "conf", "samples", sarConfigDIR) + sarCfg, err = config.NewCGRConfigFromPath(sarCfgPath) + if err != nil { + t.Error(err) + } +} + +func testSarSInitDataDb(t *testing.T) { + if err := engine.InitDataDb(sarCfg); err != nil { + t.Fatal(err) + } +} + +// Wipe out the cdr database +func testSarSResetStorDb(t *testing.T) { + if err := engine.InitStorDb(sarCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func testSarSStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(sarCfgPath, *utils.WaitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func testSarSRPCConn(t *testing.T) { + var err error + sarRPC, err = newRPCClient(sarCfg.ListenCfg()) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} +func testSarSLoadAdd(t *testing.T) { + sarProfile := &engine.SarProfileWithAPIOpts{ + SarProfile: &engine.SarProfile{ + Tenant: "cgrates.org", + ID: "SR_AVG", + QueryInterval: 2 * time.Minute, + StatID: "Stat1", + Trend: "*average", + }, + } + + var result string + if err := sarRPC.Call(context.Background(), utils.APIerSv1SetSarProfile, sarProfile, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } + +} + +func testSarSetSarProfile(t *testing.T) { + var ( + reply *engine.SarProfileWithAPIOpts + result string + ) + sarProfile = &engine.SarProfileWithAPIOpts{ + SarProfile: &engine.SarProfile{ + Tenant: "cgrates.org", + ID: "Sar1", + QueryInterval: time.Second * 15, + ThresholdIDs: []string{"THD1", "THD2"}}, + } + if err := sarRPC.Call(context.Background(), utils.APIerSv1GetSarProfile, + &utils.TenantID{Tenant: "cgrates.org", ID: "Sar1"}, &reply); err == nil || + err.Error() != utils.ErrNotFound.Error() { + t.Fatal(err) + } + if err := sarRPC.Call(context.Background(), utils.APIerSv1SetSarProfile, sarProfile, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Errorf("Expected: %v,Received: %v", utils.OK, result) + } + if err := sarRPC.Call(context.Background(), utils.APIerSv1GetSarProfile, &utils.TenantID{Tenant: "cgrates.org", ID: "Sar1"}, &reply); err != nil { + t.Error(err) + } else if diff := cmp.Diff(sarProfile, reply, cmpopts.SortSlices(func(a, b string) bool { return a < b })); diff != utils.EmptyString { + t.Errorf("Unnexpected profile (-expected +got):\n%s", diff) + } +} +func testSarSGetSarProfileIDs(t *testing.T) { + expected := []string{"Sag1", "SG_Sum"} + var result []string + if err := sarRPC.Call(context.Background(), utils.APIerSv1GetSarProfileIDs, utils.PaginatorWithTenant{}, &result); err != nil { + t.Error(err) + } else if len(expected) != len(result) { + t.Errorf("Expecting : %+v, received: %+v", expected, result) + } +} + +func testSarSUpdateSarProfile(t *testing.T) { + var ( + reply *engine.SarProfileWithAPIOpts + result string + ) + if err := sarRPC.Call(context.Background(), utils.APIerSv1SetSarProfile, sarProfile, &result); err != nil { + t.Error(err) + } + if err := sarRPC.Call(context.Background(), utils.APIerSv1GetSarProfile, &utils.TenantID{Tenant: "cgrates.org", ID: "Sar1"}, &reply); err != nil { + t.Error(err) + } else if diff := cmp.Diff(sarProfile, reply, cmpopts.SortSlices(func(a, b string) bool { return a < b })); diff != utils.EmptyString { + t.Errorf("Unnexpected profile (-expected +got):\n%s", diff) + } +} +func testSarSRemSarProfile(t *testing.T) { + var ( + resp string + reply *engine.SarProfile + ) + if err := sarRPC.Call(context.Background(), utils.APIerSv1RemoveSarProfile, + &utils.TenantID{Tenant: "cgrates.org", ID: "Sar1"}, &resp); err != nil { + t.Error(err) + } else if resp != utils.OK { + t.Error("Unexpected reply returned", resp) + } + + if err := sarRPC.Call(context.Background(), utils.APIerSv1GetSarProfile, + &utils.TenantID{Tenant: "cgrates.org", ID: "Sar1"}, + &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } +} + +func testSarSKillEngine(t *testing.T) { + if err := engine.KillEngine(*utils.WaitRater); err != nil { + t.Error(err) + } +} diff --git a/apier/v1/tpsars.go b/apier/v1/tpsars.go new file mode 100644 index 000000000..d0226d1f9 --- /dev/null +++ b/apier/v1/tpsars.go @@ -0,0 +1,101 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package v1 + +import ( + "github.com/cgrates/birpc/context" + "github.com/cgrates/cgrates/utils" +) + +// SetTPSags creates a new stat within a tariff plan +func (apierSv1 *APIerSv1) SetTPSar(ctx *context.Context, sar *utils.TPSarsProfile, reply *string) error { + if missing := utils.MissingStructFields(sar, []string{utils.TPid, utils.ID}); len(missing) != 0 { + return utils.NewErrMandatoryIeMissing(missing...) + } + if sar.Tenant == utils.EmptyString { + sar.Tenant = apierSv1.Config.GeneralCfg().DefaultTenant + } + if err := apierSv1.StorDb.SetTPSars([]*utils.TPSarsProfile{sar}); err != nil { + return utils.APIErrorHandler(err) + } + *reply = utils.OK + return nil +} + +// GetTPSag queries specific Stat on Tariff plan +func (apierSv1 *APIerSv1) GetTPSar(ctx *context.Context, sar *utils.TPTntID, reply *utils.TPSarsProfile) error { + if missing := utils.MissingStructFields(sar, []string{utils.TPid, utils.ID}); len(missing) != 0 { //Params missing + return utils.NewErrMandatoryIeMissing(missing...) + } + if sar.Tenant == utils.EmptyString { + sar.Tenant = apierSv1.Config.GeneralCfg().DefaultTenant + } + srs, err := apierSv1.StorDb.GetTPSars(sar.TPid, sar.Tenant, sar.ID) + if err != nil { + if err.Error() != utils.ErrNotFound.Error() { + err = utils.NewErrServerError(err) + } + return err + } + *reply = *srs[0] + return nil +} + +type AttrGetTPSarIds struct { + TPid string // Tariff plan id + Tenant string + utils.PaginatorWithSearch +} + +// GetTPSagIDs queries Stat identities on specific tariff plan. +func (apierSv1 *APIerSv1) GetTPSarIDs(ctx *context.Context, attrs *AttrGetTPSarIds, reply *[]string) error { + if missing := utils.MissingStructFields(&attrs, []string{utils.TPid}); len(missing) != 0 { //Params missing + return utils.NewErrMandatoryIeMissing(missing...) + } + if attrs.Tenant == utils.EmptyString { + attrs.Tenant = apierSv1.Config.GeneralCfg().DefaultTenant + } + ids, err := apierSv1.StorDb.GetTpTableIds(attrs.TPid, utils.TBLTPSars, + utils.TPDistinctIds{utils.TenantCfg, utils.IDCfg}, nil, &attrs.PaginatorWithSearch) + if err != nil { + if err.Error() != utils.ErrNotFound.Error() { + err = utils.NewErrServerError(err) + } + return err + } + *reply = ids + return nil +} + +// RemoveTPSar removes specific Sar on Tariff plan +func (apierSv1 *APIerSv1) RemoveTPSar(ctx *context.Context, sar *utils.TPTntID, reply *string) error { + if missing := utils.MissingStructFields(sar, []string{utils.TPid, utils.ID}); len(missing) != 0 { //Params missing + return utils.NewErrMandatoryIeMissing(missing...) + } + if sar.Tenant == utils.EmptyString { + sar.Tenant = apierSv1.Config.GeneralCfg().DefaultTenant + } + if err := apierSv1.StorDb.RemTpData(utils.TBLTPSars, sar.TPid, + map[string]string{utils.TenantCfg: sar.Tenant, utils.IDCfg: sar.ID}); err != nil { + return utils.NewErrServerError(err) + } + *reply = utils.OK + return nil + +} diff --git a/apier/v1/tpsars_it_test.go b/apier/v1/tpsars_it_test.go new file mode 100644 index 000000000..08fc2df9e --- /dev/null +++ b/apier/v1/tpsars_it_test.go @@ -0,0 +1,196 @@ +//go:build offline +// +build offline + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package v1 + +import ( + "path" + "reflect" + "sort" + "testing" + + "github.com/cgrates/birpc" + "github.com/cgrates/birpc/context" + "github.com/cgrates/birpc/jsonrpc" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var ( + tpSarCfgPath string + tpSarCfg *config.CGRConfig + tpSarRPC *birpc.Client + tpSar *utils.TPSarsProfile + tpSarDelay int + tpSarConfigDIR string //run tests for specific configuration +) + +var sTestsTPSars = []func(t *testing.T){ + testTPSarsInitCfg, + testTPSarsResetStorDb, + testTPSarsStartEngine, + testTPSarsRpcConn, + testTPSarsGetTPSarBeforeSet, + testTPSarsSetTPSar, + testTPSarsGetTPSarAfterSet, + testTPSarsUpdateTPSar, + testTPSarsGetTPSarAfterUpdate, + testTPSarsRemoveTPSar, + testTPSarsGetTPSarAfterRemove, + testTPSarsKillEngine, +} + +// Test start here +func TestTPSarIT(t *testing.T) { + switch *utils.DBType { + case utils.MetaInternal: + tpSarConfigDIR = "tutinternal" + case utils.MetaMySQL: + tpSarConfigDIR = "tutmysql" + case utils.MetaMongo: + tpSarConfigDIR = "tutmongo" + case utils.MetaPostgres: + tpSarConfigDIR = "tutpostgres" + default: + t.Fatal("Unknown Database type") + } + for _, stest := range sTestsTPSars { + t.Run(tpSarConfigDIR, stest) + } +} + +func testTPSarsInitCfg(t *testing.T) { + var err error + tpSarCfgPath = path.Join(*utils.DataDir, "conf", "samples", tpSarConfigDIR) + tpSarCfg, err = config.NewCGRConfigFromPath(tpSarCfgPath) + if err != nil { + t.Error(err) + } + tpSarDelay = 1000 +} + +// Wipe out the cdr database +func testTPSarsResetStorDb(t *testing.T) { + if err := engine.InitStorDb(tpSarCfg); err != nil { + t.Fatal(err) + } +} + +// Start CGR Engine +func testTPSarsStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(tpSarCfgPath, tpSarDelay); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func testTPSarsRpcConn(t *testing.T) { + var err error + tpSarRPC, err = jsonrpc.Dial(utils.TCP, tpSarCfg.ListenCfg().RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal(err) + } +} + +func testTPSarsGetTPSarBeforeSet(t *testing.T) { + var reply *utils.TPSarsProfile + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1GetTPSag, + &utils.TPTntID{TPid: "TPS1", Tenant: "cgrates.org", ID: "Sar1"}, + &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } +} + +func testTPSarsSetTPSar(t *testing.T) { + tpSar = &utils.TPSarsProfile{ + Tenant: "cgrates.org", + TPid: "TPS1", + ID: "Sar1", + QueryInterval: "1m", + ThresholdIDs: []string{"ThreshValue", "ThreshValueTwo"}, + } + sort.Strings(tpSar.ThresholdIDs) + var result string + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1SetTPSar, tpSar, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } +} + +func testTPSarsGetTPSarAfterSet(t *testing.T) { + var respond *utils.TPSarsProfile + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1GetTPSar, + &utils.TPTntID{TPid: "TPS1", Tenant: "cgrates.org", ID: "Sar1"}, &respond); err != nil { + t.Fatal(err) + } + sort.Strings(respond.ThresholdIDs) + if !reflect.DeepEqual(tpSar, respond) { + t.Errorf("Expecting: %+v, received: %+v", tpSar, respond) + } +} + +func testTPSarsUpdateTPSar(t *testing.T) { + var result string + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1SetTPSar, tpSar, &result); err != nil { + t.Error(err) + } else if result != utils.OK { + t.Error("Unexpected reply returned", result) + } +} + +func testTPSarsGetTPSarAfterUpdate(t *testing.T) { + var expectedTPS *utils.TPSarsProfile + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1GetTPSar, + &utils.TPTntID{TPid: "TPS1", Tenant: "cgrates.org", ID: "Sar1"}, &expectedTPS); err != nil { + t.Fatal(err) + } + sort.Strings(expectedTPS.ThresholdIDs) + if !reflect.DeepEqual(tpSar, expectedTPS) { + t.Errorf("Expecting: %+v, received: %+v", tpSar, expectedTPS) + } +} + +func testTPSarsRemoveTPSar(t *testing.T) { + var resp string + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1RemoveTPSar, + &utils.TPTntID{TPid: "TPS1", Tenant: "cgrates.org", ID: "Sar1"}, &resp); err != nil { + t.Error(err) + } else if resp != utils.OK { + t.Error("Unexpected reply returned", resp) + } +} + +func testTPSarsGetTPSarAfterRemove(t *testing.T) { + var respond *utils.TPSarsProfile + if err := tpSarRPC.Call(context.Background(), utils.APIerSv1GetTPSar, + &utils.TPTntID{TPid: "TPS1", Tenant: "cgrates.org", ID: "Sar1"}, + &respond); err == nil || err.Error() != utils.ErrNotFound.Error() { + t.Error(err) + } +} + +func testTPSarsKillEngine(t *testing.T) { + if err := engine.KillEngine(tpSarDelay); err != nil { + t.Error(err) + } +} diff --git a/config/config_defaults.go b/config/config_defaults.go index dc1c8dae1..39f241889 100644 --- a/config/config_defaults.go +++ b/config/config_defaults.go @@ -114,7 +114,8 @@ const CGRATES_CFG_JSON = ` "*resource_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*resources": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*statqueue_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, - "*sag_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, + "*sag_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, + "*sar_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*statqueues": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*threshold_profiles": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*thresholds": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, @@ -194,7 +195,8 @@ const CGRATES_CFG_JSON = ` "*tp_account_actions": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*tp_resources": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*tp_stats": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, - "*tp_sags": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, + "*tp_sags": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, + "*tp_sars": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*tp_thresholds": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*tp_filters": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, "*tp_routes": {"limit": -1, "ttl": "", "static_ttl": false, "remote":false, "replicate":false}, diff --git a/config/config_json_test.go b/config/config_json_test.go index e35e94c17..edb034ee3 100644 --- a/config/config_json_test.go +++ b/config/config_json_test.go @@ -417,6 +417,13 @@ func TestDfDataDbJsonCfg(t *testing.T) { Ttl: utils.StringPointer(utils.EmptyString), Static_ttl: utils.BoolPointer(false), }, + utils.MetaSarProfiles: { + Replicate: utils.BoolPointer(false), + Remote: utils.BoolPointer(false), + Limit: utils.IntPointer(-1), + Ttl: utils.StringPointer(utils.EmptyString), + Static_ttl: utils.BoolPointer(false), + }, utils.MetaThresholds: { Replicate: utils.BoolPointer(false), Remote: utils.BoolPointer(false), @@ -670,6 +677,13 @@ func TestDfStorDBJsonCfg(t *testing.T) { Ttl: utils.StringPointer(utils.EmptyString), Static_ttl: utils.BoolPointer(false), }, + utils.CacheTBLTPSars: { + Replicate: utils.BoolPointer(false), + Remote: utils.BoolPointer(false), + Limit: utils.IntPointer(-1), + Ttl: utils.StringPointer(utils.EmptyString), + Static_ttl: utils.BoolPointer(false), + }, utils.CacheTBLTPSags: { Replicate: utils.BoolPointer(false), Remote: utils.BoolPointer(false), diff --git a/config/config_test.go b/config/config_test.go index f47ca89bb..10352185a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -4853,7 +4853,7 @@ func TestV1GetConfigAsJSONDataDB(t *testing.T) { func TestV1GetConfigAsJSONStorDB(t *testing.T) { var reply string - expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"CGRateS.org","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_account_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destination_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_sags":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","mysqlDSNParams":{},"mysqlLocation":"Local","pgSSLMode":"disable","pgSchema":"","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` + expected := `{"stor_db":{"db_host":"127.0.0.1","db_name":"cgrates","db_password":"CGRateS.org","db_port":3306,"db_type":"*mysql","db_user":"cgrates","items":{"*cdrs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*session_costs":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_account_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_action_triggers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_actions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_attributes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_chargers":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destination_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_destinations":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_hosts":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_dispatcher_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_filters":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rates":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_plans":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_rating_profiles":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_resources":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_routes":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_sags":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_sars":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_shared_groups":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_stats":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_thresholds":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*tp_timings":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false},"*versions":{"limit":-1,"remote":false,"replicate":false,"static_ttl":false}},"opts":{"mongoConnScheme":"mongodb","mongoQueryTimeout":"10s","mysqlDSNParams":{},"mysqlLocation":"Local","pgSSLMode":"disable","pgSchema":"","sqlConnMaxLifetime":"0s","sqlMaxIdleConns":10,"sqlMaxOpenConns":100},"prefix_indexed_fields":[],"remote_conns":null,"replication_conns":null,"string_indexed_fields":[]}}` cfgCgr := NewDefaultCGRConfig() if err := cfgCgr.V1GetConfigAsJSON(context.Background(), &SectionWithAPIOpts{Section: STORDB_JSN}, &reply); err != nil { t.Error(err) diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 7d0f07a8d..fbdf9d0c3 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -317,6 +317,30 @@ CREATE TABLE tp_sags( UNIQUE KEY `unique_tp_sags` (`tpid`,`tenant`,`id`,`stat_ids`) ); +-- +-- Table structure for tabls `tp_sars` +-- + +DROP TABLE IF EXISTS tp_sars; +CREATE TABLE tp_sars( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `query_interval` varchar(64) NOT NULL, + `stat_id` varchar(64) NOT NULL, + `queue_length` int(11) NOT NULL, + `ttl` varchar(32) NOT NULL, + `purge_filter_ids` varchar(64) NOT NULL, + `trend` varchar(64) NOT NULL, + `threshold_ids` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_sars` (`tpid`,`tenant`,`id`,`stat_id`) + ); + + -- -- Table structure for table `tp_threshold_cfgs` -- diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index 73e5bb310..06cdd06d4 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -310,6 +310,28 @@ CREATE TABLE tp_sags( CREATE INDEX tp_sags_idx ON tp_sags (tpid); CREATE INDEX tp_sags_unique ON tp_sags ("tpid","tenant", "id","stat_ids"); +-- +-- Table structure for tabls `tp_sars` +-- + +DROP TABLE IF EXISTS tp_sars; +CREATE TABLE tp_sars( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "query_interval" varchar(64) NOT NULL, + "stat_id" varchar(64) NOT NULL, + "queue_length" INTEGER NOT NULL, + "ttl" varchar(32) NOT NULL, + "purge_filter_ids" varchar(64) NOT NULL, + "trend" varchar(32) NOT NULL, + "threshold_ids" varchar(64) NOT NULL, + "created_at" TIMESTAMP +); + CREATE INDEX tp_sars_idx ON tp_sars(tpid); + CREATE INDEX tp_sars_unique ON tp_sars("tpid","tenant","id","stat_id"); + -- -- Table structure for table `tp_threshold_cfgs` -- diff --git a/engine/datadbmock.go b/engine/datadbmock.go index 3e587014a..ac4ca1c49 100644 --- a/engine/datadbmock.go +++ b/engine/datadbmock.go @@ -41,6 +41,9 @@ type DataDBMock struct { SetSagProfileDrvF func(sq *SagProfile) (err error) GetSagProfileDrvF func(tenant string, id string) (sq *SagProfile, err error) RemSagProfileDrvF func(tenant string, id string) (err error) + SetSarProfileDrvF func(sq *SarProfile) (err error) + GetSarProfileDrvF func(tenant string, id string) (sq *SarProfile, err error) + RemSarProfileDrvF func(tenant string, id string) (err error) GetSagsProfileDrvF func(tenant, id string) (sg *SagProfile, err error) GetActionPlanDrvF func(key string) (ap *ActionPlan, err error) SetActionPlanDrvF func(key string, ap *ActionPlan) (err error) @@ -354,6 +357,26 @@ func (dbM *DataDBMock) RemSagProfileDrv(tenant string, id string) (err error) { } return utils.ErrNotImplemented } +func (dbM *DataDBMock) GetSarProfileDrv(tenant, id string) (sg *SarProfile, err error) { + if dbM.GetStatQueueProfileDrvF != nil { + return dbM.GetSarProfileDrvF(tenant, id) + } + return nil, utils.ErrNotImplemented +} + +func (dbM *DataDBMock) SetSarProfileDrv(sar *SarProfile) (err error) { + if dbM.SetSarProfileDrvF(sar) != nil { + return dbM.SetSarProfileDrvF(sar) + } + return utils.ErrNotImplemented +} + +func (dbM *DataDBMock) RemSarProfileDrv(tenant string, id string) (err error) { + if dbM.RemSarProfileDrvF != nil { + return dbM.RemSarProfileDrvF(tenant, id) + } + return utils.ErrNotImplemented +} func (dbM *DataDBMock) GetStatQueueDrv(tenant, id string) (sq *StatQueue, err error) { return nil, utils.ErrNotImplemented diff --git a/engine/datamanager.go b/engine/datamanager.go index 18f2cf591..b1c911c73 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -1286,6 +1286,70 @@ func (dm *DataManager) RemoveStatQueueProfile(tenant, id string, withIndex bool) return dm.RemoveStatQueue(tenant, id) } +func (dm *DataManager) GetSarProfile(tenant, id string) (srp *SarProfile, err error) { + if dm == nil { + err = utils.ErrNoDatabaseConn + return + } + srp, err = dm.dataDB.GetSarProfileDrv(tenant, id) + if err != nil { + if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaSarProfiles]; err == utils.ErrNotFound && itm.Remote { + if err = dm.connMgr.Call(context.TODO(), config.CgrConfig().DataDbCfg().RmtConns, + utils.ReplicatorSv1GetSarProfile, + &utils.TenantIDWithAPIOpts{ + TenantID: &utils.TenantID{Tenant: tenant, ID: id}, + APIOpts: utils.GenerateDBItemOpts(itm.APIKey, itm.RouteID, utils.EmptyString, + utils.FirstNonEmpty(config.CgrConfig().DataDbCfg().RmtConnID, + config.CgrConfig().GeneralCfg().NodeID)), + }, &srp); err == nil { + err = dm.dataDB.SetSarProfileDrv(srp) + } + } + } + return +} + +func (dm *DataManager) SetSarProfile(srp *SarProfile) (err error) { + if err = dm.DataDB().SetSarProfileDrv(srp); err != nil { + return + } + if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaSarProfiles]; itm.Replicate { + err = replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, + config.CgrConfig().DataDbCfg().RplFiltered, + utils.SarsProfilePrefix, srp.TenantID(), + utils.ReplicatorSv1SetSarProfile, + &SarProfileWithAPIOpts{ + SarProfile: srp, + APIOpts: utils.GenerateDBItemOpts(itm.APIKey, itm.RouteID, + config.CgrConfig().DataDbCfg().RplCache, utils.EmptyString)}) + } + return +} + +func (dm *DataManager) RemoveSarProfile(tenant, id string) (err error) { + oldSgs, err := dm.GetSarProfile(tenant, id) + if err != nil && err != utils.ErrNotFound { + return err + } + if err = dm.DataDB().RemSarProfileDrv(tenant, id); err != nil { + return + } + if oldSgs == nil { + return utils.ErrNotFound + } + if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaSarProfiles]; itm.Replicate { + replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, + config.CgrConfig().DataDbCfg().RplFiltered, + utils.SarsProfilePrefix, utils.ConcatenatedKey(tenant, id), // this are used to get the host IDs from cache + utils.ReplicatorSv1RemoveSarProfile, + &utils.TenantIDWithAPIOpts{ + TenantID: &utils.TenantID{Tenant: tenant, ID: id}, + APIOpts: utils.GenerateDBItemOpts(itm.APIKey, itm.RouteID, + config.CgrConfig().DataDbCfg().RplCache, utils.EmptyString)}) + } + return +} + func (dm *DataManager) GetSagProfile(tenant, id string, cacheRead, cacheWrite bool, transactionID string) (sgp *SagProfile, err error) { tntID := utils.ConcatenatedKey(tenant, id) if cacheRead { @@ -1372,7 +1436,7 @@ func (dm *DataManager) RemoveSagProfile(tenant, id string) (err error) { if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaSagProfiles]; itm.Replicate { replicate(dm.connMgr, config.CgrConfig().DataDbCfg().RplConns, config.CgrConfig().DataDbCfg().RplFiltered, - utils.ChargerProfilePrefix, utils.ConcatenatedKey(tenant, id), // this are used to get the host IDs from cache + utils.SagsProfilePrefix, utils.ConcatenatedKey(tenant, id), // this are used to get the host IDs from cache utils.ReplicatorSv1RemoveSagProfile, &utils.TenantIDWithAPIOpts{ TenantID: &utils.TenantID{Tenant: tenant, ID: id}, diff --git a/engine/sags.go b/engine/libsags.go similarity index 100% rename from engine/sags.go rename to engine/libsags.go diff --git a/engine/libsars.go b/engine/libsars.go new file mode 100644 index 000000000..fb01fd398 --- /dev/null +++ b/engine/libsars.go @@ -0,0 +1,46 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package engine + +import ( + "time" + + "github.com/cgrates/cgrates/utils" +) + +type SarProfile struct { + Tenant string + ID string + QueryInterval time.Duration + StatID string + QueueLength int + TTL time.Duration + PurgeFilterIDs []string + Trend string + ThresholdIDs []string +} + +type SarProfileWithAPIOpts struct { + *SarProfile + APIOpts map[string]any +} + +func (srp *SarProfile) TenantID() string { + return utils.ConcatenatedKey(srp.Tenant, srp.ID) +} diff --git a/engine/libtest.go b/engine/libtest.go index 912e146af..d04424556 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -245,6 +245,10 @@ cgrates.org,TestStats2,,,,,2,*sum#~*req.Cost;*average#~*req.Cost,,true,true,20, SagsCSVContent = ` #Tenant[0],Id[1],QueryInterval[2],StatIDs[2],MetricIDs[3],Sorting[4],SortingParameters[5],ThresholdIDs[6] cgrates.org,SAGS1,15m,Stats2;Stats3;Stats4,Metric1;Metric3,*asc,,THD1;THD2 +` + SarsCSVContent = ` +#Tenant[0],Id[1],QueryInterval[2],StatID[3],QueueLength[4],TTL[5],PurgeFilterIDs[6],Trend[7],ThresholdIDs[7] +cgrates.org,SARS1,5m,Stats2,-1,-1,,*average,TD1;THD2 ` ThresholdsCSVContent = ` #Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index f199b6a0a..13537ce54 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -42,7 +42,7 @@ func init() { DestinationsCSVContent, TimingsCSVContent, RatesCSVContent, DestinationRatesCSVContent, RatingPlansCSVContent, RatingProfilesCSVContent, SharedGroupsCSVContent, ActionsCSVContent, ActionPlansCSVContent, ActionTriggersCSVContent, AccountActionsCSVContent, - ResourcesCSVContent, StatsCSVContent, SagsCSVContent, ThresholdsCSVContent, FiltersCSVContent, + ResourcesCSVContent, StatsCSVContent, SarsCSVContent, SagsCSVContent, ThresholdsCSVContent, FiltersCSVContent, RoutesCSVContent, AttributesCSVContent, ChargersCSVContent, DispatcherCSVContent, DispatcherHostCSVContent), testTPID, "", nil, nil, false) if err != nil { diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 2f96e8a6a..18065a93c 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1615,11 +1615,9 @@ func APItoSags(tpSG *utils.TPSagsProfile) (sg *SagProfile, err error) { return nil, err } } - copy(sg.StatIDs, tpSG.StatIDs) copy(sg.ThresholdIDs, tpSG.ThresholdIDs) copy(sg.MetricIDs, tpSG.MetricIDs) - return sg, nil } @@ -1642,6 +1640,165 @@ func SagProfileToAPI(sg *SagProfile) (tpSG *utils.TPSagsProfile) { return } +type SarsMdls []*SarsMdl + +func (tps SarsMdls) CSVHeader() (result []string) { + return []string{"#" + utils.Tenant, utils.ID, utils.QueryInterval, utils.StatID, + utils.TTL, utils.PurgeFilterIDs, utils.Trend, utils.ThresholdIDs} +} + +func (models SarsMdls) AsTPSars() (result []*utils.TPSarsProfile) { + thresholdsMap := make(map[string]utils.StringSet) + purgeFiltersIDsMap := make(map[string]utils.StringSet) + msr := make(map[string]*utils.TPSarsProfile) + for _, model := range models { + key := &utils.TenantID{Tenant: model.Tenant, ID: model.ID} + sr, found := msr[key.TenantID()] + if !found { + sr = &utils.TPSarsProfile{ + Tenant: model.Tenant, + TPid: model.Tpid, + ID: model.ID, + QueryInterval: model.QueryInterval, + StatID: model.StatID, + Trend: model.Trend, + TTL: model.TTL, + QueueLength: model.QueueLength, + } + } + if model.QueryInterval != utils.EmptyString { + sr.QueryInterval = model.QueryInterval + } + if model.StatID != utils.EmptyString { + sr.StatID = model.StatID + } + if model.Trend != utils.EmptyString { + sr.Trend = model.Trend + } + if model.TTL != utils.EmptyString { + sr.TTL = model.TTL + } + if model.QueueLength != 0 { + sr.QueueLength = model.QueueLength + } + if model.PurgeFilterIDs != utils.EmptyString { + if _, has := purgeFiltersIDsMap[key.TenantID()]; !has { + purgeFiltersIDsMap[key.TenantID()] = make(utils.StringSet) + } + purgeFiltersIDsMap[key.TenantID()].AddSlice(strings.Split(model.PurgeFilterIDs, utils.InfieldSep)) + } + if model.ThresholdIDs != utils.EmptyString { + if _, has := thresholdsMap[key.TenantID()]; !has { + thresholdsMap[key.TenantID()] = make(utils.StringSet) + } + thresholdsMap[key.TenantID()].AddSlice(strings.Split(model.ThresholdIDs, utils.InfieldSep)) + } + msr[key.TenantID()] = sr + } + result = make([]*utils.TPSarsProfile, len(msr)) + i := 0 + for tntId, sr := range msr { + result[i] = sr + result[i].PurgeFilterIDs = purgeFiltersIDsMap[tntId].AsSlice() + result[i].ThresholdIDs = thresholdsMap[tntId].AsSlice() + i++ + } + return +} + +func APItoModelSars(tpSR *utils.TPSarsProfile) (mdls SarsMdls) { + if tpSR == nil { + return + } + if len(tpSR.PurgeFilterIDs) == 0 { + mdl := &SarsMdl{ + Tpid: tpSR.TPid, + Tenant: tpSR.Tenant, + ID: tpSR.ID, + QueryInterval: tpSR.QueryInterval, + StatID: tpSR.StatID, + QueueLength: tpSR.QueueLength, + Trend: tpSR.Trend, + } + for i, threshold := range tpSR.ThresholdIDs { + if i != 0 { + mdl.ThresholdIDs += utils.InfieldSep + } + mdl.ThresholdIDs += threshold + } + mdls = append(mdls, mdl) + } + for i, filterID := range tpSR.PurgeFilterIDs { + mdl := &SarsMdl{ + Tpid: tpSR.TPid, + Tenant: tpSR.Tenant, + ID: tpSR.ID, + } + if i == 0 { + mdl.QueueLength = tpSR.QueueLength + mdl.QueryInterval = tpSR.QueryInterval + mdl.StatID = tpSR.StatID + mdl.TTL = tpSR.TTL + mdl.Trend = tpSR.Trend + for i, threshold := range tpSR.ThresholdIDs { + if i != 0 { + mdl.ThresholdIDs += utils.InfieldSep + } + mdl.ThresholdIDs += threshold + } + } + mdl.PurgeFilterIDs = filterID + mdls = append(mdls, mdl) + } + return +} + +func APItoSars(tpSR *utils.TPSarsProfile) (sr *SarProfile, err error) { + sr = &SarProfile{ + Tenant: tpSR.Tenant, + ID: tpSR.ID, + StatID: tpSR.StatID, + QueueLength: tpSR.QueueLength, + Trend: tpSR.Trend, + PurgeFilterIDs: make([]string, len(tpSR.PurgeFilterIDs)), + ThresholdIDs: make([]string, len(tpSR.ThresholdIDs)), + } + if tpSR.TTL != utils.EmptyString { + if sr.TTL, err = utils.ParseDurationWithNanosecs(tpSR.TTL); err != nil { + return + } + } + if tpSR.QueryInterval != utils.EmptyString { + if sr.QueryInterval, err = utils.ParseDurationWithNanosecs(tpSR.QueryInterval); err != nil { + return + } + } + copy(sr.ThresholdIDs, tpSR.ThresholdIDs) + copy(sr.PurgeFilterIDs, tpSR.PurgeFilterIDs) + return +} + +func SarProfileToAPI(sr *SarProfile) (tpSR *utils.TPSarsProfile) { + tpSR = &utils.TPSarsProfile{ + Tenant: sr.Tenant, + ID: sr.ID, + PurgeFilterIDs: make([]string, len(sr.PurgeFilterIDs)), + ThresholdIDs: make([]string, len(sr.ThresholdIDs)), + StatID: sr.StatID, + QueueLength: sr.QueueLength, + Trend: sr.Trend, + } + if sr.TTL != time.Duration(0) { + tpSR.TTL = sr.TTL.String() + } + if sr.QueryInterval != time.Duration(0) { + tpSR.QueryInterval = sr.QueryInterval.String() + } + copy(tpSR.ThresholdIDs, sr.ThresholdIDs) + copy(tpSR.PurgeFilterIDs, sr.PurgeFilterIDs) + return +} + type ThresholdMdls []*ThresholdMdl // CSVHeader return the header for csv fields as a slice of string diff --git a/engine/models.go b/engine/models.go index e02711bf2..85b4b3219 100644 --- a/engine/models.go +++ b/engine/models.go @@ -301,6 +301,25 @@ func (SagsMdl) TableName() string { return utils.TBLTPSags } +type SarsMdl struct { + PK uint `gorm:"primary_key"` + Tpid string + Tenant string `index:"0" re:".*"` + ID string `index:"1" re:".*"` + QueryInterval string `index:"2" re:".*"` + StatID string `index:"3" re:".*"` + QueueLength int `index:"4" re:".*"` + TTL string `index:"5" re:".*"` + PurgeFilterIDs string `index:"6" re:".*"` + Trend string `index:"7" re:".*"` + ThresholdIDs string `index:"8" re:".*"` + CreatedAt time.Time +} + +func (SarsMdl) TableName() string { + return utils.TBLTPSars +} + type ThresholdMdl struct { PK uint `gorm:"primary_key"` Tpid string diff --git a/engine/storage_csv.go b/engine/storage_csv.go index 596e7de07..9500e4d62 100644 --- a/engine/storage_csv.go +++ b/engine/storage_csv.go @@ -58,6 +58,7 @@ type CSVStorage struct { accountactionsFn []string resProfilesFn []string statsFn []string + sarsFn []string sagsFn []string thresholdsFn []string filterFn []string @@ -68,12 +69,12 @@ type CSVStorage struct { dispatcherHostsFn []string } -// NewCSVStorage creates a CSV storege that takes the data from the paths specified +// NewCSVStorage creates a CSV storage that takes the data from the paths specified func NewCSVStorage(sep rune, destinationsFn, timingsFn, ratesFn, destinationratesFn, destinationratetimingsFn, ratingprofilesFn, sharedgroupsFn, actionsFn, actiontimingsFn, actiontriggersFn, accountactionsFn, - resProfilesFn, statsFn, sagsFn, thresholdsFn, filterFn, routeProfilesFn, + resProfilesFn, statsFn, sarsFn, sagsFn, thresholdsFn, filterFn, routeProfilesFn, attributeProfilesFn, chargerProfilesFn, dispatcherProfilesFn, dispatcherHostsFn []string) *CSVStorage { return &CSVStorage{ sep: sep, @@ -91,6 +92,7 @@ func NewCSVStorage(sep rune, accountactionsFn: accountactionsFn, resProfilesFn: resProfilesFn, statsFn: statsFn, + sarsFn: sarsFn, sagsFn: sagsFn, thresholdsFn: thresholdsFn, filterFn: filterFn, @@ -121,6 +123,7 @@ func NewFileCSVStorage(sep rune, dataPath string) (*CSVStorage, error) { accountActionsPaths := appendName(allFoldersPath, utils.AccountActionsCsv) resourcesPaths := appendName(allFoldersPath, utils.ResourcesCsv) statsPaths := appendName(allFoldersPath, utils.StatsCsv) + sarsPaths := appendName(allFoldersPath, utils.SarsCsv) sagsPaths := appendName(allFoldersPath, utils.SagsCsv) thresholdsPaths := appendName(allFoldersPath, utils.ThresholdsCsv) filtersPaths := appendName(allFoldersPath, utils.FiltersCsv) @@ -143,6 +146,7 @@ func NewFileCSVStorage(sep rune, dataPath string) (*CSVStorage, error) { accountActionsPaths, resourcesPaths, statsPaths, + sarsPaths, sagsPaths, thresholdsPaths, filtersPaths, @@ -159,13 +163,13 @@ func NewStringCSVStorage(sep rune, destinationsFn, timingsFn, ratesFn, destinationratesFn, destinationratetimingsFn, ratingprofilesFn, sharedgroupsFn, actionsFn, actiontimingsFn, actiontriggersFn, accountactionsFn, - resProfilesFn, statsFn, sagsFn, thresholdsFn, filterFn, routeProfilesFn, + resProfilesFn, statsFn, sarsFn, sagsFn, thresholdsFn, filterFn, routeProfilesFn, attributeProfilesFn, chargerProfilesFn, dispatcherProfilesFn, dispatcherHostsFn string) *CSVStorage { c := NewCSVStorage(sep, []string{destinationsFn}, []string{timingsFn}, []string{ratesFn}, []string{destinationratesFn}, []string{destinationratetimingsFn}, []string{ratingprofilesFn}, []string{sharedgroupsFn}, []string{actionsFn}, []string{actiontimingsFn}, []string{actiontriggersFn}, []string{accountactionsFn}, - []string{resProfilesFn}, []string{statsFn}, []string{sagsFn}, []string{thresholdsFn}, []string{filterFn}, + []string{resProfilesFn}, []string{statsFn}, []string{sarsFn}, []string{sagsFn}, []string{thresholdsFn}, []string{filterFn}, []string{routeProfilesFn}, []string{attributeProfilesFn}, []string{chargerProfilesFn}, []string{dispatcherProfilesFn}, []string{dispatcherHostsFn}) c.generator = NewCsvString @@ -202,6 +206,7 @@ func NewGoogleCSVStorage(sep rune, spreadsheetID string) (*CSVStorage, error) { getIfExist(utils.AccountActions), getIfExist(utils.Resources), getIfExist(utils.Stats), + getIfExist(utils.SarS), getIfExist(utils.Sars), getIfExist(utils.Thresholds), getIfExist(utils.Filters), @@ -235,6 +240,7 @@ func NewURLCSVStorage(sep rune, dataPath string) *CSVStorage { var accountActionsPaths []string var resourcesPaths []string var statsPaths []string + var sarsPaths []string var sagsPaths []string var thresholdsPaths []string var filtersPaths []string @@ -259,6 +265,7 @@ func NewURLCSVStorage(sep rune, dataPath string) *CSVStorage { accountActionsPaths = append(accountActionsPaths, joinURL(baseURL, utils.AccountActionsCsv)) resourcesPaths = append(resourcesPaths, joinURL(baseURL, utils.ResourcesCsv)) statsPaths = append(statsPaths, joinURL(baseURL, utils.StatsCsv)) + sarsPaths = append(sarsPaths, joinURL(baseURL, utils.SarsCsv)) sagsPaths = append(sagsPaths, joinURL(baseURL, utils.SagsCsv)) thresholdsPaths = append(thresholdsPaths, joinURL(baseURL, utils.ThresholdsCsv)) filtersPaths = append(filtersPaths, joinURL(baseURL, utils.FiltersCsv)) @@ -296,6 +303,8 @@ func NewURLCSVStorage(sep rune, dataPath string) *CSVStorage { resourcesPaths = append(resourcesPaths, baseURL) case strings.HasSuffix(baseURL, utils.StatsCsv): statsPaths = append(statsPaths, baseURL) + case strings.HasSuffix(baseURL, utils.SarsCsv): + sarsPaths = append(sarsPaths, baseURL) case strings.HasSuffix(baseURL, utils.SagsCsv): sagsPaths = append(sagsPaths, baseURL) case strings.HasSuffix(baseURL, utils.ThresholdsCsv): @@ -329,6 +338,7 @@ func NewURLCSVStorage(sep rune, dataPath string) *CSVStorage { accountActionsPaths, resourcesPaths, statsPaths, + sarsPaths, sagsPaths, thresholdsPaths, filtersPaths, @@ -567,6 +577,18 @@ func (csvs *CSVStorage) GetTPStats(tpid, tenant, id string) ([]*utils.TPStatProf return tpStats.AsTPStats(), nil } +func (csvs *CSVStorage) GetTPSars(tpid, tenant, id string) ([]*utils.TPSarsProfile, error) { + var tpSars SarsMdls + if err := csvs.proccesData(SarsMdl{}, csvs.sarsFn, func(tp any) { + tPsars := tp.(SarsMdl) + tPsars.Tpid = tpid + tpSars = append(tpSars, &tPsars) + }); err != nil { + return nil, err + } + return tpSars.AsTPSars(), nil +} + func (csvs *CSVStorage) GetTPSags(tpid, tenant, id string) ([]*utils.TPSagsProfile, error) { var tpSags SagsMdls if err := csvs.proccesData(SagsMdl{}, csvs.sagsFn, func(tp any) { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 9f79cdc5d..9ba8b94ea 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -104,6 +104,9 @@ type DataDB interface { SetSagProfileDrv(sq *SagProfile) (err error) GetSagProfileDrv(tenant string, id string) (sq *SagProfile, err error) RemSagProfileDrv(tenant string, id string) (err error) + SetSarProfileDrv(sq *SarProfile) (err error) + GetSarProfileDrv(tenant string, id string) (sq *SarProfile, err error) + RemSarProfileDrv(tenant string, id string) (err error) GetThresholdProfileDrv(tenant string, ID string) (tp *ThresholdProfile, err error) SetThresholdProfileDrv(tp *ThresholdProfile) (err error) RemThresholdProfileDrv(tenant, id string) (err error) @@ -176,6 +179,7 @@ type LoadReader interface { GetTPAccountActions(*utils.TPAccountActions) ([]*utils.TPAccountActions, error) GetTPResources(string, string, string) ([]*utils.TPResourceProfile, error) GetTPStats(string, string, string) ([]*utils.TPStatProfile, error) + GetTPSars(string, string, string) ([]*utils.TPSarsProfile, error) GetTPSags(string, string, string) ([]*utils.TPSagsProfile, error) GetTPThresholds(string, string, string) ([]*utils.TPThresholdProfile, error) GetTPFilters(string, string, string) ([]*utils.TPFilterProfile, error) @@ -201,6 +205,7 @@ type LoadWriter interface { SetTPAccountActions([]*utils.TPAccountActions) error SetTPResources([]*utils.TPResourceProfile) error SetTPStats([]*utils.TPStatProfile) error + SetTPSars([]*utils.TPSarsProfile) error SetTPSags([]*utils.TPSagsProfile) error SetTPThresholds([]*utils.TPThresholdProfile) error SetTPFilters([]*utils.TPFilterProfile) error diff --git a/engine/storage_internal_datadb.go b/engine/storage_internal_datadb.go index f93e686b9..a7150f8d3 100644 --- a/engine/storage_internal_datadb.go +++ b/engine/storage_internal_datadb.go @@ -592,15 +592,34 @@ func (iDB *InternalDB) RemStatQueueDrv(tenant, id string) (err error) { true, utils.NonTransactional) return } +func (iDB *InternalDB) SetSarProfileDrv(srp *SarProfile) (err error) { + iDB.db.Set(utils.CacheSarProfiles, srp.TenantID(), srp, nil, true, utils.NonTransactional) + return nil +} + +func (iDB *InternalDB) RemSarProfileDrv(tenant, id string) (err error) { + iDB.db.Remove(utils.CacheSarProfiles, utils.ConcatenatedKey(tenant, id), true, utils.NonTransactional) + return nil +} + +func (iDB *InternalDB) GetSarProfileDrv(tenant, id string) (sg *SarProfile, err error) { + x, ok := iDB.db.Get(utils.CacheSarProfiles, utils.ConcatenatedKey(tenant, id)) + if !ok || x == nil { + return nil, utils.ErrNotFound + } + return x.(*SarProfile), nil +} func (iDB *InternalDB) SetSagProfileDrv(sgp *SagProfile) (err error) { iDB.db.Set(utils.CacheSagProfiles, sgp.TenantID(), sgp, nil, true, utils.NonTransactional) return nil } + func (iDB *InternalDB) RemSagProfileDrv(tenant, id string) (err error) { iDB.db.Remove(utils.CacheSagProfiles, utils.ConcatenatedKey(tenant, id), true, utils.NonTransactional) return nil } + func (iDB *InternalDB) GetSagProfileDrv(tenant, id string) (sg *SagProfile, err error) { x, ok := iDB.db.Get(utils.CacheSagProfiles, utils.ConcatenatedKey(tenant, id)) if !ok || x == nil { diff --git a/engine/storage_internal_stordb.go b/engine/storage_internal_stordb.go index bff4b84ae..27547c0ea 100644 --- a/engine/storage_internal_stordb.go +++ b/engine/storage_internal_stordb.go @@ -403,6 +403,28 @@ func (iDB *InternalDB) GetTPStats(tpid, tenant, id string) (stats []*utils.TPSta return } +func (iDB *InternalDB) GetTPSars(tpid string, tenant string, id string) (sars []*utils.TPSarsProfile, err error) { + key := tpid + if tenant != utils.EmptyString { + key += utils.ConcatenatedKeySep + tenant + } + if id != utils.EmptyString { + key += utils.ConcatenatedKeySep + id + } + ids := iDB.db.GetItemIDs(utils.CacheTBLTPSars, key) + for _, id := range ids { + x, ok := iDB.db.Get(utils.CacheTBLTPSars, id) + if !ok || x == nil { + return nil, utils.ErrNotFound + } + sars = append(sars, x.(*utils.TPSarsProfile)) + } + if len(sars) == 0 { + return nil, utils.ErrNotFound + } + return +} + func (iDB *InternalDB) GetTPSags(tpid string, tenant string, id string) (sags []*utils.TPSagsProfile, err error) { key := tpid if tenant != utils.EmptyString { @@ -768,6 +790,15 @@ func (iDB *InternalDB) SetTPSags(sags []*utils.TPSagsProfile) (err error) { } return } +func (iDB *InternalDB) SetTPSars(sars []*utils.TPSarsProfile) (err error) { + if len(sars) == 0 { + return nil + } + for _, sar := range sars { + iDB.db.Set(utils.CacheTBLTPSars, utils.ConcatenatedKey(sar.TPid, sar.Tenant, sar.ID), sar, nil, cacheCommit(utils.NonTransactional), utils.NonTransactional) + } + return +} func (iDB *InternalDB) SetTPThresholds(thresholds []*utils.TPThresholdProfile) (err error) { if len(thresholds) == 0 { return nil diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 7ee492dc4..3228ba793 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -64,6 +64,7 @@ const ( ColTmg = "timings" ColRes = "resources" ColSqs = "statqueues" + ColSrs = "sar_profiles" ColSqp = "statqueue_profiles" ColSgp = "sag_profiles" ColTps = "threshold_profiles" @@ -301,7 +302,7 @@ func (ms *MongoStorage) ensureIndexesForCol(col string) error { // exported for switch col { case ColAct, ColApl, ColAAp, ColAtr, ColRpl, ColDst, ColRds, ColLht, ColIndx: err = ms.enusureIndex(col, true, "key") - case ColRsP, ColRes, ColSqs, ColSgp, ColSqp, ColTps, ColThs, ColRts, ColAttr, ColFlt, ColCpp, ColDpp, ColDph: + case ColRsP, ColRes, ColSqs, ColSgp, ColSrs, ColSqp, ColTps, ColThs, ColRts, ColAttr, ColFlt, ColCpp, ColDpp, ColDph: err = ms.enusureIndex(col, true, "tenant", "id") case ColRpf, ColShg, ColAcc: err = ms.enusureIndex(col, true, "id") @@ -347,7 +348,7 @@ func (ms *MongoStorage) EnsureIndexes(cols ...string) error { cols = []string{ ColAct, ColApl, ColAAp, ColAtr, ColRpl, ColDst, ColRds, ColLht, ColIndx, ColRsP, ColRes, ColSqs, ColSqp, ColTps, ColThs, ColRts, ColAttr, ColFlt, ColCpp, - ColDpp, ColRpf, ColShg, ColAcc, ColSgp, + ColDpp, ColRpf, ColShg, ColAcc, ColSgp, ColSrs, } } else { cols = []string{ @@ -436,6 +437,8 @@ func (ms *MongoStorage) RemoveKeysForPrefix(prefix string) error { colName = ColSqp case utils.SagsProfilePrefix: colName = ColSgp + case utils.SarsProfilePrefix: + colName = ColSrs case utils.ThresholdPrefix: colName = ColThs case utils.FilterPrefix: @@ -603,6 +606,8 @@ func (ms *MongoStorage) GetKeysForPrefix(prefix string) (keys []string, err erro keys, qryErr = ms.getAllKeysMatchingTenantID(sctx, ColSqs, utils.StatQueuePrefix, subject, tntID) case utils.SagsProfilePrefix: keys, qryErr = ms.getAllKeysMatchingTenantID(sctx, ColSgp, utils.SagsProfilePrefix, subject, tntID) + case utils.SarsProfilePrefix: + keys, qryErr = ms.getAllKeysMatchingTenantID(sctx, ColSrs, utils.SarsProfilePrefix, subject, tntID) case utils.StatQueueProfilePrefix: keys, qryErr = ms.getAllKeysMatchingTenantID(sctx, ColSqp, utils.StatQueueProfilePrefix, subject, tntID) case utils.AccountActionPlansPrefix: @@ -1556,6 +1561,39 @@ func (ms *MongoStorage) RemSagProfileDrv(tenant, id string) (err error) { } +func (ms *MongoStorage) GetSarProfileDrv(tenant, id string) (*SarProfile, error) { + srProfile := new(SarProfile) + err := ms.query(func(sctx mongo.SessionContext) error { + sr := ms.getCol(ColSrs).FindOne(sctx, bson.M{"tenant": tenant, "id": id}) + decodeErr := sr.Decode(srProfile) + if errors.Is(decodeErr, mongo.ErrNoDocuments) { + return utils.ErrNotFound + } + return decodeErr + }) + return srProfile, err +} + +func (ms *MongoStorage) SetSarProfileDrv(srp *SarProfile) (err error) { + return ms.query(func(sctx mongo.SessionContext) error { + _, err := ms.getCol(ColSrs).UpdateOne(sctx, bson.M{"tenant": srp.Tenant, "id": srp.ID}, + bson.M{"$set": srp}, + options.Update().SetUpsert(true)) + return err + }) +} + +func (ms *MongoStorage) RemSarProfileDrv(tenant, id string) (err error) { + return ms.query(func(sctx mongo.SessionContext) error { + dr, err := ms.getCol(ColSrs).DeleteOne(sctx, bson.M{"tenant": tenant, "id": id}) + if dr.DeletedCount == 0 { + return utils.ErrNotFound + } + return err + }) + +} + // GetThresholdProfileDrv retrieves a ThresholdProfile from dataDB func (ms *MongoStorage) GetThresholdProfileDrv(tenant, ID string) (*ThresholdProfile, error) { thProfile := new(ThresholdProfile) diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index 482e52edc..eb81d5ca5 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -439,6 +439,39 @@ func (ms *MongoStorage) GetTPStats(tpid, tenant, id string) ([]*utils.TPStatProf }) return results, err } + +func (ms *MongoStorage) GetTPSars(tpid string, tenant string, id string) ([]*utils.TPSarsProfile, error) { + filter := bson.M{ + "tpid": tpid, + } + if id != "" { + filter["id"] = id + } + if tenant != "" { + filter["tenant"] = tenant + } + var results []*utils.TPSarsProfile + err := ms.query(func(sctx mongo.SessionContext) error { + cur, err := ms.getCol(utils.TBLTPSars).Find(sctx, filter) + if err != nil { + return err + } + for cur.Next(sctx) { + var el utils.TPSarsProfile + err := cur.Decode(&el) + if err != nil { + return err + } + results = append(results, &el) + } + if len(results) == 0 { + return utils.ErrNotFound + } + return cur.Close(sctx) + }) + return results, err +} + func (ms *MongoStorage) GetTPSags(tpid string, tenant string, id string) ([]*utils.TPSagsProfile, error) { filter := bson.M{ "tpid": tpid, @@ -1248,6 +1281,22 @@ func (ms *MongoStorage) SetTPStats(tpSTs []*utils.TPStatProfile) (err error) { }) } +func (ms *MongoStorage) SetTPSars(tpSars []*utils.TPSarsProfile) (err error) { + if len(tpSars) == 0 { + return + } + return ms.query(func(sctx mongo.SessionContext) (err error) { + for _, tp := range tpSars { + _, err := ms.getCol(utils.TBLTPSars).UpdateOne(sctx, bson.M{"tpid": tp.TPid, "id": tp.ID}, + bson.M{"$set": tp}, options.Update().SetUpsert(true)) + if err != nil { + return err + } + } + return nil + }) +} + func (ms *MongoStorage) SetTPSags(tpSags []*utils.TPSagsProfile) (err error) { if len(tpSags) == 0 { return diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 533c9473f..e4786e580 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -939,6 +939,29 @@ func (rs *RedisStorage) RemStatQueueDrv(tenant, id string) (err error) { return rs.Cmd(nil, redis_DEL, utils.StatQueuePrefix+utils.ConcatenatedKey(tenant, id)) } +func (rs *RedisStorage) SetSarProfileDrv(sg *SarProfile) (err error) { + var result []byte + if result, err = rs.ms.Marshal(sg); err != nil { + return + } + return rs.Cmd(nil, redis_SET, utils.SarsProfilePrefix+utils.ConcatenatedKey(sg.Tenant, sg.ID), string(result)) +} + +func (rs *RedisStorage) GetSarProfileDrv(tenant string, id string) (sg *SarProfile, err error) { + var values []byte + if err = rs.Cmd(&values, redis_GET, utils.SarsProfilePrefix+utils.ConcatenatedKey(tenant, id)); err != nil { + return + } else if len(values) == 0 { + err = utils.ErrNotFound + return + } + err = rs.ms.Unmarshal(values, &sg) + return +} +func (rs *RedisStorage) RemSarProfileDrv(tenant string, id string) (err error) { + return rs.Cmd(nil, redis_DEL, utils.SarsProfilePrefix+utils.ConcatenatedKey(tenant, id)) +} + func (rs *RedisStorage) SetSagProfileDrv(sg *SagProfile) (err error) { var result []byte if result, err = rs.ms.Marshal(sg); err != nil { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 3f55e947e..a3168e4af 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -598,6 +598,27 @@ func (sqls *SQLStorage) SetTPSags(sgs []*utils.TPSagsProfile) error { return nil } +func (sqls *SQLStorage) SetTPSars(srs []*utils.TPSarsProfile) error { + if len(srs) == 0 { + return nil + } + tx := sqls.db.Begin() + for _, sg := range srs { + if err := tx.Where(&SarsMdl{Tpid: sg.TPid, ID: sg.ID}).Delete(SarsMdl{}).Error; err != nil { + tx.Rollback() + return err + } + for _, msg := range APItoModelSars(sg) { + if err := tx.Create(&msg).Error; err != nil { + tx.Rollback() + return err + } + } + } + tx.Commit() + return nil +} + func (sqls *SQLStorage) SetTPThresholds(ths []*utils.TPThresholdProfile) error { if len(ths) == 0 { return nil @@ -1440,6 +1461,25 @@ func (sqls *SQLStorage) GetTPStats(tpid, tenant, id string) ([]*utils.TPStatProf return asts, nil } +func (sqls *SQLStorage) GetTPSars(tpid, tenant, id string) ([]*utils.TPSarsProfile, error) { + var srs SarsMdls + q := sqls.db.Where("tpid = ?", tpid) + if len(id) != 0 { + q = q.Where("id = ?", id) + } + if len(tenant) != 0 { + q = q.Where("tenant = ?", tenant) + } + if err := q.Find(&srs).Error; err != nil { + return nil, err + } + asrs := srs.AsTPSars() + if len(asrs) == 0 { + return asrs, utils.ErrNotFound + } + return asrs, nil +} + func (sqls *SQLStorage) GetTPSags(tpid string, tenant string, id string) ([]*utils.TPSagsProfile, error) { var sgs SagsMdls q := sqls.db.Where("tpid = ?", tpid) diff --git a/engine/tpexporter.go b/engine/tpexporter.go index 1f21d59a3..d4ef4e633 100644 --- a/engine/tpexporter.go +++ b/engine/tpexporter.go @@ -279,6 +279,20 @@ func (tpExp *TPExporter) Run() error { } } + storDataSars, err := tpExp.storDb.GetTPSars(tpExp.tpID, "", "") + if err != nil && err.Error() != utils.ErrNotFound.Error() { + utils.Logger.Warning(fmt.Sprintf("<%s> error: %s,when getting %s from stordb for export", utils.ApierS, err, utils.TpSars)) + } + if len(storDataSars) != 0 { + toExportMap[utils.SarsCsv] = make([]any, len(storDataSars)) + for _, sd := range storDataSars { + sModels := APItoModelSars(sd) + for _, sdModel := range sModels { + toExportMap[utils.SarsCsv] = append(toExportMap[utils.SarsCsv], sdModel) + } + } + } + storDataSags, err := tpExp.storDb.GetTPSags(tpExp.tpID, "", "") if err != nil && err.Error() != utils.ErrNotFound.Error() { utils.Logger.Warning(fmt.Sprintf("<%s> error: %s,when getting %s from stordb for export", utils.ApierS, err, utils.TpSags)) diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index 40b38764f..8da2996bf 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -53,6 +53,7 @@ var fileHandlers = map[string]func(*TPCSVImporter, string) error{ utils.AccountActionsCsv: (*TPCSVImporter).importAccountActions, utils.ResourcesCsv: (*TPCSVImporter).importResources, utils.StatsCsv: (*TPCSVImporter).importStats, + utils.SarsCsv: (*TPCSVImporter).importSars, utils.SagsCsv: (*TPCSVImporter).importSags, utils.ThresholdsCsv: (*TPCSVImporter).importThresholds, utils.FiltersCsv: (*TPCSVImporter).importFilters, @@ -286,6 +287,17 @@ func (tpImp *TPCSVImporter) importStats(fn string) error { return tpImp.StorDb.SetTPStats(sts) } +func (tpImp *TPCSVImporter) importSars(fn string) error { + if tpImp.Verbose { + log.Printf("Processing file: <%s>", fn) + } + srs, err := tpImp.csvr.GetTPSars(tpImp.TPid, "", "") + if err != nil { + return err + } + return tpImp.StorDb.SetTPSars(srs) +} + func (tpImp *TPCSVImporter) importSags(fn string) error { if tpImp.Verbose { log.Printf("Processing file: <%s>", fn) diff --git a/engine/tpreader.go b/engine/tpreader.go index b01610c74..650a3434e 100644 --- a/engine/tpreader.go +++ b/engine/tpreader.go @@ -49,6 +49,7 @@ type TpReader struct { sharedGroups map[string]*SharedGroup resProfiles map[utils.TenantID]*utils.TPResourceProfile sqProfiles map[utils.TenantID]*utils.TPStatProfile + srProfiles map[utils.TenantID]*utils.TPSarsProfile sgProfiles map[utils.TenantID]*utils.TPSagsProfile thProfiles map[utils.TenantID]*utils.TPThresholdProfile filters map[utils.TenantID]*utils.TPFilterProfile @@ -1136,6 +1137,23 @@ func (tpr *TpReader) LoadStats() error { return tpr.LoadStatsFiltered("") } +func (tpr *TpReader) LoadSarsFiltered(tag string) error { + tps, err := tpr.lr.GetTPSars(tpr.tpid, "", tag) + if err != nil { + return err + } + mapSrs := make(map[utils.TenantID]*utils.TPSarsProfile) + for _, sr := range tps { + mapSrs[utils.TenantID{Tenant: sr.Tenant, ID: sr.ID}] = sr + } + tpr.srProfiles = mapSrs + return nil +} + +func (tpr *TpReader) LoadSars() error { + return tpr.LoadSarsFiltered("") +} + func (tpr *TpReader) LoadSagsFiltered(tag string) error { tps, err := tpr.lr.GetTPSags(tpr.tpid, "", tag) if err != nil { @@ -1336,6 +1354,9 @@ func (tpr *TpReader) LoadAll() (err error) { if err = tpr.LoadStats(); err != nil && err.Error() != utils.NotFoundCaps { return } + if err = tpr.LoadSars(); err != nil && err.Error() != utils.NotFoundCaps { + return + } if err = tpr.LoadSags(); err != nil && err.Error() != utils.NotFoundCaps { return } @@ -1594,6 +1615,21 @@ func (tpr *TpReader) WriteToDatabase(verbose, disableReverse bool) (err error) { loadIDs[utils.CacheStatQueues] = loadID loadIDs[utils.CacheStatQueueProfiles] = loadID } + if verbose { + log.Print("SarProfiles") + } + for _, tpSR := range tpr.srProfiles { + var sr *SarProfile + if sr, err = APItoSars(tpSR); err != nil { + return + } + if err = tpr.dm.SetSarProfile(sr); err != nil { + return + } + if verbose { + log.Print("\t", sr.TenantID()) + } + } if verbose { log.Print("SagProfiles:") } diff --git a/general_tests/acntacts_test.go b/general_tests/acntacts_test.go index 0a9a3cfcd..f0688dcd6 100644 --- a/general_tests/acntacts_test.go +++ b/general_tests/acntacts_test.go @@ -50,6 +50,7 @@ ENABLE_ACNT,*enable_account,,,,,,,,,,,,,false,false,10` accountActions := `cgrates.org,1,TOPUP10_AT,,,` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -59,7 +60,7 @@ ENABLE_ACNT,*enable_account,,,,,,,,,,,,,false,false,10` csvr, err := engine.NewTpReader(dbAcntActs.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, accountActions, - resLimits, stats, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) + resLimits, stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) } diff --git a/general_tests/auth_test.go b/general_tests/auth_test.go index 70d2fc351..39862c6fc 100644 --- a/general_tests/auth_test.go +++ b/general_tests/auth_test.go @@ -54,6 +54,7 @@ func TestAuthLoadCsvError(t *testing.T) { accountActions := `` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -62,7 +63,7 @@ func TestAuthLoadCsvError(t *testing.T) { chargerProfiles := `` csvr, err := engine.NewTpReader(dbAuth.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, accountActions, - resLimits, stats, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) + resLimits, stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) } @@ -89,6 +90,7 @@ cgrates.org,call,*any,2013-01-06T00:00:00Z,RP_ANY,` accountActions := `cgrates.org,testauthpostpaid1,TOPUP10_AT,,,` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -97,7 +99,7 @@ cgrates.org,call,*any,2013-01-06T00:00:00Z,RP_ANY,` chargerProfiles := `` csvr, err := engine.NewTpReader(dbAuth.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, accountActions, - resLimits, stats, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) + resLimits, stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) } diff --git a/general_tests/costs1_test.go b/general_tests/costs1_test.go index a3bf81638..40b7f190e 100644 --- a/general_tests/costs1_test.go +++ b/general_tests/costs1_test.go @@ -54,7 +54,7 @@ RP_SMS1,DR_SMS_1,ALWAYS,10` cgrates.org,data,*any,2012-01-01T00:00:00Z,RP_DATA1, cgrates.org,sms,*any,2012-01-01T00:00:00Z,RP_SMS1,` csvr, err := engine.NewTpReader(dataDB.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, dests, timings, - rates, destinationRates, ratingPlans, ratingProfiles, + rates, destinationRates, ratingPlans, ratingProfiles, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, diff --git a/general_tests/datachrg1_test.go b/general_tests/datachrg1_test.go index 9db036ee8..ac939e6ac 100644 --- a/general_tests/datachrg1_test.go +++ b/general_tests/datachrg1_test.go @@ -44,7 +44,7 @@ DR_DATA_2,*any,RT_DATA_1c,*up,4,0,` RP_DATA1,DR_DATA_2,TM2,10` ratingProfiles := `cgrates.org,data,*any,2012-01-01T00:00:00Z,RP_DATA1,` csvr, err := engine.NewTpReader(dataDB.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, - utils.EmptyString, timings, rates, destinationRates, ratingPlans, ratingProfiles, + utils.EmptyString, timings, rates, destinationRates, ratingPlans, ratingProfiles, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString), diff --git a/general_tests/ddazmbl1_test.go b/general_tests/ddazmbl1_test.go index 107e3ddfc..68abe8406 100644 --- a/general_tests/ddazmbl1_test.go +++ b/general_tests/ddazmbl1_test.go @@ -58,6 +58,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` accountActions := `cgrates.org,12344,TOPUP10_AT,,,` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -68,7 +69,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, accountActions, - resLimits, stats, sags, thresholds, filters, suppliers, + resLimits, stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) diff --git a/general_tests/ddazmbl2_test.go b/general_tests/ddazmbl2_test.go index 47ac3ecb4..2e638cbf7 100644 --- a/general_tests/ddazmbl2_test.go +++ b/general_tests/ddazmbl2_test.go @@ -58,6 +58,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` accountActions := `cgrates.org,12345,TOPUP10_AT,,,` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -67,7 +68,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` csvr, err := engine.NewTpReader(dataDB2.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, accountActions, resLimits, - stats, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) + stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) } diff --git a/general_tests/ddazmbl3_test.go b/general_tests/ddazmbl3_test.go index a3371f1c2..fb223459f 100644 --- a/general_tests/ddazmbl3_test.go +++ b/general_tests/ddazmbl3_test.go @@ -56,6 +56,7 @@ cgrates.org,call,discounted_minutes,2013-01-06T00:00:00Z,RP_UK_Mobile_BIG5_PKG,` accountActions := `cgrates.org,12346,TOPUP10_AT,,,` resLimits := `` stats := `` + sars := `` sags := `` thresholds := `` filters := `` @@ -64,7 +65,7 @@ cgrates.org,call,discounted_minutes,2013-01-06T00:00:00Z,RP_UK_Mobile_BIG5_PKG,` chargerProfiles := `` csvr, err := engine.NewTpReader(dataDB3.DataDB(), engine.NewStringCSVStorage(utils.CSVSep, destinations, timings, rates, destinationRates, ratingPlans, ratingProfiles, sharedGroups, actions, actionPlans, actionTriggers, - accountActions, resLimits, stats, sags, + accountActions, resLimits, stats, sars, sags, thresholds, filters, suppliers, attrProfiles, chargerProfiles, ``, ""), "", "", nil, nil, false) if err != nil { t.Error(err) diff --git a/general_tests/smschrg1_test.go b/general_tests/smschrg1_test.go index 68618b1f7..4d3f68fd9 100644 --- a/general_tests/smschrg1_test.go +++ b/general_tests/smschrg1_test.go @@ -47,7 +47,7 @@ func TestSMSLoadCsvTpSmsChrg1(t *testing.T) { utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString, - utils.EmptyString, utils.EmptyString, utils.EmptyString), utils.EmptyString, + utils.EmptyString, utils.EmptyString, utils.EmptyString, utils.EmptyString), utils.EmptyString, utils.EmptyString, nil, nil, false) if err != nil { t.Error(err) diff --git a/loaders/loader.go b/loaders/loader.go index 5b0071d30..965e72e1d 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -408,7 +408,35 @@ func (ldr *Loader) storeLoadedData(loaderType string, cacheArgs[utils.CacheStatQueues] = ids } } + case utils.MetaSars: + for _, lDataSet := range lds { + srsModels := make(engine.SarsMdls, len(lDataSet)) + for i, ld := range lDataSet { + srsModels[i] = new(engine.SarsMdl) + if err = utils.UpdateStructWithIfaceMap(srsModels[i], ld); err != nil { + return + } + } + for _, tpSrs := range srsModels.AsTPSars() { + srsPrf, err := engine.APItoSars(tpSrs) + if err != nil { + return err + } + if ldr.dryRun { + utils.Logger.Info( + fmt.Sprintf("<%s-%s> DRY_RUN: SarsProfile: %s", + utils.LoaderS, ldr.ldrID, utils.ToJSON(srsPrf))) + continue + } + + ids = append(ids, srsPrf.TenantID()) + if err := ldr.dm.SetSarProfile(srsPrf); err != nil { + return err + } + } + } case utils.MetaSags: + cacheIDs = []string{utils.CacheSagFilterIndexes} for _, lDataSet := range lds { stsModels := make(engine.SagsMdls, len(lDataSet)) for i, ld := range lDataSet { @@ -433,6 +461,7 @@ func (ldr *Loader) storeLoadedData(loaderType string, if err := ldr.dm.SetSagProfile(sgsPrf); err != nil { return err } + cacheArgs[utils.CacheSagFilterIndexes] = ids } } case utils.MetaThresholds: diff --git a/services/datadb_it_test.go b/services/datadb_it_test.go index 8071f0271..d93ed7555 100644 --- a/services/datadb_it_test.go +++ b/services/datadb_it_test.go @@ -123,6 +123,7 @@ func TestDataDBReload(t *testing.T) { utils.MetaResources: {Limit: -1}, utils.MetaStatQueueProfiles: {Limit: -1}, utils.MetaSagProfiles: {Limit: -1}, + utils.MetaSarProfiles: {Limit: -1}, utils.MetaThresholds: {Limit: -1}, utils.MetaThresholdProfiles: {Limit: -1}, utils.MetaFilters: {Limit: -1}, diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 8c146c852..afe2c6997 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1000,6 +1000,20 @@ type TPSagsProfile struct { ThresholdIDs []string } +// TPSarProfile is used in APIs to manage remotely offline SarProfile +type TPSarsProfile struct { + TPid string + Tenant string + ID string + QueryInterval string + StatID string + QueueLength int + TTL string + PurgeFilterIDs []string + Trend string + ThresholdIDs []string +} + // TPThresholdProfile is used in APIs to manage remotely offline ThresholdProfile type TPThresholdProfile struct { TPid string diff --git a/utils/consts.go b/utils/consts.go index f50e4286f..99b7b6a76 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -136,6 +136,7 @@ var ( TBLTPAccountActions: CacheTBLTPAccountActions, TBLTPResources: CacheTBLTPResources, TBLTPStats: CacheTBLTPStats, + TBLTPSars: CacheTBLTPSars, TBLTPSags: CacheTBLTPSags, TBLTPThresholds: CacheTBLTPThresholds, TBLTPFilters: CacheTBLTPFilters, @@ -305,6 +306,7 @@ const ( ThresholdProfilePrefix = "thp_" StatQueuePrefix = "stq_" SagsProfilePrefix = "sgp_" + SarsProfilePrefix = "srp_" LoadIDPrefix = "lid_" SessionsBackupPrefix = "sbk_" LoadInstKey = "load_history" @@ -630,6 +632,9 @@ const ( RouteFilterIDs = "RouteFilterIDs" AttributeFilterIDs = "AttributeFilterIDs" QueueLength = "QueueLength" + QueryInterval = "QueryInterval" + PurgeFilterIDs = "PurgeFilterIDs" + Trend = "Trend" TTL = "TTL" MinItems = "MinItems" MetricIDs = "MetricIDs" @@ -970,6 +975,7 @@ const ( MetaStatQueueProfiles = "*statqueue_profiles" MetaStatQueues = "*statqueues" MetaSagProfiles = "*sag_profiles" + MetaSarProfiles = "*sar_profiles" MetaThresholdProfiles = "*threshold_profiles" MetaRouteProfiles = "*route_profiles" MetaAttributeProfiles = "*attribute_profiles" @@ -1148,6 +1154,7 @@ const ( TpRoutes = "TpRoutes" TpAttributes = "TpAttributes" TpStats = "TpStats" + TpSars = "TpSars" TpSags = "TpSags" TpSharedGroups = "TpSharedGroups" TpRatingProfiles = "TpRatingProfiles" @@ -1249,6 +1256,7 @@ const ( ReplicatorSv1GetThresholdProfile = "ReplicatorSv1.GetThresholdProfile" ReplicatorSv1GetStatQueueProfile = "ReplicatorSv1.GetStatQueueProfile" ReplicatorSv1GetSagProfile = "ReplicatorSv1.GetSagProfile" + ReplicatorSv1GetSarProfile = "ReplicatorSv1.GetSarProfile" ReplicatorSv1GetTiming = "ReplicatorSv1.GetTiming" ReplicatorSv1GetResource = "ReplicatorSv1.GetResource" ReplicatorSv1GetResourceProfile = "ReplicatorSv1.GetResourceProfile" @@ -1275,6 +1283,7 @@ const ( ReplicatorSv1SetFilter = "ReplicatorSv1.SetFilter" ReplicatorSv1SetStatQueueProfile = "ReplicatorSv1.SetStatQueueProfile" ReplicatorSv1SetSagProfile = "ReplicatorSv1.SetSagProfile" + ReplicatorSv1SetSarProfile = "ReplicatorSv1.SetSarProfile" ReplicatorSv1SetTiming = "ReplicatorSv1.SetTiming" ReplicatorSv1SetResource = "ReplicatorSv1.SetResource" ReplicatorSv1SetResourceProfile = "ReplicatorSv1.SetResourceProfile" @@ -1301,6 +1310,7 @@ const ( ReplicatorSv1RemoveThresholdProfile = "ReplicatorSv1.RemoveThresholdProfile" ReplicatorSv1RemoveStatQueueProfile = "ReplicatorSv1.RemoveStatQueueProfile" ReplicatorSv1RemoveSagProfile = "ReplicatorSv1.RemoveSagProfile" + ReplicatorSv1RemoveSarProfile = "ReplicatorSv1.RemoveSarProfile" ReplicatorSv1RemoveTiming = "ReplicatorSv1.RemoveTiming" ReplicatorSv1RemoveResource = "ReplicatorSv1.RemoveResource" ReplicatorSv1RemoveResourceProfile = "ReplicatorSv1.RemoveResourceProfile" @@ -1433,6 +1443,9 @@ const ( APIerSv1SetTPSag = "APIerSv1.SetTPSag" APIerSv1GetTPSag = "APIerSv1.GetTPSag" APIerSv1RemoveTPSag = "APIerSv1.RemoveTPSag" + APIerSv1SetTPSar = "APIerSv1.SetTPSar" + APIerSv1GetTPSar = "APIerSv1.GetTPSar" + APIerSv1RemoveTPSar = "APIerSv1.RemoveTPSar" APIerSv1GetTPDestinationRate = "APIerSv1.GetTPDestinationRate" APIerSv1SetTPRouteProfile = "APIerSv1.SetTPRouteProfile" APIerSv1GetTPRouteProfile = "APIerSv1.GetTPRouteProfile" @@ -1654,6 +1667,15 @@ const ( APIerSv1GetStatQueueProfileIDs = "APIerSv1.GetStatQueueProfileIDs" ) +// SarS APIs +const ( + APIerSv1SetSarProfile = "APIerSv1.SetSagProfile" + APIerSv1RemoveSarProfile = "APIerSv1.RemoveSagProfile" + APIerSv1GetSarProfile = "APIerSv1.GetSagProfile" + APIerSv1GetSarProfileIDs = "APIerSv1.GetSagProfileIDs" + SarSv1Ping = "SagSv1.Ping" +) + // SagS APIs const ( APIerSv1SetSagProfile = "APIerSv1.SetSagProfile" @@ -1876,6 +1898,7 @@ const ( AccountActionsCsv = "AccountActions.csv" ResourcesCsv = "Resources.csv" StatsCsv = "Stats.csv" + SarsCsv = "Sars.csv" SagsCsv = "Sags.csv" ThresholdsCsv = "Thresholds.csv" FiltersCsv = "Filters.csv" @@ -1902,6 +1925,7 @@ const ( TBLTPResources = "tp_resources" TBLTPStats = "tp_stats" TBLTPSags = "tp_sags" + TBLTPSars = "tp_sars" TBLTPThresholds = "tp_thresholds" TBLTPFilters = "tp_filters" SessionCostsTBL = "session_costs" @@ -1933,6 +1957,7 @@ const ( CacheStatQueueProfiles = "*statqueue_profiles" CacheStatQueues = "*statqueues" CacheSagProfiles = "*sag_profiles" + CacheSarProfiles = "*sar_profiles" CacheThresholdProfiles = "*threshold_profiles" CacheThresholds = "*thresholds" CacheFilters = "*filters" @@ -1986,6 +2011,7 @@ const ( CacheTBLTPAccountActions = "*tp_account_actions" CacheTBLTPResources = "*tp_resources" CacheTBLTPStats = "*tp_stats" + CacheTBLTPSars = "*tp_sars" CacheTBLTPSags = "*tp_sags" CacheTBLTPThresholds = "*tp_thresholds" CacheTBLTPFilters = "*tp_filters"