From d6676866d59027a6f2de2a6902a6938d5546a1f0 Mon Sep 17 00:00:00 2001 From: gezimbll Date: Mon, 26 May 2025 15:26:58 +0200 Subject: [PATCH] added fixes for integration tests & added *none in threshold ees_ids to disable processing --- config/config_it_test.go | 1 + ...at_reset_queue.go => stats_reset_queue.go} | 2 +- ...ueue_test.go => stats_reset_queue_test.go} | 4 +-- .../mysql/create_tariffplan_tables.sql | 2 +- .../postgres/create_tariffplan_tables.sql | 2 +- data/tariffplans/tutorial/Thresholds.csv | 4 +-- engine/model_helpers.go | 29 +++++++------------ engine/model_helpers_test.go | 7 ----- engine/stats.go | 2 +- engine/thresholds.go | 23 +++++++-------- engine/z_loader_it_test.go | 2 +- .../accountsummary_thresholds_it_test.go | 4 +-- general_tests/fraud_detection_it_test.go | 4 +-- general_tests/session_disable_it_test.go | 4 +-- general_tests/session_update_stats_it_test.go | 4 +-- general_tests/thresholds_cps_it_test.go | 1 - general_tests/trends_schedule_it_test.go | 6 ++-- 17 files changed, 43 insertions(+), 58 deletions(-) rename console/{stat_reset_queue.go => stats_reset_queue.go} (97%) rename console/{stat_reset_queue_test.go => stats_reset_queue_test.go} (95%) diff --git a/config/config_it_test.go b/config/config_it_test.go index b68457356..92b296e28 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -245,6 +245,7 @@ func testCGRConfigReloadThresholdS(t *testing.T) { Opts: &ThresholdsOpts{ ProfileIDs: []string{}, }, + EEsConns: []string{}, } if !reflect.DeepEqual(expAttr, cfg.ThresholdSCfg()) { t.Errorf("Expected %s , received: %s ", utils.ToJSON(expAttr), utils.ToJSON(cfg.ThresholdSCfg())) diff --git a/console/stat_reset_queue.go b/console/stats_reset_queue.go similarity index 97% rename from console/stat_reset_queue.go rename to console/stats_reset_queue.go index 0608ea925..ef78d5712 100644 --- a/console/stat_reset_queue.go +++ b/console/stats_reset_queue.go @@ -24,7 +24,7 @@ import ( func init() { c := &CmdResetStatQueue{ - name: "stat_reset_queue", + name: "stats_reset_queue", rpcMethod: utils.StatSv1ResetStatQueue, rpcParams: &utils.TenantIDWithAPIOpts{}, } diff --git a/console/stat_reset_queue_test.go b/console/stats_reset_queue_test.go similarity index 95% rename from console/stat_reset_queue_test.go rename to console/stats_reset_queue_test.go index 2fb4da448..c206354aa 100644 --- a/console/stat_reset_queue_test.go +++ b/console/stats_reset_queue_test.go @@ -27,8 +27,8 @@ import ( "github.com/cgrates/cgrates/utils" ) -func TestCmdStatResetQueue(t *testing.T) { - command := commands["stat_reset_queue"] +func TestCmdStatsResetQueue(t *testing.T) { + command := commands["stats_reset_queue"] // verify if StatSv1 object has method on it m, ok := reflect.TypeOf(new(v1.StatSv1)).MethodByName(strings.Split(command.RpcMethod(), utils.NestingSep)[1]) if !ok { diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 853337ff5..05eb7f48b 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -364,7 +364,7 @@ CREATE TABLE tp_thresholds ( `weight` decimal(8,2) NOT NULL, `action_ids` varchar(64) NOT NULL, `async` BOOLEAN NOT NULL, - `ees_ids` varchar(64) NOT NULL, + `ee_ids` varchar(64) NOT NULL, `created_at` TIMESTAMP, PRIMARY KEY (`pk`), KEY `tpid` (`tpid`), diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index 90c583e5b..d2bdd4f38 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -355,7 +355,7 @@ CREATE TABLE tp_thresholds ( "weight" decimal(8,2) NOT NULL, "action_ids" varchar(64) NOT NULL, "async" BOOLEAN NOT NULL, - "ees_ids" varchar(64) NOT NULL, + "ee_ids" varchar(64) NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE ); CREATE INDEX tp_thresholds_idx ON tp_thresholds (tpid); diff --git a/data/tariffplans/tutorial/Thresholds.csv b/data/tariffplans/tutorial/Thresholds.csv index cdaa96680..0cc46c0e5 100644 --- a/data/tariffplans/tutorial/Thresholds.csv +++ b/data/tariffplans/tutorial/Thresholds.csv @@ -1,3 +1,3 @@ #Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] -cgrates.org,THD_ACNT_1001,FLTR_ACNT_1001,2014-07-29T15:00:00Z,1,1,1s,false,10,ACT_LOG_WARNING,true,ocs_api_event -cgrates.org,THD_ACNT_1002,FLTR_ACNT_1002,2014-07-29T15:00:00Z,-1,1,1s,false,10,ACT_LOG_WARNING,true, +cgrates.org,THD_ACNT_1001,FLTR_ACNT_1001,2014-07-29T15:00:00Z,1,1,1s,false,10,ACT_LOG_WARNING,true, +cgrates.org,THD_ACNT_1002,FLTR_ACNT_1002,2014-07-29T15:00:00Z,-1,1,1s,false,10,ACT_LOG_WARNING,true, \ No newline at end of file diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 013a1ea67..c51647da1 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1891,9 +1891,6 @@ func APItoModelTPThreshold(th *utils.TPThresholdProfile) (mdls ThresholdMdls) { if min > len(th.ActionIDs) { min = len(th.ActionIDs) } - if min > len(th.EeIDs) { - min = len(th.EeIDs) - } for i := range min { mdl := &ThresholdMdl{ Tpid: th.TPid, @@ -1915,10 +1912,16 @@ func APItoModelTPThreshold(th *utils.TPThresholdProfile) (mdls ThresholdMdls) { mdl.ActivationInterval += utils.InfieldSep + th.ActivationInterval.ExpiryTime } } + for i, val := range th.EeIDs { + if i != 0 { + mdl.EeIDs += utils.InfieldSep + } + mdl.EeIDs += val + } } mdl.FilterIDs = th.FilterIDs[i] mdl.ActionIDs = th.ActionIDs[i] - mdl.EeIDs = th.EeIDs[i] + mdls = append(mdls, mdl) } @@ -1933,17 +1936,6 @@ func APItoModelTPThreshold(th *utils.TPThresholdProfile) (mdls ThresholdMdls) { mdls = append(mdls, mdl) } } - if len(th.EeIDs)-min > 0 { - for i := min; i < len(th.EeIDs); i++ { - mdl := &ThresholdMdl{ - Tpid: th.TPid, - Tenant: th.Tenant, - ID: th.ID, - } - mdl.EeIDs = th.EeIDs[i] - mdls = append(mdls, mdl) - } - } if len(th.ActionIDs)-min > 0 { for i := min; i < len(th.ActionIDs); i++ { mdl := &ThresholdMdl{ @@ -1986,17 +1978,18 @@ func APItoThresholdProfile(tpTH *utils.TPThresholdProfile, timezone string) (th Async: tpTH.Async, ActionIDs: make([]string, len(tpTH.ActionIDs)), FilterIDs: make([]string, len(tpTH.FilterIDs)), - EeIDs: make([]string, len(tpTH.EeIDs)), } if tpTH.MinSleep != utils.EmptyString { if th.MinSleep, err = utils.ParseDurationWithNanosecs(tpTH.MinSleep); err != nil { return nil, err } } - + if len(tpTH.EeIDs) > 0 { + th.EeIDs = make([]string, len(tpTH.EeIDs)) + copy(th.EeIDs, tpTH.EeIDs) + } copy(th.ActionIDs, tpTH.ActionIDs) copy(th.FilterIDs, tpTH.FilterIDs) - copy(th.EeIDs, tpTH.EeIDs) if tpTH.ActivationInterval != nil { if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil { diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index 7beb3786d..2481045a4 100644 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -2422,15 +2422,8 @@ func TestAPItoModelTPThreshold4(t *testing.T) { Blocker: false, Weight: 20.0, ActionIDs: []string{"WARN3"}, - EeIDs: []string{"EE1"}, } models := ThresholdMdls{ - { - Tpid: "TP1", - Tenant: "cgrates.org", - ID: "TH_1", - EeIDs: "EE1", - }, { Tpid: "TP1", Tenant: "cgrates.org", diff --git a/engine/stats.go b/engine/stats.go index 02a4e401a..706e9cdff 100644 --- a/engine/stats.go +++ b/engine/stats.go @@ -306,7 +306,7 @@ func (sS *StatService) processThresholds(sQs StatQueues, opts map[string]any) (e var tIDs []string if err := sS.connMgr.Call(context.TODO(), sS.cgrcfg.StatSCfg().ThresholdSConns, utils.ThresholdSv1ProcessEvent, thEv, &tIDs); err != nil && - (len(thIDs) != 0 || err.Error() != utils.ErrNotFound.Error()) { + (len(tIDs) != 0 || err.Error() != utils.ErrNotFound.Error()) { utils.Logger.Warning( fmt.Sprintf(" error: %s processing event %+v with ThresholdS.", err.Error(), thEv)) withErrs = true diff --git a/engine/thresholds.go b/engine/thresholds.go index 2c9966416..b4dd13b4b 100644 --- a/engine/thresholds.go +++ b/engine/thresholds.go @@ -260,22 +260,21 @@ func (t *Threshold) ProcessEvent(args *utils.CGREvent, dm *DataManager, fltrS *F // processEEs processes to the EEs for this threshold func (t *Threshold) processEEs(opts map[string]any, thScfg *config.ThresholdSCfg, connMgr *ConnManager) (err error) { - if len(thScfg.EEsConns) == 0 { - return nil - } var targetEeIDs []string if len(t.tPrfl.EeIDs) > 0 { targetEeIDs = t.tPrfl.EeIDs - } else { - isNone := slices.Contains(thScfg.EEsExporterIDs, utils.MetaNone) - if isNone { + if isNone := slices.Contains(t.tPrfl.EeIDs, utils.MetaNone); isNone { targetEeIDs = []string{} - } else if len(thScfg.EEsExporterIDs) > 0 { - targetEeIDs = thScfg.EEsExporterIDs } + } else { + targetEeIDs = thScfg.EEsExporterIDs } - if targetEeIDs == nil { - return nil // Nothing to do. + if len(targetEeIDs) > 0 { + if len(thScfg.EEsConns) == 0 { + return utils.NewErrNotConnected(utils.EEs) + } + } else { + return nil // no EEs to process } if opts == nil { opts = make(map[string]any) @@ -325,7 +324,7 @@ func (t *Threshold) processEEs(opts map[string]any, thScfg *config.ThresholdSCfg cgrEventWithID, &reply); errExec != nil && errExec.Error() != utils.ErrNotFound.Error() { utils.Logger.Warning( - fmt.Sprintf(" error: %s processing event %+v with EEs.", err.Error(), cgrEv)) + fmt.Sprintf(" error: %s processing event %+v with EEs.", errExec.Error(), cgrEv)) err = utils.ErrPartiallyExecuted } return @@ -599,7 +598,7 @@ func (tS *ThresholdService) processEvent(tnt string, args *utils.CGREvent) (thre } if err = t.processEEs(args.APIOpts, tS.cgrcfg.ThresholdSCfg(), connMgr); err != nil { utils.Logger.Warning( - fmt.Sprintf(" threshold: %s processing with EEs.", err.Error())) + fmt.Sprintf(" received error: %s when processing with EEs.", err.Error())) withErrors = true } if t.dirty == nil || t.Hits == t.tPrfl.MaxHits { // one time threshold diff --git a/engine/z_loader_it_test.go b/engine/z_loader_it_test.go index c635e3e8c..026828f5a 100644 --- a/engine/z_loader_it_test.go +++ b/engine/z_loader_it_test.go @@ -469,7 +469,7 @@ func testLoaderITWriteToDatabase(t *testing.T) { t.Error(err) } if !reflect.DeepEqual(sts, rcv) { - t.Errorf("Expecting: %v, received: %v", sts, rcv) + t.Errorf("Expecting: %v, received: %v", utils.ToJSON(sts), utils.ToJSON(rcv)) } } diff --git a/general_tests/accountsummary_thresholds_it_test.go b/general_tests/accountsummary_thresholds_it_test.go index fbd59e9fc..3c27351fe 100644 --- a/general_tests/accountsummary_thresholds_it_test.go +++ b/general_tests/accountsummary_thresholds_it_test.go @@ -96,8 +96,8 @@ func TestAccountSummaryThresholds(t *testing.T) { TpFiles: map[string]string{ utils.ActionsCsv: fmt.Sprintf(`#ActionsId[0],Action[1],ExtraParameters[2],Filter[3],BalanceId[4],BalanceType[5],Categories[6],DestinationIds[7],RatingSubject[8],SharedGroup[9],ExpiryTime[10],TimingIds[11],Units[12],BalanceWeight[13],BalanceBlocker[14],BalanceDisabled[15],Weight[16] ACT_HTTP,*http_post_async,%s/balance_exhausted,,,,,,,,,,,,false,false,10`, server.URL), - utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,THD_1,*lt:~*asm.BalanceSummaries.test.Value:1,2024-07-29T15:00:00Z,1,1,,false,10,ACT_HTTP,true`, + utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] +cgrates.org,THD_1,*lt:~*asm.BalanceSummaries.test.Value:1,2024-07-29T15:00:00Z,1,1,,false,10,ACT_HTTP,true,`, }, } client, _ := ng.Run(t) diff --git a/general_tests/fraud_detection_it_test.go b/general_tests/fraud_detection_it_test.go index 795b71b42..771125742 100644 --- a/general_tests/fraud_detection_it_test.go +++ b/general_tests/fraud_detection_it_test.go @@ -239,8 +239,8 @@ cgrates.org,STATS_FRD,,,-1,24h,0,*tcc,,true,false,0,THD_FRD // Create and populate Thresholds.csv if err := writeFile(utils.ThresholdsCsv, ` -#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,THD_FRD,*gte:~*req.*tcc:2,,-1,1,0,false,0,ACT_FRD_STOP;ACT_FRD_LOG,true +#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] +cgrates.org,THD_FRD,*gte:~*req.*tcc:2,,-1,1,0,false,0,ACT_FRD_STOP;ACT_FRD_LOG,true, `); err != nil { t.Fatal(err) } diff --git a/general_tests/session_disable_it_test.go b/general_tests/session_disable_it_test.go index 5dc881d33..b2d5a8ff7 100644 --- a/general_tests/session_disable_it_test.go +++ b/general_tests/session_disable_it_test.go @@ -153,8 +153,8 @@ RT1,0.2,0.1,1s,1s,0`, cgrates.org,ATTR_ACNT,*any,,,,*opts.*accountID,*variable,~*req.Account,false,10`, utils.StatsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],QueueLength[4],TTL[5],MinItems[6],Metrics[7],MetricFilterIDs[8],Stored[9],Blocker[10],Weight[11],ThresholdIDs[12] cgrates.org,STATS_1001,*string:~*req.Account:1001,,,-1,,*sum#1,,true,,,THD_1001`, - utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,THD_1001,*string:~*req.StatID:STATS_1001,,-1,5,0,false,,DISABLE_ACC,false`, + utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] +cgrates.org,THD_1001,*string:~*req.StatID:STATS_1001,,-1,5,0,false,,DISABLE_ACC,false,`, }, LogBuffer: bytes.NewBuffer(nil), diff --git a/general_tests/session_update_stats_it_test.go b/general_tests/session_update_stats_it_test.go index b3e27fc74..36b111d00 100644 --- a/general_tests/session_update_stats_it_test.go +++ b/general_tests/session_update_stats_it_test.go @@ -118,8 +118,8 @@ RP_VOICE,DR_VOICE,*any,10`, cgrates.org,call,1001,2014-01-14T00:00:00Z,RP_VOICE,`, utils.StatsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],QueueLength[4],TTL[5],MinItems[6],Metrics[7],MetricFilterIDs[8],Stored[9],Blocker[10],Weight[11],ThresholdIDs[12] cgrates.org,Stat1,*string:~*req.Account:1001,,,,,*tcc;*acd;*tcd,,,,,`, - utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,TH1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,-1,0,0,false,10,,false`, + utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] +cgrates.org,TH1,*string:~*req.Account:1001,2014-07-29T15:00:00Z,-1,0,0,false,10,,false,`, } ng := engine.TestEngine{ ConfigJSON: content, diff --git a/general_tests/thresholds_cps_it_test.go b/general_tests/thresholds_cps_it_test.go index cbef2a2c9..7cde5d38c 100644 --- a/general_tests/thresholds_cps_it_test.go +++ b/general_tests/thresholds_cps_it_test.go @@ -78,7 +78,6 @@ func TestStressThresholdsProcessEvent(t *testing.T) { DBCfg: dbConfig, LogBuffer: bytes.NewBuffer(nil), } - defer fmt.Println(ng.LogBuffer) client, _ := ng.Run(t) t.Run("SetThresholdProfile", func(t *testing.T) { var reply string diff --git a/general_tests/trends_schedule_it_test.go b/general_tests/trends_schedule_it_test.go index 6a8a288b4..1388bf021 100644 --- a/general_tests/trends_schedule_it_test.go +++ b/general_tests/trends_schedule_it_test.go @@ -108,9 +108,9 @@ cgrates.org,TREND_2,@every 1s,Stats1_2,,-1,-1,1,*last,1,false,*none`, utils.StatsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],QueueLength[4],TTL[5],MinItems[6],Metrics[7],MetricFilterIDs[8],Stored[9],Blocker[10],Weight[11],ThresholdIDs[12] cgrates.org,Stats1_1,*string:~*req.Account:1001,,,,,*tcc;*acd;*tcd,,true,,, cgrates.org,Stats1_2,*string:~*req.Account:1002,,,,,*sum#~*req.Usage;*pdd,,true,,,`, - utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10] -cgrates.org,Threshold1,*string:~*req.Metrics.*acd.ID:*acd,2024-07-29T15:00:00Z,-1,10,1s,false,10,,true -cgrates.org,Threshold2,*string:~*req.Metrics.*pdd.ID:*pdd,2024-07-29T15:00:00Z,-1,10,1s,false,10,,true + utils.ThresholdsCsv: `#Tenant[0],Id[1],FilterIDs[2],ActivationInterval[3],MaxHits[4],MinHits[5],MinSleep[6],Blocker[7],Weight[8],ActionIDs[9],Async[10],EeIDs[11] +cgrates.org,Threshold1,*string:~*req.Metrics.*acd.ID:*acd,2024-07-29T15:00:00Z,-1,10,1s,false,10,,true, +cgrates.org,Threshold2,*string:~*req.Metrics.*pdd.ID:*pdd,2024-07-29T15:00:00Z,-1,10,1s,false,10,,true, `} ng := engine.TestEngine{