From 6e84b555be678439d2ac36c13729618f29d6f714 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 16 Sep 2024 21:01:37 +0200 Subject: [PATCH] Updating TrendS with getTrendGrowth function --- engine/libtrends.go | 113 +++++++++++++++++++++++---------------- engine/libtrends_test.go | 72 +++++++++++++++++-------- engine/model_helpers.go | 22 ++++---- engine/trends.go | 24 +++++---- utils/consts.go | 2 + utils/errors.go | 1 + 6 files changed, 148 insertions(+), 86 deletions(-) diff --git a/engine/libtrends.go b/engine/libtrends.go index f630ea6e1..24d872502 100644 --- a/engine/libtrends.go +++ b/engine/libtrends.go @@ -26,34 +26,40 @@ import ( "github.com/cgrates/cgrates/utils" ) +// A TrendProfile represents the settings of a Trend type TrendProfile struct { - Tenant string - ID string - Schedule string // Cron expression scheduling gathering of the metrics - StatID string - Metrics []*MetricWithSettings - QueueLength int - TTL time.Duration - TrendType string // *last, *average - ThresholdIDs []string + Tenant string + ID string + Schedule string // Cron expression scheduling gathering of the metrics + StatID string + Metrics []string + QueueLength int + TTL time.Duration + MinItems int // minimum number of items for building Trends + CorrelationType string // *last, *average + Tolerance float64 // allow this deviation margin for *constant trend + Stored bool // store the Trend in dataDB + ThresholdIDs []string } // Clone will clone the TrendProfile so it can be used by scheduler safely func (tP *TrendProfile) Clone() (clnTp *TrendProfile) { clnTp = &TrendProfile{ - Tenant: tP.Tenant, - ID: tP.ID, - Schedule: tP.Schedule, - StatID: tP.StatID, - QueueLength: tP.QueueLength, - TTL: tP.TTL, - TrendType: tP.TrendType, + Tenant: tP.Tenant, + ID: tP.ID, + Schedule: tP.Schedule, + StatID: tP.StatID, + QueueLength: tP.QueueLength, + TTL: tP.TTL, + MinItems: tP.MinItems, + CorrelationType: tP.CorrelationType, + Tolerance: tP.Tolerance, + Stored: tP.Stored, } if tP.Metrics != nil { - clnTp.Metrics = make([]*MetricWithSettings, len(tP.Metrics)) - for i, m := range tP.Metrics { - clnTp.Metrics[i] = &MetricWithSettings{MetricID: m.MetricID, - TrendSwingMargin: m.TrendSwingMargin} + clnTp.Metrics = make([]string, len(tP.Metrics)) + for i, mID := range tP.Metrics { + clnTp.Metrics[i] = mID } } if tP.ThresholdIDs != nil { @@ -65,12 +71,6 @@ func (tP *TrendProfile) Clone() (clnTp *TrendProfile) { return } -// MetricWithSettings adds specific settings to the Metric -type MetricWithSettings struct { - MetricID string - TrendSwingMargin float64 // allow this margin for *neutral trend -} - type TrendProfileWithAPIOpts struct { *TrendProfile APIOpts map[string]any @@ -94,15 +94,18 @@ type TrendWithAPIOpts struct { type Trend struct { *sync.RWMutex - Tenant string - ID string - RunTimes []time.Time - Metrics map[time.Time]map[string]*MetricWithTrend + Tenant string + ID string + RunTimes []time.Time + Metrics map[time.Time]map[string]*MetricWithTrend + CompressedMetrics []byte // if populated, Metrics will be emty // indexes help faster processing mLast map[string]time.Time // last time a metric was present mCounts map[string]int // number of times a metric is present in Metrics mTotals map[string]float64 // cached sum, used for average calculations + + tP *TrendProfile // cache here the settings } // computeIndexes should be called after each retrieval from DB @@ -113,6 +116,7 @@ func (t *Trend) computeIndexes() { for _, runTime := range t.RunTimes { for _, mWt := range t.Metrics[runTime] { t.indexesAppendMetric(mWt, runTime) + } } } @@ -124,27 +128,45 @@ func (t *Trend) indexesAppendMetric(mWt *MetricWithTrend, rTime time.Time) { t.mTotals[mWt.ID] += mWt.Value } +// getTrendGrowth returns the percentage growth for a specific metric +// +// @correlation parameter will define whether the comparison is against last or average value +// errors in case of previous +func (t *Trend) getTrendGrowth(mID string, mVal float64, correlation string, roundDec int) (tG float64, err error) { + var prevVal float64 + if _, has := t.mLast[mID]; !has { + return -1.0, utils.ErrNotFound + } + if _, has := t.Metrics[t.mLast[mID]][mID]; !has { + return -1.0, utils.ErrNotFound + } + + switch correlation { + case utils.MetaLast: + prevVal = t.Metrics[t.mLast[mID]][mID].Value + case utils.MetaAverage: + prevVal = t.mTotals[mID] / float64(t.mCounts[mID]) + default: + return -1.0, utils.ErrCorrelationUndefined + } + + diffVal := mVal - prevVal + return utils.Round(diffVal*100/prevVal, roundDec, utils.MetaRoundingMiddle), nil +} + // getTrendLabel identifies the trend label for the instant value of the metric // // *positive, *negative, *constant, N/A -func (t *Trend) getTrendLabel(mID string, mVal float64, swingMargin float64) (lbl string) { - var prevVal *float64 - if _, has := t.mLast[mID]; has { - prevVal = &t.Metrics[t.mLast[mID]][mID].Value - } - if prevVal == nil { - return utils.NotAvailable - } - diffVal := mVal - *prevVal +func (t *Trend) getTrendLabel(tGrowth float64, tolerance float64) (lbl string) { switch { - case diffVal > 0: + case tGrowth > 0: lbl = utils.MetaPositive - case diffVal < 0: + case tGrowth < 0: lbl = utils.MetaNegative default: lbl = utils.MetaConstant } - if math.Abs(diffVal*100/(*prevVal)) <= swingMargin { // percentage value of diff is lower than threshold + if math.Abs(tGrowth) <= tolerance { // percentage value of diff is lower than threshold lbl = utils.MetaConstant } return @@ -152,9 +174,10 @@ func (t *Trend) getTrendLabel(mID string, mVal float64, swingMargin float64) (lb // MetricWithTrend represents one read from StatS type MetricWithTrend struct { - ID string // Metric ID - Value float64 // Metric Value - Trend string // *positive, *negative, *constant, N/A + ID string // Metric ID + Value float64 // Metric Value + TrendGrowth float64 // Difference between last and previous + TrendLabel string // *positive, *negative, *constant, N/A } func (tr *Trend) TenantID() string { diff --git a/engine/libtrends_test.go b/engine/libtrends_test.go index 49b040944..00b1a521f 100644 --- a/engine/libtrends_test.go +++ b/engine/libtrends_test.go @@ -26,36 +26,66 @@ import ( "github.com/cgrates/cgrates/utils" ) -func TestTrendGetTrendLabel(t *testing.T) { +func TestTrendGetTrendGrowth(t *testing.T) { now := time.Now() - t1 := now.Add(-2 * time.Second) - t2 := now.Add(-time.Second) + t1 := now.Add(-time.Second) + t2 := now.Add(-2 * time.Second) + t3 := now.Add(-3 * time.Second) trnd1 := &Trend{ RWMutex: &sync.RWMutex{}, Tenant: "cgrates.org", ID: "TestTrendGetTrendLabel", - RunTimes: []time.Time{t1, t2, now}, + RunTimes: []time.Time{t3, t2, t1}, Metrics: map[time.Time]map[string]*MetricWithTrend{ - t1: {utils.MetaTCD: {utils.MetaTCD, float64(9 * time.Second), utils.NotAvailable}, utils.MetaTCC: {utils.MetaTCC, 9.0, utils.NotAvailable}}, - t2: {utils.MetaTCD: {utils.MetaTCD, float64(10 * time.Second), utils.MetaPositive}, utils.MetaTCC: {utils.MetaTCC, 10.0, utils.MetaPositive}}}, + t3: {utils.MetaTCD: {utils.MetaTCD, float64(41 * time.Second), -1.0, utils.NotAvailable}, utils.MetaTCC: {utils.MetaTCC, 41.0, -1.0, utils.NotAvailable}}, + t2: {utils.MetaTCD: {utils.MetaTCD, float64(9 * time.Second), -78.048, utils.MetaNegative}, utils.MetaTCC: {utils.MetaTCC, 9.0, -78.048, utils.MetaNegative}}, + t1: {utils.MetaTCD: {utils.MetaTCD, float64(10 * time.Second), 11.11111, utils.MetaPositive}, utils.MetaTCC: {utils.MetaTCC, 10.0, 11.11111, utils.MetaPositive}}}, } trnd1.computeIndexes() - if lbl := trnd1.getTrendLabel(utils.MetaTCD, float64(11*time.Second), 0.0); lbl != utils.MetaPositive { - t.Errorf("Expecting: <%q> got <%q>", utils.MetaPositive, lbl) + if _, err := trnd1.getTrendGrowth(utils.MetaTCD, float64(11*time.Second), utils.NotAvailable, 5); err != utils.ErrCorrelationUndefined { + t.Error(err) } - if lbl := trnd1.getTrendLabel(utils.MetaTCD, float64(11*time.Second), 9.0); lbl != utils.MetaPositive { - t.Errorf("Expecting: <%q> got <%q>", utils.MetaPositive, lbl) + if growth, err := trnd1.getTrendGrowth(utils.MetaTCD, float64(11*time.Second), utils.MetaLast, 5); err != nil || growth != 10.0 { + t.Errorf("Expecting: <%f> got <%f>, err: %v", 10.0, growth, err) } - if lbl := trnd1.getTrendLabel(utils.MetaTCD, float64(11*time.Second), 10.0); lbl != utils.MetaConstant { - t.Errorf("Expecting: <%q> got <%q>", utils.MetaConstant, lbl) - } - if lbl := trnd1.getTrendLabel(utils.MetaTCD, float64(9*time.Second), 9.0); lbl != utils.MetaNegative { - t.Errorf("Expecting: <%q> got <%q>", utils.MetaNegative, lbl) - } - if lbl := trnd1.getTrendLabel(utils.MetaTCD, float64(9*time.Second), 10.0); lbl != utils.MetaConstant { - t.Errorf("Expecting: <%q> got <%q>", utils.MetaConstant, lbl) - } - if lbl := trnd1.getTrendLabel(utils.MetaACD, float64(9*time.Second), 0.0); lbl != utils.NotAvailable { - t.Errorf("Expecting: <%q> got <%q>", utils.NotAvailable, lbl) + if growth, err := trnd1.getTrendGrowth(utils.MetaTCD, float64(11*time.Second), utils.MetaAverage, 5); err != nil || growth != -45.0 { + t.Errorf("Expecting: <%f> got <%f>, err: %v", -45.0, growth, err) + } +} + +func TestTrendGetTrendLabel(t *testing.T) { + now := time.Now() + t1 := now.Add(-time.Second) + t2 := now.Add(-2 * time.Second) + t3 := now.Add(-3 * time.Second) + trnd1 := &Trend{ + RWMutex: sync.RWMutex{}, + Tenant: "cgrates.org", + ID: "TestTrendGetTrendLabel", + RunTimes: []time.Time{t3, t2, t1}, + Metrics: map[time.Time]map[string]*MetricWithTrend{ + t3: {utils.MetaTCD: {utils.MetaTCD, float64(41 * time.Second), -1.0, utils.NotAvailable}, utils.MetaTCC: {utils.MetaTCC, 41.0, -1.0, utils.NotAvailable}}, + t2: {utils.MetaTCD: {utils.MetaTCD, float64(9 * time.Second), -78.048, utils.MetaNegative}, utils.MetaTCC: {utils.MetaTCC, 9.0, -78.048, utils.MetaNegative}}, + t1: {utils.MetaTCD: {utils.MetaTCD, float64(10 * time.Second), 11.11111, utils.MetaPositive}, utils.MetaTCC: {utils.MetaTCC, 10.0, 11.11111, utils.MetaPositive}}}, + } + trnd1.computeIndexes() + expct := utils.MetaPositive + if lbl := trnd1.getTrendLabel(11.0, 0.0); lbl != expct { + t.Errorf("Expecting: <%q> got <%q>", expct, lbl) + } + if lbl := trnd1.getTrendLabel(11.0, 10.0); lbl != expct { + t.Errorf("Expecting: <%q> got <%q>", expct, lbl) + } + expct = utils.MetaConstant + if lbl := trnd1.getTrendLabel(11.0, 11.0); lbl != expct { + t.Errorf("Expecting: <%q> got <%q>", expct, lbl) + } + expct = utils.MetaNegative + if lbl := trnd1.getTrendLabel(-9.0, 8.0); lbl != expct { + t.Errorf("Expecting: <%q> got <%q>", expct, lbl) + } + expct = utils.MetaConstant + if lbl := trnd1.getTrendLabel(-9.0, 10.0); lbl != expct { + t.Errorf("Expecting: <%q> got <%q>", expct, lbl) } } diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 2b5df1779..c276c1706 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1756,13 +1756,13 @@ func APItoModelTrends(tr *utils.TPTrendsProfile) (mdls TrendsMdls) { func APItoTrends(tr *utils.TPTrendsProfile) (sr *TrendProfile, err error) { sr = &TrendProfile{ - Tenant: tr.Tenant, - ID: tr.ID, - StatID: tr.StatID, - Schedule: tr.Schedule, - QueueLength: tr.QueueLength, - Metrics: make([]*MetricWithSettings, len(tr.Metrics)), - TrendType: tr.TrendType, + Tenant: tr.Tenant, + ID: tr.ID, + StatID: tr.StatID, + Schedule: tr.Schedule, + QueueLength: tr.QueueLength, + //Metrics: make([]*MetricWithSettings, len(tr.Metrics)), + //TrendType: tr.TrendType, ThresholdIDs: make([]string, len(tr.ThresholdIDs)), } if tr.TTL != utils.EmptyString { @@ -1771,12 +1771,13 @@ func APItoTrends(tr *utils.TPTrendsProfile) (sr *TrendProfile, err error) { } } copy(sr.ThresholdIDs, tr.ThresholdIDs) - for i, metric := range sr.Metrics { + /*for i, metric := range sr.Metrics { tr.Metrics[i] = utils.MetricWithSettings{ MetricID: metric.MetricID, TrendSwingMargin: metric.TrendSwingMargin, } } + */ return } @@ -1789,18 +1790,19 @@ func TrendProfileToAPI(tr *TrendProfile) (tpSR *utils.TPTrendsProfile) { ThresholdIDs: make([]string, len(tr.ThresholdIDs)), Metrics: make([]utils.MetricWithSettings, len(tr.Metrics)), QueueLength: tr.QueueLength, - TrendType: tr.TrendType, + // TrendType: tr.TrendType, } if tr.TTL != time.Duration(0) { tpSR.TTL = tr.TTL.String() } copy(tpSR.ThresholdIDs, tr.ThresholdIDs) - for i, metric := range tr.Metrics { + /*for i, metric := range tr.Metrics { tpSR.Metrics[i] = utils.MetricWithSettings{ MetricID: metric.MetricID, TrendSwingMargin: metric.TrendSwingMargin, } } + */ return } diff --git a/engine/trends.go b/engine/trends.go index 27ba2f46b..de3d6652e 100644 --- a/engine/trends.go +++ b/engine/trends.go @@ -95,29 +95,33 @@ func (tS *TrendS) computeTrend(tP *TrendProfile) { defer trend.Unlock() now := time.Now() - var metricWithSettings []*MetricWithSettings + var metrics []string if len(tP.Metrics) != 0 { - metricWithSettings = tP.Metrics // read only + metrics = tP.Metrics // read only } - if len(metricWithSettings) == 0 { // unlimited metrics in trend + if len(metrics) == 0 { // unlimited metrics in trend for mID := range floatMetrics { - metricWithSettings = append(metricWithSettings, &MetricWithSettings{MetricID: mID}) + metrics = append(metrics, mID) } } - if len(metricWithSettings) == 0 { + if len(metrics) == 0 { return // nothing to compute } trend.RunTimes = append(trend.RunTimes, now) trend.Metrics[now] = make(map[string]*MetricWithTrend) - for _, mWS := range metricWithSettings { - mWt := &MetricWithTrend{ID: mWS.MetricID} + for _, mID := range metrics { + mWt := &MetricWithTrend{ID: mID} var has bool - if mWt.Value, has = floatMetrics[mWS.MetricID]; !has { // no stats computed for metric + if mWt.Value, has = floatMetrics[mID]; !has { // no stats computed for metric mWt.Value = -1.0 - mWt.Trend = utils.NotAvailable + mWt.TrendLabel = utils.NotAvailable continue } - mWt.Trend = trend.getTrendLabel(mWt.ID, mWt.Value, mWS.TrendSwingMargin) + if mWt.TrendGrowth, err = trend.getTrendGrowth(mID, mWt.Value, tP.CorrelationType, tS.cgrcfg.GeneralCfg().RoundingDecimals); err != nil { + mWt.TrendLabel = utils.NotAvailable + } else { + mWt.TrendLabel = trend.getTrendLabel(mWt.TrendGrowth, tP.Tolerance) + } trend.Metrics[now][mWt.ID] = mWt } if err := tS.dm.SetTrend(trend); err != nil { diff --git a/utils/consts.go b/utils/consts.go index 29fbfd474..55295c506 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -319,6 +319,8 @@ const ( MetaConstant = "*constant" MetaPositive = "*positive" MetaNegative = "*negative" + MetaLast = "*last" + MetaFiller = "*filler" MetaHTTPPost = "*http_post" MetaHTTPjsonMap = "*http_json_map" diff --git a/utils/errors.go b/utils/errors.go index c04d99d14..ea3041683 100644 --- a/utils/errors.go +++ b/utils/errors.go @@ -82,6 +82,7 @@ var ( ErrNegative = errors.New("NEGATIVE") ErrCastFailed = errors.New("CAST_FAILED") ErrNoBackupFound = errors.New("NO_BACKUP_FOUND") + ErrCorrelationUndefined = errors.New("CORRELATION_UNDEFINED") ErrMap = map[string]error{ ErrNoMoreData.Error(): ErrNoMoreData,