From 24d21ba6671a7398cf8819eded3ace476e3fbdfa Mon Sep 17 00:00:00 2001 From: TeoV Date: Thu, 7 Mar 2019 15:30:55 +0200 Subject: [PATCH] Start changing for StatQueue ( add MetricFilterIDs ) --- engine/libstats.go | 15 ++-- engine/model_helpers.go | 173 +++++++++++++++---------------------- engine/models.go | 35 ++++---- engine/statmetrics.go | 148 +++++++++++++------------------ engine/statmetrics_test.go | 12 ++- utils/apitpdata.go | 27 ++++-- 6 files changed, 188 insertions(+), 222 deletions(-) diff --git a/engine/libstats.go b/engine/libstats.go index 65ff47b22..ecb57fb75 100644 --- a/engine/libstats.go +++ b/engine/libstats.go @@ -34,18 +34,23 @@ type StatQueueProfile struct { ActivationInterval *utils.ActivationInterval // Activation interval QueueLength int TTL time.Duration - Metrics []string // list of metrics to build - ThresholdIDs []string // list of thresholds to be checked after changes - Blocker bool // blocker flag to stop processing on filters matched - Stored bool - Weight float64 MinItems int + Metrics []*MetricsWithFilters // list of metrics to build + Stored bool + Blocker bool // blocker flag to stop processing on filters matched + Weight float64 + ThresholdIDs []string // list of thresholds to be checked after changes } func (sqp *StatQueueProfile) TenantID() string { return utils.ConcatenatedKey(sqp.Tenant, sqp.ID) } +type MetricsWithFilters struct { + FilterIDs []string + MetricIDs []string +} + // NewStoredStatQueue initiates a StoredStatQueue out of StatQueue func NewStoredStatQueue(sq *StatQueue, ms Marshaler) (sSQ *StoredStatQueue, err error) { sSQ = &StoredStatQueue{ diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 71f863c68..04c7659c1 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1307,64 +1307,47 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile type TpStatsS []*TpStats -//to be modify -func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) { +func (models TpStatsS) AsTPStats() (result []*utils.TPStatProfile) { filterMap := make(map[string]utils.StringMap) metricmap := make(map[string]utils.StringMap) thresholdMap := make(map[string]utils.StringMap) - mst := make(map[string]*utils.TPStats) - for _, tp := range tps { - key := &utils.TenantID{Tenant: tp.Tenant, ID: tp.ID} + mst := make(map[string]*utils.TPStatProfile) + for _, model := range models { + key := &utils.TenantID{Tenant: model.Tenant, ID: model.ID} st, found := mst[key.TenantID()] if !found { - st = &utils.TPStats{ - Tenant: tp.Tenant, - TPid: tp.Tpid, - ID: tp.ID, - Blocker: tp.Blocker, - Stored: tp.Stored, - MinItems: tp.MinItems, + st = &utils.TPStatProfile{ + Tenant: model.Tenant, + TPid: model.Tpid, + ID: model.ID, + Blocker: model.Blocker, + Stored: model.Stored, + MinItems: model.MinItems, + TTL: model.TTL, + Weight: model.Weight, } } - if tp.Blocker == false || tp.Blocker == true { - st.Blocker = tp.Blocker - } - if tp.Stored == false || tp.Stored == true { - st.Stored = tp.Stored - } - if tp.MinItems != 0 { - st.MinItems = tp.MinItems - } - if tp.QueueLength != 0 { - st.QueueLength = tp.QueueLength - } - if tp.TTL != "" { - st.TTL = tp.TTL - } - if tp.Metrics != "" { + if model.Metrics != "" { if _, has := metricmap[key.TenantID()]; !has { metricmap[key.TenantID()] = make(utils.StringMap) } - metricSplit := strings.Split(tp.Metrics, utils.INFIELD_SEP) + metricSplit := strings.Split(model.Metrics, utils.INFIELD_SEP) for _, metric := range metricSplit { metricmap[key.TenantID()][metric] = true } } - if tp.ThresholdIDs != "" { + if model.ThresholdIDs != "" { if _, has := thresholdMap[key.TenantID()]; !has { thresholdMap[key.TenantID()] = make(utils.StringMap) } - trshSplt := strings.Split(tp.ThresholdIDs, utils.INFIELD_SEP) + trshSplt := strings.Split(model.ThresholdIDs, utils.INFIELD_SEP) for _, trsh := range trshSplt { thresholdMap[key.TenantID()][trsh] = true } } - if tp.Weight != 0 { - st.Weight = tp.Weight - } - if len(tp.ActivationInterval) != 0 { + if len(model.ActivationInterval) != 0 { st.ActivationInterval = new(utils.TPActivationInterval) - aiSplt := strings.Split(tp.ActivationInterval, utils.INFIELD_SEP) + aiSplt := strings.Split(model.ActivationInterval, utils.INFIELD_SEP) if len(aiSplt) == 2 { st.ActivationInterval.ActivationTime = aiSplt[0] st.ActivationInterval.ExpiryTime = aiSplt[1] @@ -1372,18 +1355,18 @@ func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) { st.ActivationInterval.ActivationTime = aiSplt[0] } } - if tp.FilterIDs != "" { + if model.FilterIDs != "" { if _, has := filterMap[key.TenantID()]; !has { filterMap[key.TenantID()] = make(utils.StringMap) } - filterSplit := strings.Split(tp.FilterIDs, utils.INFIELD_SEP) + filterSplit := strings.Split(model.FilterIDs, utils.INFIELD_SEP) for _, filter := range filterSplit { filterMap[key.TenantID()][filter] = true } } mst[key.TenantID()] = st } - result = make([]*utils.TPStats, len(mst)) + result = make([]*utils.TPStatProfile, len(mst)) i := 0 for tntID, st := range mst { result[i] = st @@ -1401,68 +1384,20 @@ func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) { return } -func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) { - if st != nil { - // In case that TPStats don't have filter - if len(st.FilterIDs) == 0 { +func APItoModelStats(st *utils.TPStatProfile) (mdls TpStatsS) { + if st != nil && len(st.Metrics) != 0 { + for i, metric := range st.Metrics { mdl := &TpStats{ - Tenant: st.Tenant, - Tpid: st.TPid, - ID: st.ID, - MinItems: st.MinItems, - TTL: st.TTL, - Blocker: st.Blocker, - Stored: st.Stored, - Weight: st.Weight, - QueueLength: st.QueueLength, - } - for i, val := range st.Metrics { - if i != 0 { - mdl.Metrics += utils.INFIELD_SEP - } - mdl.Metrics += val - } - for i, val := range st.ThresholdIDs { - if i != 0 { - mdl.ThresholdIDs += utils.INFIELD_SEP - } - mdl.ThresholdIDs += val - } - if st.ActivationInterval != nil { - if st.ActivationInterval.ActivationTime != "" { - mdl.ActivationInterval = st.ActivationInterval.ActivationTime - } - if st.ActivationInterval.ExpiryTime != "" { - mdl.ActivationInterval += utils.INFIELD_SEP + st.ActivationInterval.ExpiryTime - } - } - mdls = append(mdls, mdl) - } - for i, fltr := range st.FilterIDs { - mdl := &TpStats{ - Tenant: st.Tenant, - Tpid: st.TPid, - ID: st.ID, - MinItems: st.MinItems, + Tpid: st.TPid, + Tenant: st.Tenant, + ID: st.ID, } if i == 0 { - mdl.TTL = st.TTL - mdl.Blocker = st.Blocker - mdl.Stored = st.Stored - mdl.Weight = st.Weight - mdl.QueueLength = st.QueueLength - mdl.MinItems = st.MinItems - for i, val := range st.Metrics { + for i, val := range st.FilterIDs { if i != 0 { - mdl.Metrics += utils.INFIELD_SEP + mdl.FilterIDs += utils.INFIELD_SEP } - mdl.Metrics += val - } - for i, val := range st.ThresholdIDs { - if i != 0 { - mdl.ThresholdIDs += utils.INFIELD_SEP - } - mdl.ThresholdIDs += val + mdl.FilterIDs += val } if st.ActivationInterval != nil { if st.ActivationInterval.ActivationTime != "" { @@ -1472,32 +1407,60 @@ func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) { mdl.ActivationInterval += utils.INFIELD_SEP + st.ActivationInterval.ExpiryTime } } + mdl.QueueLength = st.QueueLength + mdl.TTL = st.TTL + mdl.MinItems = st.MinItems + mdl.Stored = st.Stored + mdl.Blocker = st.Blocker + mdl.Weight = st.Weight + + for i, val := range st.ThresholdIDs { + if i != 0 { + mdl.ThresholdIDs += utils.INFIELD_SEP + } + mdl.ThresholdIDs += val + } + + } + for i, val := range metric.FilterIDs { + if i != 0 { + mdl.MetricFilterIDs += utils.INFIELD_SEP + } + mdl.MetricFilterIDs += val + } + for i, val := range metric.MetricIDs { + if i != 0 { + mdl.MetricIDs += utils.INFIELD_SEP + } + mdl.MetricIDs += val } - mdl.FilterIDs = fltr mdls = append(mdls, mdl) } } return } -func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err error) { +func APItoStats(tpST *utils.TPStatProfile, timezone string) (st *StatQueueProfile, err error) { st = &StatQueueProfile{ Tenant: tpST.Tenant, ID: tpST.ID, - QueueLength: tpST.QueueLength, - Metrics: tpST.Metrics, - Weight: tpST.Weight, - Blocker: tpST.Blocker, - Stored: tpST.Stored, - MinItems: tpST.MinItems, - ThresholdIDs: make([]string, len(tpST.ThresholdIDs)), FilterIDs: make([]string, len(tpST.FilterIDs)), + QueueLength: tpST.QueueLength, + MinItems: tpST.MinItems, + Metrics: make([]*MetricsWithFilters, tpST.Metrics), + Stored: tpST.Stored, + Blocker: tpST.Blocker, + Weight: tpST.Weight, + ThresholdIDs: make([]string, len(tpST.ThresholdIDs)), } if tpST.TTL != "" { if st.TTL, err = utils.ParseDurationWithNanosecs(tpST.TTL); err != nil { return nil, err } } + for i, metric := range tpST.Metrics { + st.Metrics[i] = metric + } for i, trh := range tpST.ThresholdIDs { st.ThresholdIDs[i] = trh } diff --git a/engine/models.go b/engine/models.go index 1dfd406ce..a27dbf151 100644 --- a/engine/models.go +++ b/engine/models.go @@ -238,12 +238,13 @@ type TpStats struct { ActivationInterval string `index:"3" re:""` QueueLength int `index:"4" re:""` TTL string `index:"5" re:""` - Metrics string `index:"6" re:""` - Blocker bool `index:"7" re:""` - Stored bool `index:"8" re:""` - Weight float64 `index:"9" re:"\d+\.?\d*"` - MinItems int `index:"10" re:""` - ThresholdIDs string `index:"11" re:""` + MinItems int `index:"6" re:""` + MetricFilterIDs string `index:"7" re:""` + MetricIDs string `index:"8" re:""` + Blocker bool `index:"9" re:""` + Stored bool `index:"10" re:""` + Weight float64 `index:"11" re:"\d+\.?\d*"` + ThresholdIDs string `index:"12" re:""` CreatedAt time.Time } @@ -359,17 +360,17 @@ type TpSupplier struct { type TPAttribute struct { PK uint `gorm:"primary_key"` Tpid string - Tenant string `index:"0" re:""` - ID string `index:"1" re:""` - Contexts string `index:"2" re:""` - FilterIDs string `index:"3" re:""` - ActivationInterval string `index:"4" re:""` - AttributeFilterIDs string `index:"5" re:""` - FieldName string `index:"6" re:""` - Substitute string `index:"7" re:""` - Blocker bool `index:"8" re:""` - Weight float64 `index:"9" re:"\d+\.?\d*"` - CreatedAt time.Time + Tenant string `index:"0" re:""` + ID string `index:"1" re:""` + Contexts string `index:"2" re:""` + FilterIDs string `index:"3" re:""` + ActivationInterval string `index:"4" re:""` + AttributeFilterIDs string `index:"5" re:""` + + Substitute string `index:"7" re:""` + Blocker bool `index:"8" re:""` + Weight float64 `index:"9" re:"\d+\.?\d*"` + CreatedAt time.Time } type TPCharger struct { diff --git a/engine/statmetrics.go b/engine/statmetrics.go index d0feade50..7e8ecf74d 100644 --- a/engine/statmetrics.go +++ b/engine/statmetrics.go @@ -117,10 +117,13 @@ func (asr *StatASR) GetFloat64Value() (val float64) { // AddEvent is part of StatMetric interface func (asr *StatASR) AddEvent(ev *utils.CGREvent) (err error) { var answered bool - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil && - err != utils.ErrNotFound { - return err + var at time.Time + if at, err = ev.FieldAsTime(utils.AnswerTime, + config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.AnswerTime) + } + return } else if !at.IsZero() { answered = true } @@ -205,20 +208,15 @@ func (acd *StatACD) GetFloat64Value() (v float64) { } func (acd *StatACD) AddEvent(ev *utils.CGREvent) (err error) { - var value time.Duration - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil { - return err - } else if !at.IsZero() { - if duration, err := ev.FieldAsDuration(utils.Usage); err != nil && - err != utils.ErrNotFound { - return err - } else { - value = duration - acd.Sum += duration + var dur time.Duration + if dur, err = ev.FieldAsDuration(utils.Usage); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.Usage) } + return } - acd.Events[ev.ID] = value + acd.Sum += dur + acd.Events[ev.ID] = dur acd.Count += 1 acd.val = nil return @@ -293,21 +291,15 @@ func (tcd *StatTCD) GetFloat64Value() (v float64) { } func (tcd *StatTCD) AddEvent(ev *utils.CGREvent) (err error) { - var value time.Duration - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil { - return err - } else if !at.IsZero() { - if duration, err := ev.FieldAsDuration(utils.Usage); err != nil && - err != utils.ErrNotFound { - return err - } else { - value = duration - tcd.Sum += duration + var dur time.Duration + if dur, err = ev.FieldAsDuration(utils.Usage); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.Usage) } - + return } - tcd.Events[ev.ID] = value + acd.Sum += dur + tcd.Events[ev.ID] = dur tcd.Count += 1 tcd.val = nil return @@ -380,20 +372,15 @@ func (acc *StatACC) GetFloat64Value() (v float64) { } func (acc *StatACC) AddEvent(ev *utils.CGREvent) (err error) { - var value float64 - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil { - return err - } else if !at.IsZero() { - if cost, err := ev.FieldAsFloat64(utils.COST); err != nil && - err != utils.ErrNotFound { - return err - } else if cost >= 0 { - value = cost - acc.Sum += cost + var cost float64 + if cost, err = ev.FieldAsFloat64(utils.COST); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.COST) } + return } - acc.Events[ev.ID] = value + acc.Sum += cost + acc.Events[ev.ID] = cost acc.Count += 1 acc.val = nil return @@ -404,9 +391,7 @@ func (acc *StatACC) RemEvent(evID string) (err error) { if !has { return utils.ErrNotFound } - if cost >= 0 { - acc.Sum -= cost - } + acc.Sum -= cost acc.Count -= 1 delete(acc.Events, evID) acc.val = nil @@ -466,20 +451,15 @@ func (tcc *StatTCC) GetFloat64Value() (v float64) { } func (tcc *StatTCC) AddEvent(ev *utils.CGREvent) (err error) { - var value float64 - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil { - return err - } else if !at.IsZero() { - if cost, err := ev.FieldAsFloat64(utils.COST); err != nil && - err != utils.ErrNotFound { - return err - } else if cost >= 0 { - value = cost - tcc.Sum += cost + var cost float64 + if cost, err = ev.FieldAsFloat64(utils.COST); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.COST) } + return } - tcc.Events[ev.ID] = value + acc.Sum += cost + tcc.Events[ev.ID] = cost tcc.Count += 1 tcc.val = nil return @@ -555,21 +535,15 @@ func (pdd *StatPDD) GetFloat64Value() (v float64) { } func (pdd *StatPDD) AddEvent(ev *utils.CGREvent) (err error) { - var value time.Duration - if at, err := ev.FieldAsTime(utils.AnswerTime, - config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil && - err != utils.ErrNotFound { - return err - } else if !at.IsZero() { - if duration, err := ev.FieldAsDuration(utils.PDD); err != nil && - err != utils.ErrNotFound { - return err - } else { - value = duration - pdd.Sum += duration + var dur time.Duration + if dur, err = ev.FieldAsDuration(utils.PDD); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, utils.PDD) } + return } - pdd.Events[ev.ID] = value + pdd.Sum += dur + pdd.Events[ev.ID] = dur pdd.Count += 1 pdd.val = nil return @@ -707,15 +681,15 @@ func (sum *StatSum) GetFloat64Value() (v float64) { } func (sum *StatSum) AddEvent(ev *utils.CGREvent) (err error) { - var value float64 - if val, err := ev.FieldAsFloat64(sum.FieldName); err != nil && - err != utils.ErrNotFound { - return err - } else if val >= 0 { - value = val - sum.Sum += val + var val float64 + if val, err = ev.FieldAsFloat64(sum.FieldName); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, sum.FieldName) + } + return } - sum.Events[ev.ID] = value + sum.Sum += val + sum.Events[ev.ID] = val sum.val = nil return } @@ -787,17 +761,17 @@ func (avg *StatAverage) GetFloat64Value() (v float64) { } func (avg *StatAverage) AddEvent(ev *utils.CGREvent) (err error) { - var value float64 - if val, err := ev.FieldAsFloat64(avg.FieldName); err != nil && - err != utils.ErrNotFound { - return err - } else if val > 0 { - value = val - avg.Sum += val - avg.Events[ev.ID] = value - avg.Count += 1 - avg.val = nil + var val float64 + if val, err = ev.FieldAsFloat64(avg.FieldName); err != nil { + if err == utils.ErrNotFound { + err = utils.ErrPrefix(err, sum.FieldName) + } + return } + avg.Sum += val + avg.Events[ev.ID] = val + avg.Count += 1 + avg.val = nil return } diff --git a/engine/statmetrics_test.go b/engine/statmetrics_test.go index 94cd92002..c3f6800f9 100644 --- a/engine/statmetrics_test.go +++ b/engine/statmetrics_test.go @@ -130,14 +130,20 @@ func TestACDGetStringValue(t *testing.T) { if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE { t.Errorf("wrong acd value: %s", strVal) } - acd.AddEvent(ev) + if err := acd.AddEvent(ev); err != nil { + t.Error(err) + } if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE { t.Errorf("wrong acd value: %s", strVal) } ev2 := &utils.CGREvent{Tenant: "cgrates.org", ID: "EVENT_2"} ev3 := &utils.CGREvent{Tenant: "cgrates.org", ID: "EVENT_3"} - acd.AddEvent(ev2) - acd.AddEvent(ev3) + if err := acd.AddEvent(ev2); err != nil { + t.Error(err) + } + if err := acd.AddEvent(ev3); err != nil { + t.Error(err) + } if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE { t.Errorf("wrong acd value: %s", strVal) } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index bcac2622c..f5255c0cd 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -984,7 +984,8 @@ type AttrSetBalance struct { Cdrlog *bool } -type TPResource struct { +// TPResourceProfile is used in APIs to manage remotely offline ResourceProfile +type TPResourceProfile struct { TPid string Tenant string ID string // Identifier of this limit @@ -1060,8 +1061,14 @@ type AttrDisconnectSession struct { Reason string } -// TPStats is used in APIs to manage remotely offline Stats config -type TPStats struct { +//MetricsWithFilters is used in TPStatProfile +type MetricsWithFilters struct { + FilterIDs []string + MetricIDs []string +} + +// TPStatProfile is used in APIs to manage remotely offline StatProfile +type TPStatProfile struct { TPid string Tenant string ID string @@ -1069,7 +1076,7 @@ type TPStats struct { ActivationInterval *TPActivationInterval QueueLength int TTL string - Metrics []string + Metrics []*MetricsWithFilters Blocker bool // blocker flag to stop processing on filters matched Stored bool Weight float64 @@ -1077,7 +1084,8 @@ type TPStats struct { ThresholdIDs []string } -type TPThreshold struct { +// TPThresholdProfile is used in APIs to manage remotely offline ThresholdProfile +type TPThresholdProfile struct { TPid string Tenant string ID string @@ -1092,6 +1100,7 @@ type TPThreshold struct { Async bool } +// TPFilterProfile is used in APIs to manage remotely offline FilterProfile type TPFilterProfile struct { TPid string Tenant string @@ -1100,12 +1109,14 @@ type TPFilterProfile struct { ActivationInterval *TPActivationInterval // Time when this limit becomes active and expires } +// TPFilterProfile is used in TPFilterProfile type TPFilter struct { Type string // Filter type (*string, *timing, *rsr_filters, *cdr_stats) FieldName string // Name of the field providing us the Values to check (used in case of some ) Values []string // Filter definition } +// TPSupplier is used in TPSupplierProfile type TPSupplier struct { ID string // SupplierID FilterIDs []string @@ -1118,6 +1129,7 @@ type TPSupplier struct { SupplierParameters string } +// TPSupplierProfile is used in APIs to manage remotely offline SupplierProfile type TPSupplierProfile struct { TPid string Tenant string @@ -1130,12 +1142,14 @@ type TPSupplierProfile struct { Weight float64 } +// TPAttribute is used in TPAttributeProfile type TPAttribute struct { FilterIDs []string FieldName string Substitute string } +// TPAttributeProfile is used in APIs to manage remotely offline AttributeProfile type TPAttributeProfile struct { TPid string Tenant string @@ -1148,6 +1162,7 @@ type TPAttributeProfile struct { Weight float64 } +// TPChargerProfile is used in APIs to manage remotely offline ChargerProfile type TPChargerProfile struct { TPid string Tenant string @@ -1165,6 +1180,7 @@ type TPTntID struct { ID string } +// TPDispatcherConns is used in TPDispatcherProfile type TPDispatcherConns struct { ID string FilterIDs []string @@ -1173,6 +1189,7 @@ type TPDispatcherConns struct { Blocker bool // no connection after this one } +// TPDispatcherProfile is used in APIs to manage remotely offline DispatcherProfile type TPDispatcherProfile struct { TPid string Tenant string