Start changing for StatQueue ( add MetricFilterIDs )

This commit is contained in:
TeoV
2019-03-07 15:30:55 +02:00
committed by Dan Christian Bogos
parent 85998bf4a9
commit 24d21ba667
6 changed files with 188 additions and 222 deletions

View File

@@ -34,18 +34,23 @@ type StatQueueProfile struct {
ActivationInterval *utils.ActivationInterval // Activation interval
QueueLength int
TTL time.Duration
Metrics []string // list of metrics to build
ThresholdIDs []string // list of thresholds to be checked after changes
Blocker bool // blocker flag to stop processing on filters matched
Stored bool
Weight float64
MinItems int
Metrics []*MetricsWithFilters // list of metrics to build
Stored bool
Blocker bool // blocker flag to stop processing on filters matched
Weight float64
ThresholdIDs []string // list of thresholds to be checked after changes
}
func (sqp *StatQueueProfile) TenantID() string {
return utils.ConcatenatedKey(sqp.Tenant, sqp.ID)
}
type MetricsWithFilters struct {
FilterIDs []string
MetricIDs []string
}
// NewStoredStatQueue initiates a StoredStatQueue out of StatQueue
func NewStoredStatQueue(sq *StatQueue, ms Marshaler) (sSQ *StoredStatQueue, err error) {
sSQ = &StoredStatQueue{

View File

@@ -1307,64 +1307,47 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile
type TpStatsS []*TpStats
//to be modify
func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) {
func (models TpStatsS) AsTPStats() (result []*utils.TPStatProfile) {
filterMap := make(map[string]utils.StringMap)
metricmap := make(map[string]utils.StringMap)
thresholdMap := make(map[string]utils.StringMap)
mst := make(map[string]*utils.TPStats)
for _, tp := range tps {
key := &utils.TenantID{Tenant: tp.Tenant, ID: tp.ID}
mst := make(map[string]*utils.TPStatProfile)
for _, model := range models {
key := &utils.TenantID{Tenant: model.Tenant, ID: model.ID}
st, found := mst[key.TenantID()]
if !found {
st = &utils.TPStats{
Tenant: tp.Tenant,
TPid: tp.Tpid,
ID: tp.ID,
Blocker: tp.Blocker,
Stored: tp.Stored,
MinItems: tp.MinItems,
st = &utils.TPStatProfile{
Tenant: model.Tenant,
TPid: model.Tpid,
ID: model.ID,
Blocker: model.Blocker,
Stored: model.Stored,
MinItems: model.MinItems,
TTL: model.TTL,
Weight: model.Weight,
}
}
if tp.Blocker == false || tp.Blocker == true {
st.Blocker = tp.Blocker
}
if tp.Stored == false || tp.Stored == true {
st.Stored = tp.Stored
}
if tp.MinItems != 0 {
st.MinItems = tp.MinItems
}
if tp.QueueLength != 0 {
st.QueueLength = tp.QueueLength
}
if tp.TTL != "" {
st.TTL = tp.TTL
}
if tp.Metrics != "" {
if model.Metrics != "" {
if _, has := metricmap[key.TenantID()]; !has {
metricmap[key.TenantID()] = make(utils.StringMap)
}
metricSplit := strings.Split(tp.Metrics, utils.INFIELD_SEP)
metricSplit := strings.Split(model.Metrics, utils.INFIELD_SEP)
for _, metric := range metricSplit {
metricmap[key.TenantID()][metric] = true
}
}
if tp.ThresholdIDs != "" {
if model.ThresholdIDs != "" {
if _, has := thresholdMap[key.TenantID()]; !has {
thresholdMap[key.TenantID()] = make(utils.StringMap)
}
trshSplt := strings.Split(tp.ThresholdIDs, utils.INFIELD_SEP)
trshSplt := strings.Split(model.ThresholdIDs, utils.INFIELD_SEP)
for _, trsh := range trshSplt {
thresholdMap[key.TenantID()][trsh] = true
}
}
if tp.Weight != 0 {
st.Weight = tp.Weight
}
if len(tp.ActivationInterval) != 0 {
if len(model.ActivationInterval) != 0 {
st.ActivationInterval = new(utils.TPActivationInterval)
aiSplt := strings.Split(tp.ActivationInterval, utils.INFIELD_SEP)
aiSplt := strings.Split(model.ActivationInterval, utils.INFIELD_SEP)
if len(aiSplt) == 2 {
st.ActivationInterval.ActivationTime = aiSplt[0]
st.ActivationInterval.ExpiryTime = aiSplt[1]
@@ -1372,18 +1355,18 @@ func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) {
st.ActivationInterval.ActivationTime = aiSplt[0]
}
}
if tp.FilterIDs != "" {
if model.FilterIDs != "" {
if _, has := filterMap[key.TenantID()]; !has {
filterMap[key.TenantID()] = make(utils.StringMap)
}
filterSplit := strings.Split(tp.FilterIDs, utils.INFIELD_SEP)
filterSplit := strings.Split(model.FilterIDs, utils.INFIELD_SEP)
for _, filter := range filterSplit {
filterMap[key.TenantID()][filter] = true
}
}
mst[key.TenantID()] = st
}
result = make([]*utils.TPStats, len(mst))
result = make([]*utils.TPStatProfile, len(mst))
i := 0
for tntID, st := range mst {
result[i] = st
@@ -1401,68 +1384,20 @@ func (tps TpStatsS) AsTPStats() (result []*utils.TPStats) {
return
}
func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) {
if st != nil {
// In case that TPStats don't have filter
if len(st.FilterIDs) == 0 {
func APItoModelStats(st *utils.TPStatProfile) (mdls TpStatsS) {
if st != nil && len(st.Metrics) != 0 {
for i, metric := range st.Metrics {
mdl := &TpStats{
Tenant: st.Tenant,
Tpid: st.TPid,
ID: st.ID,
MinItems: st.MinItems,
TTL: st.TTL,
Blocker: st.Blocker,
Stored: st.Stored,
Weight: st.Weight,
QueueLength: st.QueueLength,
}
for i, val := range st.Metrics {
if i != 0 {
mdl.Metrics += utils.INFIELD_SEP
}
mdl.Metrics += val
}
for i, val := range st.ThresholdIDs {
if i != 0 {
mdl.ThresholdIDs += utils.INFIELD_SEP
}
mdl.ThresholdIDs += val
}
if st.ActivationInterval != nil {
if st.ActivationInterval.ActivationTime != "" {
mdl.ActivationInterval = st.ActivationInterval.ActivationTime
}
if st.ActivationInterval.ExpiryTime != "" {
mdl.ActivationInterval += utils.INFIELD_SEP + st.ActivationInterval.ExpiryTime
}
}
mdls = append(mdls, mdl)
}
for i, fltr := range st.FilterIDs {
mdl := &TpStats{
Tenant: st.Tenant,
Tpid: st.TPid,
ID: st.ID,
MinItems: st.MinItems,
Tpid: st.TPid,
Tenant: st.Tenant,
ID: st.ID,
}
if i == 0 {
mdl.TTL = st.TTL
mdl.Blocker = st.Blocker
mdl.Stored = st.Stored
mdl.Weight = st.Weight
mdl.QueueLength = st.QueueLength
mdl.MinItems = st.MinItems
for i, val := range st.Metrics {
for i, val := range st.FilterIDs {
if i != 0 {
mdl.Metrics += utils.INFIELD_SEP
mdl.FilterIDs += utils.INFIELD_SEP
}
mdl.Metrics += val
}
for i, val := range st.ThresholdIDs {
if i != 0 {
mdl.ThresholdIDs += utils.INFIELD_SEP
}
mdl.ThresholdIDs += val
mdl.FilterIDs += val
}
if st.ActivationInterval != nil {
if st.ActivationInterval.ActivationTime != "" {
@@ -1472,32 +1407,60 @@ func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) {
mdl.ActivationInterval += utils.INFIELD_SEP + st.ActivationInterval.ExpiryTime
}
}
mdl.QueueLength = st.QueueLength
mdl.TTL = st.TTL
mdl.MinItems = st.MinItems
mdl.Stored = st.Stored
mdl.Blocker = st.Blocker
mdl.Weight = st.Weight
for i, val := range st.ThresholdIDs {
if i != 0 {
mdl.ThresholdIDs += utils.INFIELD_SEP
}
mdl.ThresholdIDs += val
}
}
for i, val := range metric.FilterIDs {
if i != 0 {
mdl.MetricFilterIDs += utils.INFIELD_SEP
}
mdl.MetricFilterIDs += val
}
for i, val := range metric.MetricIDs {
if i != 0 {
mdl.MetricIDs += utils.INFIELD_SEP
}
mdl.MetricIDs += val
}
mdl.FilterIDs = fltr
mdls = append(mdls, mdl)
}
}
return
}
func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err error) {
func APItoStats(tpST *utils.TPStatProfile, timezone string) (st *StatQueueProfile, err error) {
st = &StatQueueProfile{
Tenant: tpST.Tenant,
ID: tpST.ID,
QueueLength: tpST.QueueLength,
Metrics: tpST.Metrics,
Weight: tpST.Weight,
Blocker: tpST.Blocker,
Stored: tpST.Stored,
MinItems: tpST.MinItems,
ThresholdIDs: make([]string, len(tpST.ThresholdIDs)),
FilterIDs: make([]string, len(tpST.FilterIDs)),
QueueLength: tpST.QueueLength,
MinItems: tpST.MinItems,
Metrics: make([]*MetricsWithFilters, tpST.Metrics),
Stored: tpST.Stored,
Blocker: tpST.Blocker,
Weight: tpST.Weight,
ThresholdIDs: make([]string, len(tpST.ThresholdIDs)),
}
if tpST.TTL != "" {
if st.TTL, err = utils.ParseDurationWithNanosecs(tpST.TTL); err != nil {
return nil, err
}
}
for i, metric := range tpST.Metrics {
st.Metrics[i] = metric
}
for i, trh := range tpST.ThresholdIDs {
st.ThresholdIDs[i] = trh
}

View File

@@ -238,12 +238,13 @@ type TpStats struct {
ActivationInterval string `index:"3" re:""`
QueueLength int `index:"4" re:""`
TTL string `index:"5" re:""`
Metrics string `index:"6" re:""`
Blocker bool `index:"7" re:""`
Stored bool `index:"8" re:""`
Weight float64 `index:"9" re:"\d+\.?\d*"`
MinItems int `index:"10" re:""`
ThresholdIDs string `index:"11" re:""`
MinItems int `index:"6" re:""`
MetricFilterIDs string `index:"7" re:""`
MetricIDs string `index:"8" re:""`
Blocker bool `index:"9" re:""`
Stored bool `index:"10" re:""`
Weight float64 `index:"11" re:"\d+\.?\d*"`
ThresholdIDs string `index:"12" re:""`
CreatedAt time.Time
}
@@ -359,17 +360,17 @@ type TpSupplier struct {
type TPAttribute struct {
PK uint `gorm:"primary_key"`
Tpid string
Tenant string `index:"0" re:""`
ID string `index:"1" re:""`
Contexts string `index:"2" re:""`
FilterIDs string `index:"3" re:""`
ActivationInterval string `index:"4" re:""`
AttributeFilterIDs string `index:"5" re:""`
FieldName string `index:"6" re:""`
Substitute string `index:"7" re:""`
Blocker bool `index:"8" re:""`
Weight float64 `index:"9" re:"\d+\.?\d*"`
CreatedAt time.Time
Tenant string `index:"0" re:""`
ID string `index:"1" re:""`
Contexts string `index:"2" re:""`
FilterIDs string `index:"3" re:""`
ActivationInterval string `index:"4" re:""`
AttributeFilterIDs string `index:"5" re:""`
Substitute string `index:"7" re:""`
Blocker bool `index:"8" re:""`
Weight float64 `index:"9" re:"\d+\.?\d*"`
CreatedAt time.Time
}
type TPCharger struct {

View File

@@ -117,10 +117,13 @@ func (asr *StatASR) GetFloat64Value() (val float64) {
// AddEvent is part of StatMetric interface
func (asr *StatASR) AddEvent(ev *utils.CGREvent) (err error) {
var answered bool
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil &&
err != utils.ErrNotFound {
return err
var at time.Time
if at, err = ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.AnswerTime)
}
return
} else if !at.IsZero() {
answered = true
}
@@ -205,20 +208,15 @@ func (acd *StatACD) GetFloat64Value() (v float64) {
}
func (acd *StatACD) AddEvent(ev *utils.CGREvent) (err error) {
var value time.Duration
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil {
return err
} else if !at.IsZero() {
if duration, err := ev.FieldAsDuration(utils.Usage); err != nil &&
err != utils.ErrNotFound {
return err
} else {
value = duration
acd.Sum += duration
var dur time.Duration
if dur, err = ev.FieldAsDuration(utils.Usage); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.Usage)
}
return
}
acd.Events[ev.ID] = value
acd.Sum += dur
acd.Events[ev.ID] = dur
acd.Count += 1
acd.val = nil
return
@@ -293,21 +291,15 @@ func (tcd *StatTCD) GetFloat64Value() (v float64) {
}
func (tcd *StatTCD) AddEvent(ev *utils.CGREvent) (err error) {
var value time.Duration
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil {
return err
} else if !at.IsZero() {
if duration, err := ev.FieldAsDuration(utils.Usage); err != nil &&
err != utils.ErrNotFound {
return err
} else {
value = duration
tcd.Sum += duration
var dur time.Duration
if dur, err = ev.FieldAsDuration(utils.Usage); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.Usage)
}
return
}
tcd.Events[ev.ID] = value
acd.Sum += dur
tcd.Events[ev.ID] = dur
tcd.Count += 1
tcd.val = nil
return
@@ -380,20 +372,15 @@ func (acc *StatACC) GetFloat64Value() (v float64) {
}
func (acc *StatACC) AddEvent(ev *utils.CGREvent) (err error) {
var value float64
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil {
return err
} else if !at.IsZero() {
if cost, err := ev.FieldAsFloat64(utils.COST); err != nil &&
err != utils.ErrNotFound {
return err
} else if cost >= 0 {
value = cost
acc.Sum += cost
var cost float64
if cost, err = ev.FieldAsFloat64(utils.COST); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.COST)
}
return
}
acc.Events[ev.ID] = value
acc.Sum += cost
acc.Events[ev.ID] = cost
acc.Count += 1
acc.val = nil
return
@@ -404,9 +391,7 @@ func (acc *StatACC) RemEvent(evID string) (err error) {
if !has {
return utils.ErrNotFound
}
if cost >= 0 {
acc.Sum -= cost
}
acc.Sum -= cost
acc.Count -= 1
delete(acc.Events, evID)
acc.val = nil
@@ -466,20 +451,15 @@ func (tcc *StatTCC) GetFloat64Value() (v float64) {
}
func (tcc *StatTCC) AddEvent(ev *utils.CGREvent) (err error) {
var value float64
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil {
return err
} else if !at.IsZero() {
if cost, err := ev.FieldAsFloat64(utils.COST); err != nil &&
err != utils.ErrNotFound {
return err
} else if cost >= 0 {
value = cost
tcc.Sum += cost
var cost float64
if cost, err = ev.FieldAsFloat64(utils.COST); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.COST)
}
return
}
tcc.Events[ev.ID] = value
acc.Sum += cost
tcc.Events[ev.ID] = cost
tcc.Count += 1
tcc.val = nil
return
@@ -555,21 +535,15 @@ func (pdd *StatPDD) GetFloat64Value() (v float64) {
}
func (pdd *StatPDD) AddEvent(ev *utils.CGREvent) (err error) {
var value time.Duration
if at, err := ev.FieldAsTime(utils.AnswerTime,
config.CgrConfig().GeneralCfg().DefaultTimezone); err != nil &&
err != utils.ErrNotFound {
return err
} else if !at.IsZero() {
if duration, err := ev.FieldAsDuration(utils.PDD); err != nil &&
err != utils.ErrNotFound {
return err
} else {
value = duration
pdd.Sum += duration
var dur time.Duration
if dur, err = ev.FieldAsDuration(utils.PDD); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, utils.PDD)
}
return
}
pdd.Events[ev.ID] = value
pdd.Sum += dur
pdd.Events[ev.ID] = dur
pdd.Count += 1
pdd.val = nil
return
@@ -707,15 +681,15 @@ func (sum *StatSum) GetFloat64Value() (v float64) {
}
func (sum *StatSum) AddEvent(ev *utils.CGREvent) (err error) {
var value float64
if val, err := ev.FieldAsFloat64(sum.FieldName); err != nil &&
err != utils.ErrNotFound {
return err
} else if val >= 0 {
value = val
sum.Sum += val
var val float64
if val, err = ev.FieldAsFloat64(sum.FieldName); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, sum.FieldName)
}
return
}
sum.Events[ev.ID] = value
sum.Sum += val
sum.Events[ev.ID] = val
sum.val = nil
return
}
@@ -787,17 +761,17 @@ func (avg *StatAverage) GetFloat64Value() (v float64) {
}
func (avg *StatAverage) AddEvent(ev *utils.CGREvent) (err error) {
var value float64
if val, err := ev.FieldAsFloat64(avg.FieldName); err != nil &&
err != utils.ErrNotFound {
return err
} else if val > 0 {
value = val
avg.Sum += val
avg.Events[ev.ID] = value
avg.Count += 1
avg.val = nil
var val float64
if val, err = ev.FieldAsFloat64(avg.FieldName); err != nil {
if err == utils.ErrNotFound {
err = utils.ErrPrefix(err, sum.FieldName)
}
return
}
avg.Sum += val
avg.Events[ev.ID] = val
avg.Count += 1
avg.val = nil
return
}

View File

@@ -130,14 +130,20 @@ func TestACDGetStringValue(t *testing.T) {
if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE {
t.Errorf("wrong acd value: %s", strVal)
}
acd.AddEvent(ev)
if err := acd.AddEvent(ev); err != nil {
t.Error(err)
}
if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE {
t.Errorf("wrong acd value: %s", strVal)
}
ev2 := &utils.CGREvent{Tenant: "cgrates.org", ID: "EVENT_2"}
ev3 := &utils.CGREvent{Tenant: "cgrates.org", ID: "EVENT_3"}
acd.AddEvent(ev2)
acd.AddEvent(ev3)
if err := acd.AddEvent(ev2); err != nil {
t.Error(err)
}
if err := acd.AddEvent(ev3); err != nil {
t.Error(err)
}
if strVal := acd.GetStringValue(""); strVal != utils.NOT_AVAILABLE {
t.Errorf("wrong acd value: %s", strVal)
}

View File

@@ -984,7 +984,8 @@ type AttrSetBalance struct {
Cdrlog *bool
}
type TPResource struct {
// TPResourceProfile is used in APIs to manage remotely offline ResourceProfile
type TPResourceProfile struct {
TPid string
Tenant string
ID string // Identifier of this limit
@@ -1060,8 +1061,14 @@ type AttrDisconnectSession struct {
Reason string
}
// TPStats is used in APIs to manage remotely offline Stats config
type TPStats struct {
//MetricsWithFilters is used in TPStatProfile
type MetricsWithFilters struct {
FilterIDs []string
MetricIDs []string
}
// TPStatProfile is used in APIs to manage remotely offline StatProfile
type TPStatProfile struct {
TPid string
Tenant string
ID string
@@ -1069,7 +1076,7 @@ type TPStats struct {
ActivationInterval *TPActivationInterval
QueueLength int
TTL string
Metrics []string
Metrics []*MetricsWithFilters
Blocker bool // blocker flag to stop processing on filters matched
Stored bool
Weight float64
@@ -1077,7 +1084,8 @@ type TPStats struct {
ThresholdIDs []string
}
type TPThreshold struct {
// TPThresholdProfile is used in APIs to manage remotely offline ThresholdProfile
type TPThresholdProfile struct {
TPid string
Tenant string
ID string
@@ -1092,6 +1100,7 @@ type TPThreshold struct {
Async bool
}
// TPFilterProfile is used in APIs to manage remotely offline FilterProfile
type TPFilterProfile struct {
TPid string
Tenant string
@@ -1100,12 +1109,14 @@ type TPFilterProfile struct {
ActivationInterval *TPActivationInterval // Time when this limit becomes active and expires
}
// TPFilterProfile is used in TPFilterProfile
type TPFilter struct {
Type string // Filter type (*string, *timing, *rsr_filters, *cdr_stats)
FieldName string // Name of the field providing us the Values to check (used in case of some )
Values []string // Filter definition
}
// TPSupplier is used in TPSupplierProfile
type TPSupplier struct {
ID string // SupplierID
FilterIDs []string
@@ -1118,6 +1129,7 @@ type TPSupplier struct {
SupplierParameters string
}
// TPSupplierProfile is used in APIs to manage remotely offline SupplierProfile
type TPSupplierProfile struct {
TPid string
Tenant string
@@ -1130,12 +1142,14 @@ type TPSupplierProfile struct {
Weight float64
}
// TPAttribute is used in TPAttributeProfile
type TPAttribute struct {
FilterIDs []string
FieldName string
Substitute string
}
// TPAttributeProfile is used in APIs to manage remotely offline AttributeProfile
type TPAttributeProfile struct {
TPid string
Tenant string
@@ -1148,6 +1162,7 @@ type TPAttributeProfile struct {
Weight float64
}
// TPChargerProfile is used in APIs to manage remotely offline ChargerProfile
type TPChargerProfile struct {
TPid string
Tenant string
@@ -1165,6 +1180,7 @@ type TPTntID struct {
ID string
}
// TPDispatcherConns is used in TPDispatcherProfile
type TPDispatcherConns struct {
ID string
FilterIDs []string
@@ -1173,6 +1189,7 @@ type TPDispatcherConns struct {
Blocker bool // no connection after this one
}
// TPDispatcherProfile is used in APIs to manage remotely offline DispatcherProfile
type TPDispatcherProfile struct {
TPid string
Tenant string