Refactoring filters

This commit is contained in:
TeoV
2017-10-12 11:09:02 +03:00
committed by Dan Christian Bogos
parent 68ca42c4c0
commit 12262fb11f
12 changed files with 158 additions and 125 deletions

View File

@@ -127,11 +127,15 @@ func testFilterGetFilterBeforeSet(t *testing.T) {
func testFilterSetFilter(t *testing.T) {
filter = &engine.Filter{
Tenant: "cgrates.org",
ID: "Filter1",
Type: "*string_prefix",
FieldName: "Account",
Values: []string{"10", "20"},
Tenant: "cgrates.org",
ID: "Filter1",
Filters: []*engine.RequestFilter{
&engine.RequestFilter{
FieldName: "*string",
Type: "Account",
Values: []string{"1001", "1002"},
},
},
}
var result string
@@ -152,12 +156,17 @@ func testFilterGetFilterAfterSet(t *testing.T) {
}
func testFilterUpdateFilter(t *testing.T) {
filter = &engine.Filter{
Tenant: "cgrates.org",
ID: "Filter1",
Type: "*string_prefix",
FieldName: "Destination",
Values: []string{"1001", "1002"},
filter.Filters = []*engine.RequestFilter{
&engine.RequestFilter{
FieldName: "*string",
Type: "Account",
Values: []string{"1001", "1002"},
},
&engine.RequestFilter{
FieldName: "*string_prefix",
Type: "Destination",
Values: []string{"10", "20"},
},
}
var result string
if err := filterRPC.Call("ApierV1.SetFilter", filter, &result); err != nil {

View File

@@ -507,8 +507,8 @@ func testV1RsSetResourceProfile(t *testing.T) {
rlsConfig = &engine.ResourceProfile{
Tenant: "cgrates.org",
ID: "RCFG1",
Filters: []*engine.Filter{
&engine.Filter{
Filters: []*engine.RequestFilter{
&engine.RequestFilter{
Type: "type",
FieldName: "Name",
Values: []string{"FilterValue1", "FilterValue2"},
@@ -546,13 +546,13 @@ func testV1RsGetResourceProfileAfterSet(t *testing.T) {
func testV1RsUpdateResourceProfile(t *testing.T) {
var result string
rlsConfig.Filters = []*engine.Filter{
&engine.Filter{
rlsConfig.Filters = []*engine.RequestFilter{
&engine.RequestFilter{
Type: "type",
FieldName: "Name",
Values: []string{"FilterValue1", "FilterValue2"},
},
&engine.Filter{
&engine.RequestFilter{
Type: "*string",
FieldName: "Accout",
Values: []string{"1001", "1002"},

View File

@@ -277,8 +277,8 @@ func testV1STSSetStatQueueProfile(t *testing.T) {
statConfig = &engine.StatQueueProfile{
Tenant: "cgrates.org",
ID: "TEST_PROFILE1",
Filters: []*engine.Filter{
&engine.Filter{
Filters: []*engine.RequestFilter{
&engine.RequestFilter{
Type: "type",
FieldName: "Name",
Values: []string{"FilterValue1", "FilterValue2"},
@@ -313,18 +313,18 @@ func testV1STSSetStatQueueProfile(t *testing.T) {
func testV1STSUpdateStatQueueProfile(t *testing.T) {
var result string
statConfig.Filters = []*engine.Filter{
&engine.Filter{
statConfig.Filters = []*engine.RequestFilter{
&engine.RequestFilter{
Type: "type",
FieldName: "Name",
Values: []string{"FilterValue1", "FilterValue2"},
},
&engine.Filter{
&engine.RequestFilter{
Type: "*string",
FieldName: "Accout",
Values: []string{"1001", "1002"},
},
&engine.Filter{
&engine.RequestFilter{
Type: "*string_prefix",
FieldName: "Destination",
Values: []string{"10", "20"},

View File

@@ -1,4 +1,4 @@
#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5]
#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4]
cgrates.org,FLTR_1,*string,Account,1001;1002
cgrates.org,FLTR_1,*string_prefix,Destination,10;20
cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002)
1 #Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5] #Tenant[0] ID[1] FilterType[2] FilterFieldName[3] FilterFieldValues[4]
2 cgrates.org,FLTR_1,*string,Account,1001;1002 cgrates.org FLTR_1 *string Account 1001;1002
3 cgrates.org,FLTR_1,*string_prefix,Destination,10;20 cgrates.org FLTR_1 *string_prefix Destination 10;20
4 cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002) cgrates.org FLTR_1 *rsr_fields Subject(~^1.*1$);Destination(1002)

View File

@@ -1,17 +1,14 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
@@ -89,7 +86,7 @@ func TestReqFilterPassRSRFields(t *testing.T) {
cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963",
TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC),
DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}}
rf, err := NewFilter(MetaRSRFields, "", []string{"Tenant(~^cgr.*\\.org$)"})
rf, err := NewRequestFilter(MetaRSRFields, "", []string{"Tenant(~^cgr.*\\.org$)"})
if err != nil {
t.Error(err)
}
@@ -98,7 +95,7 @@ func TestReqFilterPassRSRFields(t *testing.T) {
} else if !passes {
t.Error("Not passing")
}
rf, err = NewFilter(MetaRSRFields, "", []string{"navigation(on)"})
rf, err = NewRequestFilter(MetaRSRFields, "", []string{"navigation(on)"})
if err != nil {
t.Error(err)
}
@@ -107,7 +104,7 @@ func TestReqFilterPassRSRFields(t *testing.T) {
} else if passes {
t.Error("Passing")
}
rf, err = NewFilter(MetaRSRFields, "", []string{"navigation(off)"})
rf, err = NewRequestFilter(MetaRSRFields, "", []string{"navigation(off)"})
if err != nil {
t.Error(err)
}
@@ -123,7 +120,7 @@ func TestReqFilterPassDestinations(t *testing.T) {
cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963",
TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC),
DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}}
rf, err := NewFilter(MetaDestinations, "Destination", []string{"DE"})
rf, err := NewRequestFilter(MetaDestinations, "Destination", []string{"DE"})
if err != nil {
t.Error(err)
}
@@ -132,7 +129,7 @@ func TestReqFilterPassDestinations(t *testing.T) {
} else if !passes {
t.Error("Not passing")
}
rf, err = NewFilter(MetaDestinations, "Destination", []string{"RO"})
rf, err = NewRequestFilter(MetaDestinations, "Destination", []string{"RO"})
if err != nil {
t.Error(err)
}

View File

@@ -72,9 +72,13 @@ type RequestFilter struct {
}
type Filter struct {
Tenant string
ID string
ReqFilters []*RequestFilter
Tenant string
ID string
Filters []*RequestFilter
}
func (flt *Filter) TenantID() string {
return utils.ConcatenatedKey(flt.Tenant, flt.ID)
}
// Separate method to compile RSR fields

View File

@@ -2211,10 +2211,9 @@ func APItoModelTPThreshold(th *utils.TPThreshold) (mdls TpThresholdS) {
mdl.FilterFieldName = fltr.FieldName
for i, val := range fltr.Values {
if i != 0 {
mdl.FilterFieldValues = mdl.FilterFieldValues + utils.INFIELD_SEP + val
} else {
mdl.FilterFieldValues = val
mdl.FilterFieldValues += utils.INFIELD_SEP
}
mdl.FilterFieldValues += val
}
mdls = append(mdls, mdl)
}
@@ -2268,14 +2267,11 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) {
ID: tp.ID,
}
}
if tp.Type != "" {
th.FilterType = tp.Type
}
if tp.Name != "" {
th.FilterFieldName = tp.Name
}
if tp.Values != "" {
th.FilterFielValues = append(th.FilterFielValues, strings.Split(tp.Values, utils.INFIELD_SEP)...)
if tp.FilterType != "" {
th.Filters = append(th.Filters, &utils.TPRequestFilter{
Type: tp.FilterType,
FieldName: tp.FilterFieldName,
Values: strings.Split(tp.FilterFieldValues, utils.INFIELD_SEP)})
}
mst[tp.ID] = th
}
@@ -2289,27 +2285,41 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) {
}
func APItoModelTPFilter(th *utils.TPFilter) (mdls TpFilterS) {
mdl := &TpFilter{
Tpid: th.TPid,
Tenant: th.Tenant,
ID: th.ID,
Name: th.FilterFieldName,
Type: th.FilterType,
if len(th.Filters) == 0 {
return
}
for i, val := range th.FilterFielValues {
if i != 0 {
mdl.Values += utils.INFIELD_SEP
for _, fltr := range th.Filters {
mdl := &TpFilter{
Tpid: th.TPid,
Tenant: th.Tenant,
ID: th.ID,
}
mdl.Values += val
mdl.FilterType = fltr.Type
mdl.FilterFieldName = fltr.FieldName
for i, val := range fltr.Values {
if i != 0 {
mdl.FilterFieldValues += utils.INFIELD_SEP
}
mdl.FilterFieldValues += val
}
mdls = append(mdls, mdl)
}
mdls = append(mdls, mdl)
return
}
func APItoFilter(tpTH *utils.TPFilter) (th *Filter, err error) {
th = &Filter{
Tenant: tpTH.Tenant,
ID: tpTH.ID,
Tenant: tpTH.Tenant,
ID: tpTH.ID,
Filters: make([]*RequestFilter, len(tpTH.Filters)),
}
for i, f := range tpTH.Filters {
rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values}
if err := rf.CompileValues(); err != nil {
return nil, err
}
th.Filters[i] = rf
}
return th, nil
}

View File

@@ -818,18 +818,18 @@ func TestAPItoResource(t *testing.T) {
Stored: tpRL.Stored,
Blocker: tpRL.Blocker,
Weight: tpRL.Weight,
Filters: make([]*Filter, len(tpRL.Filters))}
eRL.Filters[0] = &Filter{Type: MetaString,
Filters: make([]*RequestFilter, len(tpRL.Filters))}
eRL.Filters[0] = &RequestFilter{Type: MetaString,
FieldName: "Account", Values: []string{"1001", "1002"}}
eRL.Filters[1] = &Filter{Type: MetaStringPrefix,
eRL.Filters[1] = &RequestFilter{Type: MetaStringPrefix,
FieldName: "Destination", Values: []string{"10", "20"}}
eRL.Filters[2] = &Filter{Type: MetaStatS,
eRL.Filters[2] = &RequestFilter{Type: MetaStatS,
Values: []string{"CDRST1:*min_asr:34", "CDRST_1001:*min_asr:20"},
statSThresholds: []*RFStatSThreshold{
&RFStatSThreshold{QueueID: "CDRST1", ThresholdType: "*min_asr", ThresholdValue: 34},
&RFStatSThreshold{QueueID: "CDRST_1001", ThresholdType: "*min_asr", ThresholdValue: 20},
}}
eRL.Filters[3] = &Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
eRL.Filters[3] = &RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),
}
at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC")
@@ -913,7 +913,7 @@ func TestAPItoTPStats(t *testing.T) {
QueueLength: tps.QueueLength,
Metrics: []string{"*asr", "*acd", "*acc"},
Thresholds: []string{"THRESH1", "THRESH2"},
Filters: make([]*Filter, len(tps.Filters)),
Filters: make([]*RequestFilter, len(tps.Filters)),
Stored: tps.Stored,
Blocker: tps.Blocker,
Weight: 20.0,
@@ -923,7 +923,7 @@ func TestAPItoTPStats(t *testing.T) {
t.Errorf("Got error: %+v", err)
}
eTPs.Filters[0] = &Filter{Type: MetaString,
eTPs.Filters[0] = &RequestFilter{Type: MetaString,
FieldName: "Account", Values: []string{"1001", "1002"}}
at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC")
eTPs.ActivationInterval = &utils.ActivationInterval{ActivationTime: at}
@@ -995,7 +995,7 @@ func TestAPItoTPThreshold(t *testing.T) {
eTPs := &ThresholdProfile{
ID: tps.ID,
Filters: make([]*Filter, len(tps.Filters)),
Filters: make([]*RequestFilter, len(tps.Filters)),
Recurrent: tps.Recurrent,
Blocker: tps.Blocker,
Weight: tps.Weight,
@@ -1004,7 +1004,7 @@ func TestAPItoTPThreshold(t *testing.T) {
if eTPs.MinSleep, err = utils.ParseDurationWithSecs(tps.MinSleep); err != nil {
t.Errorf("Got error: %+v", err)
}
eTPs.Filters[0] = &Filter{Type: MetaString,
eTPs.Filters[0] = &RequestFilter{Type: MetaString,
FieldName: "Account", Values: []string{"1001", "1002"}}
at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC")
eTPs.ActivationInterval = &utils.ActivationInterval{ActivationTime: at}
@@ -1018,20 +1018,24 @@ func TestAPItoTPThreshold(t *testing.T) {
func TestTPFilterAsTPFilter(t *testing.T) {
tps := []*TpFilter{
&TpFilter{
Tpid: "TEST_TPID",
ID: "Filter1",
Type: MetaStringPrefix,
Name: "Account",
Values: "1001;1002",
Tpid: "TEST_TPID",
ID: "Filter1",
FilterType: MetaStringPrefix,
FilterFieldName: "Account",
FilterFieldValues: "1001;1002",
},
}
eTPs := []*utils.TPFilter{
&utils.TPFilter{
TPid: tps[0].Tpid,
ID: tps[0].ID,
FilterType: tps[0].Type,
FilterFieldName: tps[0].Name,
FilterFielValues: []string{"1001", "1002"},
TPid: tps[0].Tpid,
ID: tps[0].ID,
Filters: []*utils.TPRequestFilter{
&utils.TPRequestFilter{
Type: MetaStringPrefix,
FieldName: "Account",
Values: []string{"1001", "1002"},
},
},
},
}
@@ -1043,20 +1047,28 @@ func TestTPFilterAsTPFilter(t *testing.T) {
func TestAPItoTPFilter(t *testing.T) {
tps := &utils.TPFilter{
TPid: testTPID,
Tenant: "cgrates.org",
ID: "Filter1",
FilterType: "*string",
FilterFieldName: "Acount",
FilterFielValues: []string{"1001", "1002"},
TPid: testTPID,
Tenant: "cgrates.org",
ID: "Filter1",
Filters: []*utils.TPRequestFilter{
&utils.TPRequestFilter{
FieldName: "Account",
Type: "*string",
Values: []string{"1001", "1002"},
},
},
}
eTPs := &Filter{
Tenant: "cgrates.org",
ID: tps.ID,
FieldName: tps.FilterFieldName,
Type: tps.FilterType,
Values: tps.FilterFielValues,
Tenant: "cgrates.org",
ID: tps.ID,
Filters: []*RequestFilter{
&RequestFilter{
FieldName: "Account",
Type: "*string",
Values: []string{"1001", "1002"},
},
},
}
if st, err := APItoFilter(tps); err != nil {
t.Error(err)

View File

@@ -519,12 +519,12 @@ type TpThreshold struct {
}
type TpFilter struct {
PK uint `gorm:"primary_key"`
Tpid string
Tenant string `index:"0" re:""`
ID string `index:"1" re:""`
Type string `index:"2" re:"^\*[A-Za-z].*"`
Name string `index:"3" re:""`
Values string `index:"4" re:""`
CreatedAt time.Time
PK uint `gorm:"primary_key"`
Tpid string
Tenant string `index:"0" re:""`
ID string `index:"1" re:""`
FilterType string `index:"2" re:"^\*[A-Za-z].*"`
FilterFieldName string `index:"3" re:""`
FilterFieldValues string `index:"4" re:""`
CreatedAt time.Time
}

View File

@@ -797,9 +797,9 @@ func testOnStorITCacheResourceProfile(t *testing.T) {
Tenant: "cgrates.org",
ID: "RL_TEST",
Weight: 10,
Filters: []*Filter{
&Filter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}},
&Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
Filters: []*RequestFilter{
&RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}},
&RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),
}},
ActivationInterval: &utils.ActivationInterval{
@@ -1800,9 +1800,9 @@ func testOnStorITCRUDResourceProfile(t *testing.T) {
rL := &ResourceProfile{
ID: "RL_TEST2",
Weight: 10,
Filters: []*Filter{
&Filter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}},
&Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
Filters: []*RequestFilter{
&RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}},
&RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),
}},
ActivationInterval: &utils.ActivationInterval{
@@ -1967,7 +1967,7 @@ func testOnStorITCRUDStatQueueProfile(t *testing.T) {
sq := &StatQueueProfile{
ID: "test",
ActivationInterval: &utils.ActivationInterval{},
Filters: []*Filter{},
Filters: []*RequestFilter{},
QueueLength: 2,
TTL: timeTTL,
Metrics: []string{},
@@ -2055,7 +2055,7 @@ func testOnStorITCRUDThresholdProfile(t *testing.T) {
Tenant: "cgrates.org",
ID: "test",
ActivationInterval: &utils.ActivationInterval{},
Filters: []*Filter{},
Filters: []*RequestFilter{},
Recurrent: true,
MinSleep: timeMinSleep,
Blocker: true,
@@ -2122,11 +2122,15 @@ func testOnStorITCRUDThreshold(t *testing.T) {
func testOnStorITCRUDFilter(t *testing.T) {
fp := &Filter{
Tenant: "cgrates.org",
ID: "Filter1",
FieldName: "*string",
Type: "Account",
Values: []string{"1001", "1002"},
Tenant: "cgrates.org",
ID: "Filter1",
Filters: []*RequestFilter{
&RequestFilter{
FieldName: "Account",
Type: "*string",
Values: []string{"1001", "1002"},
},
},
}
if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
t.Error(rcvErr)

View File

@@ -53,13 +53,13 @@ func TestRSRecordUsage1(t *testing.T) {
rPrf: &ResourceProfile{
Tenant: "cgrates.org",
ID: "RL1",
Filters: []*Filter{
&Filter{
Filters: []*RequestFilter{
&RequestFilter{
Type: MetaString,
FieldName: "Account",
Values: []string{"1001", "1002"},
},
&Filter{
&RequestFilter{
Type: MetaRSRFields,
Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),
@@ -134,13 +134,13 @@ func TestRSRsort(t *testing.T) {
ID: "RL2",
rPrf: &ResourceProfile{
ID: "RL2",
Filters: []*Filter{
&Filter{
Filters: []*RequestFilter{
&RequestFilter{
Type: MetaString,
FieldName: "Account",
Values: []string{"1001", "1002"},
},
&Filter{
&RequestFilter{
Type: MetaRSRFields,
Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),
@@ -249,13 +249,13 @@ func TestRSCacheSetGet(t *testing.T) {
rPrf: &ResourceProfile{
Tenant: "cgrates.org",
ID: "RL",
Filters: []*Filter{
&Filter{
Filters: []*RequestFilter{
&RequestFilter{
Type: MetaString,
FieldName: "Account",
Values: []string{"1001", "1002"},
},
&Filter{
&RequestFilter{
Type: MetaRSRFields,
Values: []string{"Subject(~^1.*1$)", "Destination(1002)"},
rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP),

View File

@@ -1370,11 +1370,8 @@ type TPThreshold struct {
}
type TPFilter struct {
TPid string
Tenant string
ID string
FilterType string // Filter type (*string, *timing, *rsr_filters, *cdr_stats)
FilterFieldName string // Name of the field providing us the Values to check (used in case of some )
FilterFielValues []string // Filter definition
TPid string
Tenant string
ID string
Filters []*TPRequestFilter
}