uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Unify use of filters, fixes #445, fixes #427

Browse Source
This commit is contained in:
DanB
2016-05-12 20:28:04 +02:00
parent 98d9e7fc93
commit 180756d1a7
9 changed files with 30 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
cdre/cdrexporter.go
View File

@@ -21,6 +21,7 @@ package cdre
import (
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"os"
@@ -129,15 +130,15 @@ func (cdre *CdrExporter) getCdrCostDetails(CGRID, runId string) (string, error)
func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.CDR, cfgCdrFld *config.CfgCdrField) (string, error) {
var combinedVal string // Will result as combination of the field values, filters must match
for _, filterRule := range cfgCdrFld.FieldFilter {
fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
if !fltrPass {
return "", nil
if !filterRule.FilterPasses(processedCdr.FieldAsString(&utils.RSRField{Id: filterRule.Id})) { // Filter will activate the rule to extract the content
continue
}
pairingVal := processedCdr.FieldAsString(filterRule)
for _, cdr := range cdre.cdrs {
if cdr.CGRID != processedCdr.CGRID {
continue // We only care about cdrs with same primary cdr behind
}
if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte
if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == pairingVal { // First CDR with filte
for _, rsrRule := range cfgCdrFld.Value {
combinedVal += cdr.FieldAsString(rsrRule)
}
@@ -159,11 +160,16 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.CDR, cfgCdrFld *config.
if len(cfgCdrFld.Value) == 0 {
return "", nil
}
for _, fltrRl := range cfgCdrFld.FieldFilter {
if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
passesFilters := true
for _, cdfFltr := range cfgCdrFld.FieldFilter {
if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) {
passesFilters = false
break
}
}
if !passesFilters { // Not passes filters, ignore this replication
return "", errors.New("Not passing filters")
}
layout := cfgCdrFld.Layout
if len(layout) == 0 {
layout = time.RFC3339
@@ -177,11 +183,16 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.CDR, cfgCdrFld *config.
// Extracts the value specified by cfgHdr out of cdr
func (cdre *CdrExporter) cdrFieldValue(cdr *engine.CDR, cfgCdrFld *config.CfgCdrField) (string, error) {
for _, fltrRl := range cfgCdrFld.FieldFilter {
if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
passesFilters := true
for _, cdfFltr := range cfgCdrFld.FieldFilter {
if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) {
passesFilters = false
break
}
}
if !passesFilters { // Not passes filters, ignore this replication
return "", fmt.Errorf("Filters not passing")
}
layout := cfgCdrFld.Layout
if len(layout) == 0 {
layout = time.RFC3339

4
cdre/cdrexporter_test.go
View File

@@ -90,7 +90,7 @@ func TestGetDateTimeFieldVal(t *testing.T) {
t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal)
}
// Test filter
fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
fltr, _ := utils.ParseRSRFields("Tenant(itsyscom.com)", utils.INFIELD_SEP)
cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", FieldId: "stop_time", Value: val, FieldFilter: fltr, Layout: layout}
if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
t.Error(err)
@@ -116,7 +116,7 @@ func TestCdreCdrFieldValue(t *testing.T) {
} else if val != cdr.Destination {
t.Errorf("Expecting: %s, received: %s", cdr.Destination, val)
}
fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
fltr, _ := utils.ParseRSRFields("Tenant(itsyscom.com)", utils.INFIELD_SEP)
cfgCdrFld = &config.CfgCdrField{Tag: "destination", Type: "cdrfield", FieldId: utils.DESTINATION, Value: val, FieldFilter: fltr}
if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil {
t.Error("Failed to use filter")

20
engine/cdr.go
View File

@@ -255,26 +255,6 @@ func (cdr *CDR) FieldsAsString(rsrFlds utils.RSRFields) string {
return fldVal
}
func (cdr *CDR) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) {
if fieldFilter == nil {
return true, ""
}
if fieldFilter.IsStatic() && cdr.FieldAsString(&utils.RSRField{Id: fieldFilter.Id}) == cdr.FieldAsString(fieldFilter) {
return true, cdr.FieldAsString(&utils.RSRField{Id: fieldFilter.Id})
}
preparedFilter := &utils.RSRField{Id: fieldFilter.Id, RSRules: make([]*utils.ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter
for idx := range fieldFilter.RSRules {
// Hardcode the template with maximum of 5 groups ordered
preparedFilter.RSRules[idx] = &utils.ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: utils.FILTER_REGEXP_TPL}
}
preparedVal := cdr.FieldAsString(preparedFilter)
filteredValue := cdr.FieldAsString(fieldFilter)
if preparedFilter.RegexpMatched() && (len(preparedVal) == 0 || preparedVal == filteredValue) {
return true, filteredValue
}
return false, ""
}
func (cdr *CDR) AsStoredCdr(timezone string) *CDR {
return cdr
}

95
engine/cdr_test.go
View File

@@ -135,101 +135,6 @@ func TestFieldsAsString(t *testing.T) {
}
}
func TestPassesFieldFilter(t *testing.T) {
cdr := &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf",
OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), RunID: utils.DEFAULT_RUNID,
Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if pass, _ := cdr.PassesFieldFilter(nil); !pass {
t.Error("Not passing filter")
}
acntPrefxFltr, _ := utils.NewRSRField(`~Account:s/(.+)/1001/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass {
t.Error("Not passing filter")
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^(10)\d\d$/10/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass {
t.Error("Not passing valid filter")
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^\d(10)\d$/10/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Passing filter")
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^(10)\d\d$/010/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Passing filter")
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^1010$/1010/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Passing filter")
}
torFltr, _ := utils.NewRSRField(`^ToR::*voice/`)
if pass, _ := cdr.PassesFieldFilter(torFltr); !pass {
t.Error("Not passing filter")
}
torFltr, _ = utils.NewRSRField(`^TOR/*data/`)
if pass, _ := cdr.PassesFieldFilter(torFltr); pass {
t.Error("Passing filter")
}
inexistentFieldFltr, _ := utils.NewRSRField(`^fakefield/fakevalue/`)
if pass, _ := cdr.PassesFieldFilter(inexistentFieldFltr); pass {
t.Error("Passing filter")
}
}
func TestPassesFieldFilterDn1(t *testing.T) {
cdr := &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem0005",
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
acntPrefxFltr, _ := utils.NewRSRField(`~Account:s/^\w+[shmp]\d{4}$//`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass {
t.Error("Not passing valid filter")
}
cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem00005",
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Should not pass filter")
}
cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0402129281",
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^0\d{9}$//`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass {
t.Error("Not passing valid filter")
}
acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^0(\d{9})$/placeholder/`)
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Should not pass filter")
}
cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "04021292812",
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Should not pass filter")
}
cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0162447222",
ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
}
if acntPrefxFltr, err := utils.NewRSRField(`~Account:s/^0\d{9}$//`); err != nil {
t.Error("Unexpected parse error", err)
} else if acntPrefxFltr == nil {
t.Error("Failed parsing rule")
} else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass {
t.Error("Not passing valid filter")
}
if acntPrefxFltr, err := utils.NewRSRField(`~Account:s/^\w+[shmp]\d{4}$//`); err != nil {
t.Error("Unexpected parse error", err)
} else if acntPrefxFltr == nil {
t.Error("Failed parsing rule")
} else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass {
t.Error("Should not pass filter")
}
}
func TestUsageMultiply(t *testing.T) {
cdr := CDR{Usage: time.Duration(10) * time.Second}
if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second {

4
engine/cdrs.go
View File

@@ -303,7 +303,7 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) {
runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP)
matchingAllFilters := true
for _, dcRunFilter := range runFilters {
if fltrPass, _ := cdr.PassesFieldFilter(dcRunFilter); !fltrPass {
if !dcRunFilter.FilterPasses(cdr.FieldAsString(dcRunFilter)) {
matchingAllFilters = false
break
}
@@ -429,7 +429,7 @@ func (self *CdrServer) replicateCdr(cdr *CDR) error {
for _, rplCfg := range self.cgrCfg.CDRSCdrReplication {
passesFilters := true
for _, cdfFltr := range rplCfg.CdrFilter {
if fltrPass, _ := cdr.PassesFieldFilter(cdfFltr); !fltrPass {
if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) {
passesFilters = false
break
}

1
engine/event.go
View File

@@ -47,7 +47,6 @@ type Event interface {
GetExtraFields() map[string]string
MissingParameter(string) bool
ParseEventValue(*utils.RSRField, string) string
PassesFieldFilter(*utils.RSRField) (bool, string)
AsStoredCdr(timezone string) *CDR
String() string
AsEvent(string) Event

2
engine/responder.go
View File

@@ -358,7 +358,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error {
runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP)
matchingAllFilters := true
for _, dcRunFilter := range runFilters {
if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass {
if !dcRunFilter.FilterPasses(ev.FieldAsString(dcRunFilter)) {
matchingAllFilters = false
break
}

2
sessionmanager/fsevent.go
View File

@@ -316,6 +316,7 @@ func (fsev FSEvent) ParseEventValue(rsrFld *utils.RSRField, timezone string) str
}
}
/*
func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) {
// Keep in sync (or merge) with StoredCdr.PassesFieldFielter()
if fieldFilter == nil {
@@ -336,6 +337,7 @@ func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string
}
return false, ""
}
*/
func (fsev FSEvent) AsStoredCdr(timezone string) *engine.CDR {
storCdr := new(engine.CDR)

2
sessionmanager/fsevent_test.go
View File

@@ -578,6 +578,7 @@ func TestParseEventValue(t *testing.T) {
}
}
/*
func TestPassesFieldFilterDn1(t *testing.T) {
body := `Event-Name: RE_SCHEDULE
Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d
@@ -626,6 +627,7 @@ Caller-Username: 04021292812`
t.Error("Should not pass filter")
}
}
*/
func TestFsEvAsStoredCdr(t *testing.T) {
cfg, _ := config.NewDefaultCGRConfig()