From 180756d1a7f76c7bcafcf0d0bbc6d287208add0f Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 12 May 2016 20:28:04 +0200 Subject: [PATCH] Unify use of filters, fixes #445, fixes #427 --- cdre/cdrexporter.go | 31 +++++++---- cdre/cdrexporter_test.go | 4 +- engine/cdr.go | 20 ------- engine/cdr_test.go | 95 ---------------------------------- engine/cdrs.go | 4 +- engine/event.go | 1 - engine/responder.go | 2 +- sessionmanager/fsevent.go | 2 + sessionmanager/fsevent_test.go | 2 + 9 files changed, 30 insertions(+), 131 deletions(-) diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index c269385e1..39436b05d 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -21,6 +21,7 @@ package cdre import ( "encoding/csv" "encoding/json" + "errors" "fmt" "io" "os" @@ -129,15 +130,15 @@ func (cdre *CdrExporter) getCdrCostDetails(CGRID, runId string) (string, error) func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.CDR, cfgCdrFld *config.CfgCdrField) (string, error) { var combinedVal string // Will result as combination of the field values, filters must match for _, filterRule := range cfgCdrFld.FieldFilter { - fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule) - if !fltrPass { - return "", nil + if !filterRule.FilterPasses(processedCdr.FieldAsString(&utils.RSRField{Id: filterRule.Id})) { // Filter will activate the rule to extract the content + continue } + pairingVal := processedCdr.FieldAsString(filterRule) for _, cdr := range cdre.cdrs { if cdr.CGRID != processedCdr.CGRID { continue // We only care about cdrs with same primary cdr behind } - if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte + if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == pairingVal { // First CDR with filte for _, rsrRule := range cfgCdrFld.Value { combinedVal += cdr.FieldAsString(rsrRule) } @@ -159,11 +160,16 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.CDR, cfgCdrFld *config. if len(cfgCdrFld.Value) == 0 { return "", nil } - for _, fltrRl := range cfgCdrFld.FieldFilter { - if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { - return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) + passesFilters := true + for _, cdfFltr := range cfgCdrFld.FieldFilter { + if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) { + passesFilters = false + break } } + if !passesFilters { // Not passes filters, ignore this replication + return "", errors.New("Not passing filters") + } layout := cfgCdrFld.Layout if len(layout) == 0 { layout = time.RFC3339 @@ -177,11 +183,16 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.CDR, cfgCdrFld *config. // Extracts the value specified by cfgHdr out of cdr func (cdre *CdrExporter) cdrFieldValue(cdr *engine.CDR, cfgCdrFld *config.CfgCdrField) (string, error) { - for _, fltrRl := range cfgCdrFld.FieldFilter { - if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { - return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) + passesFilters := true + for _, cdfFltr := range cfgCdrFld.FieldFilter { + if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) { + passesFilters = false + break } } + if !passesFilters { // Not passes filters, ignore this replication + return "", fmt.Errorf("Filters not passing") + } layout := cfgCdrFld.Layout if len(layout) == 0 { layout = time.RFC3339 diff --git a/cdre/cdrexporter_test.go b/cdre/cdrexporter_test.go index 045fc0e4b..2ac8f55b0 100644 --- a/cdre/cdrexporter_test.go +++ b/cdre/cdrexporter_test.go @@ -90,7 +90,7 @@ func TestGetDateTimeFieldVal(t *testing.T) { t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal) } // Test filter - fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP) + fltr, _ := utils.ParseRSRFields("Tenant(itsyscom.com)", utils.INFIELD_SEP) cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", FieldId: "stop_time", Value: val, FieldFilter: fltr, Layout: layout} if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil { t.Error(err) @@ -116,7 +116,7 @@ func TestCdreCdrFieldValue(t *testing.T) { } else if val != cdr.Destination { t.Errorf("Expecting: %s, received: %s", cdr.Destination, val) } - fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP) + fltr, _ := utils.ParseRSRFields("Tenant(itsyscom.com)", utils.INFIELD_SEP) cfgCdrFld = &config.CfgCdrField{Tag: "destination", Type: "cdrfield", FieldId: utils.DESTINATION, Value: val, FieldFilter: fltr} if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil { t.Error("Failed to use filter") diff --git a/engine/cdr.go b/engine/cdr.go index bbdb56f84..69fc2df32 100644 --- a/engine/cdr.go +++ b/engine/cdr.go @@ -255,26 +255,6 @@ func (cdr *CDR) FieldsAsString(rsrFlds utils.RSRFields) string { return fldVal } -func (cdr *CDR) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) { - if fieldFilter == nil { - return true, "" - } - if fieldFilter.IsStatic() && cdr.FieldAsString(&utils.RSRField{Id: fieldFilter.Id}) == cdr.FieldAsString(fieldFilter) { - return true, cdr.FieldAsString(&utils.RSRField{Id: fieldFilter.Id}) - } - preparedFilter := &utils.RSRField{Id: fieldFilter.Id, RSRules: make([]*utils.ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter - for idx := range fieldFilter.RSRules { - // Hardcode the template with maximum of 5 groups ordered - preparedFilter.RSRules[idx] = &utils.ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: utils.FILTER_REGEXP_TPL} - } - preparedVal := cdr.FieldAsString(preparedFilter) - filteredValue := cdr.FieldAsString(fieldFilter) - if preparedFilter.RegexpMatched() && (len(preparedVal) == 0 || preparedVal == filteredValue) { - return true, filteredValue - } - return false, "" -} - func (cdr *CDR) AsStoredCdr(timezone string) *CDR { return cdr } diff --git a/engine/cdr_test.go b/engine/cdr_test.go index af2529958..76957e9f6 100644 --- a/engine/cdr_test.go +++ b/engine/cdr_test.go @@ -135,101 +135,6 @@ func TestFieldsAsString(t *testing.T) { } } -func TestPassesFieldFilter(t *testing.T) { - cdr := &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderID: 123, ToR: utils.VOICE, OriginID: "dsafdsaf", - OriginHost: "192.168.1.1", Source: "test", RequestType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", - Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), - AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), RunID: utils.DEFAULT_RUNID, - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - if pass, _ := cdr.PassesFieldFilter(nil); !pass { - t.Error("Not passing filter") - } - acntPrefxFltr, _ := utils.NewRSRField(`~Account:s/(.+)/1001/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { - t.Error("Not passing filter") - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^(10)\d\d$/10/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { - t.Error("Not passing valid filter") - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^\d(10)\d$/10/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Passing filter") - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^(10)\d\d$/010/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Passing filter") - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^1010$/1010/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Passing filter") - } - torFltr, _ := utils.NewRSRField(`^ToR::*voice/`) - if pass, _ := cdr.PassesFieldFilter(torFltr); !pass { - t.Error("Not passing filter") - } - torFltr, _ = utils.NewRSRField(`^TOR/*data/`) - if pass, _ := cdr.PassesFieldFilter(torFltr); pass { - t.Error("Passing filter") - } - inexistentFieldFltr, _ := utils.NewRSRField(`^fakefield/fakevalue/`) - if pass, _ := cdr.PassesFieldFilter(inexistentFieldFltr); pass { - t.Error("Passing filter") - } -} - -func TestPassesFieldFilterDn1(t *testing.T) { - cdr := &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem0005", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - acntPrefxFltr, _ := utils.NewRSRField(`~Account:s/^\w+[shmp]\d{4}$//`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { - t.Error("Not passing valid filter") - } - - cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem00005", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Should not pass filter") - } - cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0402129281", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^0\d{9}$//`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { - t.Error("Not passing valid filter") - } - acntPrefxFltr, _ = utils.NewRSRField(`~Account:s/^0(\d{9})$/placeholder/`) - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Should not pass filter") - } - cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "04021292812", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Should not pass filter") - } - cdr = &CDR{CGRID: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0162447222", - ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, - } - if acntPrefxFltr, err := utils.NewRSRField(`~Account:s/^0\d{9}$//`); err != nil { - t.Error("Unexpected parse error", err) - } else if acntPrefxFltr == nil { - t.Error("Failed parsing rule") - } else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { - t.Error("Not passing valid filter") - } - if acntPrefxFltr, err := utils.NewRSRField(`~Account:s/^\w+[shmp]\d{4}$//`); err != nil { - t.Error("Unexpected parse error", err) - } else if acntPrefxFltr == nil { - t.Error("Failed parsing rule") - } else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { - t.Error("Should not pass filter") - } -} - func TestUsageMultiply(t *testing.T) { cdr := CDR{Usage: time.Duration(10) * time.Second} if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second { diff --git a/engine/cdrs.go b/engine/cdrs.go index 55145e549..968ec230a 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -303,7 +303,7 @@ func (self *CdrServer) deriveCdrs(cdr *CDR) ([]*CDR, error) { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) matchingAllFilters := true for _, dcRunFilter := range runFilters { - if fltrPass, _ := cdr.PassesFieldFilter(dcRunFilter); !fltrPass { + if !dcRunFilter.FilterPasses(cdr.FieldAsString(dcRunFilter)) { matchingAllFilters = false break } @@ -429,7 +429,7 @@ func (self *CdrServer) replicateCdr(cdr *CDR) error { for _, rplCfg := range self.cgrCfg.CDRSCdrReplication { passesFilters := true for _, cdfFltr := range rplCfg.CdrFilter { - if fltrPass, _ := cdr.PassesFieldFilter(cdfFltr); !fltrPass { + if !cdfFltr.FilterPasses(cdr.FieldAsString(cdfFltr)) { passesFilters = false break } diff --git a/engine/event.go b/engine/event.go index 91803856f..2f7fb12d6 100644 --- a/engine/event.go +++ b/engine/event.go @@ -47,7 +47,6 @@ type Event interface { GetExtraFields() map[string]string MissingParameter(string) bool ParseEventValue(*utils.RSRField, string) string - PassesFieldFilter(*utils.RSRField) (bool, string) AsStoredCdr(timezone string) *CDR String() string AsEvent(string) Event diff --git a/engine/responder.go b/engine/responder.go index 751d3299c..f85daa733 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -358,7 +358,7 @@ func (rs *Responder) GetDerivedMaxSessionTime(ev *CDR, reply *float64) error { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) matchingAllFilters := true for _, dcRunFilter := range runFilters { - if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass { + if !dcRunFilter.FilterPasses(ev.FieldAsString(dcRunFilter)) { matchingAllFilters = false break } diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index 4ba9bdd4c..c955125e2 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -316,6 +316,7 @@ func (fsev FSEvent) ParseEventValue(rsrFld *utils.RSRField, timezone string) str } } +/* func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) { // Keep in sync (or merge) with StoredCdr.PassesFieldFielter() if fieldFilter == nil { @@ -336,6 +337,7 @@ func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string } return false, "" } +*/ func (fsev FSEvent) AsStoredCdr(timezone string) *engine.CDR { storCdr := new(engine.CDR) diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index 3dbb6c7e7..62cb21970 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -578,6 +578,7 @@ func TestParseEventValue(t *testing.T) { } } +/* func TestPassesFieldFilterDn1(t *testing.T) { body := `Event-Name: RE_SCHEDULE Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d @@ -626,6 +627,7 @@ Caller-Username: 04021292812` t.Error("Should not pass filter") } } +*/ func TestFsEvAsStoredCdr(t *testing.T) { cfg, _ := config.NewDefaultCGRConfig()