diff --git a/cdrc/fwv_test.go b/cdrc/fwv_test.go index 39f7c6a9c..a5a1175cd 100644 --- a/cdrc/fwv_test.go +++ b/cdrc/fwv_test.go @@ -46,7 +46,7 @@ func TestFwvRecordPassesCfgFilter(t *testing.T) { //record, configKey string) bool { cgrConfig, _ := config.NewDefaultCGRConfig() cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters - cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(+49123123120)`, utils.INFIELD_SEP) + cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(^+49123123120)`, utils.INFIELD_SEP) fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]} cdrLine := "CDR0000010 0 20120708181506000123451234 0040123123120 004 000018009980010001ISDN ABC 10Buiten uw regio EHV 00000009190000000009" if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter { diff --git a/engine/pubsub_test.go b/engine/pubsub_test.go index 803409e1d..5345a06c0 100644 --- a/engine/pubsub_test.go +++ b/engine/pubsub_test.go @@ -215,7 +215,7 @@ func TestCgrEventPassFilters(t *testing.T) { if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("^EventName::DUMMY", utils.INFIELD_SEP)) { // Should pass since we have no filter defined t.Error("Not passing no filter") } - if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(\\w*)_/$1/(TEST)", utils.INFIELD_SEP)) { + if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(.*)_/$1/(TEST)", utils.INFIELD_SEP)) { t.Error("Not passing filter") } if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(\\w*)_/$1/:s/^(\\w)(\\w)(\\w)(\\w)/$1$3$4/(TST)", utils.INFIELD_SEP)) { diff --git a/utils/consts.go b/utils/consts.go index 66c21ecad..1dcebf439 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -258,6 +258,9 @@ const ( TRIGGER_BALANCE_EXPIRED = "*balance_expired" HIERARCHY_SEP = ">" META_COMPOSED = "*composed" + NegativePrefix = "!" + MatchStartPrefix = "^" + MatchEndPrefix = "$" ) var ( diff --git a/utils/researchreplace_test.go b/utils/researchreplace_test.go index d055f9e12..3b1dfaad3 100644 --- a/utils/researchreplace_test.go +++ b/utils/researchreplace_test.go @@ -59,3 +59,10 @@ func TestProcessReSearchReplace4(t *testing.T) { t.Error("Unexpected output from SearchReplace: ", outStr) } } + +func TestProcessReSearchReplace5(t *testing.T) { + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^(.*)_`), ReplaceTemplate: "$1"} + if outStr := rsr.Process("TEST_EVENT"); outStr != "TEST" { + t.Error("Unexpected output from SearchReplace: ", outStr) + } +} diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 1b34101d4..b454500fb 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -28,13 +28,19 @@ func NewRSRField(fldStr string) (*RSRField, error) { if len(fldStr) == 0 { return nil, nil } - var filterVal string + var filters []*RSRFilter if strings.HasSuffix(fldStr, FILTER_VAL_END) { // Has filter, populate the var fltrStart := strings.LastIndex(fldStr, FILTER_VAL_START) if fltrStart < 1 { return nil, fmt.Errorf("Invalid FilterStartValue in string: %s", fldStr) } - filterVal = fldStr[fltrStart+1 : len(fldStr)-1] + for _, fltrVal := range strings.Split(fldStr[fltrStart+1:len(fldStr)-1], INFIELD_SEP) { + if rsrFltr, err := NewRSRFilter(fltrVal); err != nil { + return nil, fmt.Errorf("Invalid FilterValue in string: %s, err: %s", fltrVal, err.Error()) + } else { + filters = append(filters, rsrFltr) + } + } fldStr = fldStr[:fltrStart] // Take the filter part out before compiling further } @@ -51,17 +57,17 @@ func NewRSRField(fldStr string) (*RSRField, error) { } else { staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix } - return &RSRField{Id: staticHdr, staticValue: staticVal, filterValue: filterVal}, nil + return &RSRField{Id: staticHdr, staticValue: staticVal, filters: filters}, nil } if !strings.HasPrefix(fldStr, REGEXP_PREFIX) { - return &RSRField{Id: fldStr, filterValue: filterVal}, nil + return &RSRField{Id: fldStr, filters: filters}, nil } spltRgxp := regexp.MustCompile(`:s\/`) spltRules := spltRgxp.Split(fldStr, -1) if len(spltRules) < 2 { return nil, fmt.Errorf("Invalid Split of Search&Replace field rule. %s", fldStr) } - rsrField := &RSRField{Id: spltRules[0][1:], filterValue: filterVal} // Original id in form ~hdr_name + rsrField := &RSRField{Id: spltRules[0][1:], filters: filters} // Original id in form ~hdr_name rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.*[^\\])*\/){1,}`) for _, ruleStr := range spltRules[1:] { // :s/ already removed through split allMatches := rulesRgxp.FindStringSubmatch(ruleStr) @@ -81,7 +87,7 @@ type RSRField struct { Id string // Identifier RSRules []*ReSearchReplace // Rules to use when processing field value staticValue string // If defined, enforces parsing always to this value - filterValue string // The value to compare when used as filter + filters []*RSRFilter // The value to compare when used as filter } // Parse the field value from a string @@ -111,7 +117,63 @@ func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a reg } func (rsrf *RSRField) FilterPasses(value string) bool { - return len(rsrf.filterValue) == 0 || rsrf.ParseValue(value) == rsrf.filterValue + if len(rsrf.filters) == 0 { // No filters + return true + } + parsedVal := rsrf.ParseValue(value) + filterPasses := false + for _, fltr := range rsrf.filters { + if fltr.Pass(parsedVal) { + filterPasses = true + } + } + return filterPasses +} + +// NewRSRFilter instantiates a new RSRFilter, setting it's properties +func NewRSRFilter(fltrVal string) (rsrFltr *RSRFilter, err error) { + rsrFltr = new(RSRFilter) + if fltrVal == "" { + return rsrFltr, nil + } + if fltrVal[:1] == NegativePrefix { + rsrFltr.negative = true + fltrVal = fltrVal[1:] + if fltrVal == "" { + return rsrFltr, nil + } + } + rsrFltr.filterRule = fltrVal + if fltrVal[:1] == REGEXP_PREFIX { + if rsrFltr.fltrRgxp, err = regexp.Compile(fltrVal[1:]); err != nil { + return nil, err + } + } + return rsrFltr, nil +} + +// One filter rule +type RSRFilter struct { + filterRule string // Value in raw format + fltrRgxp *regexp.Regexp + negative bool // Rule should not match +} + +func (rsrFltr *RSRFilter) Pass(val string) bool { + if rsrFltr.filterRule == "" { + return !rsrFltr.negative + } + if rsrFltr.filterRule[:1] == REGEXP_PREFIX { + return rsrFltr.fltrRgxp.MatchString(val) != rsrFltr.negative + } + if rsrFltr.filterRule[:1] == MatchStartPrefix { + return strings.HasPrefix(val, rsrFltr.filterRule[1:]) != rsrFltr.negative + } + lastIdx := len(rsrFltr.filterRule) - 1 + if rsrFltr.filterRule[lastIdx:] == MatchEndPrefix { + return strings.HasSuffix(val, rsrFltr.filterRule[:lastIdx]) != rsrFltr.negative + } + return val == rsrFltr.filterRule != rsrFltr.negative } // Parses list of RSRFields, used for example as multiple filters in derived charging diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index cc03bd21f..65841d0ee 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -34,7 +34,8 @@ func TestNewRSRField1(t *testing.T) { t.Errorf("Expecting: %v, received: %v", expRSRField1, rsrField) } // With filter - expRSRField2 := &RSRField{Id: "sip_redirected_to", filterValue: "086517174963", + filter, _ := NewRSRFilter("086517174963") + expRSRField2 := &RSRField{Id: "sip_redirected_to", filters: []*RSRFilter{filter}, RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}} if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)@/0$1/(086517174963)`); err != nil { t.Error("Unexpected error: ", err.Error()) @@ -232,3 +233,109 @@ func TestRSRCostDetails(t *testing.T) { t.Errorf("Expecting: Canada, received: %s", parsedVal) } } + +func TestRSRFilterPass(t *testing.T) { + fltr, err := NewRSRFilter("") // Pass any + if err != nil { + t.Error(err) + } + if !fltr.Pass("") { + t.Error("Not passing!") + } + if !fltr.Pass("any") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("!") // Pass nothing + if err != nil { + t.Error(err) + } + if fltr.Pass("") { + t.Error("Passing!") + } + if fltr.Pass("any") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("full_match") // Full string pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("full_match") { + t.Error("Not passing!") + } + if fltr.Pass("full_match1") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("^prefixMatch") // Prefix pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("prefixMatch") { + t.Error("Not passing!") + } + if !fltr.Pass("prefixMatch12345") { + t.Error("Not passing!") + } + if fltr.Pass("1prefixMatch") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("suffixMatch$") // Suffix pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("suffixMatch") { + t.Error("Not passing!") + } + if !fltr.Pass("12345suffixMatch") { + t.Error("Not passing!") + } + if fltr.Pass("suffixMatch1") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!fullMatch") // Negative full pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("ShouldMatch") { + t.Error("Not passing!") + } + if fltr.Pass("fullMatch") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!^prefixMatch") // Negative prefix pass + if err != nil { + t.Error(err) + } + if fltr.Pass("prefixMatch123") { + t.Error("Passing!") + } + if !fltr.Pass("123prefixMatch") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("!suffixMatch$") // Negative suffix pass + if err != nil { + t.Error(err) + } + if fltr.Pass("123suffixMatch") { + t.Error("Passing!") + } + if !fltr.Pass("suffixMatch123") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("~^C.+S$") // Regexp pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("CGRateS") { + t.Error("Not passing!") + } + if fltr.Pass("1CGRateS") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!~^C.*S$") // Negative regexp pass + if err != nil { + t.Error(err) + } + if fltr.Pass("CGRateS") { + t.Error("Passing!") + } +}