Advanced RSRFilters, fixes #241

cdrc/fwv_test.go

@@ -46,7 +46,7 @@ func TestFwvRecordPassesCfgFilter(t *testing.T) {
 	//record, configKey string) bool {
 	cgrConfig, _ := config.NewDefaultCGRConfig()
 	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters
-	cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(+49123123120)`, utils.INFIELD_SEP)
+	cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(^+49123123120)`, utils.INFIELD_SEP)
 	fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]}
 	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
 	if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter {

engine/pubsub_test.go

@@ -215,7 +215,7 @@ func TestCgrEventPassFilters(t *testing.T) {
 	if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("^EventName::DUMMY", utils.INFIELD_SEP)) { // Should pass since we have no filter defined
 		t.Error("Not passing no filter")
 	}
-	if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(\\w*)_/$1/(TEST)", utils.INFIELD_SEP)) {
+	if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(.*)_/$1/(TEST)", utils.INFIELD_SEP)) {
 		t.Error("Not passing filter")
 	}
 	if !ev.PassFilters(utils.ParseRSRFieldsMustCompile("~EventName:s/^(\\w*)_/$1/:s/^(\\w)(\\w)(\\w)(\\w)/$1$3$4/(TST)", utils.INFIELD_SEP)) {

utils/consts.go

@@ -258,6 +258,9 @@ const (
 	TRIGGER_BALANCE_EXPIRED     = "*balance_expired"
 	HIERARCHY_SEP               = ">"
 	META_COMPOSED               = "*composed"
+	NegativePrefix              = "!"
+	MatchStartPrefix            = "^"
+	MatchEndPrefix              = "$"
 )
 
 var (

utils/researchreplace_test.go

@@ -59,3 +59,10 @@ func TestProcessReSearchReplace4(t *testing.T) {
 		t.Error("Unexpected output from SearchReplace: ", outStr)
 	}
 }
+
+func TestProcessReSearchReplace5(t *testing.T) {
+	rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^(.*)_`), ReplaceTemplate: "$1"}
+	if outStr := rsr.Process("TEST_EVENT"); outStr != "TEST" {
+		t.Error("Unexpected output from SearchReplace: ", outStr)
+	}
+}

utils/rsrfield.go

@@ -28,13 +28,19 @@ func NewRSRField(fldStr string) (*RSRField, error) {
 	if len(fldStr) == 0 {
 		return nil, nil
 	}
-	var filterVal string
+	var filters []*RSRFilter
 	if strings.HasSuffix(fldStr, FILTER_VAL_END) { // Has filter, populate the var
 		fltrStart := strings.LastIndex(fldStr, FILTER_VAL_START)
 		if fltrStart < 1 {
 			return nil, fmt.Errorf("Invalid FilterStartValue in string: %s", fldStr)
 		}
-		filterVal = fldStr[fltrStart+1 : len(fldStr)-1]
+		for _, fltrVal := range strings.Split(fldStr[fltrStart+1:len(fldStr)-1], INFIELD_SEP) {
+			if rsrFltr, err := NewRSRFilter(fltrVal); err != nil {
+				return nil, fmt.Errorf("Invalid FilterValue in string: %s, err: %s", fltrVal, err.Error())
+			} else {
+				filters = append(filters, rsrFltr)
+			}
+		}
 		fldStr = fldStr[:fltrStart] // Take the filter part out before compiling further
 
 	}
@@ -51,17 +57,17 @@ func NewRSRField(fldStr string) (*RSRField, error) {
 		} else {
 			staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix
 		}
-		return &RSRField{Id: staticHdr, staticValue: staticVal, filterValue: filterVal}, nil
+		return &RSRField{Id: staticHdr, staticValue: staticVal, filters: filters}, nil
 	}
 	if !strings.HasPrefix(fldStr, REGEXP_PREFIX) {
-		return &RSRField{Id: fldStr, filterValue: filterVal}, nil
+		return &RSRField{Id: fldStr, filters: filters}, nil
 	}
 	spltRgxp := regexp.MustCompile(`:s\/`)
 	spltRules := spltRgxp.Split(fldStr, -1)
 	if len(spltRules) < 2 {
 		return nil, fmt.Errorf("Invalid Split of Search&Replace field rule. %s", fldStr)
 	}
-	rsrField := &RSRField{Id: spltRules[0][1:], filterValue: filterVal} // Original id in form ~hdr_name
+	rsrField := &RSRField{Id: spltRules[0][1:], filters: filters} // Original id in form ~hdr_name
 	rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.*[^\\])*\/){1,}`)
 	for _, ruleStr := range spltRules[1:] { // :s/ already removed through split
 		allMatches := rulesRgxp.FindStringSubmatch(ruleStr)
@@ -81,7 +87,7 @@ type RSRField struct {
 	Id          string             //  Identifier
 	RSRules     []*ReSearchReplace // Rules to use when processing field value
 	staticValue string             // If defined, enforces parsing always to this value
-	filterValue string             // The value to compare when used as filter
+	filters     []*RSRFilter       // The value to compare when used as filter
 }
 
 // Parse the field value from a string
@@ -111,7 +117,63 @@ func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a reg
 }
 
 func (rsrf *RSRField) FilterPasses(value string) bool {
-	return len(rsrf.filterValue) == 0 || rsrf.ParseValue(value) == rsrf.filterValue
+	if len(rsrf.filters) == 0 { // No filters
+		return true
+	}
+	parsedVal := rsrf.ParseValue(value)
+	filterPasses := false
+	for _, fltr := range rsrf.filters {
+		if fltr.Pass(parsedVal) {
+			filterPasses = true
+		}
+	}
+	return filterPasses
+}
+
+// NewRSRFilter instantiates a new RSRFilter, setting it's properties
+func NewRSRFilter(fltrVal string) (rsrFltr *RSRFilter, err error) {
+	rsrFltr = new(RSRFilter)
+	if fltrVal == "" {
+		return rsrFltr, nil
+	}
+	if fltrVal[:1] == NegativePrefix {
+		rsrFltr.negative = true
+		fltrVal = fltrVal[1:]
+		if fltrVal == "" {
+			return rsrFltr, nil
+		}
+	}
+	rsrFltr.filterRule = fltrVal
+	if fltrVal[:1] == REGEXP_PREFIX {
+		if rsrFltr.fltrRgxp, err = regexp.Compile(fltrVal[1:]); err != nil {
+			return nil, err
+		}
+	}
+	return rsrFltr, nil
+}
+
+// One filter rule
+type RSRFilter struct {
+	filterRule string // Value in raw format
+	fltrRgxp   *regexp.Regexp
+	negative   bool // Rule should not match
+}
+
+func (rsrFltr *RSRFilter) Pass(val string) bool {
+	if rsrFltr.filterRule == "" {
+		return !rsrFltr.negative
+	}
+	if rsrFltr.filterRule[:1] == REGEXP_PREFIX {
+		return rsrFltr.fltrRgxp.MatchString(val) != rsrFltr.negative
+	}
+	if rsrFltr.filterRule[:1] == MatchStartPrefix {
+		return strings.HasPrefix(val, rsrFltr.filterRule[1:]) != rsrFltr.negative
+	}
+	lastIdx := len(rsrFltr.filterRule) - 1
+	if rsrFltr.filterRule[lastIdx:] == MatchEndPrefix {
+		return strings.HasSuffix(val, rsrFltr.filterRule[:lastIdx]) != rsrFltr.negative
+	}
+	return val == rsrFltr.filterRule != rsrFltr.negative
 }
 
 // Parses list of RSRFields, used for example as multiple filters in derived charging

utils/rsrfield_test.go

@@ -34,7 +34,8 @@ func TestNewRSRField1(t *testing.T) {
 		t.Errorf("Expecting: %v, received: %v", expRSRField1, rsrField)
 	}
 	// With filter
-	expRSRField2 := &RSRField{Id: "sip_redirected_to", filterValue: "086517174963",
+	filter, _ := NewRSRFilter("086517174963")
+	expRSRField2 := &RSRField{Id: "sip_redirected_to", filters: []*RSRFilter{filter},
 		RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}
 	if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)@/0$1/(086517174963)`); err != nil {
 		t.Error("Unexpected error: ", err.Error())
@@ -232,3 +233,109 @@ func TestRSRCostDetails(t *testing.T) {
 		t.Errorf("Expecting: Canada, received: %s", parsedVal)
 	}
 }
+
+func TestRSRFilterPass(t *testing.T) {
+	fltr, err := NewRSRFilter("") // Pass any
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("") {
+		t.Error("Not passing!")
+	}
+	if !fltr.Pass("any") {
+		t.Error("Not passing!")
+	}
+	fltr, err = NewRSRFilter("!") // Pass nothing
+	if err != nil {
+		t.Error(err)
+	}
+	if fltr.Pass("") {
+		t.Error("Passing!")
+	}
+	if fltr.Pass("any") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("full_match") // Full string pass
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("full_match") {
+		t.Error("Not passing!")
+	}
+	if fltr.Pass("full_match1") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("^prefixMatch") // Prefix pass
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("prefixMatch") {
+		t.Error("Not passing!")
+	}
+	if !fltr.Pass("prefixMatch12345") {
+		t.Error("Not passing!")
+	}
+	if fltr.Pass("1prefixMatch") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("suffixMatch$") // Suffix pass
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("suffixMatch") {
+		t.Error("Not passing!")
+	}
+	if !fltr.Pass("12345suffixMatch") {
+		t.Error("Not passing!")
+	}
+	if fltr.Pass("suffixMatch1") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("!fullMatch") // Negative full pass
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("ShouldMatch") {
+		t.Error("Not passing!")
+	}
+	if fltr.Pass("fullMatch") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("!^prefixMatch") // Negative prefix pass
+	if err != nil {
+		t.Error(err)
+	}
+	if fltr.Pass("prefixMatch123") {
+		t.Error("Passing!")
+	}
+	if !fltr.Pass("123prefixMatch") {
+		t.Error("Not passing!")
+	}
+	fltr, err = NewRSRFilter("!suffixMatch$") // Negative suffix pass
+	if err != nil {
+		t.Error(err)
+	}
+	if fltr.Pass("123suffixMatch") {
+		t.Error("Passing!")
+	}
+	if !fltr.Pass("suffixMatch123") {
+		t.Error("Not passing!")
+	}
+	fltr, err = NewRSRFilter("~^C.+S$") // Regexp pass
+	if err != nil {
+		t.Error(err)
+	}
+	if !fltr.Pass("CGRateS") {
+		t.Error("Not passing!")
+	}
+	if fltr.Pass("1CGRateS") {
+		t.Error("Passing!")
+	}
+	fltr, err = NewRSRFilter("!~^C.*S$") // Negative regexp pass
+	if err != nil {
+		t.Error(err)
+	}
+	if fltr.Pass("CGRateS") {
+		t.Error("Passing!")
+	}
+}