RegexpSearchReplace rule has now Matched field to confirm matching, DerivedChargers has now Regexp support in filters

cdrc/cdrc.go

@@ -196,7 +196,9 @@ func (self *Cdrc) processFile(filePath string) error {
 	}
 	csvReader := csv.NewReader(bufio.NewReader(file))
 	csvReader.Comma = self.csvSep
+	procRowNr := 0
 	for {
+		procRowNr += 1
 		record, err := csvReader.Read()
 		if err != nil && err == io.EOF {
 			break // End of file
@@ -211,12 +213,12 @@ func (self *Cdrc) processFile(filePath string) error {
 		}
 		if self.cdrsAddress == utils.INTERNAL {
 			if err := self.cdrServer.ProcessRawCdr(storedCdr); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, error: %s", err.Error()))
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, row: %d, error: %s", procRowNr, err.Error()))
 				continue
 			}
 		} else { // CDRs listening on IP
 			if _, err := self.httpClient.PostForm(fmt.Sprintf("http://%s/cgr", self.cdrsAddress), storedCdr.AsHttpForm()); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, error: %s", err.Error()))
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, row: %d, error: %s", procRowNr, err.Error()))
 				continue
 			}
 		}

cdrs/fscdr_test.go

@@ -95,7 +95,7 @@ func TestSearchExtraFieldInSlice(t *testing.T) {
 func TestSearchReplaceInExtraFields(t *testing.T) {
 	cfg, _ = config.NewDefaultCGRConfig()
 	cfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "read_codec"},
-		&utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`([A-Za-z]*).+`), "$1"}}},
+		&utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`([A-Za-z]*).+`), ReplaceTemplate: "$1"}}},
 		&utils.RSRField{Id: "write_codec"}}
 	fsCdr, _ := NewFSCdr(body)
 	extraFields := fsCdr.getExtraFields()
@@ -148,7 +148,7 @@ extra_fields =  ~effective_caller_id_number:s/(\d+)/+$1/
 	if err != nil {
 		t.Error("Could not parse the config", err.Error())
 	} else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number",
-		RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) {
+		RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) {
 		t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields)
 	}
 	fsCdr, err := NewFSCdr(simpleJsonCdr)

config/config_test.go

@@ -290,7 +290,7 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/
 	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
 		t.Error("Could not parse the config", err.Error())
 	} else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number",
-		RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) {
+		RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) {
 		t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields)
 	}
 	eFieldsCfg = []byte(`[cdrs]

config/helpers_test.go

@@ -38,7 +38,8 @@ func TestConfigSlice(t *testing.T) {
 func TestParseRSRFields(t *testing.T) {
 	fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination`
 	expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"},
-		&utils.RSRField{Id: "sip_redirected_to", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}},
+		&utils.RSRField{Id: "sip_redirected_to",
+			RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}},
 		&utils.RSRField{Id: "destination"}}
 	if parsedFields, err := ParseRSRFields(fields); err != nil {
 		t.Error("Unexpected error: ", err.Error())

utils/researchreplace.go

@@ -26,6 +26,7 @@ import (
 type ReSearchReplace struct {
 	SearchRegexp    *regexp.Regexp
 	ReplaceTemplate string
+	Matched         bool
 }
 
 func (rsr *ReSearchReplace) Process(source string) string {
@@ -36,6 +37,8 @@ func (rsr *ReSearchReplace) Process(source string) string {
 	match := rsr.SearchRegexp.FindStringSubmatchIndex(source)
 	if match == nil {
 		return source // No match returns unaltered source, so we can play with national vs international dialing
+	} else {
+		rsr.Matched = true
 	}
 	res = rsr.SearchRegexp.ExpandString(res, rsr.ReplaceTemplate, source, match)
 	return string(res)

utils/researchreplace_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestProcessReSearchReplace(t *testing.T) {
-	rsr := &ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), "0$1@$2"}
+	rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), ReplaceTemplate: "0$1@$2"}
 	source := "<sip:+4986517174963@127.0.0.1;transport=tcp>"
 	expectOut := "086517174963@127.0.0.1"
 	if outStr := rsr.Process(source); outStr != expectOut {
@@ -33,7 +33,7 @@ func TestProcessReSearchReplace(t *testing.T) {
 }
 
 func TestProcessReSearchReplace2(t *testing.T) {
-	rsr := &ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}
+	rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}
 	source := "4986517174963"
 	expectOut := "+4986517174963"
 	if outStr := rsr.Process(source); outStr != expectOut {
@@ -42,7 +42,7 @@ func TestProcessReSearchReplace2(t *testing.T) {
 }
 
 func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE080"
-	rsr := &ReSearchReplace{regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), "$1"}
+	rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), ReplaceTemplate: "$1"}
 	source := `[{"TimeStart":"2014-04-15T22:17:57+02:00","TimeEnd":"2014-04-15T22:18:01+02:00","Cost":0,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"Rates":[{"GroupIntervalStart":0,"Value":0,"RateIncrement":1000000000,"RateUnit":60000000000}],"RoundingMethod":"*middle","RoundingDecimals":4},"Weight":10},"CallDuration":4000000000,"Increments":null,"MatchedSubject":"*out:sip.test.cgrates.org:call:*any","MatchedPrefix":"+49800","MatchedDestId":"CST_31800_DE080"}]`
 	expectOut := "DE080"
 	if outStr := rsr.Process(source); outStr != expectOut {
@@ -51,7 +51,7 @@ func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE
 }
 
 func TestProcessReSearchReplace4(t *testing.T) {
-	rsr := &ReSearchReplace{regexp.MustCompile(`^\+49(\d+)`), "0$1"}
+	rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^\+49(\d+)`), ReplaceTemplate: "0$1"}
 	if outStr := rsr.Process("+4986517174963"); outStr != "086517174963" {
 		t.Error("Unexpected output from SearchReplace: ", outStr)
 	}

utils/rsrfield.go

@@ -82,3 +82,16 @@ func (rsrf *RSRField) ParseValue(value string) string {
 	}
 	return value
 }
+
+func (rsrf *RSRField) IsStatic() bool {
+	return len(rsrf.staticValue) != 0
+}
+
+func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a regexp match through the rules
+	for _, rsrule := range rsrf.RSRules {
+		if rsrule.Matched {
+			return true
+		}
+	}
+	return false
+}

utils/rsrfield_test.go

@@ -26,7 +26,8 @@ import (
 
 func TestNewRSRField1(t *testing.T) {
 	// Normal case
-	expRSRField1 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}}
+	expRSRField1 := &RSRField{Id: "sip_redirected_to",
+		RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}
 	if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)@/0$1/`); err != nil {
 		t.Error("Unexpected error: ", err.Error())
 	} else if !reflect.DeepEqual(expRSRField1, rsrField) {
@@ -37,7 +38,8 @@ func TestNewRSRField1(t *testing.T) {
 		t.Error("Parse error, field rule does not contain correct number of separators, received: %v", rsrField)
 	}
 	// One extra separator but escaped
-	expRSRField3 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)\/@`), "0$1"}}}
+	expRSRField3 := &RSRField{Id: "sip_redirected_to",
+		RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)\/@`), ReplaceTemplate: "0$1"}}}
 	if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)\/@/0$1/`); err != nil {
 		t.Error("Unexpected error: ", err.Error())
 	} else if !reflect.DeepEqual(expRSRField3, rsrField) {
@@ -46,7 +48,8 @@ func TestNewRSRField1(t *testing.T) {
 }
 
 func TestNewRSRFieldDDz(t *testing.T) {
-	expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}
+	expectRSRField := &RSRField{Id: "effective_caller_id_number",
+		RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}
 	if rsrField, err := NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(rsrField, expectRSRField) {
@@ -55,7 +58,8 @@ func TestNewRSRFieldDDz(t *testing.T) {
 }
 
 func TestNewRSRFieldIvo(t *testing.T) {
-	expectRSRField := &RSRField{Id: "cost_details", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), "$1"}}}
+	expectRSRField := &RSRField{Id: "cost_details",
+		RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), ReplaceTemplate: "$1"}}}
 	if rsrField, err := NewRSRField(`~cost_details:s/MatchedDestId":".+_(\s\s\s\s\s)"/$1/`); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(rsrField, expectRSRField) {
@@ -65,8 +69,8 @@ func TestNewRSRFieldIvo(t *testing.T) {
 
 func TestConvertPlusNationalAnd00(t *testing.T) {
 	expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{
-		&ReSearchReplace{regexp.MustCompile(`\+49(\d+)`), "0$1"},
-		&ReSearchReplace{regexp.MustCompile(`\+(\d+)`), "00$1"}}}
+		&ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+49(\d+)`), ReplaceTemplate: "0$1"},
+		&ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+(\d+)`), ReplaceTemplate: "00$1"}}}
 	rsrField, err := NewRSRField(`~effective_caller_id_number:s/\+49(\d+)/0$1/:s/\+(\d+)/00$1/`)
 	if err != nil {
 		t.Error(err)
@@ -100,7 +104,7 @@ func TestRSRParseStatic(t *testing.T) {
 
 func TestConvertDurToSecs(t *testing.T) {
 	expectRSRField := &RSRField{Id: "9", RSRules: []*ReSearchReplace{
-		&ReSearchReplace{regexp.MustCompile(`^(\d+)$`), "${1}s"}}}
+		&ReSearchReplace{SearchRegexp: regexp.MustCompile(`^(\d+)$`), ReplaceTemplate: "${1}s"}}}
 	rsrField, err := NewRSRField(`~9:s/^(\d+)$/${1}s/`)
 	if err != nil {
 		t.Error(err)
@@ -114,7 +118,7 @@ func TestConvertDurToSecs(t *testing.T) {
 
 func TestPrefix164(t *testing.T) {
 	expectRSRField := &RSRField{Id: "0", RSRules: []*ReSearchReplace{
-		&ReSearchReplace{regexp.MustCompile(`^([1-9]\d+)$`), "+$1"}}}
+		&ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}}
 	rsrField, err := NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`)
 	if err != nil {
 		t.Error(err)
@@ -125,3 +129,14 @@ func TestPrefix164(t *testing.T) {
 		t.Errorf("Expecting: +4986517174960, received: %s", parsedVal)
 	}
 }
+
+func TestIsStatic(t *testing.T) {
+	rsr1 := &RSRField{Id: "0", staticValue: "0"}
+	if !rsr1.IsStatic() {
+		t.Error("Failed to detect static value.")
+	}
+	rsr2 := &RSRField{Id: "0", RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}}
+	if rsr2.IsStatic() {
+		t.Error("Non static detected as static value")
+	}
+}

utils/storedcdr.go

@@ -137,8 +137,15 @@ func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool {
 	if fieldFilter == nil {
 		return true
 	}
-	if storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) && len(storedCdr.FieldAsString(fieldFilter)) != 0 {
-		// Field value must be non empty in order to declare it filtered, otherwise filter makes no sense
+	if fieldFilter.IsStatic() && storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) {
+		return true
+	}
+	preparedFilter := &RSRField{Id: fieldFilter.Id, RSRules: make([]*ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter
+	for idx := range fieldFilter.RSRules {
+		// Hardcode the template with maximum of 5 groups ordered
+		preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"}
+	}
+	if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() {
 		return true
 	}
 	return false

utils/storedcdr_test.go

@@ -79,8 +79,10 @@ func TestFieldAsString(t *testing.T) {
 }
 
 func TestPassesFieldFilter(t *testing.T) {
-	cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
-		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID,
+	cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf",
+		CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID,
 		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
 	}
 	if !cdr.PassesFieldFilter(nil) {
@@ -90,6 +92,22 @@ func TestPassesFieldFilter(t *testing.T) {
 	if !cdr.PassesFieldFilter(acntPrefxFltr) {
 		t.Error("Not passing filter")
 	}
+	acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/10/`)
+	if !cdr.PassesFieldFilter(acntPrefxFltr) {
+		t.Error("Not passing valid filter")
+	}
+	acntPrefxFltr, _ = NewRSRField(`~account:s/^\d(10)\d$/10/`)
+	if cdr.PassesFieldFilter(acntPrefxFltr) {
+		t.Error("Passing filter")
+	}
+	acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/010/`)
+	if cdr.PassesFieldFilter(acntPrefxFltr) {
+		t.Error("Passing filter")
+	}
+	acntPrefxFltr, _ = NewRSRField(`~account:s/^1010$/1010/`)
+	if cdr.PassesFieldFilter(acntPrefxFltr) {
+		t.Error("Passing filter")
+	}
 	torFltr, _ := NewRSRField(`^tor/*voice`)
 	if !cdr.PassesFieldFilter(torFltr) {
 		t.Error("Not passing filter")