From 28c27c88635cbbc2acaf580a022ac748518282a3 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 17:48:59 +0200 Subject: [PATCH] RegexpSearchReplace rule has now Matched field to confirm matching, DerivedChargers has now Regexp support in filters --- cdrc/cdrc.go | 6 ++++-- cdrs/fscdr_test.go | 4 ++-- config/config_test.go | 2 +- config/helpers_test.go | 3 ++- utils/researchreplace.go | 3 +++ utils/researchreplace_test.go | 8 ++++---- utils/rsrfield.go | 13 +++++++++++++ utils/rsrfield_test.go | 31 +++++++++++++++++++++++-------- utils/storedcdr.go | 11 +++++++++-- utils/storedcdr_test.go | 22 ++++++++++++++++++++-- 10 files changed, 81 insertions(+), 22 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index f485dc61f..c9b641092 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -196,7 +196,9 @@ func (self *Cdrc) processFile(filePath string) error { } csvReader := csv.NewReader(bufio.NewReader(file)) csvReader.Comma = self.csvSep + procRowNr := 0 for { + procRowNr += 1 record, err := csvReader.Read() if err != nil && err == io.EOF { break // End of file @@ -211,12 +213,12 @@ func (self *Cdrc) processFile(filePath string) error { } if self.cdrsAddress == utils.INTERNAL { if err := self.cdrServer.ProcessRawCdr(storedCdr); err != nil { - engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, error: %s", err.Error())) + engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, row: %d, error: %s", procRowNr, err.Error())) continue } } else { // CDRs listening on IP if _, err := self.httpClient.PostForm(fmt.Sprintf("http://%s/cgr", self.cdrsAddress), storedCdr.AsHttpForm()); err != nil { - engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, error: %s", err.Error())) + engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, row: %d, error: %s", procRowNr, err.Error())) continue } } diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 4ac83b1b8..1c4ffe2fb 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -95,7 +95,7 @@ func TestSearchExtraFieldInSlice(t *testing.T) { func TestSearchReplaceInExtraFields(t *testing.T) { cfg, _ = config.NewDefaultCGRConfig() cfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "read_codec"}, - &utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`([A-Za-z]*).+`), "$1"}}}, + &utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`([A-Za-z]*).+`), ReplaceTemplate: "$1"}}}, &utils.RSRField{Id: "write_codec"}} fsCdr, _ := NewFSCdr(body) extraFields := fsCdr.getExtraFields() @@ -148,7 +148,7 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/ if err != nil { t.Error("Could not parse the config", err.Error()) } else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number", - RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) { + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) { t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields) } fsCdr, err := NewFSCdr(simpleJsonCdr) diff --git a/config/config_test.go b/config/config_test.go index dc90fd857..1c4eaab56 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -290,7 +290,7 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/ if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { t.Error("Could not parse the config", err.Error()) } else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number", - RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) { + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) { t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields) } eFieldsCfg = []byte(`[cdrs] diff --git a/config/helpers_test.go b/config/helpers_test.go index 13aaeff36..31aa4747c 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -38,7 +38,8 @@ func TestConfigSlice(t *testing.T) { func TestParseRSRFields(t *testing.T) { fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"}, - &utils.RSRField{Id: "sip_redirected_to", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}}, + &utils.RSRField{Id: "sip_redirected_to", + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, &utils.RSRField{Id: "destination"}} if parsedFields, err := ParseRSRFields(fields); err != nil { t.Error("Unexpected error: ", err.Error()) diff --git a/utils/researchreplace.go b/utils/researchreplace.go index 8cb8e6ea6..66e226877 100644 --- a/utils/researchreplace.go +++ b/utils/researchreplace.go @@ -26,6 +26,7 @@ import ( type ReSearchReplace struct { SearchRegexp *regexp.Regexp ReplaceTemplate string + Matched bool } func (rsr *ReSearchReplace) Process(source string) string { @@ -36,6 +37,8 @@ func (rsr *ReSearchReplace) Process(source string) string { match := rsr.SearchRegexp.FindStringSubmatchIndex(source) if match == nil { return source // No match returns unaltered source, so we can play with national vs international dialing + } else { + rsr.Matched = true } res = rsr.SearchRegexp.ExpandString(res, rsr.ReplaceTemplate, source, match) return string(res) diff --git a/utils/researchreplace_test.go b/utils/researchreplace_test.go index 28c113ce8..44f5c52fd 100644 --- a/utils/researchreplace_test.go +++ b/utils/researchreplace_test.go @@ -24,7 +24,7 @@ import ( ) func TestProcessReSearchReplace(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), "0$1@$2"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), ReplaceTemplate: "0$1@$2"} source := "" expectOut := "086517174963@127.0.0.1" if outStr := rsr.Process(source); outStr != expectOut { @@ -33,7 +33,7 @@ func TestProcessReSearchReplace(t *testing.T) { } func TestProcessReSearchReplace2(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"} source := "4986517174963" expectOut := "+4986517174963" if outStr := rsr.Process(source); outStr != expectOut { @@ -42,7 +42,7 @@ func TestProcessReSearchReplace2(t *testing.T) { } func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE080" - rsr := &ReSearchReplace{regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), "$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), ReplaceTemplate: "$1"} source := `[{"TimeStart":"2014-04-15T22:17:57+02:00","TimeEnd":"2014-04-15T22:18:01+02:00","Cost":0,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"Rates":[{"GroupIntervalStart":0,"Value":0,"RateIncrement":1000000000,"RateUnit":60000000000}],"RoundingMethod":"*middle","RoundingDecimals":4},"Weight":10},"CallDuration":4000000000,"Increments":null,"MatchedSubject":"*out:sip.test.cgrates.org:call:*any","MatchedPrefix":"+49800","MatchedDestId":"CST_31800_DE080"}]` expectOut := "DE080" if outStr := rsr.Process(source); outStr != expectOut { @@ -51,7 +51,7 @@ func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE } func TestProcessReSearchReplace4(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`^\+49(\d+)`), "0$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^\+49(\d+)`), ReplaceTemplate: "0$1"} if outStr := rsr.Process("+4986517174963"); outStr != "086517174963" { t.Error("Unexpected output from SearchReplace: ", outStr) } diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 4af340f7f..56731428c 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -82,3 +82,16 @@ func (rsrf *RSRField) ParseValue(value string) string { } return value } + +func (rsrf *RSRField) IsStatic() bool { + return len(rsrf.staticValue) != 0 +} + +func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a regexp match through the rules + for _, rsrule := range rsrf.RSRules { + if rsrule.Matched { + return true + } + } + return false +} diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 7b078bb89..7b215f161 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -26,7 +26,8 @@ import ( func TestNewRSRField1(t *testing.T) { // Normal case - expRSRField1 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}} + expRSRField1 := &RSRField{Id: "sip_redirected_to", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}} if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)@/0$1/`); err != nil { t.Error("Unexpected error: ", err.Error()) } else if !reflect.DeepEqual(expRSRField1, rsrField) { @@ -37,7 +38,8 @@ func TestNewRSRField1(t *testing.T) { t.Error("Parse error, field rule does not contain correct number of separators, received: %v", rsrField) } // One extra separator but escaped - expRSRField3 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)\/@`), "0$1"}}} + expRSRField3 := &RSRField{Id: "sip_redirected_to", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)\/@`), ReplaceTemplate: "0$1"}}} if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)\/@/0$1/`); err != nil { t.Error("Unexpected error: ", err.Error()) } else if !reflect.DeepEqual(expRSRField3, rsrField) { @@ -46,7 +48,8 @@ func TestNewRSRField1(t *testing.T) { } func TestNewRSRFieldDDz(t *testing.T) { - expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}} + expectRSRField := &RSRField{Id: "effective_caller_id_number", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}} if rsrField, err := NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, expectRSRField) { @@ -55,7 +58,8 @@ func TestNewRSRFieldDDz(t *testing.T) { } func TestNewRSRFieldIvo(t *testing.T) { - expectRSRField := &RSRField{Id: "cost_details", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), "$1"}}} + expectRSRField := &RSRField{Id: "cost_details", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), ReplaceTemplate: "$1"}}} if rsrField, err := NewRSRField(`~cost_details:s/MatchedDestId":".+_(\s\s\s\s\s)"/$1/`); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, expectRSRField) { @@ -65,8 +69,8 @@ func TestNewRSRFieldIvo(t *testing.T) { func TestConvertPlusNationalAnd00(t *testing.T) { expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`\+49(\d+)`), "0$1"}, - &ReSearchReplace{regexp.MustCompile(`\+(\d+)`), "00$1"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+49(\d+)`), ReplaceTemplate: "0$1"}, + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+(\d+)`), ReplaceTemplate: "00$1"}}} rsrField, err := NewRSRField(`~effective_caller_id_number:s/\+49(\d+)/0$1/:s/\+(\d+)/00$1/`) if err != nil { t.Error(err) @@ -100,7 +104,7 @@ func TestRSRParseStatic(t *testing.T) { func TestConvertDurToSecs(t *testing.T) { expectRSRField := &RSRField{Id: "9", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`^(\d+)$`), "${1}s"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^(\d+)$`), ReplaceTemplate: "${1}s"}}} rsrField, err := NewRSRField(`~9:s/^(\d+)$/${1}s/`) if err != nil { t.Error(err) @@ -114,7 +118,7 @@ func TestConvertDurToSecs(t *testing.T) { func TestPrefix164(t *testing.T) { expectRSRField := &RSRField{Id: "0", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`^([1-9]\d+)$`), "+$1"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}} rsrField, err := NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`) if err != nil { t.Error(err) @@ -125,3 +129,14 @@ func TestPrefix164(t *testing.T) { t.Errorf("Expecting: +4986517174960, received: %s", parsedVal) } } + +func TestIsStatic(t *testing.T) { + rsr1 := &RSRField{Id: "0", staticValue: "0"} + if !rsr1.IsStatic() { + t.Error("Failed to detect static value.") + } + rsr2 := &RSRField{Id: "0", RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}} + if rsr2.IsStatic() { + t.Error("Non static detected as static value") + } +} diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 733b84d80..2f5f13412 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -137,8 +137,15 @@ func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { if fieldFilter == nil { return true } - if storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) && len(storedCdr.FieldAsString(fieldFilter)) != 0 { - // Field value must be non empty in order to declare it filtered, otherwise filter makes no sense + if fieldFilter.IsStatic() && storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) { + return true + } + preparedFilter := &RSRField{Id: fieldFilter.Id, RSRules: make([]*ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter + for idx := range fieldFilter.RSRules { + // Hardcode the template with maximum of 5 groups ordered + preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"} + } + if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() { return true } return false diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 6b5c9022a..d6b2f7a84 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -79,8 +79,10 @@ func TestFieldAsString(t *testing.T) { } func TestPassesFieldFilter(t *testing.T) { - cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", - Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } if !cdr.PassesFieldFilter(nil) { @@ -90,6 +92,22 @@ func TestPassesFieldFilter(t *testing.T) { if !cdr.PassesFieldFilter(acntPrefxFltr) { t.Error("Not passing filter") } + acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/10/`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing valid filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^\d(10)\d$/10/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/010/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^1010$/1010/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } torFltr, _ := NewRSRField(`^tor/*voice`) if !cdr.PassesFieldFilter(torFltr) { t.Error("Not passing filter")