From 6fd40f72961d81b8c4d4ee42a06e894e0de27b3c Mon Sep 17 00:00:00 2001 From: Trial97 Date: Wed, 18 Nov 2020 13:05:36 +0200 Subject: [PATCH] Replaced RSRField with RSRParser --- config/cdrscfg.go | 8 +- config/cdrscfg_test.go | 11 +- config/config_it_test.go | 12 +- config/config_test.go | 1 + config/rsrparser_test.go | 298 ++++++++ console/parse.go | 8 +- data/conf/samples/tutmongo2/cgrates.json | 2 +- data/conf/samples/tutmongo2_gob/cgrates.json | 2 +- data/conf/samples/tutmysql2/cgrates.json | 2 +- data/conf/samples/tutmysql2_gob/cgrates.json | 2 +- docs/ers.rst | 2 +- engine/fscdr.go | 39 +- engine/fscdr_test.go | 61 +- packages/debian/changelog | 1 + utils/rsrfield.go | 393 ---------- utils/rsrfield_test.go | 730 ------------------- utils/rsrfilters.go | 180 +++++ utils/rsrfilters_test.go | 353 +++++++++ 18 files changed, 907 insertions(+), 1198 deletions(-) delete mode 100644 utils/rsrfield.go delete mode 100644 utils/rsrfield_test.go create mode 100644 utils/rsrfilters.go create mode 100644 utils/rsrfilters_test.go diff --git a/config/cdrscfg.go b/config/cdrscfg.go index af4889725..b4f9ed284 100644 --- a/config/cdrscfg.go +++ b/config/cdrscfg.go @@ -25,9 +25,9 @@ import ( ) type CdrsCfg struct { - Enabled bool // Enable CDR Server service - ExtraFields []*utils.RSRField // Extra fields to store in CDRs - StoreCdrs bool // store cdrs in storDb + Enabled bool // Enable CDR Server service + ExtraFields RSRParsers // Extra fields to store in CDRs + StoreCdrs bool // store cdrs in storDb SMCostRetries int ChargerSConns []string RaterConns []string @@ -48,7 +48,7 @@ func (cdrscfg *CdrsCfg) loadFromJsonCfg(jsnCdrsCfg *CdrsJsonCfg) (err error) { cdrscfg.Enabled = *jsnCdrsCfg.Enabled } if jsnCdrsCfg.Extra_fields != nil { - if cdrscfg.ExtraFields, err = utils.ParseRSRFieldsFromSlice(*jsnCdrsCfg.Extra_fields); err != nil { + if cdrscfg.ExtraFields, err = NewRSRParsersFromSlice(*jsnCdrsCfg.Extra_fields); err != nil { return err } } diff --git a/config/cdrscfg_test.go b/config/cdrscfg_test.go index 379dd8878..7d73e02f6 100644 --- a/config/cdrscfg_test.go +++ b/config/cdrscfg_test.go @@ -50,6 +50,7 @@ func TestCdrsCfgloadFromJsonCfg(t *testing.T) { OnlineCDRExports: []string{"randomVal"}, SchedulerConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaScheduler), "*conn1"}, EEsConns: []string{utils.ConcatenatedKey(utils.MetaInternal, utils.MetaEEs), "*conn1"}, + ExtraFields: RSRParsers{}, } if jsnCfg, err := NewDefaultCGRConfig(); err != nil { t.Error(err) @@ -64,7 +65,7 @@ func TestExtraFieldsinloadFromJsonCfg(t *testing.T) { cfgJSON := &CdrsJsonCfg{ Extra_fields: &[]string{utils.EmptyString}, } - expectedErrMessage := "Empty RSRField in rule: " + expectedErrMessage := "emtpy RSRParser in rule: <>" if jsonCfg, err := NewDefaultCGRConfig(); err != nil { t.Error(err) } else if err = jsonCfg.cdrsCfg.loadFromJsonCfg(cfgJSON); err == nil || err.Error() != expectedErrMessage { @@ -76,7 +77,7 @@ func TestCdrsCfgAsMapInterface(t *testing.T) { cfgJSONStr := `{ "cdrs": { "enabled": true, - "extra_fields": ["PayPalAccount", "LCRProfile", "ResourceID"], + "extra_fields": ["~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"], "store_cdrs": true, "session_cost_retries": 5, "chargers_conns":["*internal:*chargers","*conn1"], @@ -91,7 +92,7 @@ func TestCdrsCfgAsMapInterface(t *testing.T) { }` eMap := map[string]interface{}{ utils.EnabledCfg: true, - utils.ExtraFieldsCfg: []string{"PayPalAccount", "LCRProfile", "ResourceID"}, + utils.ExtraFieldsCfg: []string{"~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"}, utils.StoreCdrsCfg: true, utils.SessionCostRetires: 5, utils.ChargerSConnsCfg: []string{utils.MetaInternal, "*conn1"}, @@ -111,7 +112,7 @@ func TestCdrsCfgAsMapInterface(t *testing.T) { } func TestCdrsCfgAsMapInterface2(t *testing.T) { - cfgJsonStr := `{ + cfgJSONStr := `{ "cdrs": { "enabled":true, "chargers_conns": ["conn1", "conn2"], @@ -133,7 +134,7 @@ func TestCdrsCfgAsMapInterface2(t *testing.T) { utils.SchedulerConnsCfg: []string{}, utils.EEsConnsCfg: []string{"conn1"}, } - if cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJsonStr); err != nil { + if cgrCfg, err := NewCGRConfigFromJSONStringWithDefaults(cfgJSONStr); err != nil { t.Error(err) } else if rcv := cgrCfg.cdrsCfg.AsMapInterface(); !reflect.DeepEqual(rcv, eMap) { t.Errorf("Expected %+v \n, recieved %+v", eMap, rcv) diff --git a/config/config_it_test.go b/config/config_it_test.go index e2d960710..1deaca6e4 100644 --- a/config/config_it_test.go +++ b/config/config_it_test.go @@ -328,13 +328,13 @@ func testCGRConfigReloadCDRs(t *testing.T) { } else if reply != utils.OK { t.Errorf("Expected OK received: %s", reply) } + rsr, err := NewRSRParsersFromSlice([]string{"~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"}) + if err != nil { + t.Fatal(err) + } expAttr := &CdrsCfg{ - Enabled: true, - ExtraFields: utils.RSRFields{ - utils.NewRSRFieldMustCompile("PayPalAccount"), - utils.NewRSRFieldMustCompile("LCRProfile"), - utils.NewRSRFieldMustCompile("ResourceID"), - }, + Enabled: true, + ExtraFields: rsr, ChargerSConns: []string{utils.MetaLocalHost}, RaterConns: []string{}, AttributeSConns: []string{}, diff --git a/config/config_test.go b/config/config_test.go index c2b366812..2536391e8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -430,6 +430,7 @@ func TestCgrCfgJSONDefaultsCDRS(t *testing.T) { StatSConns: []string{}, SchedulerConns: []string{}, EEsConns: []string{}, + ExtraFields: RSRParsers{}, } if !reflect.DeepEqual(eCdrsCfg, cgrCfg.cdrsCfg) { t.Errorf("Expecting: %+v , received: %+v", eCdrsCfg, cgrCfg.cdrsCfg) diff --git a/config/rsrparser_test.go b/config/rsrparser_test.go index 05559e082..51ff4b251 100644 --- a/config/rsrparser_test.go +++ b/config/rsrparser_test.go @@ -587,3 +587,301 @@ func TestRSRParsersGetIfaceFromValues(t *testing.T) { t.Error(err) } } + +func TestNewRSRParser(t *testing.T) { + // Normal case + rulesStr := `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/` + expRSRField1 := &RSRParser{ + path: "~sip_redirected_to", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{ + { + SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), + ReplaceTemplate: "0$1", + }, + }, + converters: nil, + } + if rsrField, err := NewRSRParser(rulesStr); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !reflect.DeepEqual(expRSRField1, rsrField) { + t.Errorf("Expecting: %+v, received: %+v", + expRSRField1, rsrField) + } + + // with dataConverters + rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/{*duration_seconds&*round:5:*middle}` + expRSRField := &RSRParser{ + path: "~sip_redirected_to", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), + ReplaceTemplate: "0$1", + }}, + converters: []utils.DataConverter{ + new(utils.DurationSecondsConverter), + &utils.RoundConverter{Decimals: 5, Method: "*middle"}, + }, + } + if rsrField, err := NewRSRParser(rulesStr); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !reflect.DeepEqual(expRSRField, rsrField) { + t.Errorf("Expecting: %+v, received: %+v", expRSRField, rsrField) + } + // One extra separator but escaped + rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)\/@/0$1/` + expRSRField3 := &RSRParser{ + path: "~sip_redirected_to", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)\/@`), + ReplaceTemplate: "0$1", + }}, + } + if rsrField, err := NewRSRParser(rulesStr); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !reflect.DeepEqual(expRSRField3, rsrField) { + t.Errorf("Expecting: %v, received: %v", expRSRField3, rsrField) + } + +} + +func TestNewRSRParserDDz(t *testing.T) { + rulesStr := `~effective_caller_id_number:s/(\d+)/+$1/` + expectRSRField := &RSRParser{ + path: "~effective_caller_id_number", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`(\d+)`), + ReplaceTemplate: "+$1", + }}, + } + if rsrField, err := NewRSRParser(rulesStr); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rsrField, expectRSRField) { + t.Errorf("Unexpected RSRField received: %v", rsrField) + } +} + +func TestNewRSRParserIvo(t *testing.T) { + rulesStr := `~cost_details:s/MatchedDestId":".+_(\s\s\s\s\s)"/$1/` + expectRSRField := &RSRParser{ + path: "~cost_details", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), + ReplaceTemplate: "$1", + }}, + } + if rsrField, err := NewRSRParser(rulesStr); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rsrField, expectRSRField) { + t.Errorf("Unexpected RSRField received: %v", rsrField) + } + if _, err := NewRSRParser(`~account:s/^[A-Za-z0-9]*[c|a]\d{4}$/S/:s/^[A-Za-z0-9]*n\d{4}$/C/:s/^\d{10}$//`); err != nil { + t.Error(err) + } +} + +func TestConvertPlusNationalAnd00(t *testing.T) { + rulesStr := `~effective_caller_id_number:s/\+49(\d+)/0$1/:s/\+(\d+)/00$1/` + expectRSRField := &RSRParser{ + path: "~effective_caller_id_number", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{ + { + SearchRegexp: regexp.MustCompile(`\+49(\d+)`), + ReplaceTemplate: "0$1", + }, + { + SearchRegexp: regexp.MustCompile(`\+(\d+)`), + ReplaceTemplate: "00$1", + }, + }, + } + rsrField, err := NewRSRParser(rulesStr) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rsrField, expectRSRField) { + t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) + } + if parsedVal, err := rsrField.ParseValue("+4986517174963"); err != nil { + t.Error(err) + } else if parsedVal != "086517174963" { + t.Errorf("Expecting: 086517174963, received: %s", parsedVal) + } + if parsedVal, err := rsrField.ParseValue("+3186517174963"); err != nil { + t.Error(err) + } else if parsedVal != "003186517174963" { + t.Errorf("Expecting: 003186517174963, received: %s", parsedVal) + } +} + +func TestConvertDurToSecs(t *testing.T) { + rulesStr := `~9:s/^(\d+)$/${1}s/` + expectRSRField := &RSRParser{ + path: "~9", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`^(\d+)$`), + ReplaceTemplate: "${1}s", + }}, + } + rsrField, err := NewRSRParser(rulesStr) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rsrField, expectRSRField) { + t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) + } + if parsedVal, err := rsrField.ParseValue("640113"); err != nil { + t.Error(err) + } else if parsedVal != "640113s" { + t.Errorf("Expecting: 640113s, received: %s", parsedVal) + } +} + +func TestPrefix164(t *testing.T) { + rulesStr := `~0:s/^([1-9]\d+)$/+$1/` + expectRSRField := &RSRParser{ + path: "~0", + Rules: rulesStr, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), + ReplaceTemplate: "+$1", + }}, + } + rsrField, err := NewRSRParser(rulesStr) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(rsrField, expectRSRField) { + t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) + } + if parsedVal, err := rsrField.ParseValue("4986517174960"); err != nil { + t.Error(err) + } else if parsedVal != "+4986517174960" { + t.Errorf("Expecting: +4986517174960, received: %s", parsedVal) + } +} + +func TestNewRSRParsers2(t *testing.T) { + fieldsStr1 := `~account:s/^\w+[mpls]\d{6}$//;~subject:s/^0\d{9}$//;~mediation_runid:s/^default$/default/` + rsrFld1, err := NewRSRParser(`~account:s/^\w+[mpls]\d{6}$//`) + if err != nil { + t.Fatal(err) + } + rsrFld2, err := NewRSRParser(`~subject:s/^0\d{9}$//`) + if err != nil { + t.Fatal(err) + } + rsrFld4, err := NewRSRParser(`~mediation_runid:s/^default$/default/`) + if err != nil { + t.Fatal(err) + } + eRSRFields := RSRParsers{rsrFld1, rsrFld2, rsrFld4} + if rsrFlds, err := NewRSRParsers(fieldsStr1, utils.INFIELD_SEP); err != nil { + t.Error("Unexpected error: ", err) + } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { + t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) + } + fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` + expectParsedFields := RSRParsers{ + { + path: "host", + Rules: "host", + }, + { + path: "~sip_redirected_to", + Rules: `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/`, + rsrRules: []*utils.ReSearchReplace{{ + SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), + ReplaceTemplate: "0$1", + }}, + }, + { + path: "destination", + Rules: "destination", + }, + } + if parsedFields, err := NewRSRParsers(fields, utils.FIELDS_SEP); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !reflect.DeepEqual(parsedFields, expectParsedFields) { + t.Errorf("Expected: %s ,received: %s ", utils.ToJSON(expectParsedFields), utils.ToJSON(parsedFields)) + } +} + +func TestParseCdrcDn1(t *testing.T) { + rl, err := NewRSRParser(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/:s/^\+49(18\d{2})$/+491400$1/`) + if err != nil { + t.Error("Unexpected error: ", err) + } + if parsed, err := rl.ParseValue("0049ABOC0630415354"); err != nil { + t.Error(err) + } else if parsed != "+49630415354" { + t.Errorf("Expecting: +49630415354, received: %s", parsed) + } + if parsed2, err := rl.ParseValue("00491888"); err != nil { + t.Error(err) + } else if parsed2 != "+4914001888" { + t.Errorf("Expecting: +4914001888, received: %s", parsed2) + } +} + +func TestRSRCostDetails(t *testing.T) { + fieldsStr1 := `{"Category":"default_route","Tenant":"demo.cgrates.org","Subject":"voxbeam_premium","Account":"6335820713","Destination":"15143606781","ToR":"*voice","Cost":0.0007,"Timespans":[{"TimeStart":"2015-08-30T21:46:54Z","TimeEnd":"2015-08-30T21:47:06Z","Cost":0.00072,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"RoundingMethod":"*middle","RoundingDecimals":5,"MaxCost":0,"MaxCostStrategy":"0","Rates":[{"GroupIntervalStart":0,"Value":0.0036,"RateIncrement":6000000000,"RateUnit":60000000000}]},"Weight":10},"DurationIndex":12000000000,"Increments":[{"Duration":6000000000,"Cost":0.00036,"BalanceInfo":{"UnitBalanceUuid":"","MoneyBalanceUuid":"40adda88-25d3-4009-b928-f39d61590439","AccountId":"*out:demo.cgrates.org:6335820713"},"BalanceRateInterval":null,"UnitInfo":null,"CompressFactor":2}],"MatchedSubject":"*out:demo.cgrates.org:default_route:voxbeam_premium","MatchedPrefix":"1514","MatchedDestId":"Canada","RatingPlanId":"RP_VOXBEAM_PREMIUM"}]}` + rsrField, err := NewRSRParser(`~cost_details:s/"MatchedDestId":"(\w+)"/${1}/`) + if err != nil { + t.Error(err) + } + if parsedVal, err := rsrField.ParseValue(fieldsStr1); err != nil { + t.Error(err) + } else if parsedVal != "Canada" { + t.Errorf("Expecting: Canada, received: %s", parsedVal) + } + fieldsStr2 := `{"Category":"call","Tenant":"sip.test.cgrates.org","Subject":"dan","Account":"dan","Destination":"+4986517174963","ToR":"*voice","Cost":0,"Timespans":[{"TimeStart":"2015-05-13T15:03:34+02:00","TimeEnd":"2015-05-13T15:03:38+02:00","Cost":0,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"RoundingMethod":"*middle","RoundingDecimals":4,"MaxCost":0,"MaxCostStrategy":"","Rates":[{"GroupIntervalStart":0,"Value":0,"RateIncrement":1000000000,"RateUnit":60000000000}]},"Weight":10},"DurationIndex":4000000000,"Increments":[{"Duration":1000000000,"Cost":0,"BalanceInfo":{"Unit":null,"Monetary":null,"AccountID":""},"CompressFactor":4}],"RoundIncrement":null,"MatchedSubject":"*out:sip.test.cgrates.org:call:*any","MatchedPrefix":"+31800","MatchedDestId":"CST_49800_DE080","RatingPlanId":"ISC_V","CompressFactor":1}],"RatedUsage":4}` + rsrField, err = NewRSRParser(`~CostDetails:s/"MatchedDestId":.*_(\w{5})/${1}/:s/"MatchedDestId":"INTERNAL"/ON010/`) + if err != nil { + t.Error(err) + } + eMatch := "DE080" + if parsedVal, err := rsrField.ParseValue(fieldsStr2); err != nil { + t.Error(err) + } else if parsedVal != eMatch { + t.Errorf("Expecting: <%s>, received: <%s>", eMatch, parsedVal) + } +} + +func TestRSRFldParse(t *testing.T) { + // with dataConverters + rulesStr := `~Usage:s/(\d+)/${1}ms/{*duration_seconds&*round:1:*middle}` + rsrField, err := NewRSRParser(rulesStr) + if err != nil { + t.Fatal(err) + } + eOut := "2.2" + if out, err := rsrField.ParseValue("2210"); err != nil { + t.Error(err) + } else if out != eOut { + t.Errorf("expecting: %s, received: %s", eOut, out) + } + rulesStr = `~Usage:s/(\d+)/${1}ms/{*duration_seconds&*round}` + if rsrField, err = NewRSRParser(rulesStr); err != nil { + t.Error(err) + } + eOut = "2" + if out, err := rsrField.ParseValue("2210"); err != nil { + t.Error(err) + } else if out != eOut { + t.Errorf("expecting: %s, received: %s", eOut, out) + } + rulesStr = `~Usage{*duration_seconds}` + rsrField, err = NewRSRParser(rulesStr) + if err != nil { + t.Error(err) + } + eOut = "10" + if out, err := rsrField.ParseValue("10000000000"); err != nil { + t.Error(err) + } else if out != eOut { + t.Errorf("expecting: %s, received: %s", eOut, out) + } +} diff --git a/console/parse.go b/console/parse.go index 632a712a4..3328b9375 100644 --- a/console/parse.go +++ b/console/parse.go @@ -18,7 +18,9 @@ along with this program. If not, see package console -import "github.com/cgrates/cgrates/utils" +import ( + "github.com/cgrates/cgrates/config" +) func init() { c := &CmdParse{ @@ -71,9 +73,9 @@ func (self *CmdParse) LocalExecute() string { if self.rpcParams.Value == "" { return "Empty value error" } - if rsrField, err := utils.NewRSRField(self.rpcParams.Expression); err != nil { + if rsrField, err := config.NewRSRParser(self.rpcParams.Expression); err != nil { return err.Error() - } else if parsed, err := rsrField.Parse(self.rpcParams.Value); err != nil { + } else if parsed, err := rsrField.ParseValue(self.rpcParams.Value); err != nil { return err.Error() } else { return parsed diff --git a/data/conf/samples/tutmongo2/cgrates.json b/data/conf/samples/tutmongo2/cgrates.json index c29ab37a4..de47c2d11 100644 --- a/data/conf/samples/tutmongo2/cgrates.json +++ b/data/conf/samples/tutmongo2/cgrates.json @@ -44,7 +44,7 @@ "cdrs": { "enabled": true, - "extra_fields": ["PayPalAccount", "LCRProfile", "ResourceID"], + "extra_fields": ["~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"], "chargers_conns":["*localhost"], "store_cdrs": true, "online_cdr_exports": [], diff --git a/data/conf/samples/tutmongo2_gob/cgrates.json b/data/conf/samples/tutmongo2_gob/cgrates.json index d9bb080de..93a6f2275 100644 --- a/data/conf/samples/tutmongo2_gob/cgrates.json +++ b/data/conf/samples/tutmongo2_gob/cgrates.json @@ -51,7 +51,7 @@ "cdrs": { "enabled": true, - "extra_fields": ["PayPalAccount", "LCRProfile", "ResourceID"], + "extra_fields": ["~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"], "chargers_conns":["conn1"], "store_cdrs": true, "online_cdr_exports": [], diff --git a/data/conf/samples/tutmysql2/cgrates.json b/data/conf/samples/tutmysql2/cgrates.json index 0e10dd27b..54a49a221 100644 --- a/data/conf/samples/tutmysql2/cgrates.json +++ b/data/conf/samples/tutmysql2/cgrates.json @@ -36,7 +36,7 @@ "cdrs": { "enabled": true, - "extra_fields": ["PayPalAccount", "LCRProfile", "ResourceID"], + "extra_fields": ["~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"], "chargers_conns":["*localhost"], "store_cdrs": true, "online_cdr_exports": [] diff --git a/data/conf/samples/tutmysql2_gob/cgrates.json b/data/conf/samples/tutmysql2_gob/cgrates.json index da0348e75..0e63c6f9a 100644 --- a/data/conf/samples/tutmysql2_gob/cgrates.json +++ b/data/conf/samples/tutmysql2_gob/cgrates.json @@ -44,7 +44,7 @@ "cdrs": { "enabled": true, - "extra_fields": ["PayPalAccount", "LCRProfile", "ResourceID"], + "extra_fields": ["~*req.PayPalAccount", "~*req.LCRProfile", "~*req.ResourceID"], "chargers_conns":["conn1"], "store_cdrs": true, "online_cdr_exports": [] diff --git a/docs/ers.rst b/docs/ers.rst index cdef64da1..8648a3fa5 100644 --- a/docs/ers.rst +++ b/docs/ers.rst @@ -168,7 +168,7 @@ xml_root_path Used in case of XML content and will specify the prefix path applied to each xml element read. tenant - Will auto-populate the Tenant within the API calls sent to CGRateS. It has the form of a RSRField. If undefined, default one from *general* section will be used. + Will auto-populate the Tenant within the API calls sent to CGRateS. It has the form of a RSRParser. If undefined, default one from *general* section will be used. timezone Defines the timezone for source content which does not carry that information. If undefined, default one from *general* section will be used. diff --git a/engine/fscdr.go b/engine/fscdr.go index 577115b76..a4b4db6bf 100644 --- a/engine/fscdr.go +++ b/engine/fscdr.go @@ -20,9 +20,7 @@ package engine import ( "encoding/json" - "fmt" "io" - "reflect" "strconv" "strings" @@ -78,16 +76,18 @@ func (fsCdr FSCdr) getCGRID() string { func (fsCdr FSCdr) getExtraFields() map[string]string { extraFields := make(map[string]string, len(fsCdr.cgrCfg.CdrsCfg().ExtraFields)) + const dynprefix string = utils.MetaDynReq + utils.NestingSep for _, field := range fsCdr.cgrCfg.CdrsCfg().ExtraFields { - origFieldVal, foundInVars := fsCdr.vars[field.Id] - if strings.HasPrefix(field.Id, utils.STATIC_VALUE_PREFIX) { // Support for static values injected in the CDRS. it will show up as {^value:value} - foundInVars = true + if !strings.HasPrefix(field.Rules, dynprefix) { + continue } + attrName := field.AttrName()[5:] + origFieldVal, foundInVars := fsCdr.vars[attrName] if !foundInVars { - origFieldVal = fsCdr.searchExtraField(field.Id, fsCdr.body) + origFieldVal = fsCdr.searchExtraField(attrName, fsCdr.body) } - if parsed, err := field.Parse(origFieldVal); err == nil { - extraFields[field.Id] = parsed + if parsed, err := field.ParseValue(origFieldVal); err == nil { + extraFields[attrName] = parsed } } return extraFields @@ -95,31 +95,22 @@ func (fsCdr FSCdr) getExtraFields() map[string]string { func (fsCdr FSCdr) searchExtraField(field string, body map[string]interface{}) (result string) { for key, value := range body { + if key == field { + return utils.IfaceAsString(value) + } switch v := value.(type) { - case string: - if key == field { - return v - } - case float64: - if key == field { - return strconv.FormatFloat(v, 'f', -1, 64) - } case map[string]interface{}: - if result = fsCdr.searchExtraField(field, v); result != "" { + if result = fsCdr.searchExtraField(field, v); len(result) != 0 { return } case []interface{}: for _, item := range v { if otherMap, ok := item.(map[string]interface{}); ok { - if result = fsCdr.searchExtraField(field, otherMap); result != "" { + if result = fsCdr.searchExtraField(field, otherMap); len(result) != 0 { return } - } else { - utils.Logger.Warning(fmt.Sprintf("Slice with no maps: %v", reflect.TypeOf(item))) } } - default: - utils.Logger.Warning(fmt.Sprintf("Unexpected type: %v", reflect.TypeOf(v))) } } return @@ -155,8 +146,8 @@ func (fsCdr FSCdr) AsCDR(timezone string) (storCdr *CDR, err error) { CostSource: fsCdr.vars["cgr_costsource"], Cost: -1, } - if orderId, hasIt := fsCdr.vars["cgr_orderid"]; hasIt { - if storCdr.OrderID, err = strconv.ParseInt(orderId, 10, 64); err != nil { + if orderID, hasIt := fsCdr.vars["cgr_orderid"]; hasIt { + if storCdr.OrderID, err = strconv.ParseInt(orderID, 10, 64); err != nil { return nil, err } } diff --git a/engine/fscdr_test.go b/engine/fscdr_test.go index 2f5de6f28..24242e04a 100644 --- a/engine/fscdr_test.go +++ b/engine/fscdr_test.go @@ -407,7 +407,7 @@ func TestFsCdrFirstNonEmpty(t *testing.T) { } func TestFsCdrCDRFields(t *testing.T) { - fsCdrCfg.CdrsCfg().ExtraFields = []*utils.RSRField{{Id: "sip_user_agent"}} + fsCdrCfg.CdrsCfg().ExtraFields = config.NewRSRParsersMustCompile("~*req.sip_user_agent", utils.FIELDS_SEP) reader := bytes.NewReader(body) fsCdr, err := NewFSCdr(reader, fsCdrCfg) if err != nil { @@ -416,15 +416,23 @@ func TestFsCdrCDRFields(t *testing.T) { setupTime, _ := utils.ParseTimeDetectLayout("1515666344", "") answerTime, _ := utils.ParseTimeDetectLayout("1515666347", "") expctCDR := &CDR{ - CGRID: "24b5766be325fa751fab5a0a06373e106f33a257", - ToR: utils.VOICE, OriginID: "3da8bf84-c133-4959-9e24-e72875cb33a1", - OriginHost: "", Source: "freeswitch_json", Category: "call", - RequestType: utils.META_RATED, Tenant: "cgrates.org", - Account: "1001", Subject: "1001", - Destination: "1002", SetupTime: setupTime, - AnswerTime: answerTime, Usage: 68 * time.Second, + CGRID: "24b5766be325fa751fab5a0a06373e106f33a257", + ToR: utils.VOICE, + OriginID: "3da8bf84-c133-4959-9e24-e72875cb33a1", + OriginHost: "", + Source: "freeswitch_json", + Category: "call", + RequestType: utils.META_RATED, + Tenant: "cgrates.org", + Account: "1001", + Subject: "1001", + Destination: "1002", + SetupTime: setupTime, + AnswerTime: answerTime, + Usage: 68 * time.Second, Cost: -1, - ExtraFields: map[string]string{"sip_user_agent": "Jitsi2.10.5550Linux"}} + ExtraFields: map[string]string{"sip_user_agent": "Jitsi2.10.5550Linux"}, + } if CDR, err := fsCdr.AsCDR(""); err != nil { t.Error(err) } else if !reflect.DeepEqual(expctCDR, CDR) { @@ -450,14 +458,13 @@ func TestFsCdrSearchExtraField(t *testing.T) { if err != nil { t.Error(err) } - rsrSt1, _ := utils.NewRSRField("^injected_value") - rsrSt2, _ := utils.NewRSRField("^injected_hdr::injected_value/") - fsCdrCfg.CdrsCfg().ExtraFields = []*utils.RSRField{{Id: "caller_id_name"}, rsrSt1, rsrSt2} + fsCdrCfg.CdrsCfg().ExtraFields, err = config.NewRSRParsersFromSlice([]string{"~*req.caller_id_name"}) + if err != nil { + t.Fatal(err) + } extraFields := fsCdr.getExtraFields() - if len(extraFields) != 3 || extraFields["caller_id_name"] != "1001" || - extraFields["injected_value"] != "injected_value" || - extraFields["injected_hdr"] != "injected_value" { - t.Error("Error parsing extra fields: ", extraFields) + if len(extraFields) != 1 || extraFields["caller_id_name"] != "1001" { + t.Error("Error parsing extra fields: ", utils.ToJSON(extraFields)) } } @@ -472,7 +479,7 @@ func TestFsCdrSearchExtraFieldInSlice(t *testing.T) { } func TestFsCdrSearchReplaceInExtraFields(t *testing.T) { - fsCdrCfg.CdrsCfg().ExtraFields = utils.ParseRSRFieldsMustCompile(`read_codec;~sip_user_agent:s/([A-Za-z]*).+/$1/;write_codec`, utils.INFIELD_SEP) + fsCdrCfg.CdrsCfg().ExtraFields = config.NewRSRParsersMustCompile(`~*req.read_codec;~*req.sip_user_agent:s/([A-Za-z]*).+/$1/;~*req.write_codec`, utils.INFIELD_SEP) newReader := bytes.NewReader(body) fsCdr, err := NewFSCdr(newReader, fsCdrCfg) if err != nil { @@ -483,15 +490,15 @@ func TestFsCdrSearchReplaceInExtraFields(t *testing.T) { t.Error("Error parsing extra fields: ", extraFields) } if extraFields["sip_user_agent"] != "Jitsi" { - t.Error("Error parsing extra fields: ", extraFields) + t.Error("Error parsing extra fields: ", utils.ToJSON(extraFields)) } } func TestFsCdrDDazRSRExtraFields(t *testing.T) { eFieldsCfg := `{"cdrs": { - "extra_fields": ["~effective_caller_id_number:s/(\\d+)/+$1/"], + "extra_fields": ["~*req.effective_caller_id_number:s/(\\d+)/+$1/"], },}` - simpleJsonCdr := []byte(`{ + simpleJSONCdr := []byte(`{ "core-uuid": "feef0b51-7fdf-4c4a-878e-aff233752de2", "channel_data": { "state": "CS_REPORTING", @@ -522,13 +529,13 @@ func TestFsCdrDDazRSRExtraFields(t *testing.T) { }`) var err error fsCdrCfg, err = config.NewCGRConfigFromJSONStringWithDefaults(eFieldsCfg) - expCdrExtra := utils.ParseRSRFieldsMustCompile(`~effective_caller_id_number:s/(\d+)/+$1/`, utils.INFIELD_SEP) + expCdrExtra := config.NewRSRParsersMustCompile(`~*req.effective_caller_id_number:s/(\d+)/+$1/`, utils.INFIELD_SEP) if err != nil { t.Error("Could not parse the config", err.Error()) } else if !reflect.DeepEqual(expCdrExtra[0], fsCdrCfg.CdrsCfg().ExtraFields[0]) { // Kinda deepEqual bug since without index does not match - t.Errorf("Expecting: %+v, received: %+v", expCdrExtra, fsCdrCfg.CdrsCfg().ExtraFields) + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(expCdrExtra), utils.ToJSON(fsCdrCfg.CdrsCfg().ExtraFields)) } - newReader := bytes.NewReader(simpleJsonCdr) + newReader := bytes.NewReader(simpleJSONCdr) fsCdr, err := NewFSCdr(newReader, fsCdrCfg) if err != nil { t.Error("Could not parse cdr", err.Error()) @@ -557,7 +564,7 @@ func TestFscdrAsCDR(t *testing.T) { if err != nil { t.Error(err) } - cgrCfg.CdrsCfg().ExtraFields, err = utils.ParseRSRFieldsFromSlice([]string{"PayPalAccount"}) + cgrCfg.CdrsCfg().ExtraFields, err = config.NewRSRParsersFromSlice([]string{"~*req.PayPalAccount"}) if err != nil { t.Error(err) } @@ -793,16 +800,14 @@ func TestGetExtraFields(t *testing.T) { if err != nil { t.Error(err) } - cgrCfg.CdrsCfg().ExtraFields, err = utils.ParseRSRFieldsFromSlice([]string{"^^PayPalAccount"}) + cgrCfg.CdrsCfg().ExtraFields, err = config.NewRSRParsersFromSlice([]string{"PayPalAccount"}) if err != nil { t.Error(err) } fsCdr := FSCdr{ cgrCfg: cgrCfg, } - expected := map[string]string{ - "^PayPalAccount": "^PayPalAccount", - } + expected := map[string]string{} if reply := fsCdr.getExtraFields(); !reflect.DeepEqual(reply, expected) { t.Errorf("Expected %+v, received %+v", utils.ToJSON(expected), utils.ToJSON(reply)) } diff --git a/packages/debian/changelog b/packages/debian/changelog index 22049183f..b3fc12d7c 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -128,6 +128,7 @@ cgrates (0.11.0~dev) UNRELEASED; urgency=medium * [ConfigS] Renamed ReloadConfigFromPath API to ReloadConfig * [ConfigS] Renamed ReloadConfig API to SetConfig * [ConfigS] Renamed ReloadConfigFromJSON API to SetConfigFromJSON + * [CDRs] Replaced RSRField with RSRParser -- DanB Wed, 19 Feb 2020 13:25:52 +0200 diff --git a/utils/rsrfield.go b/utils/rsrfield.go deleted file mode 100644 index 95b9b4f0f..000000000 --- a/utils/rsrfield.go +++ /dev/null @@ -1,393 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ - -package utils - -import ( - "fmt" - "regexp" - "strings" -) - -var ( - spltRgxp = regexp.MustCompile(`:s\/`) - rulesRgxp = regexp.MustCompile(`(?:(.+[^\\])\/(.*[^\\])*\/){1,}`) -) - -func NewRSRField(fldStr string) (fld *RSRField, err error) { - if len(fldStr) == 0 { - return nil, nil - } - rsrField := &RSRField{Rules: fldStr} - var filters []*RSRFilter - if strings.HasSuffix(fldStr, FilterValEnd) { // Has filter, populate the var - fltrStart := strings.LastIndex(fldStr, FilterValStart) - if fltrStart < 1 { - return nil, fmt.Errorf("Invalid FilterStartValue in string: %s", fldStr) - } - fltrVal := fldStr[fltrStart+1 : len(fldStr)-1] - filters, err = ParseRSRFilters(fltrVal, MetaPipe) - if err != nil { - return nil, fmt.Errorf("Invalid FilterValue in string: %s, err: %s", fltrVal, err.Error()) - } - fldStr = fldStr[:fltrStart] // Take the filter part out before compiling further - - } - if idxConverters := strings.Index(fldStr, "{*"); idxConverters != -1 { // converters in the string - if !strings.HasSuffix(fldStr, "}") { - return nil, - fmt.Errorf("Invalid converter value in string: %s, err: invalid converter terminator", - fldStr) - } - convertersStr := fldStr[idxConverters+1 : len(fldStr)-1] // strip also {} - convsSplt := strings.Split(convertersStr, INFIELD_SEP) - rsrField.converters = make([]DataConverter, len(convsSplt)) - for i, convStr := range convsSplt { - if conv, err := NewDataConverter(convStr); err != nil { - return nil, - fmt.Errorf("Invalid converter value in string: %s, err: %s", - convStr, err.Error()) - } else { - rsrField.converters[i] = conv - } - } - fldStr = fldStr[:idxConverters] - } - if strings.HasPrefix(fldStr, STATIC_VALUE_PREFIX) { // Special case when RSR is defined as static header/value - var staticHdr, staticVal string - if splt := strings.Split(fldStr, STATIC_HDRVAL_SEP); len(splt) == 2 { // Using / as separator since ':' is often use in date/time fields - staticHdr, staticVal = splt[0][1:], splt[1] // Strip the / suffix - if strings.HasSuffix(staticVal, "/") { // If value ends with /, strip it since it is a part of the definition syntax - staticVal = staticVal[:len(staticVal)-1] - } - } else if len(splt) > 2 { - return nil, fmt.Errorf("Invalid RSRField string: %s", fldStr) - } else { - staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix - } - rsrField.Id = staticHdr - rsrField.staticValue = staticVal - rsrField.filters = filters - return rsrField, nil - } - if !strings.HasPrefix(fldStr, DynamicDataPrefix) { - rsrField.Id = fldStr - rsrField.filters = filters - return rsrField, nil - } - spltRules := spltRgxp.Split(fldStr, -1) - if len(spltRules) < 2 { - return nil, fmt.Errorf("Invalid Split of Search&Replace field rule. %s", fldStr) - } - rsrField.Id = spltRules[0][1:] - rsrField.filters = filters // Original id in form ~hdr_name - for _, ruleStr := range spltRules[1:] { // :s/ already removed through split - allMatches := rulesRgxp.FindStringSubmatch(ruleStr) - if len(allMatches) != 3 { - return nil, fmt.Errorf("Not enough members in Search&Replace, ruleStr: %s, matches: %v, ", ruleStr, allMatches) - } - if srRegexp, err := regexp.Compile(allMatches[1]); err != nil { - return nil, fmt.Errorf("Invalid Search&Replace subfield rule: %s", allMatches[1]) - } else { - rsrField.RSRules = append(rsrField.RSRules, &ReSearchReplace{SearchRegexp: srRegexp, ReplaceTemplate: allMatches[2]}) - } - } - return rsrField, nil -} - -func NewRSRFieldMustCompile(fldStr string) (rsrFld *RSRField) { - var err error - if rsrFld, err = NewRSRField(fldStr); err != nil { - return nil - } - return -} - -type RSRField struct { - Id string // Identifier - Rules string // Rules container holding the string rules to be able to restore it after DB - staticValue string // If defined, enforces parsing always to this value - RSRules []*ReSearchReplace // Rules to use when processing field value - filters []*RSRFilter // The value to compare when used as filter - converters DataConverters // set of converters to apply on output -} - -// IsCompiled finds out whether this RSRField was already parsed or RAW state -func (rsrf *RSRField) IsCompiled() bool { - return rsrf.staticValue != "" || - rsrf.RSRules != nil || - rsrf.filters != nil || - rsrf.converters != nil -} - -// Compile parses Rules string and repopulates other fields -func (rsrf *RSRField) Compile() error { - if newRSRFld, err := NewRSRField(rsrf.Rules); err != nil { - return err - } else if newRSRFld != nil { - rsrf.staticValue = newRSRFld.staticValue - rsrf.RSRules = newRSRFld.RSRules - rsrf.filters = newRSRFld.filters - } - return nil -} - -// parseValue the field value from a string -func (rsrf *RSRField) parseValue(value string) string { - if len(rsrf.staticValue) != 0 { // Enforce parsing of static values - return rsrf.staticValue - } - for _, rsRule := range rsrf.RSRules { - if rsRule != nil { - value = rsRule.Process(value) - } - } - return value -} - -func (rsrf *RSRField) filtersPassing(value string) bool { - for _, fltr := range rsrf.filters { - if !fltr.Pass(value) { - return false - } - } - return true -} - -// Parse will parse the value out considering converters and filters -func (rsrf *RSRField) Parse(value interface{}) (out string, err error) { - out = rsrf.parseValue(IfaceAsString(value)) - if out, err = rsrf.converters.ConvertString(out); err != nil { - return - } - if !rsrf.filtersPassing(out) { - return "", ErrFilterNotPassingNoCaps - } - return -} - -// NewRSRFilter instantiates a new RSRFilter, setting it's properties -func NewRSRFilter(fltrVal string) (rsrFltr *RSRFilter, err error) { - rsrFltr = new(RSRFilter) - if fltrVal == "" { - return rsrFltr, nil - } - if fltrVal[:1] == NegativePrefix { - rsrFltr.negative = true - fltrVal = fltrVal[1:] - if fltrVal == "" { - return rsrFltr, nil - } - } - rsrFltr.filterRule = fltrVal - if fltrVal[:1] == DynamicDataPrefix { - if rsrFltr.fltrRgxp, err = regexp.Compile(fltrVal[1:]); err != nil { - return nil, err - } - } - return rsrFltr, nil -} - -// NewRSRFilterMustCompile is used mostly in tests -func NewRSRFilterMustCompile(fltrVal string) (rsrFltr *RSRFilter) { - var err error - if rsrFltr, err = NewRSRFilter(fltrVal); err != nil { - panic(fmt.Sprintf("parsing <%s>, err: %s", fltrVal, err.Error())) - } - return -} - -// One filter rule -type RSRFilter struct { - filterRule string // Value in raw format - fltrRgxp *regexp.Regexp - negative bool // Rule should not match -} - -func (rsrFltr *RSRFilter) FilterRule() string { - return rsrFltr.filterRule -} - -func (rsrFltr *RSRFilter) Pass(val string) bool { - if rsrFltr.filterRule == "" { - return !rsrFltr.negative - } - if rsrFltr.filterRule[:1] == DynamicDataPrefix { - return rsrFltr.fltrRgxp.MatchString(val) != rsrFltr.negative - } - if rsrFltr.filterRule == "^$" { // Special case to test empty value - return len(val) == 0 != rsrFltr.negative - } - if rsrFltr.filterRule[:1] == MatchStartPrefix { - if rsrFltr.filterRule[len(rsrFltr.filterRule)-1:] == MatchEndPrefix { // starts with ^ and ends with $, exact match - return val == rsrFltr.filterRule[1:len(rsrFltr.filterRule)-1] != rsrFltr.negative - } - return strings.HasPrefix(val, rsrFltr.filterRule[1:]) != rsrFltr.negative - } - lastIdx := len(rsrFltr.filterRule) - 1 - if rsrFltr.filterRule[lastIdx:] == MatchEndPrefix { - return strings.HasSuffix(val, rsrFltr.filterRule[:lastIdx]) != rsrFltr.negative - } - if len(rsrFltr.filterRule) > 2 && rsrFltr.filterRule[:2] == MatchGreaterThanOrEqual { - gt, err := GreaterThan(StringToInterface(val), - StringToInterface(rsrFltr.filterRule[2:]), true) - if err != nil { - Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) - return false - } - return gt != rsrFltr.negative - } - - if len(rsrFltr.filterRule) > 2 && rsrFltr.filterRule[:2] == MatchLessThanOrEqual { - gt, err := GreaterThan(StringToInterface(rsrFltr.filterRule[2:]), // compare the rule with the val - StringToInterface(val), - true) - if err != nil { - Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) - return false - } - return gt != rsrFltr.negative - } - - if rsrFltr.filterRule[:1] == MatchGreaterThan { - gt, err := GreaterThan(StringToInterface(val), - StringToInterface(rsrFltr.filterRule[1:]), false) - if err != nil { - Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) - return false - } - return gt != rsrFltr.negative - } - - if rsrFltr.filterRule[:1] == MatchLessThan { - gt, err := GreaterThan(StringToInterface(rsrFltr.filterRule[1:]), // compare the rule with the val - StringToInterface(val), - false) - if err != nil { - Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) - return false - } - return gt != rsrFltr.negative - } - return (strings.Index(val, rsrFltr.filterRule) != -1) != rsrFltr.negative // default is string index -} - -func ParseRSRFilters(fldsStr, sep string) (RSRFilters, error) { - if fldsStr == "" { - return nil, nil - } - fltrSplt := strings.Split(fldsStr, sep) - return ParseRSRFiltersFromSlice(fltrSplt) -} - -func ParseRSRFiltersFromSlice(fltrStrs []string) (RSRFilters, error) { - rsrFltrs := make(RSRFilters, len(fltrStrs)) - for i, rlStr := range fltrStrs { - if rsrFltr, err := NewRSRFilter(rlStr); err != nil { - return nil, err - } else if rsrFltr == nil { - return nil, fmt.Errorf("Empty RSRFilter in rule: %s", rlStr) - } else { - rsrFltrs[i] = rsrFltr - } - } - return rsrFltrs, nil -} - -type RSRFilters []*RSRFilter - -func (fltrs RSRFilters) FilterRules() (rls string) { - for _, fltr := range fltrs { - rls += fltr.FilterRule() - } - return -} - -// @all: specifies whether all filters should match or at least one -func (fltrs RSRFilters) Pass(val string, allMustMatch bool) (matched bool) { - if len(fltrs) == 0 { - return true - } - for _, fltr := range fltrs { - matched = fltr.Pass(val) - if allMustMatch { - if !matched { - return - } - } else if matched { - return - } - } - return -} - -func ParseRSRFieldsFromSlice(flds []string) (RSRFields, error) { - if len(flds) == 0 { - return nil, nil - } - rsrFields := make(RSRFields, len(flds)) - for idx, ruleStr := range flds { - if rsrField, err := NewRSRField(ruleStr); err != nil { - return nil, err - } else if rsrField == nil { - return nil, fmt.Errorf("Empty RSRField in rule: %s", ruleStr) - } else { - rsrFields[idx] = rsrField - } - } - return rsrFields, nil - -} - -// Parses list of RSRFields, used for example as multiple filters in derived charging -func ParseRSRFields(fldsStr, sep string) (RSRFields, error) { - //rsrRlsPattern := regexp.MustCompile(`^(~\w+:s/.+/.*/)|(\^.+(/.+/)?)(;(~\w+:s/.+/.*/)|(\^.+(/.+/)?))*$`) //ToDo:Fix here rule able to confirm the content - if len(fldsStr) == 0 { - return nil, nil - } - rulesSplt := strings.Split(fldsStr, sep) - return ParseRSRFieldsFromSlice(rulesSplt) - -} - -func ParseRSRFieldsMustCompile(fldsStr, sep string) RSRFields { - if flds, err := ParseRSRFields(fldsStr, sep); err != nil { - return nil - } else { - return flds - } -} - -type RSRFields []*RSRField - -// Return first Id of the rsrFields, used in cdre -func (flds RSRFields) Id() string { - if len(flds) == 0 { - return "" - } - return flds[0].Id -} - -func (flds RSRFields) Compile() (err error) { - for _, rsrFld := range flds { - if err = rsrFld.Compile(); err != nil { - break - } - } - return -} diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go deleted file mode 100644 index 40f0c1678..000000000 --- a/utils/rsrfield_test.go +++ /dev/null @@ -1,730 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package utils - -import ( - "reflect" - "regexp" - "testing" -) - -func TestNewRSRField(t *testing.T) { - // Empty case - if rcv, err := NewRSRField(EmptyString); err != nil { - t.Error(err) - } else if rcv != nil { - t.Error("Expecting nill, received: ", rcv) - } - // Normal case - rulesStr := `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/(someval)` - filter, _ := NewRSRFilter("someval") - expRSRField1 := &RSRField{ - Id: "sip_redirected_to", - Rules: rulesStr, - RSRules: []*ReSearchReplace{ - { - SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), - ReplaceTemplate: "0$1"}}, - filters: []*RSRFilter{filter}, - converters: nil} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(expRSRField1, rsrField) { - t.Errorf("Expecting: %+v, received: %+v", - expRSRField1, rsrField) - } - // With filter - rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/(086517174963)` - // rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/{*duration_seconds;*round:5:*middle}(086517174963)` - filter, _ = NewRSRFilter("086517174963") - expRSRField2 := &RSRField{Id: "sip_redirected_to", Rules: rulesStr, filters: []*RSRFilter{filter}, - RSRules: []*ReSearchReplace{{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(expRSRField2, rsrField) { - t.Errorf("Expecting: %v, received: %v", expRSRField2, rsrField) - } - // with dataConverters - rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/{*duration_seconds;*round:5:*middle}(086517174963)` - filter, _ = NewRSRFilter("086517174963") - expRSRField := &RSRField{ - Id: "sip_redirected_to", - Rules: rulesStr, - RSRules: []*ReSearchReplace{ - { - SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), - ReplaceTemplate: "0$1"}}, - filters: []*RSRFilter{filter}, - converters: []DataConverter{ - new(DurationSecondsConverter), &RoundConverter{Decimals: 5, Method: "*middle"}}} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(expRSRField, rsrField) { - t.Errorf("Expecting: %+v, received: %+v", expRSRField, rsrField) - } - // One extra separator but escaped - rulesStr = `~sip_redirected_to:s/sip:\+49(\d+)\/@/0$1/` - expRSRField3 := &RSRField{Id: "sip_redirected_to", Rules: rulesStr, - RSRules: []*ReSearchReplace{{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)\/@`), ReplaceTemplate: "0$1"}}} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(expRSRField3, rsrField) { - t.Errorf("Expecting: %v, received: %v", expRSRField3, rsrField) - } - if _, err := NewRSRField("(test)"); err == nil || err.Error() != "Invalid FilterStartValue in string: (test)" { - t.Error(err) - } - if _, err := NewRSRField("{*test"); err == nil || err.Error() != "Invalid converter value in string: {*test, err: invalid converter terminator" { - t.Error(err) - } - if _, err := NewRSRField("^t::e::s::t::"); err == nil || err.Error() != "Invalid RSRField string: ^t::e::s::t::" { - t.Error(err) - } - if _, err := NewRSRField("~-1"); err == nil || err.Error() != "Invalid Split of Search&Replace field rule. ~-1" { - t.Error(err) - } -} - -func TestNewRSRFieldDDz(t *testing.T) { - rulesStr := `~effective_caller_id_number:s/(\d+)/+$1/` - expectRSRField := &RSRField{Id: "effective_caller_id_number", Rules: rulesStr, - RSRules: []*ReSearchReplace{{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, expectRSRField) { - t.Errorf("Unexpected RSRField received: %v", rsrField) - } -} - -func TestNewRSRFieldIvo(t *testing.T) { - rulesStr := `~cost_details:s/MatchedDestId":".+_(\s\s\s\s\s)"/$1/` - expectRSRField := &RSRField{Id: "cost_details", Rules: rulesStr, - RSRules: []*ReSearchReplace{{SearchRegexp: regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), ReplaceTemplate: "$1"}}} - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, expectRSRField) { - t.Errorf("Unexpected RSRField received: %v", rsrField) - } - if _, err := NewRSRField(`~account:s/^[A-Za-z0-9]*[c|a]\d{4}$/S/:s/^[A-Za-z0-9]*n\d{4}$/C/:s/^\d{10}$//`); err != nil { - t.Error(err) - } -} - -func TestConvertPlusNationalAnd00(t *testing.T) { - rulesStr := `~effective_caller_id_number:s/\+49(\d+)/0$1/:s/\+(\d+)/00$1/` - expectRSRField := &RSRField{Id: "effective_caller_id_number", Rules: rulesStr, - RSRules: []*ReSearchReplace{ - {SearchRegexp: regexp.MustCompile(`\+49(\d+)`), ReplaceTemplate: "0$1"}, - {SearchRegexp: regexp.MustCompile(`\+(\d+)`), ReplaceTemplate: "00$1"}}} - rsrField, err := NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, expectRSRField) { - t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) - } - if parsedVal, err := rsrField.Parse("+4986517174963"); err != nil { - t.Error(err) - } else if parsedVal != "086517174963" { - t.Errorf("Expecting: 086517174963, received: %s", parsedVal) - } - if parsedVal, err := rsrField.Parse("+3186517174963"); err != nil { - t.Error(err) - } else if parsedVal != "003186517174963" { - t.Errorf("Expecting: 003186517174963, received: %s", parsedVal) - } -} - -func TestRSRParseStatic(t *testing.T) { - rulesStr := "^static_header::static_value/" - rsrField, err := NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, &RSRField{Id: "static_header", Rules: rulesStr, - staticValue: "static_value"}) { - t.Errorf("Unexpected RSRField received: %v", rsrField) - } - if parsed, err := rsrField.Parse("dynamic_value"); err != nil { - t.Error(err) - } else if parsed != "static_value" { - t.Errorf("Expected: %s, received: %s", "static_value", parsed) - } - rulesStr = `^static_hdrvalue` - rsrField, err = NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, &RSRField{Id: "static_hdrvalue", Rules: rulesStr, - staticValue: "static_hdrvalue"}) { - t.Errorf("Unexpected RSRField received: %v", rsrField) - } - if parsed, err := rsrField.Parse("dynamic_value"); err != nil { - t.Error(err) - } else if parsed != "static_hdrvalue" { - t.Errorf("Expected: %s, received: %s", "static_hdrvalue", parsed) - } -} - -func TestConvertDurToSecs(t *testing.T) { - rulesStr := `~9:s/^(\d+)$/${1}s/` - expectRSRField := &RSRField{Id: "9", Rules: rulesStr, - RSRules: []*ReSearchReplace{ - {SearchRegexp: regexp.MustCompile(`^(\d+)$`), ReplaceTemplate: "${1}s"}}} - rsrField, err := NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, expectRSRField) { - t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) - } - if parsedVal, err := rsrField.Parse("640113"); err != nil { - t.Error(err) - } else if parsedVal != "640113s" { - t.Errorf("Expecting: 640113s, received: %s", parsedVal) - } -} - -func TestPrefix164(t *testing.T) { - rulesStr := `~0:s/^([1-9]\d+)$/+$1/` - expectRSRField := &RSRField{Id: "0", Rules: rulesStr, - RSRules: []*ReSearchReplace{ - {SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}} - rsrField, err := NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } else if !reflect.DeepEqual(rsrField, expectRSRField) { - t.Errorf("Expecting: %v, received: %v", expectRSRField, rsrField) - } - if parsedVal, err := rsrField.Parse("4986517174960"); err != nil { - t.Error(err) - } else if parsedVal != "+4986517174960" { - t.Errorf("Expecting: +4986517174960, received: %s", parsedVal) - } -} - -func TestParseRSRFields(t *testing.T) { - fieldsStr1 := `~account:s/^\w+[mpls]\d{6}$//;~subject:s/^0\d{9}$//;^destination/+4912345/;~mediation_runid:s/^default$/default/` - rsrFld1, _ := NewRSRField(`~account:s/^\w+[mpls]\d{6}$//`) - rsrFld2, _ := NewRSRField(`~subject:s/^0\d{9}$//`) - rsrFld3, _ := NewRSRField(`^destination/+4912345/`) - rsrFld4, _ := NewRSRField(`~mediation_runid:s/^default$/default/`) - eRSRFields := RSRFields{rsrFld1, rsrFld2, rsrFld3, rsrFld4} - if rsrFlds, err := ParseRSRFields(fieldsStr1, INFIELD_SEP); err != nil { - t.Error("Unexpected error: ", err) - } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { - t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) - } - fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` - expectParsedFields := RSRFields{ - &RSRField{Id: "host", Rules: "host"}, - &RSRField{Id: "sip_redirected_to", Rules: `~sip_redirected_to:s/sip:\+49(\d+)@/0$1/`, - RSRules: []*ReSearchReplace{{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, - &RSRField{Id: "destination", Rules: "destination"}} - if parsedFields, err := ParseRSRFields(fields, FIELDS_SEP); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(parsedFields, expectParsedFields) { - t.Errorf("Unexpected value of parsed fields") - } -} - -func TestParseCdrcDn1(t *testing.T) { - rl, err := NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/:s/^\+49(18\d{2})$/+491400$1/`) - if err != nil { - t.Error("Unexpected error: ", err) - } - if parsed, err := rl.Parse("0049ABOC0630415354"); err != nil { - t.Error(err) - } else if parsed != "+49630415354" { - t.Errorf("Expecting: +49630415354, received: %s", parsed) - } - if parsed2, err := rl.Parse("00491888"); err != nil { - t.Error(err) - } else if parsed2 != "+4914001888" { - t.Errorf("Expecting: +4914001888, received: %s", parsed2) - } -} - -func TestFilterPasses(t *testing.T) { - rl, err := NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/:s/^\+49(18\d{2})$/+491400$1/(+49630415354)`) - if err != nil { - t.Error("Unexpected error: ", err) - } - if _, err = rl.Parse("0031ABOC0630415354"); err == nil || - err != ErrFilterNotPassingNoCaps { - t.Error("Passing filter") - } - rl, err = NewRSRField(`~1:s/^$/_empty_/(_empty_)`) - if err != nil { - t.Error("Unexpected error: ", err) - } - if _, err = rl.Parse(""); err == ErrFilterNotPassingNoCaps { - t.Error("Not passing filter") - } - if _, err = rl.Parse("Non empty"); err == nil || - err != ErrFilterNotPassingNoCaps { - t.Error("Passing filter") - } -} - -func TestRSRFieldsId(t *testing.T) { - fieldsStr1 := `~account:s/^\w+[mpls]\d{6}$//;~subject:s/^0\d{9}$//;^destination/+4912345/;~mediation_runid:s/^default$/default/` - if rsrFlds, err := ParseRSRFields(fieldsStr1, INFIELD_SEP); err != nil { - t.Error("Unexpected error: ", err) - } else if idRcv := rsrFlds.Id(); idRcv != "account" { - t.Errorf("Received id: %s", idRcv) - } - fieldsStr2 := "" - if rsrFlds, err := ParseRSRFields(fieldsStr2, INFIELD_SEP); err != nil { - t.Error("Unexpected error: ", err) - } else if idRcv := rsrFlds.Id(); idRcv != "" { - t.Errorf("Received id: %s", idRcv) - } -} - -func TestRSRCostDetails(t *testing.T) { - fieldsStr1 := `{"Category":"default_route","Tenant":"demo.cgrates.org","Subject":"voxbeam_premium","Account":"6335820713","Destination":"15143606781","ToR":"*voice","Cost":0.0007,"Timespans":[{"TimeStart":"2015-08-30T21:46:54Z","TimeEnd":"2015-08-30T21:47:06Z","Cost":0.00072,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"RoundingMethod":"*middle","RoundingDecimals":5,"MaxCost":0,"MaxCostStrategy":"0","Rates":[{"GroupIntervalStart":0,"Value":0.0036,"RateIncrement":6000000000,"RateUnit":60000000000}]},"Weight":10},"DurationIndex":12000000000,"Increments":[{"Duration":6000000000,"Cost":0.00036,"BalanceInfo":{"UnitBalanceUuid":"","MoneyBalanceUuid":"40adda88-25d3-4009-b928-f39d61590439","AccountId":"*out:demo.cgrates.org:6335820713"},"BalanceRateInterval":null,"UnitInfo":null,"CompressFactor":2}],"MatchedSubject":"*out:demo.cgrates.org:default_route:voxbeam_premium","MatchedPrefix":"1514","MatchedDestId":"Canada","RatingPlanId":"RP_VOXBEAM_PREMIUM"}]}` - rsrField, err := NewRSRField(`~cost_details:s/"MatchedDestId":"(\w+)"/${1}/`) - if err != nil { - t.Error(err) - } - if parsedVal, err := rsrField.Parse(fieldsStr1); err != nil { - t.Error(err) - } else if parsedVal != "Canada" { - t.Errorf("Expecting: Canada, received: %s", parsedVal) - } - fieldsStr2 := `{"Category":"call","Tenant":"sip.test.cgrates.org","Subject":"dan","Account":"dan","Destination":"+4986517174963","ToR":"*voice","Cost":0,"Timespans":[{"TimeStart":"2015-05-13T15:03:34+02:00","TimeEnd":"2015-05-13T15:03:38+02:00","Cost":0,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"RoundingMethod":"*middle","RoundingDecimals":4,"MaxCost":0,"MaxCostStrategy":"","Rates":[{"GroupIntervalStart":0,"Value":0,"RateIncrement":1000000000,"RateUnit":60000000000}]},"Weight":10},"DurationIndex":4000000000,"Increments":[{"Duration":1000000000,"Cost":0,"BalanceInfo":{"Unit":null,"Monetary":null,"AccountID":""},"CompressFactor":4}],"RoundIncrement":null,"MatchedSubject":"*out:sip.test.cgrates.org:call:*any","MatchedPrefix":"+31800","MatchedDestId":"CST_49800_DE080","RatingPlanId":"ISC_V","CompressFactor":1}],"RatedUsage":4}` - rsrField, err = NewRSRField(`~CostDetails:s/"MatchedDestId":.*_(\w{5})/${1}/:s/"MatchedDestId":"INTERNAL"/ON010/`) - if err != nil { - t.Error(err) - } - eMatch := "DE080" - if parsedVal, err := rsrField.Parse(fieldsStr2); err != nil { - t.Error(err) - } else if parsedVal != eMatch { - t.Errorf("Expecting: <%s>, received: <%s>", eMatch, parsedVal) - } -} - -func TestRSRFilterPass(t *testing.T) { - fltr, err := NewRSRFilter("") // Pass any - if err != nil { - t.Error(err) - } - if !fltr.Pass("") { - t.Error("Not passing!") - } - if !fltr.Pass("any") { - t.Error("Not passing!") - } - fltr, err = NewRSRFilter("!") // Pass nothing - if err != nil { - t.Error(err) - } - if fltr.Pass("") { - t.Error("Passing!") - } - if fltr.Pass("any") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("^full_match$") // Full string pass - if err != nil { - t.Error(err) - } - if !fltr.Pass("full_match") { - t.Error("Not passing!") - } - if fltr.Pass("full_match1") { - t.Error("Passing!") - } - if fltr.Pass("") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("^prefixMatch") // Prefix pass - if err != nil { - t.Error(err) - } - if !fltr.Pass("prefixMatch") { - t.Error("Not passing!") - } - if !fltr.Pass("prefixMatch12345") { - t.Error("Not passing!") - } - if fltr.Pass("1prefixMatch") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("suffixMatch$") // Suffix pass - if err != nil { - t.Error(err) - } - if !fltr.Pass("suffixMatch") { - t.Error("Not passing!") - } - if !fltr.Pass("12345suffixMatch") { - t.Error("Not passing!") - } - if fltr.Pass("suffixMatch1") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("!fullMatch") // Negative full pass - if err != nil { - t.Error(err) - } - if !fltr.Pass("ShouldMatch") { - t.Error("Not passing!") - } - if fltr.Pass("fullMatch") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("!^prefixMatch") // Negative prefix pass - if err != nil { - t.Error(err) - } - if fltr.Pass("prefixMatch123") { - t.Error("Passing!") - } - if !fltr.Pass("123prefixMatch") { - t.Error("Not passing!") - } - fltr, err = NewRSRFilter("!suffixMatch$") // Negative suffix pass - if err != nil { - t.Error(err) - } - if fltr.Pass("123suffixMatch") { - t.Error("Passing!") - } - if !fltr.Pass("suffixMatch123") { - t.Error("Not passing!") - } - fltr, err = NewRSRFilter("~^C.+S$") // Regexp pass - if err != nil { - t.Error(err) - } - if !fltr.Pass("CGRateS") { - t.Error("Not passing!") - } - if fltr.Pass("1CGRateS") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("!~^C.*S$") // Negative regexp pass - if err != nil { - t.Error(err) - } - if fltr.Pass("CGRateS") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("^$") // Empty value - if err != nil { - t.Error(err) - } - if fltr.Pass("CGRateS") { - t.Error("Passing!") - } - if !fltr.Pass("") { - t.Error("Not passing!") - } - fltr, err = NewRSRFilter("!^$") // Non-empty value - if err != nil { - t.Error(err) - } - if !fltr.Pass("CGRateS") { - t.Error("Not passing!") - } - if fltr.Pass("") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("indexed_match") // Indexed match - if err != nil { - t.Error(err) - } - if !fltr.Pass("indexed_match") { - t.Error("Not passing!") - } - if !fltr.Pass("suf_indexed_match") { - t.Error("Not passing!") - } - if !fltr.Pass("indexed_match_pref") { - t.Error("Not passing!") - } - if !fltr.Pass("suf_indexed_match_pref") { - t.Error("Not passing!") - } - if fltr.Pass("indexed_matc") { - t.Error("Passing!") - } - if fltr.Pass("") { - t.Error("Passing!") - } - fltr, err = NewRSRFilter("!indexed_match") // Negative indexed match - if err != nil { - t.Error(err) - } - if fltr.Pass("indexed_match") { - t.Error("passing!") - } - if fltr.Pass("suf_indexed_match") { - t.Error("passing!") - } - if fltr.Pass("indexed_match_pref") { - t.Error("passing!") - } - if fltr.Pass("suf_indexed_match_pref") { - t.Error("passing!") - } - if !fltr.Pass("indexed_matc") { - t.Error("not passing!") - } - if !fltr.Pass("") { - t.Error("Passing!") - } - - // compare greaterThan - fltr, err = NewRSRFilter(">0s") - if err != nil { - t.Error(err) - } - if fltr.Pass("0s") { - t.Error("passing!") - } - if !fltr.Pass("13") { - t.Error("not passing!") - } - if !fltr.Pass("12s") { - t.Error("not passing!") - } - - // compare not greaterThan - fltr, err = NewRSRFilter("!>0s") // !(n>0s) - if err != nil { - t.Error(err) - } - if !fltr.Pass("0s") { - t.Error("not passing!") - } - if fltr.Pass("13") { - t.Error("passing!") - } - if fltr.Pass("12s") { - t.Error("passing!") - } - - // compare greaterThanOrEqual - fltr, err = NewRSRFilter(">=0s") - if err != nil { - t.Error(err) - } - if fltr.Pass("-1s") { - t.Error("passing!") - } - if !fltr.Pass("0s") { - t.Error("not passing!") - } - if !fltr.Pass("13") { - t.Error("not passing!") - } - if !fltr.Pass("12s") { - t.Error("not passing!") - } - - // compare not greaterThanOrEqual - fltr, err = NewRSRFilter("!>=0s") - if err != nil { - t.Error(err) - } - if !fltr.Pass("-1s") { - t.Error("not passing!") - } - if fltr.Pass("0s") { - t.Error("passing!") - } - if fltr.Pass("13") { - t.Error("passing!") - } - if fltr.Pass("12s") { - t.Error("passing!") - } - - // compare lessThan - fltr, err = NewRSRFilter("<0s") - if err != nil { - t.Error(err) - } - if fltr.Pass("1ns") { - t.Error("passing!") - } - if fltr.Pass("13") { - t.Error("passing!") - } - if fltr.Pass("12s") { - t.Error("passing!") - } - if !fltr.Pass("-12s") { - t.Error("not passing!") - } - - // compare not lessThan - fltr, err = NewRSRFilter("!<0s") - if err != nil { - t.Error(err) - } - if !fltr.Pass("1ns") { - t.Error("not passing!") - } - if !fltr.Pass("13") { - t.Error("not passing!") - } - if !fltr.Pass("12s") { - t.Error("not passing!") - } - if fltr.Pass("-12s") { - t.Error("passing!") - } - - // compare lessThanOrEqual - fltr, err = NewRSRFilter("<=0s") - if err != nil { - t.Error(err) - } - if !fltr.Pass("-1s") { - t.Error("not passing!") - } - if !fltr.Pass("0s") { - t.Error("not passing!") - } - if fltr.Pass("13") { - t.Error("passing!") - } - if fltr.Pass("12s") { - t.Error("passing!") - } - - // compare not lessThanOrEqual - fltr, err = NewRSRFilter("!<=0s") - if err != nil { - t.Error(err) - } - if fltr.Pass("-1s") { - t.Error("passing!") - } - if fltr.Pass("0s") { - t.Error("passing!") - } - if !fltr.Pass("13") { - t.Error("not passing!") - } - if !fltr.Pass("12s") { - t.Error("not passing!") - } -} - -func TestRSRFiltersPass(t *testing.T) { - rlStr := "~^C.+S$;CGRateS;ateS$" - fltrs, err := ParseRSRFilters(rlStr, INFIELD_SEP) - if err != nil { - t.Error(err) - } - if !fltrs.Pass("CGRateS", true) { - t.Error("Not passing") - } - if fltrs.Pass("ateS", true) { - t.Error("Passing") - } - if !fltrs.Pass("ateS", false) { - t.Error("Not passing") - } - if fltrs.Pass("teS", false) { - t.Error("Passing") - } -} - -func TestParseDifferentMethods(t *testing.T) { - rlStr := `~effective_caller_id_number:s/(\d+)/+$1/` - resParseStr, _ := ParseRSRFields(rlStr, INFIELD_SEP) - resParseSlc, _ := ParseRSRFieldsFromSlice([]string{rlStr}) - if !reflect.DeepEqual(resParseStr, resParseSlc) { - t.Errorf("Expecting: %+v, received: %+v", resParseStr, resParseSlc) - } -} - -func TestIsParsed(t *testing.T) { - rulesStr := `^static_hdrvalue` - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error(err) - } else if !rsrField.IsCompiled() { - t.Error("Not compiled") - } - rulesStr = `~effective_caller_id_number:s/(\d+)/+$1/` - if rsrField, err := NewRSRField(rulesStr); err != nil { - t.Error(err) - } else if !rsrField.IsCompiled() { - t.Error("Not compiled") - } - rsrField := &RSRField{Rules: rulesStr} - if rsrField.IsCompiled() { - t.Error("Is compiled") - } -} - -func TestCompileRules(t *testing.T) { - rulesStr := `^static_hdrvalue` - rsrField := &RSRField{Rules: rulesStr} - if err := rsrField.Compile(); err != nil { - t.Error(err) - } - newRSRFld, _ := NewRSRField(rulesStr) - if reflect.DeepEqual(rsrField, newRSRFld) { - t.Errorf("Expecting: %+v, received: %+v", rsrField, newRSRFld) - } -} - -func TestRSRFldParse(t *testing.T) { - // with dataConverters - rulesStr := `~Usage:s/(\d+)/${1}ms/{*duration_seconds;*round:1:*middle}(2.2)` - rsrField, err := NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } - eOut := "2.2" - if out, err := rsrField.Parse("2210"); err != nil { - t.Error(err) - } else if out != eOut { - t.Errorf("expecting: %s, received: %s", eOut, out) - } - rulesStr = `~Usage:s/(\d+)/${1}ms/{*duration_seconds;*round:1:*middle}(2.21)` - rsrField, _ = NewRSRField(rulesStr) - if _, err := rsrField.Parse("2210"); err == nil || err.Error() != "filter not passing" { - t.Error(err) - } - rulesStr = `~Usage:s/(\d+)/${1}ms/{*duration_seconds;*round}` - if rsrField, err = NewRSRField(rulesStr); err != nil { - t.Error(err) - } - eOut = "2" - if out, err := rsrField.Parse("2210"); err != nil { - t.Error(err) - } else if out != eOut { - t.Errorf("expecting: %s, received: %s", eOut, out) - } - rulesStr = `Usage{*duration_seconds}` - rsrField, err = NewRSRField(rulesStr) - if err != nil { - t.Error(err) - } - eOut = "10" - if out, err := rsrField.Parse("10000000000"); err != nil { - t.Error(err) - } else if out != eOut { - t.Errorf("expecting: %s, received: %s", eOut, out) - } -} diff --git a/utils/rsrfilters.go b/utils/rsrfilters.go new file mode 100644 index 000000000..8d3bc61b3 --- /dev/null +++ b/utils/rsrfilters.go @@ -0,0 +1,180 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package utils + +import ( + "fmt" + "regexp" + "strings" +) + +// NewRSRFilter instantiates a new RSRFilter, setting it's properties +func NewRSRFilter(fltrVal string) (rsrFltr *RSRFilter, err error) { + rsrFltr = new(RSRFilter) + if fltrVal == "" { + return rsrFltr, nil + } + if fltrVal[:1] == NegativePrefix { + rsrFltr.negative = true + fltrVal = fltrVal[1:] + if fltrVal == "" { + return rsrFltr, nil + } + } + rsrFltr.filterRule = fltrVal + if fltrVal[:1] == DynamicDataPrefix { + if rsrFltr.fltrRgxp, err = regexp.Compile(fltrVal[1:]); err != nil { + return nil, err + } + } + return rsrFltr, nil +} + +// NewRSRFilterMustCompile is used mostly in tests +func NewRSRFilterMustCompile(fltrVal string) (rsrFltr *RSRFilter) { + var err error + if rsrFltr, err = NewRSRFilter(fltrVal); err != nil { + panic(fmt.Sprintf("parsing <%s>, err: %s", fltrVal, err.Error())) + } + return +} + +// One filter rule +type RSRFilter struct { + filterRule string // Value in raw format + fltrRgxp *regexp.Regexp + negative bool // Rule should not match +} + +func (rsrFltr *RSRFilter) FilterRule() string { + return rsrFltr.filterRule +} + +func (rsrFltr *RSRFilter) Pass(val string) bool { + if rsrFltr.filterRule == "" { + return !rsrFltr.negative + } + if rsrFltr.filterRule[:1] == DynamicDataPrefix { + return rsrFltr.fltrRgxp.MatchString(val) != rsrFltr.negative + } + if rsrFltr.filterRule == "^$" { // Special case to test empty value + return len(val) == 0 != rsrFltr.negative + } + if rsrFltr.filterRule[:1] == MatchStartPrefix { + if rsrFltr.filterRule[len(rsrFltr.filterRule)-1:] == MatchEndPrefix { // starts with ^ and ends with $, exact match + return val == rsrFltr.filterRule[1:len(rsrFltr.filterRule)-1] != rsrFltr.negative + } + return strings.HasPrefix(val, rsrFltr.filterRule[1:]) != rsrFltr.negative + } + lastIdx := len(rsrFltr.filterRule) - 1 + if rsrFltr.filterRule[lastIdx:] == MatchEndPrefix { + return strings.HasSuffix(val, rsrFltr.filterRule[:lastIdx]) != rsrFltr.negative + } + if len(rsrFltr.filterRule) > 2 && rsrFltr.filterRule[:2] == MatchGreaterThanOrEqual { + gt, err := GreaterThan(StringToInterface(val), + StringToInterface(rsrFltr.filterRule[2:]), true) + if err != nil { + Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) + return false + } + return gt != rsrFltr.negative + } + + if len(rsrFltr.filterRule) > 2 && rsrFltr.filterRule[:2] == MatchLessThanOrEqual { + gt, err := GreaterThan(StringToInterface(rsrFltr.filterRule[2:]), // compare the rule with the val + StringToInterface(val), + true) + if err != nil { + Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) + return false + } + return gt != rsrFltr.negative + } + + if rsrFltr.filterRule[:1] == MatchGreaterThan { + gt, err := GreaterThan(StringToInterface(val), + StringToInterface(rsrFltr.filterRule[1:]), false) + if err != nil { + Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) + return false + } + return gt != rsrFltr.negative + } + + if rsrFltr.filterRule[:1] == MatchLessThan { + gt, err := GreaterThan(StringToInterface(rsrFltr.filterRule[1:]), // compare the rule with the val + StringToInterface(val), + false) + if err != nil { + Logger.Warning(fmt.Sprintf(" rule: <%s>, err: <%s>", rsrFltr.filterRule, err.Error())) + return false + } + return gt != rsrFltr.negative + } + return (strings.Index(val, rsrFltr.filterRule) != -1) != rsrFltr.negative // default is string index +} + +func ParseRSRFilters(fldsStr, sep string) (RSRFilters, error) { + if fldsStr == "" { + return nil, nil + } + fltrSplt := strings.Split(fldsStr, sep) + return ParseRSRFiltersFromSlice(fltrSplt) +} + +func ParseRSRFiltersFromSlice(fltrStrs []string) (RSRFilters, error) { + rsrFltrs := make(RSRFilters, len(fltrStrs)) + for i, rlStr := range fltrStrs { + if rsrFltr, err := NewRSRFilter(rlStr); err != nil { + return nil, err + } else if rsrFltr == nil { + return nil, fmt.Errorf("Empty RSRFilter in rule: %s", rlStr) + } else { + rsrFltrs[i] = rsrFltr + } + } + return rsrFltrs, nil +} + +type RSRFilters []*RSRFilter + +func (fltrs RSRFilters) FilterRules() (rls string) { + for _, fltr := range fltrs { + rls += fltr.FilterRule() + } + return +} + +// @all: specifies whether all filters should match or at least one +func (fltrs RSRFilters) Pass(val string, allMustMatch bool) (matched bool) { + if len(fltrs) == 0 { + return true + } + for _, fltr := range fltrs { + matched = fltr.Pass(val) + if allMustMatch { + if !matched { + return + } + } else if matched { + return + } + } + return +} diff --git a/utils/rsrfilters_test.go b/utils/rsrfilters_test.go new file mode 100644 index 000000000..dbfb5fa2c --- /dev/null +++ b/utils/rsrfilters_test.go @@ -0,0 +1,353 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package utils + +import ( + "testing" +) + +func TestRSRFilterPass(t *testing.T) { + fltr, err := NewRSRFilter("") // Pass any + if err != nil { + t.Error(err) + } + if !fltr.Pass("") { + t.Error("Not passing!") + } + if !fltr.Pass("any") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("!") // Pass nothing + if err != nil { + t.Error(err) + } + if fltr.Pass("") { + t.Error("Passing!") + } + if fltr.Pass("any") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("^full_match$") // Full string pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("full_match") { + t.Error("Not passing!") + } + if fltr.Pass("full_match1") { + t.Error("Passing!") + } + if fltr.Pass("") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("^prefixMatch") // Prefix pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("prefixMatch") { + t.Error("Not passing!") + } + if !fltr.Pass("prefixMatch12345") { + t.Error("Not passing!") + } + if fltr.Pass("1prefixMatch") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("suffixMatch$") // Suffix pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("suffixMatch") { + t.Error("Not passing!") + } + if !fltr.Pass("12345suffixMatch") { + t.Error("Not passing!") + } + if fltr.Pass("suffixMatch1") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!fullMatch") // Negative full pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("ShouldMatch") { + t.Error("Not passing!") + } + if fltr.Pass("fullMatch") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!^prefixMatch") // Negative prefix pass + if err != nil { + t.Error(err) + } + if fltr.Pass("prefixMatch123") { + t.Error("Passing!") + } + if !fltr.Pass("123prefixMatch") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("!suffixMatch$") // Negative suffix pass + if err != nil { + t.Error(err) + } + if fltr.Pass("123suffixMatch") { + t.Error("Passing!") + } + if !fltr.Pass("suffixMatch123") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("~^C.+S$") // Regexp pass + if err != nil { + t.Error(err) + } + if !fltr.Pass("CGRateS") { + t.Error("Not passing!") + } + if fltr.Pass("1CGRateS") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!~^C.*S$") // Negative regexp pass + if err != nil { + t.Error(err) + } + if fltr.Pass("CGRateS") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("^$") // Empty value + if err != nil { + t.Error(err) + } + if fltr.Pass("CGRateS") { + t.Error("Passing!") + } + if !fltr.Pass("") { + t.Error("Not passing!") + } + fltr, err = NewRSRFilter("!^$") // Non-empty value + if err != nil { + t.Error(err) + } + if !fltr.Pass("CGRateS") { + t.Error("Not passing!") + } + if fltr.Pass("") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("indexed_match") // Indexed match + if err != nil { + t.Error(err) + } + if !fltr.Pass("indexed_match") { + t.Error("Not passing!") + } + if !fltr.Pass("suf_indexed_match") { + t.Error("Not passing!") + } + if !fltr.Pass("indexed_match_pref") { + t.Error("Not passing!") + } + if !fltr.Pass("suf_indexed_match_pref") { + t.Error("Not passing!") + } + if fltr.Pass("indexed_matc") { + t.Error("Passing!") + } + if fltr.Pass("") { + t.Error("Passing!") + } + fltr, err = NewRSRFilter("!indexed_match") // Negative indexed match + if err != nil { + t.Error(err) + } + if fltr.Pass("indexed_match") { + t.Error("passing!") + } + if fltr.Pass("suf_indexed_match") { + t.Error("passing!") + } + if fltr.Pass("indexed_match_pref") { + t.Error("passing!") + } + if fltr.Pass("suf_indexed_match_pref") { + t.Error("passing!") + } + if !fltr.Pass("indexed_matc") { + t.Error("not passing!") + } + if !fltr.Pass("") { + t.Error("Passing!") + } + + // compare greaterThan + fltr, err = NewRSRFilter(">0s") + if err != nil { + t.Error(err) + } + if fltr.Pass("0s") { + t.Error("passing!") + } + if !fltr.Pass("13") { + t.Error("not passing!") + } + if !fltr.Pass("12s") { + t.Error("not passing!") + } + + // compare not greaterThan + fltr, err = NewRSRFilter("!>0s") // !(n>0s) + if err != nil { + t.Error(err) + } + if !fltr.Pass("0s") { + t.Error("not passing!") + } + if fltr.Pass("13") { + t.Error("passing!") + } + if fltr.Pass("12s") { + t.Error("passing!") + } + + // compare greaterThanOrEqual + fltr, err = NewRSRFilter(">=0s") + if err != nil { + t.Error(err) + } + if fltr.Pass("-1s") { + t.Error("passing!") + } + if !fltr.Pass("0s") { + t.Error("not passing!") + } + if !fltr.Pass("13") { + t.Error("not passing!") + } + if !fltr.Pass("12s") { + t.Error("not passing!") + } + + // compare not greaterThanOrEqual + fltr, err = NewRSRFilter("!>=0s") + if err != nil { + t.Error(err) + } + if !fltr.Pass("-1s") { + t.Error("not passing!") + } + if fltr.Pass("0s") { + t.Error("passing!") + } + if fltr.Pass("13") { + t.Error("passing!") + } + if fltr.Pass("12s") { + t.Error("passing!") + } + + // compare lessThan + fltr, err = NewRSRFilter("<0s") + if err != nil { + t.Error(err) + } + if fltr.Pass("1ns") { + t.Error("passing!") + } + if fltr.Pass("13") { + t.Error("passing!") + } + if fltr.Pass("12s") { + t.Error("passing!") + } + if !fltr.Pass("-12s") { + t.Error("not passing!") + } + + // compare not lessThan + fltr, err = NewRSRFilter("!<0s") + if err != nil { + t.Error(err) + } + if !fltr.Pass("1ns") { + t.Error("not passing!") + } + if !fltr.Pass("13") { + t.Error("not passing!") + } + if !fltr.Pass("12s") { + t.Error("not passing!") + } + if fltr.Pass("-12s") { + t.Error("passing!") + } + + // compare lessThanOrEqual + fltr, err = NewRSRFilter("<=0s") + if err != nil { + t.Error(err) + } + if !fltr.Pass("-1s") { + t.Error("not passing!") + } + if !fltr.Pass("0s") { + t.Error("not passing!") + } + if fltr.Pass("13") { + t.Error("passing!") + } + if fltr.Pass("12s") { + t.Error("passing!") + } + + // compare not lessThanOrEqual + fltr, err = NewRSRFilter("!<=0s") + if err != nil { + t.Error(err) + } + if fltr.Pass("-1s") { + t.Error("passing!") + } + if fltr.Pass("0s") { + t.Error("passing!") + } + if !fltr.Pass("13") { + t.Error("not passing!") + } + if !fltr.Pass("12s") { + t.Error("not passing!") + } +} + +func TestRSRFiltersPass(t *testing.T) { + rlStr := "~^C.+S$;CGRateS;ateS$" + fltrs, err := ParseRSRFilters(rlStr, INFIELD_SEP) + if err != nil { + t.Error(err) + } + if !fltrs.Pass("CGRateS", true) { + t.Error("Not passing") + } + if fltrs.Pass("ateS", true) { + t.Error("Passing") + } + if !fltrs.Pass("ateS", false) { + t.Error("Not passing") + } + if fltrs.Pass("teS", false) { + t.Error("Passing") + } +}