From 455a5e9485ed619b8c00f160c3d3fcca7be3a7b3 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 2 Jul 2020 18:09:02 +0300 Subject: [PATCH] Updated RSRParser constant handling --- apier/v1/cdrs_it_test.go | 10 +++--- apier/v1/config_it_test.go | 3 +- config/rsrparser.go | 42 ++++++----------------- config/rsrparser_test.go | 64 +++++------------------------------ loaders/libloader_test.go | 68 +++++++++++++++++++------------------- 5 files changed, 60 insertions(+), 127 deletions(-) diff --git a/apier/v1/cdrs_it_test.go b/apier/v1/cdrs_it_test.go index f94b17b92..49b4a3866 100644 --- a/apier/v1/cdrs_it_test.go +++ b/apier/v1/cdrs_it_test.go @@ -280,16 +280,16 @@ func testV1CDRsRefundOutOfSessionCost(t *testing.T) { Usage: utils.DurationPointer(time.Duration(3 * time.Minute)), Cost: utils.Float64Pointer(2.3), Charges: []*engine.ChargingInterval{ - &engine.ChargingInterval{ + { RatingID: "c1a5ab9", Increments: []*engine.ChargingIncrement{ - &engine.ChargingIncrement{ + { Usage: time.Duration(2 * time.Minute), Cost: 2.0, AccountingID: "a012888", CompressFactor: 1, }, - &engine.ChargingIncrement{ + { Usage: time.Duration(1 * time.Second), Cost: 0.005, AccountingID: "44d6c02", @@ -303,7 +303,7 @@ func testV1CDRsRefundOutOfSessionCost(t *testing.T) { Tenant: "cgrates.org", ID: "testV1CDRsRefundOutOfSessionCost", BalanceSummaries: []*engine.BalanceSummary{ - &engine.BalanceSummary{ + { UUID: balanceUuid, Type: utils.MONETARY, Value: 50, @@ -364,7 +364,7 @@ func testV1CDRsRefundOutOfSessionCost(t *testing.T) { utils.Account: "testV1CDRsRefundOutOfSessionCost", utils.Destination: "+4986517174963", utils.AnswerTime: time.Date(2019, 11, 27, 12, 21, 26, 0, time.UTC), - utils.Usage: time.Duration(123) * time.Minute, + utils.Usage: 123 * time.Minute, }, }, } diff --git a/apier/v1/config_it_test.go b/apier/v1/config_it_test.go index 4ebf7e9ab..1eaea3d14 100644 --- a/apier/v1/config_it_test.go +++ b/apier/v1/config_it_test.go @@ -149,6 +149,7 @@ func testConfigSReloadConfigFromJSONSessionS(t *testing.T) { "MinCallDuration": 0., "SessionTTL": 0., "SessionTTLLastUsed": nil, + "SessionTTLLastUsage": nil, "SessionTTLMaxDelay": nil, "SessionTTLUsage": nil, "StoreSCosts": false, @@ -166,7 +167,7 @@ func testConfigSReloadConfigFromJSONSessionS(t *testing.T) { }, &rpl); err != nil { t.Error(err) } else if !reflect.DeepEqual(exp, rpl) { - t.Errorf("Expected %+v , received: %+v ", exp, rpl) + t.Errorf("Expected %+v , received: %+v ", utils.ToJSON(exp), utils.ToJSON(rpl)) } } diff --git a/config/rsrparser.go b/config/rsrparser.go index 3ad4f76da..526c2e5fc 100644 --- a/config/rsrparser.go +++ b/config/rsrparser.go @@ -141,18 +141,6 @@ func (prsrs RSRParsers) ParseValue(value interface{}) (out string, err error) { return } -// ParseEvent will parse the event values into one output -func (prsrs RSRParsers) ParseEvent(ev map[string]interface{}) (out string, err error) { - for _, prsr := range prsrs { - if outPrsr, err := prsr.ParseEvent(ev); err != nil { - return "", err - } else { - out += outPrsr - } - } - return -} - func (prsrs RSRParsers) ParseDataProvider(dP utils.DataProvider, separator string) (out string, err error) { for _, prsr := range prsrs { if outPrsr, err := prsr.ParseDataProvider(dP, separator); err != nil { @@ -212,13 +200,13 @@ func NewRSRParser(parserRules string, allFiltersMatch bool) (rsrParser *RSRParse } parserRules = parserRules[:idxConverters] } - if rsrParser.isConstant = !strings.HasPrefix(parserRules, utils.DynamicDataPrefix) || - len(parserRules) == 1; rsrParser.isConstant { // special case when RSR is defined as static attribute + if !strings.HasPrefix(parserRules, utils.DynamicDataPrefix) || + len(parserRules) == 1 { // special case when RSR is defined as static attribute return } // dynamic content via attributeNames spltRules := spltRgxp.Split(parserRules, -1) - rsrParser.attrName = spltRules[0][1:] // in form ~hdr_name + rsrParser.path = spltRules[0][1:] // in form ~hdr_name if len(spltRules) > 1 { for _, ruleStr := range spltRules[1:] { // :s/ already removed through split allMatches := rulesRgxp.FindStringSubmatch(ruleStr) @@ -251,16 +239,15 @@ type RSRParser struct { Rules string // Rules container holding the string rules, public so it can be stored AllFiltersMatch bool // all filters must match policy - attrName string // instruct extracting info out of header in event + path string // instruct extracting info out of header in event rsrRules []*utils.ReSearchReplace // rules to use when parsing value converters utils.DataConverters // set of converters to apply on output filters utils.RSRFilters // The value to compare when used as filter - isConstant bool } // AttrName exports the attribute name of the RSRParser func (prsr *RSRParser) AttrName() string { - return prsr.attrName + return prsr.path } // Compile parses Rules string and repopulates other fields @@ -287,7 +274,7 @@ func (prsr *RSRParser) RegexpMatched() bool { // parseValue the field value from a string func (prsr *RSRParser) parseValue(value string) string { - if prsr.isConstant { // Enforce parsing of static values + if prsr.path == "" { // Enforce parsing of static values return prsr.Rules } for _, rsRule := range prsr.rsrRules { @@ -308,20 +295,11 @@ func (prsr *RSRParser) ParseValue(value interface{}) (out string, err error) { return } -// ParseEvent will parse the value out considering converters and filters -func (prsr *RSRParser) ParseEvent(ev map[string]interface{}) (out string, err error) { - val, has := ev[prsr.attrName] - if !has && !prsr.isConstant { - return "", utils.ErrNotFound - } - return prsr.ParseValue(val) -} - func (prsr *RSRParser) ParseDataProvider(dP utils.DataProvider, separator string) (out string, err error) { var outStr string - if !prsr.isConstant { + if prsr.path != "" { if outStr, err = dP.FieldAsString( - strings.Split(prsr.attrName, separator)); err != nil && + strings.Split(prsr.path, separator)); err != nil && (err != utils.ErrNotFound || prsr.filters.FilterRules() != "^$") { return } @@ -331,9 +309,9 @@ func (prsr *RSRParser) ParseDataProvider(dP utils.DataProvider, separator string func (prsr *RSRParser) ParseDataProviderWithInterfaces(dP utils.DataProvider, separator string) (out string, err error) { var outIface interface{} - if !prsr.isConstant { + if prsr.path != "" { if outIface, err = dP.FieldAsInterface( - strings.Split(prsr.attrName, separator)); err != nil && + strings.Split(prsr.path, separator)); err != nil && (err != utils.ErrNotFound || prsr.filters.FilterRules() != "^$") { return } diff --git a/config/rsrparser_test.go b/config/rsrparser_test.go index ef3f6b0d4..69faf29f3 100644 --- a/config/rsrparser_test.go +++ b/config/rsrparser_test.go @@ -29,16 +29,16 @@ import ( func TestNewRSRParsers(t *testing.T) { ruleStr := `Value1;Value2;~Header3(Val3&!Val4);~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c);Value5{*duration_seconds&*round:2}` eRSRParsers := RSRParsers{ - &RSRParser{Rules: "Value1", AllFiltersMatch: true, isConstant: true}, - &RSRParser{Rules: "Value2", AllFiltersMatch: true, isConstant: true}, - &RSRParser{Rules: "~Header3(Val3&!Val4)", AllFiltersMatch: true, attrName: "Header3", + &RSRParser{Rules: "Value1", AllFiltersMatch: true}, + &RSRParser{Rules: "Value2", AllFiltersMatch: true}, + &RSRParser{Rules: "~Header3(Val3&!Val4)", AllFiltersMatch: true, path: "Header3", filters: utils.RSRFilters{utils.NewRSRFilterMustCompile("Val3"), utils.NewRSRFilterMustCompile("!Val4")}}, &RSRParser{Rules: "~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)", AllFiltersMatch: true, - attrName: "Header4", + path: "Header4", rsrRules: []*utils.ReSearchReplace{ - &utils.ReSearchReplace{ + { SearchRegexp: regexp.MustCompile(`a`), ReplaceTemplate: "${1}b"}}, converters: utils.DataConverters{utils.NewDataConverterMustCompile("*duration_seconds"), @@ -48,7 +48,6 @@ func TestNewRSRParsers(t *testing.T) { }, &RSRParser{Rules: "Value5{*duration_seconds&*round:2}", AllFiltersMatch: true, - isConstant: true, converters: utils.DataConverters{utils.NewDataConverterMustCompile("*duration_seconds"), utils.NewDataConverterMustCompile("*round:2")}, }, @@ -62,10 +61,10 @@ func TestNewRSRParsers(t *testing.T) { func TestRSRParserCompile(t *testing.T) { ePrsr := &RSRParser{ - Rules: "~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)", - attrName: "Header4", + Rules: "~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)", + path: "Header4", rsrRules: []*utils.ReSearchReplace{ - &utils.ReSearchReplace{ + { SearchRegexp: regexp.MustCompile(`a`), ReplaceTemplate: "${1}b"}}, converters: utils.DataConverters{utils.NewDataConverterMustCompile("*duration_seconds"), @@ -83,20 +82,6 @@ func TestRSRParserCompile(t *testing.T) { } } -func TestRSRParsersParseEvent(t *testing.T) { - prsrs := NewRSRParsersMustCompile("~Header1;|;~Header2", true, utils.INFIELD_SEP) - ev := map[string]interface{}{ - "Header1": "Value1", - "Header2": "Value2", - } - eOut := "Value1|Value2" - if out, err := prsrs.ParseEvent(ev); err != nil { - t.Error(err) - } else if eOut != out { - t.Errorf("expecting: %s, received: %s", eOut, out) - } -} - func TestRSRParserConstant(t *testing.T) { rule := "cgrates.org" rsrParsers, err := NewRSRParsers(rule, true, utils.INFIELD_SEP) @@ -123,36 +108,6 @@ func TestRSRParserNotConstant(t *testing.T) { } } -func TestRSRParsersParseEvent2(t *testing.T) { - prsrs := NewRSRParsersMustCompile("~Header1.Test;|;~Header2.Test", true, utils.INFIELD_SEP) - ev := map[string]interface{}{ - "Header1.Test": "Value1", - "Header2.Test": "Value2", - } - eOut := "Value1|Value2" - if out, err := prsrs.ParseEvent(ev); err != nil { - t.Error(err) - } else if eOut != out { - t.Errorf("expecting: %s, received: %s", eOut, out) - } -} - -func TestRSRParsersParseEvent3(t *testing.T) { - prsr, err := NewRSRParser("~Category:s/(.*)/${1}_suffix/", true) - if err != nil { - t.Error(err) - } - ev := map[string]interface{}{ - "Category": "call", - } - eOut := "call_suffix" - if out, err := prsr.ParseEvent(ev); err != nil { - t.Error(err) - } else if eOut != out { - t.Errorf("expecting: %s, received: %s", eOut, out) - } -} - // TestRSRParsersParseInnerBraces makes sure the inner braces are allowed in a filter rule func TestRSRParsersParseInnerBracket(t *testing.T) { rule := "~*req.Service-Information.IN-Information.CalledPartyAddress(~^(00)*(33|0)890240004$)" @@ -169,7 +124,7 @@ func TestRSRParsersParseInnerBracket(t *testing.T) { func TestNewRSRParsersConstant(t *testing.T) { ruleStr := "`>;q=0.7;expires=3600`" eRSRParsers := RSRParsers{ - &RSRParser{Rules: ">;q=0.7;expires=3600", AllFiltersMatch: true, isConstant: true}, + &RSRParser{Rules: ">;q=0.7;expires=3600", AllFiltersMatch: true}, } if rsrParsers, err := NewRSRParsers(ruleStr, true, utils.INFIELD_SEP); err != nil { t.Error("Unexpected error: ", err.Error()) @@ -215,7 +170,6 @@ func TestRSRParserCompileConstant(t *testing.T) { ePrsr := &RSRParser{ Rules: "*constant:>;q=0.7;expires=3600", AllFiltersMatch: true, - isConstant: true, } prsr := &RSRParser{ Rules: "*constant:>;q=0.7;expires=3600", diff --git a/loaders/libloader_test.go b/loaders/libloader_test.go index 0546825d4..0d867f1f4 100644 --- a/loaders/libloader_test.go +++ b/loaders/libloader_test.go @@ -28,53 +28,53 @@ import ( func TestDataUpdateFromCSVOneFile(t *testing.T) { attrSFlds := []*config.FCTemplate{ - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~0", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~1", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Contexts", + {Tag: "Contexts", Path: "Contexts", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~2", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~3", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~4", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Path", + {Tag: "Path", Path: "Path", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~5", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Initial", + {Tag: "Initial", Path: "Initial", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Substitute", + {Tag: "Substitute", Path: "Substitute", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Append", + {Tag: "Append", Path: "Append", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, } rows := [][]string{ - []string{"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, - []string{"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, + {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, + {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, } lData := make(LoaderData) if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds, @@ -118,53 +118,53 @@ func TestDataUpdateFromCSVOneFile(t *testing.T) { func TestDataUpdateFromCSVOneFile2(t *testing.T) { attrSFlds := []*config.FCTemplate{ - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~0", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~1", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Contexts", + {Tag: "Contexts", Path: "Contexts", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~2", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~3", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~4", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Path", + {Tag: "Path", Path: "Path", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~5", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Initial", + {Tag: "Initial", Path: "Initial", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~6", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Substitute", + {Tag: "Substitute", Path: "Substitute", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~7", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Append", + {Tag: "Append", Path: "Append", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~8", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.MetaVariable, Value: config.NewRSRParsersMustCompile("~9", true, utils.INFIELD_SEP)}, } rows := [][]string{ - []string{"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, - []string{"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, + {"cgrates.org", "ATTR_1", "*sessions;*cdrs", "*string:Account:1007", "2014-01-14T00:00:00Z", "Account", "*any", "1001", "false", "10"}, + {"cgrates.org", "ATTR_1", "", "", "", "Subject", "*any", "1001", "true", ""}, } lData := make(LoaderData) if err := lData.UpdateFromCSV("Attributes.csv", rows[0], attrSFlds, @@ -208,45 +208,45 @@ func TestDataUpdateFromCSVOneFile2(t *testing.T) { func TestDataUpdateFromCSVMultiFiles(t *testing.T) { attrSFlds := []*config.FCTemplate{ - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("cgrates.org", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File2.csv:1", true, utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Contexts", + {Tag: "Contexts", Path: "Contexts", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("*any", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Path", + {Tag: "Path", Path: "Path", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File1.csv:5", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Initial", + {Tag: "Initial", Path: "Initial", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File1.csv:6", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Substitute", + {Tag: "Substitute", Path: "Substitute", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~File1.csv:7", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Append", + {Tag: "Append", Path: "Append", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("true", true, utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("10", true, utils.INFIELD_SEP)}, } loadRun1 := map[string][]string{ - "File1.csv": []string{"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, - "File2.csv": []string{"ignored", "ATTR_1"}, + "File1.csv": {"ignored", "ignored", "ignored", "ignored", "ignored", "Subject", "*any", "1001", "ignored", "ignored"}, + "File2.csv": {"ignored", "ATTR_1"}, } lData := make(LoaderData) for fName, record := range loadRun1 {