From 368c0468051ac6dd1ff481aead4285d3eba2e21f Mon Sep 17 00:00:00 2001 From: gezimbll Date: Tue, 6 Feb 2024 06:03:15 -0500 Subject: [PATCH] added *http filter type --- engine/attributes.go | 4 ++ engine/attributes_test.go | 48 ++++++++++++++++++++++ engine/filterhelpers.go | 75 ++++++++++++++++++++++++++++++++-- engine/filters.go | 15 +++++++ engine/filters_test.go | 84 +++++++++++++++++++++++++++++++++++++++ engine/libattributes.go | 10 ++++- utils/consts.go | 1 + 7 files changed, 232 insertions(+), 5 deletions(-) diff --git a/engine/attributes.go b/engine/attributes.go index a524ae742..ef363d9c6 100644 --- a/engine/attributes.go +++ b/engine/attributes.go @@ -563,6 +563,10 @@ func ParseAttribute(dp utils.DataProvider, attrType, path string, value config.R sort.Strings(values[1:]) out = strings.Join(values, utils.InfieldSep) default: + if strings.HasPrefix(attrType, utils.MetaHTTP) { + out, err = externalAttributeAPI(attrType, dp) + break + } return utils.EmptyString, fmt.Errorf("unsupported type: <%s>", attrType) } return diff --git a/engine/attributes_test.go b/engine/attributes_test.go index 2b4eb9311..7ae4bf653 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -19,6 +19,9 @@ along with this program. If not, see package engine import ( + "fmt" + "net/http" + "net/http/httptest" "reflect" "testing" "time" @@ -923,3 +926,48 @@ func TestParseAtributeCCUsageNegativeReqNr(t *testing.T) { t.Errorf("Expected %v\n but received %v", exp, out) } } +func TestAttributeFromHTTP(t *testing.T) { + exp := "Account" + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, exp) + + })) + + defer testServer.Close() + attrType := utils.MetaHTTP + utils.HashtagSep + utils.IdxStart + testServer.URL + utils.IdxEnd + + attrID := attrType + ":*req.Category:*attributes" + expAttrPrf1 := &AttributeProfile{ + Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, + ID: attrType + ":*req.Category:*attributes", + Attributes: []*Attribute{ + { + Path: utils.MetaReq + utils.NestingSep + "Category", + Type: attrType, + Value: config.NewRSRParsersMustCompile("*attributes", utils.InfieldSep), + }, + }, + } + attrPrf, err := NewAttributeFromInline(config.CgrConfig().GeneralCfg().DefaultTenant, attrID) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expAttrPrf1, attrPrf) { + t.Errorf("Expecting %+v, received: %+v", utils.ToJSON(expAttrPrf1), utils.ToJSON(attrPrf)) + } + dp := utils.MapStorage{ + utils.MetaReq: utils.MapStorage{}, + } + + attr := attrPrf.Attributes[0] + if out, err := ParseAttribute(dp, attr.Type, attr.Path, attr.Value, + 0, utils.EmptyString, utils.EmptyString, utils.InfieldSep); err != nil { + t.Fatal(err) + } else if exp != out { + t.Errorf("Expected %q, Received %q", exp, out) + } +} diff --git a/engine/filterhelpers.go b/engine/filterhelpers.go index dfc770fc0..7096d62ad 100644 --- a/engine/filterhelpers.go +++ b/engine/filterhelpers.go @@ -21,10 +21,12 @@ package engine import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" + "strings" "github.com/cgrates/birpc/context" "github.com/cgrates/cgrates/config" @@ -216,7 +218,7 @@ func sentrypeerGetToken(tokenUrl, clientID, clientSecret, audience, grantType st utils.GrantTypeCfg: grantType, } jsonPayload, _ := json.Marshal(payload) - resp, err = getHTTP(http.MethodPost, tokenUrl, bytes.NewBuffer(jsonPayload), map[string][]string{"Content-Type": {"application/json"}}) + resp, err = getHTTP(http.MethodPost, tokenUrl, bytes.NewBuffer(jsonPayload), map[string]string{"Content-Type": "application/json"}) if err != nil { return } @@ -235,7 +237,7 @@ func sentrypeerGetToken(tokenUrl, clientID, clientSecret, audience, grantType st // sentrypeerHasData return a boolean based on query response on finding ip/number func sentrypeerHasData(itemId, token, url string) (found bool, err error) { var resp *http.Response - resp, err = getHTTP(http.MethodGet, url, nil, map[string][]string{"Authorization": {fmt.Sprintf("Bearer %s", token)}}) + resp, err = getHTTP(http.MethodGet, url, nil, map[string]string{"Authorization": fmt.Sprintf("Bearer %s", token)}) if err != nil { return } @@ -256,12 +258,77 @@ func sentrypeerHasData(itemId, token, url string) (found bool, err error) { } return false, err } +func filterHTTP(httpType string, dDP utils.DataProvider, fieldname, value string) (bool, error) { + var ( + parsedURL *url.URL + resp string + err error + ) + urlS, err := extractUrlFromType(httpType) + if err != nil { + return false, err + } -func getHTTP(method, url string, payload io.Reader, headers map[string][]string) (resp *http.Response, err error) { + parsedURL, err = url.Parse(urlS) + if err != nil { + return false, err + } + if fieldname != utils.MetaAny { + queryParams := parsedURL.Query() + queryParams.Set(fieldname, value) + parsedURL.RawQuery = queryParams.Encode() + resp, err = externalAPI(parsedURL.String(), nil) + } else { + resp, err = externalAPI(parsedURL.String(), bytes.NewReader([]byte(dDP.String()))) + } + if err != nil { + return false, err + } + return utils.IfaceAsBool(resp) +} + +func externalAPI(url string, rdr io.Reader) (string, error) { + hdr := map[string]string{ + "Content-Type": "application/json", + } + resp, err := getHTTP(http.MethodGet, url, rdr, hdr) + if err != nil { + return "", fmt.Errorf("error processing the request: %w", err) + } + + defer resp.Body.Close() + + if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusMultipleChoices { + body, err := io.ReadAll(resp.Body) + return "", fmt.Errorf("http request returned non-OK status code: %d ,body: %v ,err: %w", resp.StatusCode, string(body), err) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error decoding response: %w", err) + } + + return string(body), nil +} + +// constructs an request via parameters provided,url,header and payload ,uses defaultclient for sending the request +func getHTTP(method, url string, payload io.Reader, headers map[string]string) (resp *http.Response, err error) { var req *http.Request if req, err = http.NewRequest(method, url, payload); err != nil { return } - req.Header = headers + for k, hVal := range headers { + req.Header.Add(k, hVal) + } return http.DefaultClient.Do(req) } + +func extractUrlFromType(httpType string) (string, error) { + parts := strings.Split(httpType, utils.HashtagSep) + if len(parts) != 2 { + return "", errors.New("url is not specified") + } + //extracting the url from the type + url := strings.Trim(parts[1], utils.IdxStart+utils.IdxEnd) + return url, nil +} diff --git a/engine/filters.go b/engine/filters.go index 52adcb9e2..50c96cb06 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -425,6 +425,10 @@ func (fltr *FilterRule) Pass(ctx *context.Context, dDP utils.DataProvider) (resu case utils.MetaNever: result, err = fltr.passNever(dDP) default: + if strings.HasPrefix(fltr.Type, utils.MetaHTTP) && strings.Index(fltr.Type, "#") == len(utils.MetaHTTP) { + result, err = fltr.passHttp(dDP) + break + } err = utils.ErrPrefixNotErrNotImplemented(fltr.Type) } if err != nil { @@ -800,6 +804,17 @@ func (fltr *FilterRule) passRegex(dDP utils.DataProvider) (bool, error) { func (fltr *FilterRule) passNever(dDP utils.DataProvider) (bool, error) { return false, nil } +func (fltr *FilterRule) passHttp(dDP utils.DataProvider) (bool, error) { + strVal, err := fltr.rsrElement.ParseDataProvider(dDP) + if err != nil { + if err == utils.ErrNotFound { + return false, nil + } + return false, err + } + return filterHTTP(fltr.Type, dDP, fltr.Element, strVal) + +} func (fltr *Filter) Set(path []string, val any, newBranch bool, _ string) (err error) { switch len(path) { diff --git a/engine/filters_test.go b/engine/filters_test.go index 594fa73af..00103b5cd 100644 --- a/engine/filters_test.go +++ b/engine/filters_test.go @@ -18,6 +18,8 @@ along with this program. If not, see package engine import ( + "encoding/json" + "fmt" "net/http" "net/http/httptest" "reflect" @@ -2432,3 +2434,85 @@ func TestFRPassStringParseDataProviderErr(t *testing.T) { } } +func TestHttpInlineFilter(t *testing.T) { + + dP := &utils.MapStorage{ + utils.MetaReq: map[string]any{ + "Attribute": "AttributeProfile1", + "CGRID": "CGRATES_ID1", + utils.AccountField: "1002", + utils.AnswerTime: time.Date(2013, 12, 30, 14, 59, 31, 0, time.UTC), + }, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if len(r.URL.Query()) != 0 { + queryVal := r.URL.Query() + reply := queryVal.Has("~*req.Account") && queryVal.Get("~*req.Account") == "1002" + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, reply) + return + } + + var data map[string]any + if err := json.NewDecoder(r.Body).Decode(&data); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _, has := data["*req"] + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, has) + + })) + defer srv.Close() + url := "*http#" + "[" + srv.URL + "]" + + exp := &Filter{ + Tenant: "cgrates.org", + ID: url + ":" + "~*req.Account:", + Rules: []*FilterRule{ + { + Type: url, + Element: "~*req.Account", + }, + }, + } + if err := exp.Compile(); err != nil { + t.Fatal(err) + } + if fl, err := NewFilterFromInline("cgrates.org", url+":"+"~*req.Account:"); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(exp, fl) { + t.Errorf("Expected: %s , received: %s", utils.ToJSON(exp), utils.ToJSON(fl)) + } + if pass, err := exp.Rules[0].Pass(context.Background(), dP); err != nil { + t.Error(err) + } else if !pass { + t.Error("should had passed") + } + + exp2 := &Filter{ + Tenant: "cgrates.org", + ID: url + ":" + "*any:", + Rules: []*FilterRule{ + { + Type: url, + Element: "*any", + }, + }, + } + + if fl2, err := NewFilterFromInline("cgrates.org", url+":"+"*any:"); err != nil { + t.Error(err) + } else if fl2.Rules[0].Type != exp2.Rules[0].Type { + t.Errorf("Expected: %s , received: %s", utils.ToJSON(exp2), utils.ToJSON(fl2)) + } else if pass, err := fl2.Rules[0].Pass(context.Background(), dP); err != nil { + t.Error(err) + } else if !pass { + t.Error("should had passed") + } +} diff --git a/engine/libattributes.go b/engine/libattributes.go index 62eee0e24..7e2f22ad7 100644 --- a/engine/libattributes.go +++ b/engine/libattributes.go @@ -19,6 +19,7 @@ along with this program. If not, see package engine import ( + "bytes" "fmt" "strings" @@ -170,7 +171,7 @@ func NewAttributeFromInline(tenant, inlnRule string) (attr *AttributeProfile, er ID: inlnRule, } for _, rule := range strings.Split(inlnRule, utils.InfieldSep) { - ruleSplt := strings.SplitN(rule, utils.InInFieldSep, 3) + ruleSplt := utils.SplitPath(rule, utils.InInFieldSep[0], 3) if len(ruleSplt) < 3 { return nil, fmt.Errorf("inline parse error for string: <%s>", rule) } @@ -190,6 +191,13 @@ func NewAttributeFromInline(tenant, inlnRule string) (attr *AttributeProfile, er } return } +func externalAttributeAPI(httpType string, dDP utils.DataProvider) (string, error) { + urlS, err := extractUrlFromType(httpType) + if err != nil { + return "", err + } + return externalAPI(urlS, bytes.NewReader([]byte(dDP.String()))) +} func (ap *AttributeProfile) Set(path []string, val any, newBranch bool, rsrSep string) (err error) { switch len(path) { diff --git a/utils/consts.go b/utils/consts.go index bf52e5571..e7f6d0276 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -1106,6 +1106,7 @@ const ( MetaNotSentryPeer = "*notsentrypeer" MetaNotActivationInterval = "*notai" MetaNotRegex = "*notregex" + MetaHTTP = "*http" MetaEC = "*ec" )