From 6f6374abb98feb3f3b9f700dac4e58c4953e34cb Mon Sep 17 00:00:00 2001 From: gezimbll Date: Thu, 11 Jan 2024 10:57:16 -0500 Subject: [PATCH] added *http attribute type --- engine/attributes.go | 4 ++ engine/attributes_test.go | 47 +++++++++++++++++ engine/filterhelpers.go | 51 ++++++++++++++---- engine/filters.go | 9 +--- engine/filters_test.go | 1 + engine/libattributes.go | 18 ++++++- engine/z_attributes_test.go | 102 ++++++++++++++++++++++++++++++++++++ 7 files changed, 211 insertions(+), 21 deletions(-) diff --git a/engine/attributes.go b/engine/attributes.go index b69701839..35b718a2c 100644 --- a/engine/attributes.go +++ b/engine/attributes.go @@ -560,6 +560,10 @@ func ParseAttribute(dp utils.DataProvider, attrType, path string, value config.R sort.Strings(values[1:]) out = strings.Join(values, utils.InfieldSep) default: + if strings.HasPrefix(attrType, utils.MetaHTTP) { + out, err = externalAttributeAPI(attrType, dp) + break + } return utils.EmptyString, fmt.Errorf("unsupported type: <%s>", attrType) } return diff --git a/engine/attributes_test.go b/engine/attributes_test.go index 8153b9e63..b70f11758 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -1580,3 +1580,50 @@ func TestAttributesV1ProcessEventSentryPeer(t *testing.T) { } } + +func TestAttributeFromHTTP(t *testing.T) { + exp := "Account" + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, exp) + + })) + + defer testServer.Close() + attrType := utils.MetaHTTP + utils.HashtagSep + utils.IdxStart + testServer.URL + utils.IdxEnd + + attrID := attrType + ":*req.Category:*attributes" + expAttrPrf1 := &AttributeProfile{ + Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, + ID: attrType + ":*req.Category:*attributes", + Contexts: []string{utils.MetaAny}, + Attributes: []*Attribute{ + { + Path: utils.MetaReq + utils.NestingSep + "Category", + Type: attrType, + Value: config.NewRSRParsersMustCompile("*attributes", utils.InfieldSep), + }, + }, + } + attrPrf, err := NewAttributeFromInline(config.CgrConfig().GeneralCfg().DefaultTenant, attrID) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(expAttrPrf1, attrPrf) { + t.Errorf("Expecting %+v, received: %+v", utils.ToJSON(expAttrPrf1), utils.ToJSON(attrPrf)) + } + dp := utils.MapStorage{ + utils.MetaReq: utils.MapStorage{}, + } + + attr := attrPrf.Attributes[0] + if out, err := ParseAttribute(dp, attr.Type, attr.Path, attr.Value, + 0, utils.EmptyString, utils.EmptyString, utils.InfieldSep); err != nil { + t.Fatal(err) + } else if exp != out { + t.Errorf("Expected %q, Received %q", exp, out) + } +} diff --git a/engine/filterhelpers.go b/engine/filterhelpers.go index 42f08a8b4..07778dfd2 100644 --- a/engine/filterhelpers.go +++ b/engine/filterhelpers.go @@ -21,10 +21,12 @@ package engine import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" + "strings" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/guardian" @@ -242,9 +244,18 @@ func sentrypeerHasData(itemId, token, url string) (found bool, err error) { // expects a boolean reply // when element is set to *any the CGREvent is sent as JSON body // when the element is specified as a path e.g ~*req.Account is sent as query string pair ,the path being the key with the value extracted from dataprovider -func externalAPI(urlStr string, dDP any, fieldname, value string) (reply bool, err error) { - var resp *http.Response - parsedURL, err := url.Parse(urlStr) +func filterHTTP(httpType string, dDP any, fieldname, value string) (bool, error) { + var ( + parsedURL *url.URL + resp string + err error + ) + urlS, err := extractUrlFromType(httpType) + if err != nil { + return false, err + } + + parsedURL, err = url.Parse(urlS) if err != nil { return false, err } @@ -252,32 +263,40 @@ func externalAPI(urlStr string, dDP any, fieldname, value string) (reply bool, e queryParams := parsedURL.Query() queryParams.Set(fieldname, value) parsedURL.RawQuery = queryParams.Encode() - resp, err = getHTTP(http.MethodGet, parsedURL.String(), nil, nil) + resp, err = externalAPI(parsedURL.String(), nil, nil) } else { var data []byte data, err = json.Marshal(dDP) if err != nil { return false, fmt.Errorf("error marshaling data: %w", err) } - resp, err = getHTTP(http.MethodGet, parsedURL.String(), bytes.NewReader(data), nil) + resp, err = externalAPI(parsedURL.String(), bytes.NewReader(data), nil) } - if err != nil { - return false, fmt.Errorf("error processing the request: %w", err) + return false, err + } + return utils.IfaceAsBool(resp) +} + +func externalAPI(url string, rdr io.Reader, hdr map[string]string) (string, error) { + resp, err := getHTTP(http.MethodGet, url, rdr, hdr) + if err != nil { + return "", fmt.Errorf("error processing the request: %w", err) } defer resp.Body.Close() if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusMultipleChoices { body, err := io.ReadAll(resp.Body) - return false, fmt.Errorf("http request returned non-OK status code: %d ,body: %v ,err: %w", resp.StatusCode, string(body), err) + return "", fmt.Errorf("http request returned non-OK status code: %d ,body: %v ,err: %w", resp.StatusCode, string(body), err) } - if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil { - return false, fmt.Errorf("error decoding response: %w", err) + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error decoding response: %w", err) } - return + return string(body), nil } // constructs an request via parameters provided,url,header and payload ,uses defaultclient for sending the request @@ -291,3 +310,13 @@ func getHTTP(method, url string, payload io.Reader, headers map[string]string) ( } return http.DefaultClient.Do(req) } + +func extractUrlFromType(httpType string) (string, error) { + parts := strings.Split(httpType, utils.HashtagSep) + if len(parts) != 2 { + return "", errors.New("url is not specified") + } + //extracting the url from the type + url := strings.Trim(parts[1], utils.IdxStart+utils.IdxEnd) + return url, nil +} diff --git a/engine/filters.go b/engine/filters.go index 689916d88..bd7f92259 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -19,7 +19,6 @@ along with this program. If not, see package engine import ( - "errors" "fmt" "net" "reflect" @@ -789,12 +788,6 @@ func (fltr *FilterRule) passHttp(dDP utils.DataProvider) (bool, error) { return false, err } - parts := strings.Split(fltr.Type, utils.HashtagSep) - if len(parts) != 2 { - return false, errors.New("url is not specified") - } - //extracting the url from the type - url := strings.Trim(parts[1], "[]") - return externalAPI(url, dDP, fltr.Element, strVal) + return filterHTTP(fltr.Type, dDP, fltr.Element, strVal) } diff --git a/engine/filters_test.go b/engine/filters_test.go index f54850017..dc4fa5db2 100644 --- a/engine/filters_test.go +++ b/engine/filters_test.go @@ -2711,6 +2711,7 @@ func TestHttpInlineFilter(t *testing.T) { fmt.Fprint(w, has) })) + defer srv.Close() url := "*http#" + "[" + srv.URL + "]" exp := &Filter{ diff --git a/engine/libattributes.go b/engine/libattributes.go index 1ae026623..186e4550b 100644 --- a/engine/libattributes.go +++ b/engine/libattributes.go @@ -19,6 +19,8 @@ along with this program. If not, see package engine import ( + "bytes" + "encoding/json" "fmt" "sort" "strings" @@ -148,8 +150,8 @@ func NewAttributeFromInline(tenant, inlnRule string) (attr *AttributeProfile, er Contexts: []string{utils.MetaAny}, } for _, rule := range strings.Split(inlnRule, utils.InfieldSep) { - ruleSplt := strings.SplitN(rule, utils.InInFieldSep, 3) - if len(ruleSplt) < 3 { + ruleSplt := utils.SplitPath(rule, utils.InInFieldSep[0], 3) + if len(ruleSplt) != 3 { return nil, fmt.Errorf("inline parse error for string: <%s>", rule) } var vals config.RSRParsers @@ -168,3 +170,15 @@ func NewAttributeFromInline(tenant, inlnRule string) (attr *AttributeProfile, er } return } + +func externalAttributeAPI(httpType string, dDP utils.DataProvider) (string, error) { + urlS, err := extractUrlFromType(httpType) + if err != nil { + return "", err + } + data, err := json.Marshal(dDP) + if err != nil { + return "", fmt.Errorf("error marshaling data: %w", err) + } + return externalAPI(urlS, bytes.NewReader(data), nil) +} diff --git a/engine/z_attributes_test.go b/engine/z_attributes_test.go index 2f00b509d..7ffdc20b2 100644 --- a/engine/z_attributes_test.go +++ b/engine/z_attributes_test.go @@ -18,7 +18,11 @@ along with this program. If not, see package engine import ( + "fmt" + "net/http" + "net/http/httptest" "reflect" + "slices" "sort" "testing" "time" @@ -2633,6 +2637,104 @@ func TestProcessAttributeSuffix(t *testing.T) { } } +func TestProcessAttributeHTTP(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + responses := map[string]struct { + code int + reply string + }{ + "/passwd": {code: http.StatusOK, reply: "passwd1"}, + "/account": {code: http.StatusOK, reply: "1001"}, + } + if val, has := responses[r.URL.EscapedPath()]; has { + w.WriteHeader(val.code) + fmt.Fprint(w, val.reply) + return + } + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + })) + defer ts.Close() + cfg := config.NewDefaultCGRConfig() + cfg.AttributeSCfg().Opts.ProcessRuns = 1 + data := NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) + attrS = NewAttributeService(dmAtr, &FilterS{dm: dmAtr, cfg: cfg}, cfg) + + //refresh the DM + if err := dmAtr.DataDB().Flush(""); err != nil { + t.Error(err) + } + Cache.Clear(nil) + + attrPrf := &AttributeProfile{ + Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, + ID: "ATTR_HTTP", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"*string:~*req.Field1:Value1"}, + Attributes: []*Attribute{ + { + Path: utils.MetaReq + utils.NestingSep + utils.AccountField, + Type: utils.MetaHTTP + utils.HashtagSep + utils.IdxStart + ts.URL + "/account" + utils.IdxEnd, + Value: config.NewRSRParsersMustCompile("*attributes", utils.InfieldSep), + }, + { + Path: utils.MetaReq + utils.NestingSep + "Password", + Type: utils.MetaHTTP + utils.HashtagSep + utils.IdxStart + ts.URL + "/passwd" + utils.IdxEnd, + Value: config.NewRSRParsersMustCompile("*attributes", utils.InfieldSep), + }, + }, + Weight: 10, + } + // Add attribute in DM + if err := dmAtr.SetAttributeProfile(attrPrf, true); err != nil { + t.Error(err) + } + attrArgs := &utils.CGREvent{ + Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, + ID: utils.GenUUID(), + Event: map[string]any{ + "Field1": "Value1", + }, + APIOpts: map[string]any{ + utils.OptsAttributesProcessRuns: 1, + utils.OptsContext: utils.MetaSessionS, + }, + } + eRply := &AttrSProcessEventReply{ + MatchedProfiles: []string{"cgrates.org:ATTR_HTTP"}, + AlteredFields: []string{utils.MetaReq + utils.NestingSep + utils.AccountField, utils.MetaReq + utils.NestingSep + "Password"}, + CGREvent: &utils.CGREvent{ + Tenant: config.CgrConfig().GeneralCfg().DefaultTenant, + ID: "TestProcessAttributeHTTP", + Event: map[string]any{ + "Field1": "Value1", + "Account": "1001", + "Password": "passwd1", + }, + }, + } + var reply AttrSProcessEventReply + if err := attrS.V1ProcessEvent(context.Background(), attrArgs, &reply); err != nil { + t.Errorf("Error: %+v", err) + } + if !slices.Equal(eRply.MatchedProfiles, reply.MatchedProfiles) { + t.Errorf("Expecting %+v, received: %+v", eRply.MatchedProfiles, reply.MatchedProfiles) + } + if sort.Slice(reply.AlteredFields, func(i, j int) bool { + return reply.AlteredFields[i] < reply.AlteredFields[j] + }); !slices.Equal(reply.AlteredFields, eRply.AlteredFields) { + t.Errorf("Expecting %+v, received: %+v", eRply.AlteredFields, reply.AlteredFields) + } + if !reflect.DeepEqual(eRply.CGREvent.Event, reply.CGREvent.Event) { + t.Errorf("Expecting %+v, received: %+v", eRply.CGREvent.Event, reply.CGREvent.Event) + } + +} + func TestAttributeIndexSelectsFalse(t *testing.T) { // change the IndexedSelects to false cfg := config.NewDefaultCGRConfig()