http.go

Documentation: github.com/ory/x/decoderx

     1  package decoderx
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/sha256"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"net/http"
    11  	"net/url"
    12  	"strconv"
    13  	"strings"
    14  
    15  	"github.com/pkg/errors"
    16  	"github.com/tidwall/gjson"
    17  	"github.com/tidwall/sjson"
    18  
    19  	"github.com/ory/jsonschema/v3"
    20  
    21  	"github.com/ory/herodot"
    22  
    23  	"github.com/ory/x/httpx"
    24  	"github.com/ory/x/jsonschemax"
    25  	"github.com/ory/x/stringslice"
    26  )
    27  
    28  type (
    29  	// HTTP decodes json and form-data from HTTP Request Bodies.
    30  	HTTP struct{}
    31  
    32  	httpDecoderOptions struct {
    33  		keepRequestBody           bool
    34  		allowedContentTypes       []string
    35  		allowedHTTPMethods        []string
    36  		jsonSchemaRef             string
    37  		jsonSchemaCompiler        *jsonschema.Compiler
    38  		jsonSchemaValidate        bool
    39  		maxCircularReferenceDepth uint8
    40  		handleParseErrors         parseErrorStrategy
    41  		expectJSONFlattened       bool
    42  		queryAndBody              bool
    43  	}
    44  
    45  	// HTTPDecoderOption configures the HTTP decoder.
    46  	HTTPDecoderOption func(*httpDecoderOptions)
    47  
    48  	parseErrorStrategy uint8
    49  )
    50  
    51  const (
    52  	httpContentTypeMultipartForm  = "multipart/form-data"
    53  	httpContentTypeURLEncodedForm = "application/x-www-form-urlencoded"
    54  	httpContentTypeJSON           = "application/json"
    55  )
    56  
    57  const (
    58  	// ParseErrorIgnoreConversionErrors will ignore any errors caused by strconv.Parse* and use the
    59  	// raw form field value, which is a string, when such a parse error occurs.
    60  	//
    61  	// If the JSON Schema defines `{"ratio": {"type": "number"}}` but `ratio=foobar` then field
    62  	// `ratio` will be handled as a string. If the destination struct is a `json.RawMessage`, then
    63  	// the output will be `{"ratio": "foobar"}`.
    64  	ParseErrorIgnoreConversionErrors parseErrorStrategy = iota + 1
    65  
    66  	// ParseErrorUseEmptyValueOnConversionErrors will ignore any parse errors caused by strconv.Parse* and use the
    67  	// default value of the type to be casted, e.g. float64(0), string("").
    68  	//
    69  	// If the JSON Schema defines `{"ratio": {"type": "number"}}` but `ratio=foobar` then field
    70  	// `ratio` will receive the default value for the primitive type (here `0.0` for `number`).
    71  	// If the destination struct is a `json.RawMessage`, then the output will be `{"ratio": 0.0}`.
    72  	ParseErrorUseEmptyValueOnConversionErrors
    73  
    74  	// ParseErrorReturnOnConversionErrors will abort and return with an error if strconv.Parse* returns
    75  	// an error.
    76  	//
    77  	// If the JSON Schema defines `{"ratio": {"type": "number"}}` but `ratio=foobar` the parser aborts
    78  	// and returns an error, here: `strconv.ParseFloat: parsing "foobar"`.
    79  	ParseErrorReturnOnConversionErrors
    80  )
    81  
    82  // HTTPFormDecoder configures the HTTP decoder to only accept form-data
    83  // (application/x-www-form-urlencoded, multipart/form-data)
    84  func HTTPFormDecoder() HTTPDecoderOption {
    85  	return func(o *httpDecoderOptions) {
    86  		o.allowedContentTypes = []string{httpContentTypeMultipartForm, httpContentTypeURLEncodedForm}
    87  	}
    88  }
    89  
    90  // HTTPJSONDecoder configures the HTTP decoder to only accept form-data
    91  // (application/json).
    92  func HTTPJSONDecoder() HTTPDecoderOption {
    93  	return func(o *httpDecoderOptions) {
    94  		o.allowedContentTypes = []string{httpContentTypeJSON}
    95  	}
    96  }
    97  
    98  // HTTPKeepRequestBody configures the HTTP decoder to allow other
    99  // HTTP request body readers to read the body as well by keeping
   100  // the data in memory.
   101  func HTTPKeepRequestBody(keep bool) HTTPDecoderOption {
   102  	return func(o *httpDecoderOptions) {
   103  		o.keepRequestBody = keep
   104  	}
   105  }
   106  
   107  // HTTPDecoderSetValidatePayloads sets if payloads should be validated or not.
   108  func HTTPDecoderSetValidatePayloads(validate bool) HTTPDecoderOption {
   109  	return func(o *httpDecoderOptions) {
   110  		o.jsonSchemaValidate = validate
   111  	}
   112  }
   113  
   114  // HTTPDecoderJSONFollowsFormFormat if set tells the decoder that JSON follows the same conventions
   115  // as the form decoder, meaning `{"foo.bar": "..."}` is translated to `{"foo": {"bar": "..."}}`.
   116  func HTTPDecoderJSONFollowsFormFormat() HTTPDecoderOption {
   117  	return func(o *httpDecoderOptions) {
   118  		o.expectJSONFlattened = true
   119  	}
   120  }
   121  
   122  // HTTPDecoderAllowedMethods sets the allowed HTTP methods. Defaults are POST, PUT, PATCH.
   123  func HTTPDecoderAllowedMethods(method ...string) HTTPDecoderOption {
   124  	return func(o *httpDecoderOptions) {
   125  		o.allowedHTTPMethods = method
   126  	}
   127  }
   128  
   129  // HTTPDecoderUseQueryAndBody will check both the HTTP body and the HTTP query params when decoding.
   130  // Only relevant for non-GET operations.
   131  func HTTPDecoderUseQueryAndBody() HTTPDecoderOption {
   132  	return func(o *httpDecoderOptions) {
   133  		o.queryAndBody = true
   134  	}
   135  }
   136  
   137  // HTTPDecoderSetIgnoreParseErrorsStrategy sets a strategy for dealing with strconv.Parse* errors:
   138  //
   139  // - decoderx.ParseErrorIgnoreConversionErrors will ignore any parse errors caused by strconv.Parse* and use the
   140  // raw form field value, which is a string, when such a parse error occurs. (default)
   141  // - decoderx.ParseErrorUseEmptyValueOnConversionErrors will ignore any parse errors caused by strconv.Parse* and use the
   142  // default value of the type to be casted, e.g. float64(0), string("").
   143  // - decoderx.ParseErrorReturnOnConversionErrors will abort and return with an error if strconv.Parse* returns
   144  // an error.
   145  func HTTPDecoderSetIgnoreParseErrorsStrategy(strategy parseErrorStrategy) HTTPDecoderOption {
   146  	return func(o *httpDecoderOptions) {
   147  		o.handleParseErrors = strategy
   148  	}
   149  }
   150  
   151  // HTTPDecoderSetMaxCircularReferenceDepth sets the maximum recursive reference resolution depth.
   152  func HTTPDecoderSetMaxCircularReferenceDepth(depth uint8) HTTPDecoderOption {
   153  	return func(o *httpDecoderOptions) {
   154  		o.maxCircularReferenceDepth = depth
   155  	}
   156  }
   157  
   158  // HTTPJSONSchemaCompiler sets a JSON schema to be used for validation and type assertion of
   159  // incoming requests.
   160  func HTTPJSONSchemaCompiler(ref string, compiler *jsonschema.Compiler) HTTPDecoderOption {
   161  	return func(o *httpDecoderOptions) {
   162  		if compiler == nil {
   163  			compiler = jsonschema.NewCompiler()
   164  		}
   165  		compiler.ExtractAnnotations = true
   166  		o.jsonSchemaCompiler = compiler
   167  		o.jsonSchemaRef = ref
   168  		o.jsonSchemaValidate = true
   169  	}
   170  }
   171  
   172  // HTTPRawJSONSchemaCompiler uses a JSON Schema Compiler with the provided JSON Schema in raw byte form.
   173  func HTTPRawJSONSchemaCompiler(raw []byte) (HTTPDecoderOption, error) {
   174  	compiler := jsonschema.NewCompiler()
   175  	id := fmt.Sprintf("%x.json", sha256.Sum256(raw))
   176  	if err := compiler.AddResource(id, bytes.NewReader(raw)); err != nil {
   177  		return nil, err
   178  	}
   179  	compiler.ExtractAnnotations = true
   180  
   181  	return func(o *httpDecoderOptions) {
   182  		o.jsonSchemaCompiler = compiler
   183  		o.jsonSchemaRef = id
   184  		o.jsonSchemaValidate = true
   185  	}, nil
   186  }
   187  
   188  // MustHTTPRawJSONSchemaCompiler uses HTTPRawJSONSchemaCompiler and panics on error.
   189  func MustHTTPRawJSONSchemaCompiler(raw []byte) HTTPDecoderOption {
   190  	f, err := HTTPRawJSONSchemaCompiler(raw)
   191  	if err != nil {
   192  		panic(err)
   193  	}
   194  	return f
   195  }
   196  
   197  func newHTTPDecoderOptions(fs []HTTPDecoderOption) *httpDecoderOptions {
   198  	o := &httpDecoderOptions{
   199  		allowedContentTypes: []string{
   200  			httpContentTypeMultipartForm, httpContentTypeURLEncodedForm, httpContentTypeJSON,
   201  		},
   202  		allowedHTTPMethods:        []string{"POST", "PUT", "PATCH"},
   203  		maxCircularReferenceDepth: 5,
   204  		handleParseErrors:         ParseErrorIgnoreConversionErrors,
   205  	}
   206  
   207  	for _, f := range fs {
   208  		f(o)
   209  	}
   210  
   211  	return o
   212  }
   213  
   214  // NewHTTP creates a new HTTP decoder.
   215  func NewHTTP() *HTTP {
   216  	return new(HTTP)
   217  }
   218  
   219  func (t *HTTP) validateRequest(r *http.Request, c *httpDecoderOptions) error {
   220  	method := strings.ToUpper(r.Method)
   221  
   222  	if !stringslice.Has(c.allowedHTTPMethods, method) {
   223  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf(`Unable to decode body because HTTP Request Method was "%s" but only %v are supported.`, method, c.allowedHTTPMethods))
   224  	}
   225  
   226  	if method != "GET" {
   227  		if r.ContentLength == 0 && method != "GET" {
   228  			return errors.WithStack(herodot.ErrBadRequest.WithReasonf(`Unable to decode HTTP Request Body because its HTTP Header "Content-Length" is zero.`))
   229  		}
   230  
   231  		if !httpx.HasContentType(r, c.allowedContentTypes...) {
   232  			return errors.WithStack(herodot.ErrBadRequest.WithReasonf(`HTTP %s Request used unknown HTTP Header "Content-Type: %s", only %v are supported.`, method, r.Header.Get("Content-Type"), c.allowedContentTypes))
   233  		}
   234  	}
   235  
   236  	return nil
   237  }
   238  
   239  func (t *HTTP) validatePayload(raw json.RawMessage, c *httpDecoderOptions) error {
   240  	if !c.jsonSchemaValidate {
   241  		return nil
   242  	}
   243  
   244  	if c.jsonSchemaCompiler == nil {
   245  		return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("JSON Schema Validation is required but no compiler was provided."))
   246  	}
   247  
   248  	schema, err := c.jsonSchemaCompiler.Compile(c.jsonSchemaRef)
   249  	if err != nil {
   250  		return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("Unable to load JSON Schema from location: %s", c.jsonSchemaRef).WithDebug(err.Error()))
   251  	}
   252  
   253  	if err := schema.Validate(bytes.NewBuffer(raw)); err != nil {
   254  		if _, ok := err.(*jsonschema.ValidationError); ok {
   255  			return errors.WithStack(err)
   256  		}
   257  		return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("Unable to process JSON Schema and input: %s", err).WithDebug(err.Error()))
   258  	}
   259  
   260  	return nil
   261  }
   262  
   263  // Decode takes a HTTP Request Body and decodes it into destination.
   264  func (t *HTTP) Decode(r *http.Request, destination interface{}, opts ...HTTPDecoderOption) error {
   265  	c := newHTTPDecoderOptions(opts)
   266  	if err := t.validateRequest(r, c); err != nil {
   267  		return err
   268  	}
   269  
   270  	if r.Method == "GET" {
   271  		return t.decodeForm(r, destination, c)
   272  	} else if httpx.HasContentType(r, httpContentTypeJSON) {
   273  		if c.expectJSONFlattened {
   274  			return t.decodeJSONForm(r, destination, c)
   275  		}
   276  		return t.decodeJSON(r, destination, c)
   277  	} else if httpx.HasContentType(r, httpContentTypeMultipartForm, httpContentTypeURLEncodedForm) {
   278  		return t.decodeForm(r, destination, c)
   279  	}
   280  
   281  	return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("Unable to determine decoder for content type: %s", r.Header.Get("Content-Type")))
   282  }
   283  
   284  func (t *HTTP) requestBody(r *http.Request, o *httpDecoderOptions) (reader io.ReadCloser, err error) {
   285  	if strings.ToUpper(r.Method) == "GET" {
   286  		return ioutil.NopCloser(bytes.NewBufferString(r.URL.Query().Encode())), nil
   287  	}
   288  
   289  	if !o.keepRequestBody {
   290  		return r.Body, nil
   291  	}
   292  
   293  	bodyBytes, err := ioutil.ReadAll(r.Body)
   294  	if err != nil {
   295  		return nil, errors.Wrapf(err, "unable to read body")
   296  	}
   297  
   298  	_ = r.Body.Close() //  must close
   299  	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
   300  
   301  	return ioutil.NopCloser(bytes.NewBuffer(bodyBytes)), nil
   302  }
   303  
   304  func (t *HTTP) decodeJSONForm(r *http.Request, destination interface{}, o *httpDecoderOptions) error {
   305  	if o.jsonSchemaCompiler == nil {
   306  		return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("Unable to decode HTTP Form Body because no validation schema was provided. This is a code bug."))
   307  	}
   308  
   309  	paths, err := jsonschemax.ListPathsWithRecursion(o.jsonSchemaRef, o.jsonSchemaCompiler, o.maxCircularReferenceDepth)
   310  	if err != nil {
   311  		return errors.WithStack(herodot.ErrInternalServerError.WithTrace(err).WithReasonf("Unable to prepare JSON Schema for HTTP Post Body Form parsing: %s", err).WithDebugf("%+v", err))
   312  	}
   313  
   314  	reader, err := t.requestBody(r, o)
   315  	if err != nil {
   316  		return err
   317  	}
   318  
   319  	var interim json.RawMessage
   320  	if err := json.NewDecoder(reader).Decode(&interim); err != nil {
   321  		return err
   322  	}
   323  
   324  	parsed := gjson.ParseBytes(interim)
   325  	if !parsed.IsObject() {
   326  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected JSON sent in request body to be an object but got: %s", parsed.Type.String()))
   327  	}
   328  
   329  	values := url.Values{}
   330  	parsed.ForEach(func(k, v gjson.Result) bool {
   331  		values.Set(k.String(), v.String())
   332  		return true
   333  	})
   334  
   335  	if o.queryAndBody {
   336  		_ = r.ParseForm()
   337  		for k := range r.Form {
   338  			values.Set(k, r.Form.Get(k))
   339  		}
   340  	}
   341  
   342  	raw, err := t.decodeURLValues(values, paths, o)
   343  	if err != nil {
   344  		return err
   345  	}
   346  
   347  	if err := json.Unmarshal(raw, destination); err != nil {
   348  		return errors.WithStack(err)
   349  	}
   350  
   351  	if err := t.validatePayload(raw, o); err != nil {
   352  		return err
   353  	}
   354  
   355  	return nil
   356  }
   357  
   358  func (t *HTTP) decodeForm(r *http.Request, destination interface{}, o *httpDecoderOptions) error {
   359  	if o.jsonSchemaCompiler == nil {
   360  		return errors.WithStack(herodot.ErrInternalServerError.WithReasonf("Unable to decode HTTP Form Body because no validation schema was provided. This is a code bug."))
   361  	}
   362  
   363  	reader, err := t.requestBody(r, o)
   364  	if err != nil {
   365  		return err
   366  	}
   367  
   368  	defer func() {
   369  		r.Body = reader
   370  	}()
   371  
   372  	if err := r.ParseForm(); err != nil {
   373  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf("Unable to decode HTTP %s form body: %s", strings.ToUpper(r.Method), err).WithDebug(err.Error()))
   374  	}
   375  
   376  	paths, err := jsonschemax.ListPathsWithRecursion(o.jsonSchemaRef, o.jsonSchemaCompiler, o.maxCircularReferenceDepth)
   377  	if err != nil {
   378  		return errors.WithStack(herodot.ErrInternalServerError.WithTrace(err).WithReasonf("Unable to prepare JSON Schema for HTTP Post Body Form parsing: %s", err).WithDebugf("%+v", err))
   379  	}
   380  
   381  	values := r.PostForm
   382  	if r.Method == "GET" || o.queryAndBody {
   383  		values = r.Form
   384  	}
   385  
   386  	raw, err := t.decodeURLValues(values, paths, o)
   387  	if err != nil {
   388  		return err
   389  	}
   390  
   391  	if err := json.NewDecoder(bytes.NewReader(raw)).Decode(destination); err != nil {
   392  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf("Unable to decode JSON payload: %s", err))
   393  	}
   394  
   395  	if err := t.validatePayload(raw, o); err != nil {
   396  		return err
   397  	}
   398  
   399  	return nil
   400  }
   401  
   402  func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *httpDecoderOptions) (json.RawMessage, error) {
   403  	raw := json.RawMessage(`{}`)
   404  	for key := range values {
   405  		for _, path := range paths {
   406  			if key == path.Name {
   407  				var err error
   408  				switch path.Type.(type) {
   409  				case []string:
   410  					raw, err = sjson.SetBytes(raw, path.Name, values[key])
   411  				case []float64:
   412  					for k, v := range values[key] {
   413  						if f, err := strconv.ParseFloat(v, 64); err != nil {
   414  							switch o.handleParseErrors {
   415  							case ParseErrorIgnoreConversionErrors:
   416  								raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), v)
   417  							case ParseErrorUseEmptyValueOnConversionErrors:
   418  								raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f)
   419  							case ParseErrorReturnOnConversionErrors:
   420  								return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a number.").
   421  									WithDetail("parse_error", err.Error()).
   422  									WithDetail("name", key).
   423  									WithDetailf("index", "%d", k).
   424  									WithDetail("value", v))
   425  							}
   426  						} else {
   427  							raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f)
   428  						}
   429  					}
   430  				case []bool:
   431  					for k, v := range values[key] {
   432  						if f, err := strconv.ParseBool(v); err != nil {
   433  							switch o.handleParseErrors {
   434  							case ParseErrorIgnoreConversionErrors:
   435  								raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), v)
   436  							case ParseErrorUseEmptyValueOnConversionErrors:
   437  								raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f)
   438  							case ParseErrorReturnOnConversionErrors:
   439  								return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a boolean.").
   440  									WithDetail("parse_error", err.Error()).
   441  									WithDetail("name", key).
   442  									WithDetailf("index", "%d", k).
   443  									WithDetail("value", v))
   444  							}
   445  						} else {
   446  							raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f)
   447  						}
   448  					}
   449  				case []interface{}:
   450  					raw, err = sjson.SetBytes(raw, path.Name, values[key])
   451  				case bool:
   452  					v := values[key][len(values[key])-1]
   453  					if f, err := strconv.ParseBool(v); err != nil {
   454  						switch o.handleParseErrors {
   455  						case ParseErrorIgnoreConversionErrors:
   456  							raw, err = sjson.SetBytes(raw, path.Name, v)
   457  						case ParseErrorUseEmptyValueOnConversionErrors:
   458  							raw, err = sjson.SetBytes(raw, path.Name, f)
   459  						case ParseErrorReturnOnConversionErrors:
   460  							return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a boolean.").
   461  								WithDetail("parse_error", err.Error()).
   462  								WithDetail("name", key).
   463  								WithDetail("value", values.Get(key)))
   464  						}
   465  					} else {
   466  						raw, err = sjson.SetBytes(raw, path.Name, f)
   467  					}
   468  				case float64:
   469  					v := values.Get(key)
   470  					if f, err := strconv.ParseFloat(v, 64); err != nil {
   471  						switch o.handleParseErrors {
   472  						case ParseErrorIgnoreConversionErrors:
   473  							raw, err = sjson.SetBytes(raw, path.Name, v)
   474  						case ParseErrorUseEmptyValueOnConversionErrors:
   475  							raw, err = sjson.SetBytes(raw, path.Name, f)
   476  						case ParseErrorReturnOnConversionErrors:
   477  							return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a number.").
   478  								WithDetail("parse_error", err.Error()).
   479  								WithDetail("name", key).
   480  								WithDetail("value", values.Get(key)))
   481  						}
   482  					} else {
   483  						raw, err = sjson.SetBytes(raw, path.Name, f)
   484  					}
   485  				case string:
   486  					raw, err = sjson.SetBytes(raw, path.Name, values.Get(key))
   487  				case map[string]interface{}:
   488  					raw, err = sjson.SetBytes(raw, path.Name, values.Get(key))
   489  				case []map[string]interface{}:
   490  					raw, err = sjson.SetBytes(raw, path.Name, values[key])
   491  				}
   492  
   493  				if err != nil {
   494  					return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Unable to type assert values from HTTP Post Body: %s", err))
   495  				}
   496  				break
   497  			}
   498  		}
   499  	}
   500  	return raw, nil
   501  }
   502  
   503  func (t *HTTP) decodeJSON(r *http.Request, destination interface{}, o *httpDecoderOptions) error {
   504  	reader, err := t.requestBody(r, o)
   505  	if err != nil {
   506  		return err
   507  	}
   508  
   509  	raw, err := ioutil.ReadAll(reader)
   510  	if err != nil {
   511  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf("Unable to read HTTP POST body: %s", err))
   512  	}
   513  
   514  	if err := json.NewDecoder(bytes.NewReader(raw)).Decode(destination); err != nil {
   515  		return errors.WithStack(herodot.ErrBadRequest.WithReasonf("Unable to decode JSON payload: %s", err))
   516  	}
   517  
   518  	if err := t.validatePayload(raw, o); err != nil {
   519  		return err
   520  	}
   521  
   522  	return nil
   523  }
   524
View as plain text