upgradeaware.go

Documentation: k8s.io/apimachinery/pkg/util/proxy

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package proxy
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"fmt"
    23  	"io"
    24  	"log"
    25  	"net"
    26  	"net/http"
    27  	"net/http/httputil"
    28  	"net/url"
    29  	"os"
    30  	"strings"
    31  	"time"
    32  
    33  	"k8s.io/apimachinery/pkg/api/errors"
    34  	"k8s.io/apimachinery/pkg/util/httpstream"
    35  	utilnet "k8s.io/apimachinery/pkg/util/net"
    36  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    37  
    38  	"github.com/mxk/go-flowrate/flowrate"
    39  
    40  	"k8s.io/klog/v2"
    41  )
    42  
    43  // UpgradeRequestRoundTripper provides an additional method to decorate a request
    44  // with any authentication or other protocol level information prior to performing
    45  // an upgrade on the server. Any response will be handled by the intercepting
    46  // proxy.
    47  type UpgradeRequestRoundTripper interface {
    48  	http.RoundTripper
    49  	// WrapRequest takes a valid HTTP request and returns a suitably altered version
    50  	// of request with any HTTP level values required to complete the request half of
    51  	// an upgrade on the server. It does not get a chance to see the response and
    52  	// should bypass any request side logic that expects to see the response.
    53  	WrapRequest(*http.Request) (*http.Request, error)
    54  }
    55  
    56  // UpgradeAwareHandler is a handler for proxy requests that may require an upgrade
    57  type UpgradeAwareHandler struct {
    58  	// UpgradeRequired will reject non-upgrade connections if true.
    59  	UpgradeRequired bool
    60  	// Location is the location of the upstream proxy. It is used as the location to Dial on the upstream server
    61  	// for upgrade requests unless UseRequestLocationOnUpgrade is true.
    62  	Location *url.URL
    63  	// AppendLocationPath determines if the original path of the Location should be appended to the upstream proxy request path
    64  	AppendLocationPath bool
    65  	// Transport provides an optional round tripper to use to proxy. If nil, the default proxy transport is used
    66  	Transport http.RoundTripper
    67  	// UpgradeTransport, if specified, will be used as the backend transport when upgrade requests are provided.
    68  	// This allows clients to disable HTTP/2.
    69  	UpgradeTransport UpgradeRequestRoundTripper
    70  	// WrapTransport indicates whether the provided Transport should be wrapped with default proxy transport behavior (URL rewriting, X-Forwarded-* header setting)
    71  	WrapTransport bool
    72  	// UseRequestLocation will use the incoming request URL when talking to the backend server.
    73  	UseRequestLocation bool
    74  	// UseLocationHost overrides the HTTP host header in requests to the backend server to use the Host from Location.
    75  	// This will override the req.Host field of a request, while UseRequestLocation will override the req.URL field
    76  	// of a request. The req.URL.Host specifies the server to connect to, while the req.Host field
    77  	// specifies the Host header value to send in the HTTP request. If this is false, the incoming req.Host header will
    78  	// just be forwarded to the backend server.
    79  	UseLocationHost bool
    80  	// FlushInterval controls how often the standard HTTP proxy will flush content from the upstream.
    81  	FlushInterval time.Duration
    82  	// MaxBytesPerSec controls the maximum rate for an upstream connection. No rate is imposed if the value is zero.
    83  	MaxBytesPerSec int64
    84  	// Responder is passed errors that occur while setting up proxying.
    85  	Responder ErrorResponder
    86  	// Reject to forward redirect response
    87  	RejectForwardingRedirects bool
    88  }
    89  
    90  const defaultFlushInterval = 200 * time.Millisecond
    91  
    92  // ErrorResponder abstracts error reporting to the proxy handler to remove the need to hardcode a particular
    93  // error format.
    94  type ErrorResponder interface {
    95  	Error(w http.ResponseWriter, req *http.Request, err error)
    96  }
    97  
    98  // SimpleErrorResponder is the legacy implementation of ErrorResponder for callers that only
    99  // service a single request/response per proxy.
   100  type SimpleErrorResponder interface {
   101  	Error(err error)
   102  }
   103  
   104  func NewErrorResponder(r SimpleErrorResponder) ErrorResponder {
   105  	return simpleResponder{r}
   106  }
   107  
   108  type simpleResponder struct {
   109  	responder SimpleErrorResponder
   110  }
   111  
   112  func (r simpleResponder) Error(w http.ResponseWriter, req *http.Request, err error) {
   113  	r.responder.Error(err)
   114  }
   115  
   116  // upgradeRequestRoundTripper implements proxy.UpgradeRequestRoundTripper.
   117  type upgradeRequestRoundTripper struct {
   118  	http.RoundTripper
   119  	upgrader http.RoundTripper
   120  }
   121  
   122  var (
   123  	_ UpgradeRequestRoundTripper  = &upgradeRequestRoundTripper{}
   124  	_ utilnet.RoundTripperWrapper = &upgradeRequestRoundTripper{}
   125  )
   126  
   127  // WrappedRoundTripper returns the round tripper that a caller would use.
   128  func (rt *upgradeRequestRoundTripper) WrappedRoundTripper() http.RoundTripper {
   129  	return rt.RoundTripper
   130  }
   131  
   132  // WriteToRequest calls the nested upgrader and then copies the returned request
   133  // fields onto the passed request.
   134  func (rt *upgradeRequestRoundTripper) WrapRequest(req *http.Request) (*http.Request, error) {
   135  	resp, err := rt.upgrader.RoundTrip(req)
   136  	if err != nil {
   137  		return nil, err
   138  	}
   139  	return resp.Request, nil
   140  }
   141  
   142  // onewayRoundTripper captures the provided request - which is assumed to have
   143  // been modified by other round trippers - and then returns a fake response.
   144  type onewayRoundTripper struct{}
   145  
   146  // RoundTrip returns a simple 200 OK response that captures the provided request.
   147  func (onewayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
   148  	return &http.Response{
   149  		Status:     "200 OK",
   150  		StatusCode: http.StatusOK,
   151  		Body:       io.NopCloser(&bytes.Buffer{}),
   152  		Request:    req,
   153  	}, nil
   154  }
   155  
   156  // MirrorRequest is a round tripper that can be called to get back the calling request as
   157  // the core round tripper in a chain.
   158  var MirrorRequest http.RoundTripper = onewayRoundTripper{}
   159  
   160  // NewUpgradeRequestRoundTripper takes two round trippers - one for the underlying TCP connection, and
   161  // one that is able to write headers to an HTTP request. The request rt is used to set the request headers
   162  // and that is written to the underlying connection rt.
   163  func NewUpgradeRequestRoundTripper(connection, request http.RoundTripper) UpgradeRequestRoundTripper {
   164  	return &upgradeRequestRoundTripper{
   165  		RoundTripper: connection,
   166  		upgrader:     request,
   167  	}
   168  }
   169  
   170  // normalizeLocation returns the result of parsing the full URL, with scheme set to http if missing
   171  func normalizeLocation(location *url.URL) *url.URL {
   172  	normalized, _ := url.Parse(location.String())
   173  	if len(normalized.Scheme) == 0 {
   174  		normalized.Scheme = "http"
   175  	}
   176  	return normalized
   177  }
   178  
   179  // NewUpgradeAwareHandler creates a new proxy handler with a default flush interval. Responder is required for returning
   180  // errors to the caller.
   181  func NewUpgradeAwareHandler(location *url.URL, transport http.RoundTripper, wrapTransport, upgradeRequired bool, responder ErrorResponder) *UpgradeAwareHandler {
   182  	return &UpgradeAwareHandler{
   183  		Location:        normalizeLocation(location),
   184  		Transport:       transport,
   185  		WrapTransport:   wrapTransport,
   186  		UpgradeRequired: upgradeRequired,
   187  		FlushInterval:   defaultFlushInterval,
   188  		Responder:       responder,
   189  	}
   190  }
   191  
   192  func proxyRedirectsforRootPath(path string, w http.ResponseWriter, req *http.Request) bool {
   193  	redirect := false
   194  	method := req.Method
   195  
   196  	// From pkg/genericapiserver/endpoints/handlers/proxy.go#ServeHTTP:
   197  	// Redirect requests with an empty path to a location that ends with a '/'
   198  	// This is essentially a hack for https://issue.k8s.io/4958.
   199  	// Note: Keep this code after tryUpgrade to not break that flow.
   200  	if len(path) == 0 && (method == http.MethodGet || method == http.MethodHead) {
   201  		var queryPart string
   202  		if len(req.URL.RawQuery) > 0 {
   203  			queryPart = "?" + req.URL.RawQuery
   204  		}
   205  		w.Header().Set("Location", req.URL.Path+"/"+queryPart)
   206  		w.WriteHeader(http.StatusMovedPermanently)
   207  		redirect = true
   208  	}
   209  	return redirect
   210  }
   211  
   212  // ServeHTTP handles the proxy request
   213  func (h *UpgradeAwareHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   214  	if h.tryUpgrade(w, req) {
   215  		return
   216  	}
   217  	if h.UpgradeRequired {
   218  		h.Responder.Error(w, req, errors.NewBadRequest("Upgrade request required"))
   219  		return
   220  	}
   221  
   222  	loc := *h.Location
   223  	loc.RawQuery = req.URL.RawQuery
   224  
   225  	// If original request URL ended in '/', append a '/' at the end of the
   226  	// of the proxy URL
   227  	if !strings.HasSuffix(loc.Path, "/") && strings.HasSuffix(req.URL.Path, "/") {
   228  		loc.Path += "/"
   229  	}
   230  
   231  	proxyRedirect := proxyRedirectsforRootPath(loc.Path, w, req)
   232  	if proxyRedirect {
   233  		return
   234  	}
   235  
   236  	if h.Transport == nil || h.WrapTransport {
   237  		h.Transport = h.defaultProxyTransport(req.URL, h.Transport)
   238  	}
   239  
   240  	// WithContext creates a shallow clone of the request with the same context.
   241  	newReq := req.WithContext(req.Context())
   242  	newReq.Header = utilnet.CloneHeader(req.Header)
   243  	if !h.UseRequestLocation {
   244  		newReq.URL = &loc
   245  	}
   246  	if h.UseLocationHost {
   247  		// exchanging req.Host with the backend location is necessary for backends that act on the HTTP host header (e.g. API gateways),
   248  		// because req.Host has preference over req.URL.Host in filling this header field
   249  		newReq.Host = h.Location.Host
   250  	}
   251  
   252  	// create the target location to use for the reverse proxy
   253  	reverseProxyLocation := &url.URL{Scheme: h.Location.Scheme, Host: h.Location.Host}
   254  	if h.AppendLocationPath {
   255  		reverseProxyLocation.Path = h.Location.Path
   256  	}
   257  
   258  	proxy := httputil.NewSingleHostReverseProxy(reverseProxyLocation)
   259  	proxy.Transport = h.Transport
   260  	proxy.FlushInterval = h.FlushInterval
   261  	proxy.ErrorLog = log.New(noSuppressPanicError{}, "", log.LstdFlags)
   262  	if h.RejectForwardingRedirects {
   263  		oldModifyResponse := proxy.ModifyResponse
   264  		proxy.ModifyResponse = func(response *http.Response) error {
   265  			code := response.StatusCode
   266  			if code >= 300 && code <= 399 && len(response.Header.Get("Location")) > 0 {
   267  				// close the original response
   268  				response.Body.Close()
   269  				msg := "the backend attempted to redirect this request, which is not permitted"
   270  				// replace the response
   271  				*response = http.Response{
   272  					StatusCode:    http.StatusBadGateway,
   273  					Status:        fmt.Sprintf("%d %s", response.StatusCode, http.StatusText(response.StatusCode)),
   274  					Body:          io.NopCloser(strings.NewReader(msg)),
   275  					ContentLength: int64(len(msg)),
   276  				}
   277  			} else {
   278  				if oldModifyResponse != nil {
   279  					if err := oldModifyResponse(response); err != nil {
   280  						return err
   281  					}
   282  				}
   283  			}
   284  			return nil
   285  		}
   286  	}
   287  	if h.Responder != nil {
   288  		// if an optional error interceptor/responder was provided wire it
   289  		// the custom responder might be used for providing a unified error reporting
   290  		// or supporting retry mechanisms by not sending non-fatal errors to the clients
   291  		proxy.ErrorHandler = h.Responder.Error
   292  	}
   293  	proxy.ServeHTTP(w, newReq)
   294  }
   295  
   296  type noSuppressPanicError struct{}
   297  
   298  func (noSuppressPanicError) Write(p []byte) (n int, err error) {
   299  	// skip "suppressing panic for copyResponse error in test; copy error" error message
   300  	// that ends up in CI tests on each kube-apiserver termination as noise and
   301  	// everybody thinks this is fatal.
   302  	if strings.Contains(string(p), "suppressing panic") {
   303  		return len(p), nil
   304  	}
   305  	return os.Stderr.Write(p)
   306  }
   307  
   308  // tryUpgrade returns true if the request was handled.
   309  func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Request) bool {
   310  	if !httpstream.IsUpgradeRequest(req) {
   311  		klog.V(6).Infof("Request was not an upgrade")
   312  		return false
   313  	}
   314  
   315  	var (
   316  		backendConn net.Conn
   317  		rawResponse []byte
   318  		err         error
   319  	)
   320  
   321  	location := *h.Location
   322  	if h.UseRequestLocation {
   323  		location = *req.URL
   324  		location.Scheme = h.Location.Scheme
   325  		location.Host = h.Location.Host
   326  		if h.AppendLocationPath {
   327  			location.Path = singleJoiningSlash(h.Location.Path, location.Path)
   328  		}
   329  	}
   330  
   331  	clone := utilnet.CloneRequest(req)
   332  	// Only append X-Forwarded-For in the upgrade path, since httputil.NewSingleHostReverseProxy
   333  	// handles this in the non-upgrade path.
   334  	utilnet.AppendForwardedForHeader(clone)
   335  	klog.V(6).Infof("Connecting to backend proxy (direct dial) %s\n  Headers: %v", &location, clone.Header)
   336  	if h.UseLocationHost {
   337  		clone.Host = h.Location.Host
   338  	}
   339  	clone.URL = &location
   340  	klog.V(6).Infof("UpgradeAwareProxy: dialing for SPDY upgrade with headers: %v", clone.Header)
   341  	backendConn, err = h.DialForUpgrade(clone)
   342  	if err != nil {
   343  		klog.V(6).Infof("Proxy connection error: %v", err)
   344  		h.Responder.Error(w, req, err)
   345  		return true
   346  	}
   347  	defer backendConn.Close()
   348  
   349  	// determine the http response code from the backend by reading from rawResponse+backendConn
   350  	backendHTTPResponse, headerBytes, err := getResponse(io.MultiReader(bytes.NewReader(rawResponse), backendConn))
   351  	if err != nil {
   352  		klog.V(6).Infof("Proxy connection error: %v", err)
   353  		h.Responder.Error(w, req, err)
   354  		return true
   355  	}
   356  	if len(headerBytes) > len(rawResponse) {
   357  		// we read beyond the bytes stored in rawResponse, update rawResponse to the full set of bytes read from the backend
   358  		rawResponse = headerBytes
   359  	}
   360  
   361  	// If the backend did not upgrade the request, return an error to the client. If the response was
   362  	// an error, the error is forwarded directly after the connection is hijacked. Otherwise, just
   363  	// return a generic error here.
   364  	if backendHTTPResponse.StatusCode != http.StatusSwitchingProtocols && backendHTTPResponse.StatusCode < 400 {
   365  		err := fmt.Errorf("invalid upgrade response: status code %d", backendHTTPResponse.StatusCode)
   366  		klog.Errorf("Proxy upgrade error: %v", err)
   367  		h.Responder.Error(w, req, err)
   368  		return true
   369  	}
   370  
   371  	// Once the connection is hijacked, the ErrorResponder will no longer work, so
   372  	// hijacking should be the last step in the upgrade.
   373  	requestHijacker, ok := w.(http.Hijacker)
   374  	if !ok {
   375  		klog.Errorf("Unable to hijack response writer: %T", w)
   376  		h.Responder.Error(w, req, fmt.Errorf("request connection cannot be hijacked: %T", w))
   377  		return true
   378  	}
   379  	requestHijackedConn, _, err := requestHijacker.Hijack()
   380  	if err != nil {
   381  		klog.Errorf("Unable to hijack response: %v", err)
   382  		h.Responder.Error(w, req, fmt.Errorf("error hijacking connection: %v", err))
   383  		return true
   384  	}
   385  	defer requestHijackedConn.Close()
   386  
   387  	if backendHTTPResponse.StatusCode != http.StatusSwitchingProtocols {
   388  		// If the backend did not upgrade the request, echo the response from the backend to the client and return, closing the connection.
   389  		klog.V(6).Infof("Proxy upgrade error, status code %d", backendHTTPResponse.StatusCode)
   390  		// set read/write deadlines
   391  		deadline := time.Now().Add(10 * time.Second)
   392  		backendConn.SetReadDeadline(deadline)
   393  		requestHijackedConn.SetWriteDeadline(deadline)
   394  		// write the response to the client
   395  		err := backendHTTPResponse.Write(requestHijackedConn)
   396  		if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
   397  			klog.Errorf("Error proxying data from backend to client: %v", err)
   398  		}
   399  		// Indicate we handled the request
   400  		return true
   401  	}
   402  
   403  	// Forward raw response bytes back to client.
   404  	if len(rawResponse) > 0 {
   405  		klog.V(6).Infof("Writing %d bytes to hijacked connection", len(rawResponse))
   406  		if _, err = requestHijackedConn.Write(rawResponse); err != nil {
   407  			utilruntime.HandleError(fmt.Errorf("Error proxying response from backend to client: %v", err))
   408  		}
   409  	}
   410  
   411  	// Proxy the connection. This is bidirectional, so we need a goroutine
   412  	// to copy in each direction. Once one side of the connection exits, we
   413  	// exit the function which performs cleanup and in the process closes
   414  	// the other half of the connection in the defer.
   415  	writerComplete := make(chan struct{})
   416  	readerComplete := make(chan struct{})
   417  
   418  	go func() {
   419  		var writer io.WriteCloser
   420  		if h.MaxBytesPerSec > 0 {
   421  			writer = flowrate.NewWriter(backendConn, h.MaxBytesPerSec)
   422  		} else {
   423  			writer = backendConn
   424  		}
   425  		_, err := io.Copy(writer, requestHijackedConn)
   426  		if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
   427  			klog.Errorf("Error proxying data from client to backend: %v", err)
   428  		}
   429  		close(writerComplete)
   430  	}()
   431  
   432  	go func() {
   433  		var reader io.ReadCloser
   434  		if h.MaxBytesPerSec > 0 {
   435  			reader = flowrate.NewReader(backendConn, h.MaxBytesPerSec)
   436  		} else {
   437  			reader = backendConn
   438  		}
   439  		_, err := io.Copy(requestHijackedConn, reader)
   440  		if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
   441  			klog.Errorf("Error proxying data from backend to client: %v", err)
   442  		}
   443  		close(readerComplete)
   444  	}()
   445  
   446  	// Wait for one half the connection to exit. Once it does the defer will
   447  	// clean up the other half of the connection.
   448  	select {
   449  	case <-writerComplete:
   450  	case <-readerComplete:
   451  	}
   452  	klog.V(6).Infof("Disconnecting from backend proxy %s\n  Headers: %v", &location, clone.Header)
   453  
   454  	return true
   455  }
   456  
   457  // FIXME: Taken from net/http/httputil/reverseproxy.go as singleJoiningSlash is not exported to be re-used.
   458  // See-also: https://github.com/golang/go/issues/44290
   459  func singleJoiningSlash(a, b string) string {
   460  	aslash := strings.HasSuffix(a, "/")
   461  	bslash := strings.HasPrefix(b, "/")
   462  	switch {
   463  	case aslash && bslash:
   464  		return a + b[1:]
   465  	case !aslash && !bslash:
   466  		return a + "/" + b
   467  	}
   468  	return a + b
   469  }
   470  
   471  func (h *UpgradeAwareHandler) DialForUpgrade(req *http.Request) (net.Conn, error) {
   472  	if h.UpgradeTransport == nil {
   473  		return dial(req, h.Transport)
   474  	}
   475  	updatedReq, err := h.UpgradeTransport.WrapRequest(req)
   476  	if err != nil {
   477  		return nil, err
   478  	}
   479  	return dial(updatedReq, h.UpgradeTransport)
   480  }
   481  
   482  // getResponseCode reads a http response from the given reader, returns the response,
   483  // the bytes read from the reader, and any error encountered
   484  func getResponse(r io.Reader) (*http.Response, []byte, error) {
   485  	rawResponse := bytes.NewBuffer(make([]byte, 0, 256))
   486  	// Save the bytes read while reading the response headers into the rawResponse buffer
   487  	resp, err := http.ReadResponse(bufio.NewReader(io.TeeReader(r, rawResponse)), nil)
   488  	if err != nil {
   489  		return nil, nil, err
   490  	}
   491  	// return the http response and the raw bytes consumed from the reader in the process
   492  	return resp, rawResponse.Bytes(), nil
   493  }
   494  
   495  // dial dials the backend at req.URL and writes req to it.
   496  func dial(req *http.Request, transport http.RoundTripper) (net.Conn, error) {
   497  	conn, err := DialURL(req.Context(), req.URL, transport)
   498  	if err != nil {
   499  		return nil, fmt.Errorf("error dialing backend: %v", err)
   500  	}
   501  
   502  	if err = req.Write(conn); err != nil {
   503  		conn.Close()
   504  		return nil, fmt.Errorf("error sending request: %v", err)
   505  	}
   506  
   507  	return conn, err
   508  }
   509  
   510  func (h *UpgradeAwareHandler) defaultProxyTransport(url *url.URL, internalTransport http.RoundTripper) http.RoundTripper {
   511  	scheme := url.Scheme
   512  	host := url.Host
   513  	suffix := h.Location.Path
   514  	if strings.HasSuffix(url.Path, "/") && !strings.HasSuffix(suffix, "/") {
   515  		suffix += "/"
   516  	}
   517  	pathPrepend := strings.TrimSuffix(url.Path, suffix)
   518  	rewritingTransport := &Transport{
   519  		Scheme:       scheme,
   520  		Host:         host,
   521  		PathPrepend:  pathPrepend,
   522  		RoundTripper: internalTransport,
   523  	}
   524  	return &corsRemovingTransport{
   525  		RoundTripper: rewritingTransport,
   526  	}
   527  }
   528  
   529  // corsRemovingTransport is a wrapper for an internal transport. It removes CORS headers
   530  // from the internal response.
   531  // Implements pkg/util/net.RoundTripperWrapper
   532  type corsRemovingTransport struct {
   533  	http.RoundTripper
   534  }
   535  
   536  var _ = utilnet.RoundTripperWrapper(&corsRemovingTransport{})
   537  
   538  func (rt *corsRemovingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
   539  	resp, err := rt.RoundTripper.RoundTrip(req)
   540  	if err != nil {
   541  		return nil, err
   542  	}
   543  	removeCORSHeaders(resp)
   544  	return resp, nil
   545  }
   546  
   547  func (rt *corsRemovingTransport) WrappedRoundTripper() http.RoundTripper {
   548  	return rt.RoundTripper
   549  }
   550  
   551  // removeCORSHeaders strip CORS headers sent from the backend
   552  // This should be called on all responses before returning
   553  func removeCORSHeaders(resp *http.Response) {
   554  	resp.Header.Del("Access-Control-Allow-Credentials")
   555  	resp.Header.Del("Access-Control-Allow-Headers")
   556  	resp.Header.Del("Access-Control-Allow-Methods")
   557  	resp.Header.Del("Access-Control-Allow-Origin")
   558  }
   559
View as plain text