From f5437632f486b7d0a0a181c58f113c86d032b02c Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 24 Apr 2017 20:11:36 -0400 Subject: Upgrading server dependancies (#6215) --- vendor/github.com/NYTimes/gziphandler/.travis.yml | 1 + vendor/github.com/NYTimes/gziphandler/README.md | 4 +- vendor/github.com/NYTimes/gziphandler/gzip.go | 127 +++++++++++++++++---- vendor/github.com/NYTimes/gziphandler/gzip_go18.go | 43 +++++++ .../NYTimes/gziphandler/gzip_go18_test.go | 70 ++++++++++++ vendor/github.com/NYTimes/gziphandler/gzip_test.go | 69 +++++++++-- 6 files changed, 281 insertions(+), 33 deletions(-) create mode 100644 vendor/github.com/NYTimes/gziphandler/gzip_go18.go create mode 100644 vendor/github.com/NYTimes/gziphandler/gzip_go18_test.go (limited to 'vendor/github.com/NYTimes') diff --git a/vendor/github.com/NYTimes/gziphandler/.travis.yml b/vendor/github.com/NYTimes/gziphandler/.travis.yml index f918f8d76..d2b67f69c 100644 --- a/vendor/github.com/NYTimes/gziphandler/.travis.yml +++ b/vendor/github.com/NYTimes/gziphandler/.travis.yml @@ -2,4 +2,5 @@ language: go go: - 1.7 + - 1.8 - tip diff --git a/vendor/github.com/NYTimes/gziphandler/README.md b/vendor/github.com/NYTimes/gziphandler/README.md index b1d55e26e..6d7246070 100644 --- a/vendor/github.com/NYTimes/gziphandler/README.md +++ b/vendor/github.com/NYTimes/gziphandler/README.md @@ -38,12 +38,12 @@ func main() { ## Documentation -The docs can be found at [godoc.org] [docs], as usual. +The docs can be found at [godoc.org][docs], as usual. ## License -[Apache 2.0] [license]. +[Apache 2.0][license]. diff --git a/vendor/github.com/NYTimes/gziphandler/gzip.go b/vendor/github.com/NYTimes/gziphandler/gzip.go index 23efacc45..cccf79de7 100644 --- a/vendor/github.com/NYTimes/gziphandler/gzip.go +++ b/vendor/github.com/NYTimes/gziphandler/gzip.go @@ -4,6 +4,7 @@ import ( "bufio" "compress/gzip" "fmt" + "io" "net" "net/http" "strconv" @@ -21,10 +22,16 @@ const ( type codings map[string]float64 -// The default qvalue to assign to an encoding if no explicit qvalue is set. -// This is actually kind of ambiguous in RFC 2616, so hopefully it's correct. -// The examples seem to indicate that it is. -const DEFAULT_QVALUE = 1.0 +const ( + // DefaultQValue is the default qvalue to assign to an encoding if no explicit qvalue is set. + // This is actually kind of ambiguous in RFC 2616, so hopefully it's correct. + // The examples seem to indicate that it is. + DefaultQValue = 1.0 + + // DefaultMinSize defines the minimum size to reach to enable compression. + // It's 512 bytes. + DefaultMinSize = 512 +) // gzipWriterPools stores a sync.Pool for each compression level for reuse of // gzip.Writers. Use poolIndex to covert a compression level to an index into @@ -63,35 +70,88 @@ func addLevelPool(level int) { // GzipResponseWriter provides an http.ResponseWriter interface, which gzips // bytes before writing them to the underlying response. This doesn't close the // writers, so don't forget to do that. +// It can be configured to skip response smaller than minSize. type GzipResponseWriter struct { http.ResponseWriter index int // Index for gzipWriterPools. gw *gzip.Writer + + code int // Saves the WriteHeader value. + + minSize int // Specifed the minimum response size to gzip. If the response length is bigger than this value, it is compressed. + buf []byte // Holds the first part of the write before reaching the minSize or the end of the write. } // Write appends data to the gzip writer. func (w *GzipResponseWriter) Write(b []byte) (int, error) { - // Lazily create the gzip.Writer, this allows empty bodies to be actually - // empty, for example in the case of status code 204 (no content). - if w.gw == nil { - w.init() - } - + // If content type is not set. if _, ok := w.Header()[contentType]; !ok { - // If content type is not set, infer it from the uncompressed body. + // It infer it from the uncompressed body. w.Header().Set(contentType, http.DetectContentType(b)) } - return w.gw.Write(b) + + // GZIP responseWriter is initialized. Use the GZIP responseWriter. + if w.gw != nil { + n, err := w.gw.Write(b) + return n, err + } + + // Save the write into a buffer for later use in GZIP responseWriter (if content is long enough) or at close with regular responseWriter. + w.buf = append(w.buf, b...) + + // If the global writes are bigger than the minSize, compression is enable. + if len(w.buf) >= w.minSize { + err := w.startGzip() + if err != nil { + return 0, err + } + } + + return len(b), nil +} + +// startGzip initialize any GZIP specific informations. +func (w *GzipResponseWriter) startGzip() error { + + // Set the GZIP header. + w.Header().Set(contentEncoding, "gzip") + + // if the Content-Length is already set, then calls to Write on gzip + // will fail to set the Content-Length header since its already set + // See: https://github.com/golang/go/issues/14975. + w.Header().Del(contentLength) + + // Write the header to gzip response. + w.writeHeader() + + // Initialize the GZIP response. + w.init() + + // Flush the buffer into the gzip reponse. + n, err := w.gw.Write(w.buf) + + // This should never happen (per io.Writer docs), but if the write didn't + // accept the entire buffer but returned no specific error, we have no clue + // what's going on, so abort just to be safe. + if err == nil && n < len(w.buf) { + return io.ErrShortWrite + } + + w.buf = nil + return err } -// WriteHeader will check if the gzip writer needs to be lazily initiated and -// then pass the code along to the underlying ResponseWriter. +// WriteHeader just saves the response code until close or GZIP effective writes. func (w *GzipResponseWriter) WriteHeader(code int) { - if w.gw == nil && - code != http.StatusNotModified && code != http.StatusNoContent { - w.init() + w.code = code +} + +// writeHeader uses the saved code to send it to the ResponseWriter. +func (w *GzipResponseWriter) writeHeader() { + if w.code == 0 { + w.code = http.StatusOK } - w.ResponseWriter.WriteHeader(code) + w.ResponseWriter.WriteHeader(w.code) } // init graps a new gzip writer from the gzipWriterPool and writes the correct @@ -102,15 +162,22 @@ func (w *GzipResponseWriter) init() { gzw := gzipWriterPools[w.index].Get().(*gzip.Writer) gzw.Reset(w.ResponseWriter) w.gw = gzw - w.ResponseWriter.Header().Set(contentEncoding, "gzip") - // if the Content-Length is already set, then calls to Write on gzip - // will fail to set the Content-Length header since its already set - // See: https://github.com/golang/go/issues/14975 - w.ResponseWriter.Header().Del(contentLength) } // Close will close the gzip.Writer and will put it back in the gzipWriterPool. func (w *GzipResponseWriter) Close() error { + // Buffer not nil means the regular response must be returned. + if w.buf != nil { + w.writeHeader() + // Make the write into the regular response. + _, writeErr := w.ResponseWriter.Write(w.buf) + // Returns the error if any at write. + if writeErr != nil { + return fmt.Errorf("gziphandler: write to regular responseWriter at close gets error: %q", writeErr.Error()) + } + } + + // If the GZIP responseWriter is not set no needs to close it. if w.gw == nil { return nil } @@ -163,9 +230,18 @@ func MustNewGzipLevelHandler(level int) func(http.Handler) http.Handler { // if an invalid gzip compression level is given, so if one can ensure the level // is valid, the returned error can be safely ignored. func NewGzipLevelHandler(level int) (func(http.Handler) http.Handler, error) { + return NewGzipLevelAndMinSize(level, DefaultMinSize) +} + +// NewGzipLevelAndMinSize behave as NewGzipLevelHandler except it let the caller +// specify the minimum size before compression. +func NewGzipLevelAndMinSize(level, minSize int) (func(http.Handler) http.Handler, error) { if level != gzip.DefaultCompression && (level < gzip.BestSpeed || level > gzip.BestCompression) { return nil, fmt.Errorf("invalid compression level requested: %d", level) } + if minSize < 0 { + return nil, fmt.Errorf("minimum size must be more than zero") + } return func(h http.Handler) http.Handler { index := poolIndex(level) @@ -176,6 +252,9 @@ func NewGzipLevelHandler(level int) (func(http.Handler) http.Handler, error) { gw := &GzipResponseWriter{ ResponseWriter: w, index: index, + minSize: minSize, + + buf: []byte{}, } defer gw.Close() @@ -238,7 +317,7 @@ func parseEncodings(s string) (codings, error) { func parseCoding(s string) (coding string, qvalue float64, err error) { for n, part := range strings.Split(s, ";") { part = strings.TrimSpace(part) - qvalue = DEFAULT_QVALUE + qvalue = DefaultQValue if n == 0 { coding = strings.ToLower(part) diff --git a/vendor/github.com/NYTimes/gziphandler/gzip_go18.go b/vendor/github.com/NYTimes/gziphandler/gzip_go18.go new file mode 100644 index 000000000..fa9665b7e --- /dev/null +++ b/vendor/github.com/NYTimes/gziphandler/gzip_go18.go @@ -0,0 +1,43 @@ +// +build go1.8 + +package gziphandler + +import "net/http" + +// Push initiates an HTTP/2 server push. +// Push returns ErrNotSupported if the client has disabled push or if push +// is not supported on the underlying connection. +func (w *GzipResponseWriter) Push(target string, opts *http.PushOptions) error { + pusher, ok := w.ResponseWriter.(http.Pusher) + if ok && pusher != nil { + return pusher.Push(target, setAcceptEncodingForPushOptions(opts)) + } + return http.ErrNotSupported +} + +// setAcceptEncodingForPushOptions sets "Accept-Encoding" : "gzip" for PushOptions without overriding existing headers. +func setAcceptEncodingForPushOptions(opts *http.PushOptions) *http.PushOptions { + + if opts == nil { + opts = &http.PushOptions{ + Header: http.Header{ + acceptEncoding: []string{"gzip"}, + }, + } + return opts + } + + if opts.Header == nil { + opts.Header = http.Header{ + acceptEncoding: []string{"gzip"}, + } + return opts + } + + if encoding := opts.Header.Get(acceptEncoding); encoding == "" { + opts.Header.Add(acceptEncoding, "gzip") + return opts + } + + return opts +} diff --git a/vendor/github.com/NYTimes/gziphandler/gzip_go18_test.go b/vendor/github.com/NYTimes/gziphandler/gzip_go18_test.go new file mode 100644 index 000000000..412b2918e --- /dev/null +++ b/vendor/github.com/NYTimes/gziphandler/gzip_go18_test.go @@ -0,0 +1,70 @@ +// +build go1.8 + +package gziphandler + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSetAcceptEncodingForPushOptionsWithoutHeaders(t *testing.T) { + var opts *http.PushOptions + opts = setAcceptEncodingForPushOptions(opts) + + assert.NotNil(t, opts) + assert.NotNil(t, opts.Header) + + for k, v := range opts.Header { + assert.Equal(t, "Accept-Encoding", k) + assert.Len(t, v, 1) + assert.Equal(t, "gzip", v[0]) + } + + opts = &http.PushOptions{} + opts = setAcceptEncodingForPushOptions(opts) + + assert.NotNil(t, opts) + assert.NotNil(t, opts.Header) + + for k, v := range opts.Header { + assert.Equal(t, "Accept-Encoding", k) + assert.Len(t, v, 1) + assert.Equal(t, "gzip", v[0]) + } +} + +func TestSetAcceptEncodingForPushOptionsWithHeaders(t *testing.T) { + opts := &http.PushOptions{ + Header: http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36"}, + }, + } + opts = setAcceptEncodingForPushOptions(opts) + + assert.NotNil(t, opts) + assert.NotNil(t, opts.Header) + + assert.Equal(t, "gzip", opts.Header.Get("Accept-Encoding")) + assert.Equal(t, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", opts.Header.Get("User-Agent")) + + opts = &http.PushOptions{ + Header: http.Header{ + "User-Agent": []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36"}, + acceptEncoding: []string{"deflate"}, + }, + } + opts = setAcceptEncodingForPushOptions(opts) + + assert.NotNil(t, opts) + assert.NotNil(t, opts.Header) + + e, found := opts.Header["Accept-Encoding"] + if !found { + assert.Fail(t, "Missing Accept-Encoding header value") + } + assert.Len(t, e, 1) + assert.Equal(t, "deflate", e[0]) + assert.Equal(t, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", opts.Header.Get("User-Agent")) +} diff --git a/vendor/github.com/NYTimes/gziphandler/gzip_test.go b/vendor/github.com/NYTimes/gziphandler/gzip_test.go index 80f54c77d..b9e687c8e 100644 --- a/vendor/github.com/NYTimes/gziphandler/gzip_test.go +++ b/vendor/github.com/NYTimes/gziphandler/gzip_test.go @@ -16,8 +16,12 @@ import ( "github.com/stretchr/testify/assert" ) -func TestParseEncodings(t *testing.T) { +const ( + smallTestBody = "aaabbcaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbccc" + testBody = "aaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbcccaaabbbccc" +) +func TestParseEncodings(t *testing.T) { examples := map[string]codings{ // Examples from RFC 2616 @@ -39,8 +43,6 @@ func TestParseEncodings(t *testing.T) { } func TestGzipHandler(t *testing.T) { - testBody := "aaabbbccc" - // This just exists to provide something for GzipHandler to wrap. handler := newTestHandler(testBody) @@ -80,7 +82,6 @@ func TestGzipHandler(t *testing.T) { } func TestNewGzipLevelHandler(t *testing.T) { - testBody := "aaabbbccc" handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) io.WriteString(w, testBody) @@ -102,7 +103,6 @@ func TestNewGzipLevelHandler(t *testing.T) { assert.Equal(t, "gzip", res.Header.Get("Content-Encoding")) assert.Equal(t, "Accept-Encoding", res.Header.Get("Vary")) assert.Equal(t, gzipStrLevel(testBody, lvl), resp.Body.Bytes()) - } } @@ -135,7 +135,7 @@ func TestGzipHandlerNoBody(t *testing.T) { {http.StatusNoContent, "", 0}, {http.StatusNotModified, "", 0}, // Body is going to get gzip'd no matter what. - {http.StatusOK, "gzip", 23}, + {http.StatusOK, "", 0}, } for num, test := range tests { @@ -170,7 +170,7 @@ func TestGzipHandlerNoBody(t *testing.T) { } func TestGzipHandlerContentLength(t *testing.T) { - b := []byte("testtesttesttesttesttesttesttesttesttesttesttesttest") + b := []byte(testBody) handler := GzipHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", strconv.Itoa(len(b))) w.Write(b) @@ -215,6 +215,48 @@ func TestGzipHandlerContentLength(t *testing.T) { assert.NotEqual(t, b, body) } +func TestGzipHandlerMinSizeMustBePositive(t *testing.T) { + _, err := NewGzipLevelAndMinSize(gzip.DefaultCompression, -1) + assert.Error(t, err) +} + +func TestGzipHandlerMinSize(t *testing.T) { + responseLength := 0 + b := []byte{'x'} + + wrapper, _ := NewGzipLevelAndMinSize(gzip.DefaultCompression, 128) + handler := wrapper(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + // Write responses one byte at a time to ensure that the flush + // mechanism, if used, is working properly. + for i := 0; i < responseLength; i++ { + n, err := w.Write(b) + assert.Equal(t, 1, n) + assert.Nil(t, err) + } + }, + )) + + r, _ := http.NewRequest("GET", "/whatever", &bytes.Buffer{}) + r.Header.Add("Accept-Encoding", "gzip") + + // Short response is not compressed + responseLength = 127 + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Result().Header.Get(contentEncoding) == "gzip" { + t.Error("Expected uncompressed response, got compressed") + } + + // Long response is not compressed + responseLength = 128 + w = httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Result().Header.Get(contentEncoding) != "gzip" { + t.Error("Expected compressed response, got uncompressed") + } +} + func TestGzipDoubleClose(t *testing.T) { // reset the pool for the default compression so we can make sure duplicates // aren't added back by double close @@ -240,6 +282,19 @@ func TestGzipDoubleClose(t *testing.T) { assert.False(t, w1 == w2) } +func TestStatusCodes(t *testing.T) { + handler := GzipHandler(http.NotFoundHandler()) + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Accept-Encoding", "gzip") + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + + result := w.Result() + if result.StatusCode != 404 { + t.Errorf("StatusCode should have been 404 but was %d", result.StatusCode) + } +} + // -------------------------------------------------------------------- func BenchmarkGzipHandler_S2k(b *testing.B) { benchmark(b, false, 2048) } -- cgit v1.2.3-1-g7c22