Make repository response support HTTP range request (#24592)

Replace #20480 Replace #18448 Close #16414
go-gitea · May 9, 2023 · 023a048 · 023a048
1 parent c090f87
commit 023a048
Show file tree

Hide file tree

Showing 12 changed files with 434 additions and 212 deletions.
diff --git a/modules/context/context_serve.go b/modules/context/context_serve.go
@@ -4,71 +4,20 @@
 package context
 
 import (
-	"fmt"
 	"io"
 	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
 
-	"code.gitea.io/gitea/modules/httpcache"
-	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/httplib"
 )
 
-type ServeHeaderOptions struct {
-	ContentType        string // defaults to "application/octet-stream"
-	ContentTypeCharset string
-	ContentLength      *int64
-	Disposition        string // defaults to "attachment"
-	Filename           string
-	CacheDuration      time.Duration // defaults to 5 minutes
-	LastModified       time.Time
-}
-
-// SetServeHeaders sets necessary content serve headers
-func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) {
-	header := ctx.Resp.Header()
-
-	contentType := typesniffer.ApplicationOctetStream
-	if opts.ContentType != "" {
-		if opts.ContentTypeCharset != "" {
-			contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset)
-		} else {
-			contentType = opts.ContentType
-		}
-	}
-	header.Set("Content-Type", contentType)
-	header.Set("X-Content-Type-Options", "nosniff")
-
-	if opts.ContentLength != nil {
-		header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10))
-	}
-
-	if opts.Filename != "" {
-		disposition := opts.Disposition
-		if disposition == "" {
-			disposition = "attachment"
-		}
-
-		backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \"
-		header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename)))
-		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
-	}
-
-	duration := opts.CacheDuration
-	if duration == 0 {
-		duration = 5 * time.Minute
-	}
-	httpcache.SetCacheControlInHeader(header, duration)
+type ServeHeaderOptions httplib.ServeHeaderOptions
 
-	if !opts.LastModified.IsZero() {
-		header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
-	}
+func (ctx *Context) SetServeHeaders(opt *ServeHeaderOptions) {
+	httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opt))
 }
 
 // ServeContent serves content to http request
 func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) {
-	ctx.SetServeHeaders(opts)
+	httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opts))
 	http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r)
 }
diff --git a/modules/context/context_test.go b/modules/context/context_test.go
@@ -7,32 +7,16 @@ import (
 	"net/http"
 	"testing"
 
+	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/stretchr/testify/assert"
 )
 
-type mockResponseWriter struct {
-	header http.Header
-}
-
-func (m *mockResponseWriter) Header() http.Header {
-	return m.header
-}
-
-func (m *mockResponseWriter) Write(bytes []byte) (int, error) {
-	panic("implement me")
-}
-
-func (m *mockResponseWriter) WriteHeader(statusCode int) {
-	panic("implement me")
-}
-
 func TestRemoveSessionCookieHeader(t *testing.T) {
-	w := &mockResponseWriter{}
-	w.header = http.Header{}
-	w.header.Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String())
-	w.header.Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String())
+	w := httplib.NewMockResponseWriter()
+	w.Header().Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String())
+	w.Header().Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String())
 	assert.Len(t, w.Header().Values("Set-Cookie"), 2)
 	removeSessionCookieHeader(w)
 	assert.Len(t, w.Header().Values("Set-Cookie"), 1)

diff --git a/modules/httplib/mock.go b/modules/httplib/mock.go
@@ -0,0 +1,35 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httplib
+
+import (
+	"bytes"
+	"net/http"
+)
+
+type MockResponseWriter struct {
+	header http.Header
+
+	StatusCode int
+	BodyBuffer bytes.Buffer
+}
+
+func (m *MockResponseWriter) Header() http.Header {
+	return m.header
+}
+
+func (m *MockResponseWriter) Write(bytes []byte) (int, error) {
+	if m.StatusCode == 0 {
+		m.StatusCode = http.StatusOK
+	}
+	return m.BodyBuffer.Write(bytes)
+}
+
+func (m *MockResponseWriter) WriteHeader(statusCode int) {
+	m.StatusCode = statusCode
+}
+
+func NewMockResponseWriter() *MockResponseWriter {
+	return &MockResponseWriter{header: http.Header{}}
+}
diff --git a/modules/httplib/httplib.go → modules/httplib/request.go b/modules/httplib/httplib.go → modules/httplib/request.go
diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go
@@ -0,0 +1,225 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httplib
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	charsetModule "code.gitea.io/gitea/modules/charset"
+	"code.gitea.io/gitea/modules/httpcache"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
+)
+
+type ServeHeaderOptions struct {
+	ContentType        string // defaults to "application/octet-stream"
+	ContentTypeCharset string
+	ContentLength      *int64
+	Disposition        string // defaults to "attachment"
+	Filename           string
+	CacheDuration      time.Duration // defaults to 5 minutes
+	LastModified       time.Time
+}
+
+// ServeSetHeaders sets necessary content serve headers
+func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
+	header := w.Header()
+
+	contentType := typesniffer.ApplicationOctetStream
+	if opts.ContentType != "" {
+		if opts.ContentTypeCharset != "" {
+			contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset)
+		} else {
+			contentType = opts.ContentType
+		}
+	}
+	header.Set("Content-Type", contentType)
+	header.Set("X-Content-Type-Options", "nosniff")
+
+	if opts.ContentLength != nil {
+		header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10))
+	}
+
+	if opts.Filename != "" {
+		disposition := opts.Disposition
+		if disposition == "" {
+			disposition = "attachment"
+		}
+
+		backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \"
+		header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename)))
+		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
+	}
+
+	duration := opts.CacheDuration
+	if duration == 0 {
+		duration = 5 * time.Minute
+	}
+	httpcache.SetCacheControlInHeader(header, duration)
+
+	if !opts.LastModified.IsZero() {
+		header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
+	}
+}
+
+// ServeData download file from io.Reader
+func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) {
+	// do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests
+	opts := &ServeHeaderOptions{
+		Filename: path.Base(filePath),
+	}
+
+	sniffedType := typesniffer.DetectContentType(mineBuf)
+
+	// the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later
+	isPlain := sniffedType.IsText() || r.FormValue("render") != ""
+
+	if setting.MimeTypeMap.Enabled {
+		fileExtension := strings.ToLower(filepath.Ext(filePath))
+		opts.ContentType = setting.MimeTypeMap.Map[fileExtension]
+	}
+
+	if opts.ContentType == "" {
+		if sniffedType.IsBrowsableBinaryType() {
+			opts.ContentType = sniffedType.GetMimeType()
+		} else if isPlain {
+			opts.ContentType = "text/plain"
+		} else {
+			opts.ContentType = typesniffer.ApplicationOctetStream
+		}
+	}
+
+	if isPlain {
+		charset, err := charsetModule.DetectEncoding(mineBuf)
+		if err != nil {
+			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err)
+			charset = "utf-8"
+		}
+		opts.ContentTypeCharset = strings.ToLower(charset)
+	}
+
+	isSVG := sniffedType.IsSvgImage()
+
+	// serve types that can present a security risk with CSP
+	if isSVG {
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
+	} else if sniffedType.IsPDF() {
+		// no sandbox attribute for pdf as it breaks rendering in at least safari. this
+		// should generally be safe as scripts inside PDF can not escape the PDF document
+		// see https://bugs.chromium.org/p/chromium/issues/detail?id=413851 for more discussion
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'")
+	}
+
+	opts.Disposition = "inline"
+	if isSVG && !setting.UI.SVG.Enabled {
+		opts.Disposition = "attachment"
+	}
+
+	ServeSetHeaders(w, opts)
+}
+
+const mimeDetectionBufferLen = 1024
+
+func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) {
+	buf := make([]byte, mimeDetectionBufferLen)
+	n, err := util.ReadAtMost(reader, buf)
+	if err != nil {
+		http.Error(w, "serve content: unable to pre-read", http.StatusRequestedRangeNotSatisfiable)
+		return
+	}
+	if n >= 0 {
+		buf = buf[:n]
+	}
+	setServeHeadersByFile(r, w, filePath, buf)
+
+	// reset the reader to the beginning
+	reader = io.MultiReader(bytes.NewReader(buf), reader)
+
+	rangeHeader := r.Header.Get("Range")
+
+	// if no size or no supported range, serve as 200 (complete response)
+	if size <= 0 || !strings.HasPrefix(rangeHeader, "bytes=") {
+		if size >= 0 {
+			w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+		}
+		_, _ = io.Copy(w, reader) // just like http.ServeContent, not necessary to handle the error
+		return
+	}
+
+	// do our best to support the minimal "Range" request (no support for multiple range: "Range: bytes=0-50, 100-150")
+	//
+	// GET /...
+	// Range: bytes=0-1023
+	//
+	// HTTP/1.1 206 Partial Content
+	// Content-Range: bytes 0-1023/146515
+	// Content-Length: 1024
+
+	_, rangeParts, _ := strings.Cut(rangeHeader, "=")
+	rangeBytesStart, rangeBytesEnd, found := strings.Cut(rangeParts, "-")
+	start, err := strconv.ParseInt(rangeBytesStart, 10, 64)
+	if start < 0 || start >= size {
+		err = errors.New("invalid start range")
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
+		return
+	}
+	end, err := strconv.ParseInt(rangeBytesEnd, 10, 64)
+	if rangeBytesEnd == "" && found {
+		err = nil
+		end = size - 1
+	}
+	if end >= size {
+		end = size - 1
+	}
+	if end < start {
+		err = errors.New("invalid end range")
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	partialLength := end - start + 1
+	w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size))
+	w.Header().Set("Content-Length", strconv.FormatInt(partialLength, 10))
+	if _, err = io.CopyN(io.Discard, reader, start); err != nil {
+		http.Error(w, "serve content: unable to skip", http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(http.StatusPartialContent)
+	_, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error
+}
+
+func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime time.Time, reader io.ReadSeeker) {
+	buf := make([]byte, mimeDetectionBufferLen)
+	n, err := util.ReadAtMost(reader, buf)
+	if err != nil {
+		http.Error(w, "serve content: unable to read", http.StatusInternalServerError)
+		return
+	}
+	if _, err = reader.Seek(0, io.SeekStart); err != nil {
+		http.Error(w, "serve content: unable to seek", http.StatusInternalServerError)
+		return
+	}
+	if n >= 0 {
+		buf = buf[:n]
+	}
+	setServeHeadersByFile(r, w, filePath, buf)
+	http.ServeContent(w, r, path.Base(filePath), modTime, reader)
+}