Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix HTTPS #1159

Merged
merged 3 commits into from
May 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,9 @@ jobs:
- lang: cpp
docker-image: alpine:3.15.4
entrypoint: /bin/sh
- lang: curl
docker-image: alpine
entrypoint: /bin/sh

name: test-${{ matrix.lang }}-example
runs-on: ubuntu-latest
Expand Down
10 changes: 10 additions & 0 deletions ci/run-curl-example.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Copyright 2023 Francisco Souza. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

set -e

./fake-gcs-server -backend memory -port 4443 -data ${PWD}/examples/data &

apk add --update curl
curl --silent --fail --insecure https://0.0.0.0:4443/storage/v1/b
84 changes: 44 additions & 40 deletions fakestorage/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ type Server struct {
uploads sync.Map
transport http.RoundTripper
ts *httptest.Server
mux *mux.Router
handler http.Handler
options Options
externalURL string
publicHost string
Expand Down Expand Up @@ -119,8 +119,6 @@ type Options struct {
CertificateLocation string

PrivateKeyLocation string

Listener net.Listener
}

// NewServerWithOptions creates a new server configured according to the
Expand Down Expand Up @@ -149,30 +147,29 @@ func NewServerWithOptions(options Options) (*Server, error) {
handlers.ExposedHeaders([]string{"Location"}),
)

handler := cors(s.mux)
s.handler = cors(s.handler)
if options.Writer != nil {
handler = handlers.LoggingHandler(options.Writer, handler)
}
handler = requestCompressHandler(handler)
s.transport = &muxTransport{handler: handler}
if options.NoListener {
return s, nil
s.handler = handlers.LoggingHandler(options.Writer, s.handler)
}
s.handler = requestCompressHandler(s.handler)
s.transport = &muxTransport{handler: s.handler}

s.eventManager, err = notification.NewPubsubEventManager(options.EventOptions, options.Writer)
if err != nil {
return nil, err
}

s.ts = httptest.NewUnstartedServer(handler)
if options.NoListener {
return s, nil
}

s.ts = httptest.NewUnstartedServer(s.handler)
startFunc := s.ts.StartTLS
if options.Scheme == "http" {
startFunc = s.ts.Start
}
if options.Listener != nil {
s.ts.Listener.Close()
s.ts.Listener = options.Listener
} else if options.Port != 0 {

if options.Port != 0 {
addr := fmt.Sprintf("%s:%d", options.Host, options.Port)
l, err := net.Listen("tcp", addr)
if err != nil {
Expand Down Expand Up @@ -237,14 +234,14 @@ func unescapeMuxVars(vars map[string]string) map[string]string {

func (s *Server) buildMuxer() {
const apiPrefix = "/storage/v1"
s.mux = mux.NewRouter().SkipClean(true).UseEncodedPath()
handler := mux.NewRouter().SkipClean(true).UseEncodedPath()

// healthcheck
s.mux.Path("/_internal/healthcheck").Methods(http.MethodGet).HandlerFunc(s.healthcheck)
handler.Path("/_internal/healthcheck").Methods(http.MethodGet).HandlerFunc(s.healthcheck)

routers := []*mux.Router{
s.mux.PathPrefix(apiPrefix).Subrouter(),
s.mux.MatcherFunc(s.publicHostMatcher).PathPrefix(apiPrefix).Subrouter(),
handler.PathPrefix(apiPrefix).Subrouter(),
handler.MatcherFunc(s.publicHostMatcher).PathPrefix(apiPrefix).Subrouter(),
}

for _, r := range routers {
Expand All @@ -266,44 +263,46 @@ func (s *Server) buildMuxer() {
}

// Internal / update server configuration
s.mux.Path("/_internal/config").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.updateServerConfig))
s.mux.MatcherFunc(s.publicHostMatcher).Path("/_internal/config").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.updateServerConfig))
s.mux.Path("/_internal/reseed").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.reseedServer))
handler.Path("/_internal/config").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.updateServerConfig))
handler.MatcherFunc(s.publicHostMatcher).Path("/_internal/config").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.updateServerConfig))
handler.Path("/_internal/reseed").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.reseedServer))
// Internal - end

// XML API
xmlApiRouters := []*mux.Router{
s.mux.Host(fmt.Sprintf("{bucketName}.%s", s.publicHost)).Subrouter(),
s.mux.MatcherFunc(s.publicHostMatcher).PathPrefix(`/{bucketName}`).Subrouter(),
handler.Host(fmt.Sprintf("{bucketName}.%s", s.publicHost)).Subrouter(),
handler.MatcherFunc(s.publicHostMatcher).PathPrefix(`/{bucketName}`).Subrouter(),
}
for _, r := range xmlApiRouters {
r.Path("/").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects))
r.Path("").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects))
}

bucketHost := fmt.Sprintf("{bucketName}.%s", s.publicHost)
s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
s.mux.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods(http.MethodGet).HandlerFunc(s.downloadObject)
s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject))
s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent))
s.mux.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent))
handler.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
handler.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods(http.MethodGet).HandlerFunc(s.downloadObject)
handler.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.insertObject))
handler.Path("/upload/storage/v1/b/{bucketName}/o").Methods(http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent))
handler.Path("/upload/resumable/{uploadId}").Methods(http.MethodPut, http.MethodPost).HandlerFunc(jsonToHTTPHandler(s.uploadFileContent))

// Batch endpoint
s.mux.MatcherFunc(s.publicHostMatcher).Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall)
s.mux.Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall)
handler.MatcherFunc(s.publicHostMatcher).Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall)
handler.Path("/batch/storage/v1").Methods(http.MethodPost).HandlerFunc(s.handleBatchCall)

s.mux.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
s.mux.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)
handler.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject)

// Form Uploads
s.mux.Host(s.publicHost).Path("/{bucketName}").MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))
s.mux.Host(bucketHost).MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))
handler.Host(s.publicHost).Path("/{bucketName}").MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))
handler.Host(bucketHost).MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject))

// Signed URLs (upload and download)
s.mux.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
s.mux.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.getObject)
s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
s.mux.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.getObject)
handler.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))
handler.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject))

s.handler = handler
}

func (s *Server) reseedServer(r *http.Request) jsonResponse {
Expand Down Expand Up @@ -441,6 +440,11 @@ func (s *Server) HTTPClient() *http.Client {
return &http.Client{Transport: s.transport}
}

// HTTPHandler returns an HTTP handler that behaves like GCS.
func (s *Server) HTTPHandler() http.Handler {
return s.handler
}

// Client returns a GCS client configured to talk to the server.
func (s *Server) Client() *storage.Client {
client, err := storage.NewClient(context.Background(), option.WithHTTPClient(s.HTTPClient()), option.WithCredentials(&google.Credentials{}))
Expand Down Expand Up @@ -501,7 +505,7 @@ func (s *Server) handleBatchCall(w http.ResponseWriter, r *http.Request) {
continue
}

s.mux.ServeHTTP(partResponseWriter, partRequest)
s.handler.ServeHTTP(partResponseWriter, partRequest)
writeMultipartResponse(partResponseWriter.Result(), partWriter, contentID)
}
mw.Close()
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ require (
github.com/minio/minio-go/v7 v7.0.52
github.com/pkg/xattr v0.4.9
github.com/sirupsen/logrus v1.9.0
github.com/soheilhy/cmux v0.1.5
github.com/stretchr/testify v1.8.2
golang.org/x/oauth2 v0.8.0
google.golang.org/api v0.122.0
Expand Down
3 changes: 0 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,6 @@ github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand Down Expand Up @@ -155,7 +153,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
Expand Down
57 changes: 29 additions & 28 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,21 @@ const (
)

type Config struct {
Scheme string
Seed string
publicHost string
externalURL string
allowedCORSHeaders []string
scheme string
host string
port uint
backend string
fsRoot string
event EventConfig
bucketLocation string
certificateLocation string
privateKeyLocation string
LogLevel logrus.Level
Host string
Port uint
CertificateLocation string
PrivateKeyLocation string

publicHost string
externalURL string
allowedCORSHeaders []string
backend string
fsRoot string
event EventConfig
bucketLocation string
LogLevel logrus.Level
}

type EventConfig struct {
Expand All @@ -63,18 +64,18 @@ func Load(args []string) (Config, error) {
fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist")
fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host")
fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running")
fs.StringVar(&cfg.scheme, "scheme", "https", "using http or https")
fs.StringVar(&cfg.host, "host", "0.0.0.0", "host to bind to")
fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https")
fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to")
fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)")
fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist")
fs.UintVar(&cfg.port, "port", 4443, "port to bind to")
fs.UintVar(&cfg.Port, "port", 4443, "port to bind to")
fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic")
fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on")
fs.StringVar(&cfg.event.prefix, "event.object-prefix", "", "if not empty, only objects having this prefix will generate trigger events")
fs.StringVar(&eventList, "event.list", eventFinalize, "comma separated list of events to publish on cloud function URl. Options are: finalize, delete, and metadataUpdate")
fs.StringVar(&cfg.bucketLocation, "location", "US-CENTRAL1", "location for buckets")
fs.StringVar(&cfg.certificateLocation, "cert-location", "", "location for server certificate")
fs.StringVar(&cfg.privateKeyLocation, "private-key-location", "", "location for private key")
fs.StringVar(&cfg.CertificateLocation, "cert-location", "", "location for server certificate")
fs.StringVar(&cfg.PrivateKeyLocation, "private-key-location", "", "location for private key")
fs.StringVar(&givenLogLevel, "log-level", "info", "level for logging. Options same as for logrus: trace, debug, info, warn, error, fatal, and panic")

err := fs.Parse(args)
Expand Down Expand Up @@ -104,11 +105,11 @@ func (c *Config) validate() error {
if c.backend == filesystemBackend && c.fsRoot == "" {
return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend)
}
if c.scheme != "http" && c.scheme != "https" {
return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.scheme)
if c.Scheme != "http" && c.Scheme != "https" {
return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.Scheme)
}
if c.port > math.MaxUint16 {
return fmt.Errorf("port %d is too high, maximum value is %d", c.port, math.MaxUint16)
if c.Port > math.MaxUint16 {
return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16)
}

return c.event.validate()
Expand Down Expand Up @@ -170,18 +171,18 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options {
opts := fakestorage.Options{
StorageRoot: storageRoot,
Seed: c.Seed,
Scheme: c.scheme,
Host: c.host,
Port: uint16(c.port),
Scheme: c.Scheme,
Host: c.Host,
Port: uint16(c.Port),
PublicHost: c.publicHost,
ExternalURL: c.externalURL,
AllowedCORSHeaders: c.allowedCORSHeaders,
Writer: logger.Writer(),
EventOptions: eventOptions,
BucketsLocation: c.bucketLocation,
CertificateLocation: c.certificateLocation,
PrivateKeyLocation: c.privateKeyLocation,
Listener: nil,
CertificateLocation: c.CertificateLocation,
PrivateKeyLocation: c.PrivateKeyLocation,
NoListener: true,
}
return opts
}
22 changes: 12 additions & 10 deletions internal/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ func TestLoadConfig(t *testing.T) {
publicHost: "127.0.0.1.nip.io:8443",
externalURL: "https://myhost.example.com:8443",
allowedCORSHeaders: []string{"X-Goog-Meta-Uploader"},
host: "127.0.0.1",
port: 443,
scheme: "http",
Host: "127.0.0.1",
Port: 443,
Scheme: "http",
event: EventConfig{
pubsubProjectID: "test-project",
pubsubTopic: "gcs-events",
Expand All @@ -70,9 +70,9 @@ func TestLoadConfig(t *testing.T) {
publicHost: "storage.googleapis.com",
externalURL: "",
allowedCORSHeaders: nil,
host: "0.0.0.0",
port: 4443,
scheme: "https",
Host: "0.0.0.0",
Port: 4443,
Scheme: "https",
event: EventConfig{
list: []string{"finalize"},
},
Expand Down Expand Up @@ -152,8 +152,8 @@ func TestToFakeGcsOptions(t *testing.T) {
fsRoot: "/tmp/something",
publicHost: "127.0.0.1.nip.io:8443",
externalURL: "https://myhost.example.com:8443",
host: "0.0.0.0",
port: 443,
Host: "0.0.0.0",
Port: 443,
event: EventConfig{
pubsubProjectID: "test-project",
pubsubTopic: "gcs-events",
Expand All @@ -179,6 +179,7 @@ func TestToFakeGcsOptions(t *testing.T) {
},
},
BucketsLocation: "US-EAST1",
NoListener: true,
},
},
{
Expand All @@ -188,15 +189,16 @@ func TestToFakeGcsOptions(t *testing.T) {
fsRoot: "/tmp/something",
publicHost: "127.0.0.1.nip.io:8443",
externalURL: "https://myhost.example.com:8443",
host: "0.0.0.0",
port: 443,
Host: "0.0.0.0",
Port: 443,
},
fakestorage.Options{
StorageRoot: "",
PublicHost: "127.0.0.1.nip.io:8443",
ExternalURL: "https://myhost.example.com:8443",
Host: "0.0.0.0",
Port: 443,
NoListener: true,
},
},
}
Expand Down
5 changes: 2 additions & 3 deletions grpc/server.go → internal/grpc/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package grpc
import (
"context"
"io"
"net"

pb "github.com/fsouza/fake-gcs-server/genproto/googleapis/storage/v1"
"github.com/fsouza/fake-gcs-server/internal/backend"
Expand All @@ -22,11 +21,11 @@ func InitServer(backend backend.Storage) *Server {
return &Server{backend: backend}
}

func NewServerWithBackend(backend backend.Storage, grpcListener net.Listener) error {
func NewServerWithBackend(backend backend.Storage) *grpc.Server {
grpcServer := grpc.NewServer()
pb.RegisterStorageServerServer(grpcServer, InitServer(backend))
reflection.Register(grpcServer)
return grpcServer.Serve(grpcListener)
return grpcServer
}

func (g *Server) GetBucket(ctx context.Context, req *pb.GetBucketRequest) (*pb.Bucket, error) {
Expand Down
File renamed without changes.
Loading