feat(storage/gcs): add support for Google Cloud Storage

build/internal/cmd/gcs/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+
+	gstorage "cloud.google.com/go/storage"
+)
+
+func main() {
+	var testdataDir, bucket string
+	flag.StringVar(&testdataDir, "testdata-dir", "", "Directory path to testdata")
+	flag.StringVar(&bucket, "bucket", "testdata", "Google Cloud Storage bucket")
+	flag.Parse()
+
+	fatalOnError := func(err error) {
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	blobURL := os.Getenv("STORAGE_EMULATOR_HOST")
+
+	if blobURL == "" {
+		log.Fatal("Must supply non-empty env STORAGE_EMULATOR_HOST value.")
+	}
+
+	fmt.Fprintln(os.Stderr, "Syncing data to gcs blob at", blobURL)
+
+	ctx := context.Background()
+	client, err := gstorage.NewClient(ctx)
+	fatalOnError(err)
+	defer client.Close()
+	fmt.Fprintln(os.Stderr, "Using GCS bucket", bucket)
+	bkt := client.Bucket(bucket)
+	err = bkt.Create(ctx, "", nil)
+	fatalOnError(err)
+
+	dir := os.DirFS(testdataDir)
+	fatalOnError(err)
+
+	// copy testdata into target s3 bucket
+	err = fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		fmt.Fprintln(os.Stderr, "Copying", path)
+
+		f, err := dir.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		w := bkt.Object(path).NewWriter(ctx)
+		_, err = io.Copy(w, f)
+		if err != nil {
+			return err
+		}
+
+		return w.Close()
+	})
+	fatalOnError(err)
+}

build/testing/integration.go

@@ -36,6 +36,7 @@ var (
 		"fs/s3":         s3,
 		"fs/oci":        oci,
 		"fs/azblob":     azblob,
+		"fs/gcs":        gcs,
 		"import/export": importExport,
 	}
 )
@@ -529,3 +530,32 @@ func azblob(ctx context.Context, client *dagger.Client, base, flipt *dagger.Cont
 
 	return suite(ctx, "readonly", base, flipt.WithExec(nil), conf)
 }
+
+// gcs simulates the Google Cloud Storage service
+func gcs(ctx context.Context, client *dagger.Client, base, flipt *dagger.Container, conf testConfig) func() error {
+	gcs := client.Container().
+		From("fsouza/fake-gcs-server").
+		WithExposedPort(4443).
+		WithExec([]string{"-scheme", "http", "-public-host", "gcs:4443"}).
+		AsService()
+
+	_, err := base.
+		WithServiceBinding("gcs", gcs).
+		WithEnvVariable("STORAGE_EMULATOR_HOST", "gcs:4443").
+		WithExec([]string{"go", "run", "./build/internal/cmd/gcs/...", "-testdata-dir", testdataDir}).
+		Sync(ctx)
+	if err != nil {
+		return func() error { return err }
+	}
+
+	flipt = flipt.
+		WithServiceBinding("gcs", gcs).
+		WithEnvVariable("FLIPT_LOG_LEVEL", "DEBUG").
+		WithEnvVariable("FLIPT_STORAGE_TYPE", "object").
+		WithEnvVariable("FLIPT_STORAGE_OBJECT_TYPE", "googlecloud").
+		WithEnvVariable("FLIPT_STORAGE_OBJECT_GOOGLECLOUD_BUCKET", "testdata").
+		WithEnvVariable("STORAGE_EMULATOR_HOST", "gcs:4443").
+		WithEnvVariable("UNIQUE", uuid.New().String())
+
+	return suite(ctx, "readonly", base, flipt.WithExec(nil), conf)
+}

build/testing/test.go

@@ -47,6 +47,12 @@ func Unit(ctx context.Context, client *dagger.Client, flipt *dagger.Container) e
 		WithExec([]string{"azurite-blob", "--blobHost", "0.0.0.0", "--silent"}).
 		AsService()
 
+	gcs := client.Container().
+		From("fsouza/fake-gcs-server").
+		WithExposedPort(4443).
+		WithExec([]string{"-scheme", "http", "-public-host", "gcs:4443"}).
+		AsService()
+
 	flipt = flipt.
 		WithServiceBinding("minio", minio.AsService()).
 		WithEnvVariable("AWS_ACCESS_KEY_ID", "user").
@@ -56,13 +62,15 @@ func Unit(ctx context.Context, client *dagger.Client, flipt *dagger.Container) e
 	flipt, err = flipt.
 		WithServiceBinding("redis", redisSrv.AsService()).
 		WithServiceBinding("azurite", azurite).
+		WithServiceBinding("gcs", gcs).
 		WithEnvVariable("REDIS_HOST", "redis:6379").
 		WithEnvVariable("TEST_GIT_REPO_URL", "http://gitea:3000/root/features.git").
 		WithEnvVariable("TEST_GIT_REPO_HEAD", push["HEAD"]).
 		WithEnvVariable("TEST_S3_ENDPOINT", "http://minio:9009").
 		WithEnvVariable("TEST_AZURE_ENDPOINT", "http://azurite:10000/devstoreaccount1").
 		WithEnvVariable("AZURE_STORAGE_ACCOUNT", "devstoreaccount1").
 		WithEnvVariable("AZURE_STORAGE_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==").
+		WithEnvVariable("STORAGE_EMULATOR_HOST", "gcs:4443").
 		WithExec([]string{"go", "test", "-race", "-p", "1", "-coverprofile=coverage.txt", "-covermode=atomic", "./..."}).
 		Sync(ctx)
 	if err != nil {

config/flipt.schema.cue

@@ -171,7 +171,7 @@ import "strings"
 			})
 		}
 		object?: {
-			type: "s3" | "azblob" | *""
+			type: "s3" | "azblob" | "googlecloud" | *""
 			s3?: {
 				region:         string
 				bucket:         string
@@ -184,6 +184,11 @@ import "strings"
 				endpoint?: string
 				poll_interval?: =~#duration | *"1m"
 			}
+			googlecloud?: {
+				bucket:         string
+				prefix?:        string
+				poll_interval?: =~#duration | *"1m"
+			}
 		}
 		oci?: {
 			repository:         string

config/flipt.schema.json

@@ -614,7 +614,7 @@
           "properties": {
             "type": {
               "type": "string",
-              "enum": ["s3", "azblob"],
+              "enum": ["s3", "azblob", "googlecloud"],
               "default": "s3"
             },
             "s3": {
@@ -672,6 +672,31 @@
                 }
               },
               "title": "Azure Blob Storage"
+            },
+            "googlecloud": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "bucket": {
+                  "type": "string"
+                },
+                "prefix": {
+                  "type": "string"
+                },
+                "poll_interval": {
+                  "oneOf": [
+                    {
+                      "type": "string",
+                      "pattern": "^([0-9]+(ns|us|µs|ms|s|m|h))+$"
+                    },
+                    {
+                      "type": "integer"
+                    }
+                  ],
+                  "default": "1m"
+                }
+              },
+              "title": "Google Cloud Storage"
             }
           },
           "title": "Object"

go.mod

@@ -3,6 +3,7 @@ module go.flipt.io/flipt
 go 1.21
 
 require (
+	cloud.google.com/go/storage v1.35.1
 	cuelang.org/go v0.7.0
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1
@@ -85,8 +86,10 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.110.10 // indirect
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
@@ -99,10 +102,12 @@ require (
 	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/aws/aws-sdk-go v1.48.3 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.24.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
@@ -160,6 +165,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
@@ -224,6 +230,7 @@ require (
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/api v0.153.0 // indirect

go.sum

@@ -417,6 +417,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=