-
Notifications
You must be signed in to change notification settings - Fork 225
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(storage/gcs): add support for Google Cloud Storage (#2589)
* feat(storage/gcs): add support for Google Cloud Storage fixes #2288 * address PR feedback, cleanup tests and better context for blob FS * cleanup docs * fix tests and schema * Update internal/config/storage.go Co-authored-by: Mark Phelps <[email protected]> * cleanup go.mod * Update internal/storage/fs/object/blob/blob_fs.go Co-authored-by: George <[email protected]> * remove unused import * Update internal/storage/fs/object/blob/blob_fs_test.go Co-authored-by: George <[email protected]> * fix test prefix * remove prefixedBucket * refactor(storage/fs/object): move all implementations into single store backed by gcblob (#9) Co-authored-by: George MacRorie <[email protected]> * fix linter issue and remove panic call * cleanup * added new integration test to github actions and improve test coverage --------- Co-authored-by: Mark Phelps <[email protected]> Co-authored-by: George <[email protected]>
- Loading branch information
3 people
authored
Jan 7, 2024
1 parent
c2e602a
commit 4b6b9b3
Showing
35 changed files
with
1,388 additions
and
1,400 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -58,6 +58,7 @@ jobs: | |
| "fs/s3", | ||
| "fs/oci", | ||
| "fs/azblob", | ||
| "fs/gcs", | ||
| "import/export", | ||
| ] | ||
| steps: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "log" | ||
| "os" | ||
|
|
||
| gstorage "cloud.google.com/go/storage" | ||
| ) | ||
|
|
||
| func main() { | ||
| var testdataDir, bucket string | ||
| flag.StringVar(&testdataDir, "testdata-dir", "", "Directory path to testdata") | ||
| flag.StringVar(&bucket, "bucket", "testdata", "Google Cloud Storage bucket") | ||
| flag.Parse() | ||
|
|
||
| fatalOnError := func(err error) { | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| blobURL := os.Getenv("STORAGE_EMULATOR_HOST") | ||
|
|
||
| if blobURL == "" { | ||
| log.Fatal("Must supply non-empty env STORAGE_EMULATOR_HOST value.") | ||
| } | ||
|
|
||
| fmt.Fprintln(os.Stderr, "Syncing data to gcs blob at", blobURL) | ||
|
|
||
| ctx := context.Background() | ||
| client, err := gstorage.NewClient(ctx) | ||
| fatalOnError(err) | ||
| defer client.Close() | ||
| fmt.Fprintln(os.Stderr, "Using GCS bucket", bucket) | ||
| bkt := client.Bucket(bucket) | ||
| err = bkt.Create(ctx, "", nil) | ||
| fatalOnError(err) | ||
|
|
||
| dir := os.DirFS(testdataDir) | ||
| fatalOnError(err) | ||
|
|
||
| // copy testdata into target s3 bucket | ||
| err = fs.WalkDir(dir, ".", func(path string, d fs.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if d.IsDir() { | ||
| return nil | ||
| } | ||
|
|
||
| fmt.Fprintln(os.Stderr, "Copying", path) | ||
|
|
||
| f, err := dir.Open(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer f.Close() | ||
|
|
||
| w := bkt.Object(path).NewWriter(ctx) | ||
| _, err = io.Copy(w, f) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return w.Close() | ||
| }) | ||
| fatalOnError(err) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.