Add GCS storage provider (#90)

Add GCS storage provider backend.

go.mod

@@ -3,20 +3,26 @@ module github.com/discentem/pantri_but_go
 go 1.18
 
 require (
+	cloud.google.com/go/storage v1.27.0
 	github.com/aws/aws-sdk-go-v2 v1.16.16
 	github.com/aws/aws-sdk-go-v2/config v1.17.8
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.34
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
+	github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/google/logger v1.1.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/afero v1.9.3
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.1
-	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
+	google.golang.org/api v0.107.0
 )
 
 require (
+	cloud.google.com/go v0.105.0 // indirect
+	cloud.google.com/go/compute v1.14.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect
@@ -34,7 +40,12 @@ require (
 	github.com/aws/smithy-go v1.13.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -46,8 +57,16 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
+	google.golang.org/grpc v1.52.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/cli/helpers.go

@@ -2,8 +2,10 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
+	"path/filepath"
 	"strings"
 
 	"github.com/discentem/pantri_but_go/internal/config"
@@ -12,9 +14,26 @@ import (
 	"github.com/spf13/afero"
 )
 
-func removePathPrefix(pathPrefix string, objects []string) ([]string, error) {
+func rootOfSourceRepo() (*string, error) {
+	absPathOfConfig, err := filepath.Abs(".pantri/config")
+	if err != nil {
+		return nil, errors.New(".pantri/config not detected, not in sourceRepo root")
+	}
+	logger.V(2).Infof("absPathOfconfig: %q", absPathOfConfig)
+	root := filepath.Dir(filepath.Dir(absPathOfConfig))
+	return &root, nil
+}
+
+func removePathPrefix(objects []string, prefix string) ([]string, error) {
 	for i, object := range objects {
-		objects[i] = strings.TrimPrefix(object, fmt.Sprintf("%s/", pathPrefix))
+		absObject, err := filepath.Abs(object)
+		if err != nil {
+			return nil, err
+		}
+		if !strings.HasPrefix(absObject, prefix) {
+			return nil, fmt.Errorf("%q does not exist relative to source_repo: %q", object, prefix)
+		}
+		objects[i] = strings.TrimPrefix(absObject, fmt.Sprintf("%s/", prefix))
 	}
 
 	return objects, nil
@@ -29,6 +48,12 @@ func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs,
 			return nil, fmt.Errorf("improper stores.S3Client init: %v", err)
 		}
 		s = stores.Store(s3)
+	case stores.StoreTypeGCS:
+		gcs, err := stores.NewGCSStoreClient(ctx, fsys, opts)
+		if err != nil {
+			return nil, fmt.Errorf("improper stores.GCSClient init: %v", err)
+		}
+		s = stores.Store(gcs)
 	default:
 		return nil, fmt.Errorf("type %s is not supported", cfg.StoreType)
 	}

internal/cli/helpers_test.go

@@ -20,11 +20,12 @@ func TestRemovePathPrefix(t *testing.T) {
 
 	expectedRemovePathPrefixes := []string{"foo/foo.dmg", "bar/bar.pkg"}
 	testRemovePathPrefixes, err := removePathPrefix(
-		pathPrefix,
 		[]string{
 			fmt.Sprintf("%s/%s", pathPrefix, "foo/foo.dmg"),
 			fmt.Sprintf("%s/%s", pathPrefix, "bar/bar.pkg"),
-		})
+		},
+		pathPrefix,
+	)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedRemovePathPrefixes, testRemovePathPrefixes)
 }

internal/cli/init.go

@@ -89,6 +89,14 @@ func initFn(cmd *cobra.Command, args []string) error {
 			region,
 			opts,
 		)
+	case stores.StoreTypeGCS:
+		cfg = config.InitializeStoreTypeGCS(
+			cmd.Context(),
+			fsys,
+			repoToInit,
+			pantriAddress,
+			opts,
+		)
 	default:
 		return config.ErrUnsupportedStore
 	}

internal/cli/retrieve.go

@@ -1,6 +1,9 @@
 package cli
 
 import (
+	"errors"
+	"fmt"
+
 	"github.com/google/logger"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -56,6 +59,20 @@ func retrieveFn(cmd *cobra.Command, objects []string) error {
 		return err
 	}
 
+	sourceRepoRoot, err := rootOfSourceRepo()
+	if err != nil {
+		return err
+	}
+	if sourceRepoRoot == nil {
+		return errors.New("sourceRepoRoot cannot be nil")
+	}
+
+	// We need to remove the prefix from the path so it is relative
+	objects, err = removePathPrefix(objects, *sourceRepoRoot)
+	if err != nil {
+		return fmt.Errorf("retrieve error: %w", err)
+	}
+
 	logger.Infof("Downloading files from: %s", s.GetOptions().PantriAddress)
 	logger.Infof("Downloading file: %s", objects)
 	if err := s.Retrieve(cmd.Context(), objects...); err != nil {

internal/cli/upload.go

@@ -1,6 +1,9 @@
 package cli
 
 import (
+	"errors"
+	"fmt"
+
 	"github.com/google/logger"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -56,6 +59,20 @@ func uploadFn(cmd *cobra.Command, objects []string) error {
 		return err
 	}
 
+	sourceRepoRoot, err := rootOfSourceRepo()
+	if err != nil {
+		return err
+	}
+	if sourceRepoRoot == nil {
+		return errors.New("sourceRepoRoot cannot be nil")
+	}
+
+	// We need to remove the prefix from the path so it is relative
+	objects, err = removePathPrefix(objects, *sourceRepoRoot)
+	if err != nil {
+		return fmt.Errorf("upload error: %w", err)
+	}
+
 	logger.Infof("Uploading to: %s", s.GetOptions().PantriAddress)
 	logger.Infof("Uploading file: %s", objects)
 	if err := s.Upload(cmd.Context(), objects...); err != nil {

internal/config/config.go

@@ -49,6 +49,21 @@ func InitializeStoreTypeS3(
 	}
 }
 
+func InitializeStoreTypeGCS(
+	ctx context.Context,
+	fsys afero.Fs,
+	sourceRepo, pantriAddress string,
+	opts stores.Options,
+) Config {
+	return Config{
+		StoreType: stores.StoreTypeGCS,
+		Options:   opts,
+		Validate: func() error {
+			return nil
+		},
+	}
+}
+
 func (c *Config) Write(fsys afero.Fs, sourceRepo string) error {
 	if c.Validate == nil {
 		return ErrValidateNil

internal/config/config_test.go

@@ -108,3 +108,30 @@ func TestInitializeStoreTypeS3(t *testing.T) {
 
 	assert.NoError(t, cfg.Validate())
 }
+
+func TestInitializeStoreTypeGCS(t *testing.T) {
+	ctx := context.Background()
+	fsys := afero.NewMemMapFs()
+
+	opts := stores.Options{
+		PantriAddress:         "my-test-bucket",
+		MetaDataFileExtension: "pfile",
+		Region:                "us-east-9876",
+	}
+
+	cfg := InitializeStoreTypeGCS(
+		ctx,
+		fsys,
+		"~/some_repo_root",
+		opts.PantriAddress,
+		opts,
+	)
+
+	// Assert the S3Store Config matches all of the inputs
+	assert.Equal(t, cfg.StoreType, stores.StoreTypeGCS)
+	assert.Equal(t, cfg.Options.PantriAddress, opts.PantriAddress)
+	assert.Equal(t, cfg.Options.MetaDataFileExtension, opts.MetaDataFileExtension)
+	assert.Equal(t, cfg.Options.Region, opts.Region)
+
+	assert.NoError(t, cfg.Validate())
+}

internal/metadata/metadata.go

@@ -22,7 +22,7 @@ type ObjectMetaData struct {
 func SHA256FromReader(r io.Reader) (string, error) {
 	h := sha256.New()
 	if _, err := io.Copy(h, r); err != nil {
-		return "", err
+		return "", fmt.Errorf("%v: %s", err, "could not generate sha256 due to io.Copy error")
 	}
 	return hex.EncodeToString(h.Sum(nil)), nil
 }

internal/stores/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "stores",
     srcs = [
+        "gcs.go",
         "options.go",
         "s3.go",
         "stores.go",
@@ -17,5 +18,7 @@ go_library(
         "@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
         "@com_github_google_logger//:logger",
         "@com_github_spf13_afero//:afero",
+        "@com_google_cloud_go_storage//:storage",
+        "@org_golang_google_api//option",
     ],
 )