skaffold render --output takes GCS file path (#3979)

* render --output takes gcs path

* fix linter errors

* Code review changes.

* Update pkg/skaffold/deploy/util.go

* More code review changes.

* More code review changes.

Co-authored-by: David Gageot <[email protected]>

pkg/skaffold/deploy/deploy_mux.go

@@ -109,5 +109,5 @@ func (m DeployerMux) Render(ctx context.Context, w io.Writer, as []build.Artifac
 	}
 
 	allResources := strings.Join(resources, "\n---\n")
-	return dumpToFileOrWriter(allResources, filepath, w)
+	return outputRenderedManifests(allResources, filepath, w)
 }

pkg/skaffold/deploy/kubectl.go

@@ -217,7 +217,7 @@ func (k *KubectlDeployer) Render(ctx context.Context, out io.Writer, builds []bu
 		return err
 	}
 
-	return dumpToFileOrWriter(manifests.String(), filepath, out)
+	return outputRenderedManifests(manifests.String(), filepath, out)
 }
 
 func (k *KubectlDeployer) renderManifests(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller) (deploy.ManifestList, error) {

pkg/skaffold/deploy/util.go

@@ -19,15 +19,25 @@ package deploy
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/kubernetes/scheme"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/kubectl"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+)
+
+const (
+	renderedManifestsStagingDir  = "render_temp_"
+	renderedManifestsStagingFile = "rendered_manifest.yaml"
 )
 
 func parseRuntimeObject(namespace string, b []byte) (*Artifact, error) {
@@ -83,12 +93,33 @@ func getObjectNamespaceIfDefined(doc []byte, ns string) (string, error) {
 	return ns, nil
 }
 
-func dumpToFileOrWriter(renderedManifests string, filepath string, manifestOut io.Writer) error {
-	if filepath == "" {
+// Outputs rendered manifests to a file, a writer or a GCS bucket.
+func outputRenderedManifests(renderedManifests string, output string, manifestOut io.Writer) error {
+	switch {
+	case output == "":
 		_, err := fmt.Fprintln(manifestOut, renderedManifests)
 		return err
+	case strings.HasPrefix(output, "gs://"):
+		tempDir, err := ioutil.TempDir("", renderedManifestsStagingDir)
+		if err != nil {
+			return fmt.Errorf("failed to create tmp directory: %w", err)
+		}
+		defer os.RemoveAll(tempDir)
+		tempFile := filepath.Join(tempDir, renderedManifestsStagingFile)
+		if err := dumpToFile(renderedManifests, tempFile); err != nil {
+			return err
+		}
+		gcs := util.Gsutil{}
+		if err := gcs.Copy(context.Background(), tempFile, output, false); err != nil {
+			return fmt.Errorf("failed to copy rendered manifests to GCS: %w", err)
+		}
+		return nil
+	default:
+		return dumpToFile(renderedManifests, output)
 	}
+}
 
+func dumpToFile(renderedManifests string, filepath string) error {
 	f, err := os.OpenFile(filepath, os.O_RDWR|os.O_CREATE, 0666)
 	if err != nil {
 		return fmt.Errorf("opening file for writing manifests: %w", err)

pkg/skaffold/util/gsutil.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+const GsutilExec = "gsutil"
+
+type Gsutil struct{}
+
+// Copy calls `gsutil cp [-r] <source_url> <destination_url>
+func (g *Gsutil) Copy(ctx context.Context, src, dst string, recursive bool) error {
+	args := []string{"cp"}
+	if recursive {
+		args = append(args, "-r")
+	}
+	args = append(args, src, dst)
+	cmd := exec.CommandContext(ctx, GsutilExec, args...)
+	out, err := RunCmdOut(cmd)
+	if err != nil {
+		return fmt.Errorf("copy file(s) with %s failed: %w", GsutilExec, err)
+	}
+	logrus.Info(out)
+	return nil
+}

pkg/skaffold/util/gsutil_test.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2020 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/GoogleContainerTools/skaffold/testutil"
+)
+
+const (
+	file      = "source/file"
+	gcsFile   = "gs://bucket/file"
+	folder    = "source/"
+	gcsFolder = "gs://bucket/folder/"
+)
+
+func TestCopy(t *testing.T) {
+	tests := []struct {
+		description string
+		src         string
+		dst         string
+		commands    Command
+		recursive   bool
+		shouldErr   bool
+	}{
+		{
+			description: "copy single file",
+			src:         file,
+			dst:         gcsFile,
+			commands:    testutil.CmdRunOut(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs"),
+		},
+		{
+			description: "copy recursively",
+			src:         folder,
+			dst:         gcsFolder,
+			commands:    testutil.CmdRunOut(fmt.Sprintf("gsutil cp -r %s %s", folder, gcsFolder), "logs"),
+			recursive:   true,
+		},
+		{
+			description: "copy failed",
+			src:         file,
+			dst:         gcsFile,
+			commands:    testutil.CmdRunOutErr(fmt.Sprintf("gsutil cp %s %s", file, gcsFile), "logs", fmt.Errorf("file not found")),
+			shouldErr:   true,
+		},
+	}
+	for _, test := range tests {
+		testutil.Run(t, test.description, func(t *testutil.T) {
+			t.Override(&DefaultExecCommand, test.commands)
+
+			gcs := Gsutil{}
+			err := gcs.Copy(context.Background(), test.src, test.dst, test.recursive)
+
+			t.CheckError(test.shouldErr, err)
+		})
+	}
+}