Parallels isolation support (#211)

* Move RetrieveAgentBinary() to a separate package

* Move RunConfig{} to a separate package

* .golangci.yml: temporarily disable new linters

* Parallels isolation support

* Persistent worker: notify the server about failed tasks

* Ensure that the VM is isolated from the host

* Ignore lll warning for URL in the comment

.golangci.yml

@@ -56,6 +56,11 @@ linters:
     # Enough parallelism for now.
     - paralleltest
 
+    # https://github.com/cirruslabs/cirrus-cli/issues/210
+    - makezero
+    - thelper
+    - forbidigo
+
 issues:
   # Don't hide multiple issues that belong to one class since GitHub annotations can handle them all nicely.
   max-issues-per-linter: 0

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/bmatcuk/doublestar v1.3.2
 	github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054
-	github.com/cirruslabs/cirrus-ci-agent v1.24.0
+	github.com/cirruslabs/cirrus-ci-agent v1.25.1
 	github.com/cirruslabs/echelon v1.4.0
 	github.com/cirruslabs/podmanapi v0.1.0
 	github.com/containerd/containerd v1.4.1 // indirect
@@ -41,6 +41,7 @@ require (
 	github.com/onsi/ginkgo v1.14.2 // indirect
 	github.com/onsi/gomega v1.10.3 // indirect
 	github.com/otiai10/copy v1.2.0
+	github.com/pkg/sftp v1.12.0
 	github.com/qri-io/starlib v0.4.2
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
@@ -52,6 +53,7 @@ require (
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
 	github.com/yudai/pp v2.0.1+incompatible // indirect
 	go.starlark.net v0.0.0-20201118183435-e55f603d8c79
+	golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
 	golang.org/x/net v0.0.0-20201022231255-08b38378de70 // indirect
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
 	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd

go.sum

@@ -90,8 +90,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
 github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s=
-github.com/cirruslabs/cirrus-ci-agent v1.24.0 h1:GiK7hYgRP94Sz2pEDiy5oBFS/EaAjynns9EyKfmoOBc=
-github.com/cirruslabs/cirrus-ci-agent v1.24.0/go.mod h1:ga2zCGBfC+/+tnlHWa5cHwEuBPnhFuaf1PgTpWPGJWo=
+github.com/cirruslabs/cirrus-ci-agent v1.25.1 h1:9dg5s1JTA2KC5kvpbGzna4s/ESe16thiH/CbuZy7/6s=
+github.com/cirruslabs/cirrus-ci-agent v1.25.1/go.mod h1:ga2zCGBfC+/+tnlHWa5cHwEuBPnhFuaf1PgTpWPGJWo=
 github.com/cirruslabs/cirrus-ci-annotations v0.0.0-20200908203753-b813f63941d7/go.mod h1:98qD7HLlBx5aNqWiCH80OTTqTTsbXT69wxnlnrnoL0E=
 github.com/cirruslabs/echelon v1.4.0 h1:xubCf8BLFEBl1kamBZ1zjBrcw5p4z4anvJUBeR3E5YY=
 github.com/cirruslabs/echelon v1.4.0/go.mod h1:1jFBACMy3tzodXyTtNNLN9bw6UUU7Xpq9tYMRydehtY=
@@ -322,6 +322,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -425,6 +427,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.12.0 h1:/f3b24xrDhkhddlaobPe2JgBqfdt+gC/NYl0QY9IOuI=
+github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -557,6 +561,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

internal/executor/instance/persistentworker.go → internal/executor/agent/agent.go

@@ -1,95 +1,20 @@
-package instance
+package agent
 
 import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/otiai10/copy"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 )
 
-var (
-	ErrPopulateFailed      = errors.New("failed to populate working directory")
-	ErrAgentDownloadFailed = errors.New("failed to download agent")
-)
-
-type PersistentWorkerInstance struct {
-	tempDir string
-	cleanup func() error
-}
-
-func NewPersistentWorkerInstance() (*PersistentWorkerInstance, error) {
-	// Create a working directory that will be used if no dirty mode is requested in Run()
-	tempDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		return nil, err
-	}
-
-	return &PersistentWorkerInstance{
-		tempDir: tempDir,
-		cleanup: func() error {
-			return os.RemoveAll(tempDir)
-		},
-	}, nil
-}
-
-func (pwi *PersistentWorkerInstance) Run(ctx context.Context, config *RunConfig) (err error) {
-	// Retrieve the agent's binary
-	agentPath, err := RetrieveAgentBinary(ctx, config.GetAgentVersion(), runtime.GOOS, runtime.GOARCH)
-	if err != nil {
-		return err
-	}
-
-	cmd := exec.CommandContext(ctx, agentPath,
-		"-api-endpoint",
-		config.DirectEndpoint,
-		"-server-token",
-		config.ServerSecret,
-		"-client-token",
-		config.ClientSecret,
-		"-task-id",
-		strconv.FormatInt(config.TaskID, 10),
-	)
-
-	// Determine the working directory for the agent
-	if config.DirtyMode {
-		cmd.Dir = config.ProjectDir
-	} else {
-		// Populate the working directory
-		if config.ProjectDir != "" {
-			if err := copy.Copy(config.ProjectDir, pwi.tempDir); err != nil {
-				return fmt.Errorf("%w: while copying %s's contents into %s: %v",
-					ErrPopulateFailed, config.ProjectDir, pwi.tempDir, err)
-			}
-		}
-
-		cmd.Dir = pwi.tempDir
-	}
-
-	// Run the agent
-	if err := cmd.Run(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (pwi *PersistentWorkerInstance) WorkingDirectory(projectDir string, dirtyMode bool) string {
-	if dirtyMode {
-		return projectDir
-	}
-
-	return pwi.tempDir
-}
+var ErrAgentDownloadFailed = errors.New("failed to download agent")
 
-func RetrieveAgentBinary(
+func RetrieveBinary(
 	ctx context.Context,
 	agentVersion string,
 	agentOS string,

internal/executor/instance/persistentworker_test.go → internal/executor/agent/agent_test.go

@@ -1,8 +1,8 @@
-package instance_test
+package agent_test
 
 import (
 	"context"
-	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/agent"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/platform"
 	"github.com/stretchr/testify/assert"
 	"os"
@@ -13,7 +13,7 @@ import (
 
 func TestRetrieveAgentBinary(t *testing.T) {
 	// Does it work?
-	firstPath, err := instance.RetrieveAgentBinary(context.Background(),
+	firstPath, err := agent.RetrieveBinary(context.Background(),
 		platform.DefaultAgentVersion, runtime.GOOS, runtime.GOARCH)
 	if err != nil {
 		t.Fatal()
@@ -26,7 +26,7 @@ func TestRetrieveAgentBinary(t *testing.T) {
 	// Does it cache the agent?
 	cachedRetrievalStart := time.Now()
 
-	secondPath, err := instance.RetrieveAgentBinary(context.Background(),
+	secondPath, err := agent.RetrieveBinary(context.Background(),
 		platform.DefaultAgentVersion, runtime.GOOS, runtime.GOARCH)
 	if err != nil {
 		t.Fatal()

internal/executor/build/task.go

@@ -6,6 +6,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/internal/executor/build/commandstatus"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/build/taskstatus"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/abstract"
 	"strconv"
 	"strings"
 	"sync"
@@ -22,7 +23,7 @@ type Task struct {
 	Name        string
 	Labels      []string
 	status      taskstatus.Status
-	Instance    instance.Instance
+	Instance    abstract.Instance
 	Timeout     time.Duration
 	Environment map[string]string
 	Commands    []*Command

internal/executor/executor.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/internal/executor/environment"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/containerbackend"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/runconfig"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/options"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/rpc"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/taskfilter"
@@ -142,7 +143,7 @@ func (e *Executor) Run(ctx context.Context) error {
 
 		// Prepare task's instance
 		taskInstance := task.Instance
-		instanceRunOpts := instance.RunConfig{
+		instanceRunOpts := runconfig.RunConfig{
 			ContainerBackend:  e.containerBackend,
 			ProjectDir:        e.build.ProjectDir,
 			ContainerEndpoint: e.rpc.ContainerEndpoint(),

internal/executor/instance/abstract/abstract.go

@@ -0,0 +1,11 @@
+package abstract
+
+import (
+	"context"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/runconfig"
+)
+
+type Instance interface {
+	Run(context.Context, *runconfig.RunConfig) error
+	WorkingDirectory(projectDir string, dirtyMode bool) string
+}

internal/executor/instance/container.go

@@ -3,6 +3,7 @@ package instance
 import (
 	"context"
 	"github.com/cirruslabs/cirrus-ci-agent/api"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/runconfig"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/platform"
 	"path"
 )
@@ -15,7 +16,7 @@ type ContainerInstance struct {
 	Platform             platform.Platform
 }
 
-func (inst *ContainerInstance) Run(ctx context.Context, config *RunConfig) (err error) {
+func (inst *ContainerInstance) Run(ctx context.Context, config *runconfig.RunConfig) (err error) {
 	workingVolume, err := CreateWorkingVolumeFromConfig(ctx, config, inst.Platform)
 	if err != nil {
 		return err

internal/executor/instance/instance.go

@@ -6,7 +6,10 @@ import (
 	"fmt"
 	"github.com/cirruslabs/cirrus-ci-agent/api"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/heuristic"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/abstract"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/containerbackend"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/persistentworker"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/runconfig"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/options"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/platform"
 	"github.com/cirruslabs/echelon"
@@ -32,12 +35,7 @@ const (
 	nano = 1_000_000_000
 )
 
-type Instance interface {
-	Run(context.Context, *RunConfig) error
-	WorkingDirectory(projectDir string, dirtyMode bool) string
-}
-
-func NewFromProto(anyInstance *any.Any, commands []*api.Command) (Instance, error) {
+func NewFromProto(anyInstance *any.Any, commands []*api.Command) (abstract.Instance, error) {
 	var dynamicInstance ptypes.DynamicAny
 	if err := ptypes.UnmarshalAny(anyInstance, &dynamicInstance); err != nil {
 		return nil, err
@@ -89,7 +87,7 @@ func NewFromProto(anyInstance *any.Any, commands []*api.Command) (Instance, erro
 			Arguments:  instance.Arguments,
 		}, nil
 	case *api.PersistentWorkerInstance:
-		return NewPersistentWorkerInstance()
+		return persistentworker.New(instance.Isolation)
 	case *api.DockerBuilder:
 		// Ensures that we're not trying to run e.g. Windows-specific scripts on macOS
 		instanceOS := strings.ToLower(instance.Platform.String())
@@ -98,37 +96,16 @@ func NewFromProto(anyInstance *any.Any, commands []*api.Command) (Instance, erro
 				ErrFailedToCreateInstance, strings.Title(instanceOS))
 		}
 
-		return NewPersistentWorkerInstance()
+		return persistentworker.New(&api.Isolation{
+			Type: &api.Isolation_None_{
+				None: &api.Isolation_None{},
+			},
+		})
 	default:
 		return nil, fmt.Errorf("%w: %T", ErrUnsupportedInstance, instance)
 	}
 }
 
-type RunConfig struct {
-	ContainerBackend           containerbackend.ContainerBackend
-	ProjectDir                 string
-	ContainerEndpoint          string
-	DirectEndpoint             string
-	ServerSecret, ClientSecret string
-	TaskID                     int64
-	Logger                     *echelon.Logger
-	DirtyMode                  bool
-	ContainerOptions           options.ContainerOptions
-	agentVersion               string
-}
-
-func (rc *RunConfig) GetAgentVersion() string {
-	if rc.agentVersion == "" {
-		return platform.DefaultAgentVersion
-	}
-
-	return rc.agentVersion
-}
-
-func (rc *RunConfig) SetAgentVersion(agentVersion string) {
-	rc.agentVersion = agentVersion
-}
-
 type Params struct {
 	Image                  string
 	CPU                    float32
@@ -139,7 +116,7 @@ type Params struct {
 	WorkingVolumeName      string
 }
 
-func RunContainerizedAgent(ctx context.Context, config *RunConfig, params *Params) error {
+func RunContainerizedAgent(ctx context.Context, config *runconfig.RunConfig, params *Params) error {
 	logger := config.Logger
 	backend := config.ContainerBackend