Podman support (#151)

* Abstract away Docker into a ContainerBackend interface

* Swagger-generated Podman API package

* Fix podmanapi package compilation and document how it was generated

* Further abstract Docker away

...and introduce ContainerBackendFromEnv test helper to pick up the
backend from CIRRUS_CONTAINER_BACKEND environment variable.

* Re-generate podmanapi using Swagger Codegen 3.0.23

* Podman support

* Cleanup additional containers first

* cirrus run: support multiple container backends

* Rename internal/podmanapi/model_plugin_config_linux.go

...to fix the compilation on non-Linux platforms.

* Rename DockerOptions to ContainerOptions

* Rename "docker pull" scope to "image pull"

* Run a single "podman system service" subprocess per each backend instance

* Don't swallow error

* .cirrus.yml: re-use alias name

* cirrus run: rename --docker-no-pull to --container-no-pull

* cirrus run: introduce --container-backend argument

* Only include Podman support on Linux

* Fix inverted logic

.cirrus.yml

@@ -10,21 +10,41 @@ task:
       format: golangci
 
 docker_builder:
-  name: Test
+  name: Test (Docker)
+  alias: Tests
   test_script:
     - wget --no-verbose -O - https://golang.org/dl/go1.15.linux-amd64.tar.gz | tar -C /usr/local -xz
     - export PATH=$PATH:/usr/local/go/bin
     - go test ./...
   env:
     HOME: /root
 
+docker_builder:
+  name: Test (Podman)
+  alias: Tests
+  install_podman_script:
+    - . /etc/os-release
+    - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+    - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add -
+    - sudo apt-get update
+    - sudo apt-get -y upgrade
+    - sudo apt-get -y install podman
+  run_podman_background_script:
+    - podman system service -t 0 unix:///tmp/podman.sock
+  test_script:
+    - wget --no-verbose -O - https://golang.org/dl/go1.15.linux-amd64.tar.gz | tar -C /usr/local -xz
+    - export PATH=$PATH:/usr/local/go/bin
+    - go test ./...
+  env:
+    HOME: /root
+    CIRRUS_CONTAINER_BACKEND: podman
 
 task:
   name: Release (Dry Run)
   only_if: $CIRRUS_PR != ''
   depends_on:
     - Lint
-    - Test
+    - Tests
   container:
     image: goreleaser/goreleaser:latest
   release_script: goreleaser build --snapshot

.golangci.yml

@@ -1,6 +1,9 @@
 run:
   timeout: 5m
 
+  skip-dirs:
+    - internal/podmanapi
+
 linters-settings:
   # Even in Rust you can get away with partial matching,
   # so make sure that the linter respects the programmer's

go.mod

@@ -8,6 +8,8 @@ replace gopkg.in/yaml.v2 => github.com/cirruslabs/yaml v0.0.0-20201005110149-09a
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/PaesslerAG/gval v1.1.0
+	github.com/antihax/optional v1.0.0
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/bmatcuk/doublestar v1.3.2
 	github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054
 	github.com/cirruslabs/cirrus-ci-agent v1.16.0

go.sum

@@ -53,11 +53,15 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=

internal/PODMANAPI.md

@@ -0,0 +1,16 @@
+# podmanapi
+
+The `podmanapi` package was generated using the Swagger Codegen 3.0.23 [downloaded from the Maven Repository](https://mvnrepository.com/artifact/io.swagger.codegen.v3/swagger-codegen-cli):
+
+```
+java -jar swagger-codegen-cli-3.0.23.jar generate -l go -i https://storage.googleapis.com/libpod-master-releases/swagger-latest-master.yaml -o podmanapi
+```
+
+The link to the [`swagger-latest-master.yaml`](https://storage.googleapis.com/libpod-master-releases/swagger-latest-master.yaml) was found in the auto-generated [API documentation page](https://podman.readthedocs.io/en/latest/_static/api.html).
+
+Afterwards:
+
+* the missing `os` imports were added
+* `model_plugin_config_linux.go` was renamed to `model_plugin_config_linux_.go`
+
+... to fix the compilation.

internal/commands/run.go

@@ -7,13 +7,13 @@ import (
 	"github.com/cirruslabs/cirrus-cli/internal/commands/logs"
 	"github.com/cirruslabs/cirrus-cli/internal/executor"
 	eenvironment "github.com/cirruslabs/cirrus-cli/internal/executor/environment"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/containerbackend"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/options"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/taskfilter"
 	"github.com/cirruslabs/cirrus-cli/pkg/larker"
 	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/local"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser"
 	"github.com/cirruslabs/cirrus-cli/pkg/rpcparser"
-	"github.com/docker/docker/client"
 	"github.com/spf13/cobra"
 	"io/ioutil"
 	"log"
@@ -29,8 +29,9 @@ var output string
 var environment []string
 var verbose bool
 
-// Docker-related flags.
-var dockerNoPull bool
+// Container-related flags.
+var containerBackend string
+var containerNoPull bool
 
 // Flags useful for debugging.
 var debugNoCleanup bool
@@ -93,24 +94,12 @@ func readStarlarkConfig(ctx context.Context, env map[string]string) (string, err
 	return lrk.Main(ctx, string(starlarkSource))
 }
 
-func preflightCheck() error {
-	// Since all of the instance types we currently support use Docker,
-	// check that it's actually installed as early as possible
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("%w: cannot connect to Docker daemon: %v, make sure the Docker is installed",
-			ErrRun, err)
-	}
-	defer cli.Close()
-
-	return nil
-}
-
 func run(cmd *cobra.Command, args []string) error {
 	// https://github.com/spf13/cobra/issues/340#issuecomment-374617413
 	cmd.SilenceUsage = true
 
-	if err := preflightCheck(); err != nil {
+	backend, err := containerbackend.New(containerBackend)
+	if err != nil {
 		return err
 	}
 
@@ -182,9 +171,9 @@ func run(cmd *cobra.Command, args []string) error {
 		executorOpts = append(executorOpts, executor.WithDirtyMode())
 	}
 
-	// Docker-related options
-	executorOpts = append(executorOpts, executor.WithDockerOptions(options.DockerOptions{
-		NoPull:    dockerNoPull,
+	// Container-related options
+	executorOpts = append(executorOpts, executor.WithContainerOptions(options.ContainerOptions{
+		NoPull:    containerNoPull,
 		NoCleanup: debugNoCleanup,
 	}))
 
@@ -194,6 +183,9 @@ func run(cmd *cobra.Command, args []string) error {
 		executor.WithUserSpecifiedEnvironment(userSpecifiedEnvironment),
 	)
 
+	// Container backend
+	executorOpts = append(executorOpts, executor.WithContainerBackend(backend))
+
 	// Run
 	e, err := executor.New(projectDir, result.Tasks, executorOpts...)
 	if err != nil {
@@ -220,9 +212,17 @@ func newRunCmd() *cobra.Command {
 	cmd.PersistentFlags().StringVarP(&output, "output", "o", logs.DefaultFormat(), fmt.Sprintf("output format of logs, "+
 		"supported values: %s", strings.Join(logs.Formats(), ", ")))
 
-	// Docker-related flags
-	cmd.PersistentFlags().BoolVar(&dockerNoPull, "docker-no-pull", false,
+	// Container-related flags
+	cmd.PersistentFlags().StringVar(&containerBackend, "container-backend", containerbackend.BackendAuto,
+		fmt.Sprintf("container engine backend to use, either \"%s\", \"%s\" or \"%s\"",
+			containerbackend.BackendDocker, containerbackend.BackendPodman, containerbackend.BackendAuto))
+	cmd.PersistentFlags().BoolVar(&containerNoPull, "container-no-pull", false,
+		"don't attempt to pull the images before starting containers")
+
+	// Deprecated flags
+	cmd.PersistentFlags().BoolVar(&containerNoPull, "docker-no-pull", false,
 		"don't attempt to pull the images before starting containers")
+	_ = cmd.PersistentFlags().MarkDeprecated("docker-no-pull", "use --container-no-pull instead")
 
 	// Flags useful for debugging
 	cmd.PersistentFlags().BoolVar(&debugNoCleanup, "debug-no-cleanup", false,

internal/commands/run_test.go

@@ -5,8 +5,6 @@ import (
 	"context"
 	"github.com/cirruslabs/cirrus-cli/internal/commands"
 	"github.com/cirruslabs/cirrus-cli/internal/testutil"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/stretchr/testify/assert"
@@ -223,7 +221,7 @@ func TestRunDockerNoPull(t *testing.T) {
 
 	require.NotNil(t, err)
 	assert.NotContains(t, buf.String(), "pulling image")
-	assert.Contains(t, buf.String(), "No such image")
+	assert.Contains(t, strings.ToLower(buf.String()), "no such image")
 }
 
 // TestRunTaskFilteringByLabel ensures that task filtering logic is label-aware.
@@ -266,10 +264,7 @@ func TestRunNoCleanup(t *testing.T) {
 	assert.Contains(t, buf.String(), "not cleaning up working volume")
 
 	// The fun ends here since now we have to cleanup containers and volumes ourselves
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		t.Fatal(err)
-	}
+	backend := testutil.ContainerBackendFromEnv(t)
 
 	containerRegex := regexp.MustCompile("not cleaning up (?:container|additional container) (?P<container_id>[^,]+)")
 	volumeRegex := regexp.MustCompile("not cleaning up working volume (?P<volume_id>[^,]+)")
@@ -278,18 +273,15 @@ func TestRunNoCleanup(t *testing.T) {
 		matches := containerRegex.FindStringSubmatch(line)
 		if matches != nil {
 			containerID := matches[containerRegex.SubexpIndex("container_id")]
-			if err := cli.ContainerRemove(context.Background(), containerID, types.ContainerRemoveOptions{
-				RemoveVolumes: true,
-				Force:         true,
-			}); err != nil {
+			if err := backend.ContainerDelete(context.Background(), containerID); err != nil {
 				t.Fatal(err)
 			}
 		}
 
 		matches = volumeRegex.FindStringSubmatch(line)
 		if matches != nil {
 			volumeID := matches[volumeRegex.SubexpIndex("volume_id")]
-			if err := cli.VolumeRemove(context.Background(), volumeID, false); err != nil {
+			if err := backend.VolumeDelete(context.Background(), volumeID); err != nil {
 				t.Fatal(err)
 			}
 		}

internal/executor/executor.go

@@ -9,6 +9,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/internal/executor/build/taskstatus"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/environment"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/containerbackend"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/options"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/rpc"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/taskfilter"
@@ -29,7 +30,8 @@ type Executor struct {
 	baseEnvironment          map[string]string
 	userSpecifiedEnvironment map[string]string
 	dirtyMode                bool
-	dockerOptions            options.DockerOptions
+	containerBackend         containerbackend.ContainerBackend
+	containerOptions         options.ContainerOptions
 }
 
 func New(projectDir string, tasks []*api.Task, opts ...Option) (*Executor, error) {
@@ -53,6 +55,13 @@ func New(projectDir string, tasks []*api.Task, opts ...Option) (*Executor, error
 		renderer := renderers.NewSimpleRenderer(ioutil.Discard, nil)
 		e.logger = echelon.NewLogger(echelon.InfoLevel, renderer)
 	}
+	if e.containerBackend == nil {
+		backend, err := containerbackend.NewDocker()
+		if err != nil {
+			return nil, err
+		}
+		e.containerBackend = backend
+	}
 
 	// Filter tasks (e.g. if a user wants to run only a specific task without dependencies)
 	tasks = e.taskFilter(tasks)
@@ -87,7 +96,7 @@ func New(projectDir string, tasks []*api.Task, opts ...Option) (*Executor, error
 	for _, task := range b.Tasks() {
 		// Collect images that shouldn't be pulled under any circumstances
 		if prebuiltInstance, ok := task.Instance.(*instance.PrebuiltInstance); ok {
-			e.dockerOptions.NoPullImages = append(e.dockerOptions.NoPullImages, prebuiltInstance.Image)
+			e.containerOptions.NoPullImages = append(e.containerOptions.NoPullImages, prebuiltInstance.Image)
 		}
 
 		// Set task's working directory based on it's instance (if not overridden by the user)
@@ -120,6 +129,7 @@ func (e *Executor) Run(ctx context.Context) error {
 		// Prepare task's instance
 		taskInstance := task.Instance
 		instanceRunOpts := instance.RunConfig{
+			ContainerBackend:  e.containerBackend,
 			ProjectDir:        e.build.ProjectDir,
 			ContainerEndpoint: e.rpc.ContainerEndpoint(),
 			DirectEndpoint:    e.rpc.DirectEndpoint(),
@@ -128,7 +138,7 @@ func (e *Executor) Run(ctx context.Context) error {
 			TaskID:            task.ID,
 			Logger:            taskLogger,
 			DirtyMode:         e.dirtyMode,
-			DockerOptions:     e.dockerOptions,
+			ContainerOptions:  e.containerOptions,
 		}
 
 		// Wrap the context to enforce a timeout for this task

internal/executor/executor_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/cirruslabs/cirrus-ci-agent/api"
 	"github.com/cirruslabs/cirrus-cli/internal/executor"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
+	"github.com/cirruslabs/cirrus-cli/internal/executor/instance/containerbackend"
 	"github.com/cirruslabs/cirrus-cli/internal/testutil"
 	"github.com/cirruslabs/cirrus-cli/pkg/rpcparser"
 	"github.com/cirruslabs/echelon"
@@ -24,7 +25,7 @@ import (
 func TestExecutorEmpty(t *testing.T) {
 	dir := testutil.TempDir(t)
 
-	e, err := executor.New(dir, []*api.Task{})
+	e, err := executor.New(dir, []*api.Task{}, executor.WithContainerBackend(testutil.ContainerBackendFromEnv(t)))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -70,7 +71,7 @@ func TestExecutorClone(t *testing.T) {
 			},
 			Instance: testutil.GetBasicContainerInstance(t, "debian:latest"),
 		},
-	})
+	}, executor.WithContainerBackend(testutil.ContainerBackendFromEnv(t)))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -115,7 +116,7 @@ func TestExecutorScript(t *testing.T) {
 			},
 			Instance: testutil.GetBasicContainerInstance(t, "debian:latest"),
 		},
-	}, executor.WithLogger(logger))
+	}, executor.WithLogger(logger), executor.WithContainerBackend(testutil.ContainerBackendFromEnv(t)))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -148,7 +149,7 @@ func TestExecutorFails(t *testing.T) {
 			},
 			Instance: testutil.GetBasicContainerInstance(t, "debian:latest"),
 		},
-	})
+	}, executor.WithContainerBackend(testutil.ContainerBackendFromEnv(t)))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -160,6 +161,11 @@ func TestExecutorFails(t *testing.T) {
 
 // TestResourceLimits ensures that the desired CPU and memory limits are enforced for instances.
 func TestResourceLimits(t *testing.T) {
+	// Skip this test on Podman due to https://github.com/containers/podman/issues/7959
+	if _, ok := testutil.ContainerBackendFromEnv(t).(*containerbackend.Podman); ok {
+		return
+	}
+
 	dir := testutil.TempDirPopulatedWith(t, "testdata/resource-limits")
 	err := testutil.Execute(t, dir)
 	assert.NoError(t, err)
@@ -168,6 +174,11 @@ func TestResourceLimits(t *testing.T) {
 // TestAdditionalContainers ensures that the services created in the additional containers
 // are reachable from the main container.
 func TestAdditionalContainers(t *testing.T) {
+	// Skip this test on Podman
+	if _, ok := testutil.ContainerBackendFromEnv(t).(*containerbackend.Podman); ok {
+		return
+	}
+
 	dir := testutil.TempDirPopulatedWith(t, "testdata/additional-containers")
 	err := testutil.Execute(t, dir)
 	assert.NoError(t, err)

internal/executor/instance/container.go

@@ -19,14 +19,14 @@ func (inst *ContainerInstance) Run(ctx context.Context, config *RunConfig) (err
 		return err
 	}
 	defer func() {
-		if config.DockerOptions.NoCleanup {
+		if config.ContainerOptions.NoCleanup {
 			config.Logger.Infof("not cleaning up working volume %s, don't forget to remove it with \"docker volume rm %s\"",
 				workingVolume.Name(), workingVolume.Name())
 
 			return
 		}
 
-		cleanupErr := workingVolume.Close()
+		cleanupErr := workingVolume.Close(config.ContainerBackend)
 		if err == nil {
 			err = cleanupErr
 		}
@@ -40,7 +40,7 @@ func (inst *ContainerInstance) Run(ctx context.Context, config *RunConfig) (err
 		WorkingVolumeName:    workingVolume.Name(),
 	}
 
-	if err := RunDockerizedAgent(ctx, config, params); err != nil {
+	if err := RunContainerizedAgent(ctx, config, params); err != nil {
 		return err
 	}