cirrus run: introduce --lazy-pull flag

internal/commands/run.go

@@ -32,6 +32,9 @@ var affectedFilesGitRevision string
 var affectedFilesGitCachedRevision string
 var verbose bool
 
+// Common instance-related flags.
+var lazyPull bool
+
 // Container-related flags.
 var containerBackend string
 var containerLazyPull bool
@@ -41,6 +44,9 @@ var containerLazyPull bool
 var dockerfileImageTemplate string
 var dockerfileImagePush bool
 
+// Tart-related flags.
+var tartLazyPull bool
+
 // Flags useful for debugging.
 var debugNoCleanup bool
 
@@ -120,13 +126,18 @@ func run(cmd *cobra.Command, args []string) error {
 
 	// Container-related options
 	executorOpts = append(executorOpts, executor.WithContainerOptions(options.ContainerOptions{
-		EagerPull: !containerLazyPull,
+		LazyPull:  lazyPull || containerLazyPull,
 		NoCleanup: debugNoCleanup,
 
 		DockerfileImageTemplate: dockerfileImageTemplate,
 		DockerfileImagePush:     dockerfileImagePush,
 	}))
 
+	// Tart-related options
+	executorOpts = append(executorOpts, executor.WithTartOptions(options.TartOptions{
+		LazyPull: lazyPull || tartLazyPull,
+	}))
+
 	// Environment
 	executorOpts = append(executorOpts,
 		executor.WithBaseEnvironmentOverride(baseEnvironment),
@@ -171,6 +182,11 @@ func newRunCmd() *cobra.Command {
 	cmd.PersistentFlags().StringVarP(&output, "output", "o", logs.DefaultFormat(), fmt.Sprintf("output format of logs, "+
 		"supported values: %s", strings.Join(logs.Formats(), ", ")))
 
+	// Common instance-related flags
+	cmd.PersistentFlags().BoolVar(&lazyPull, "lazy-pull", false,
+		"attempt to pull container and VM images only if they are missing locally "+
+			"(helpful in case of registry rate limits; enables --container-lazy-pull and --tart-lazy-pull)")
+
 	// Container-related flags
 	cmd.PersistentFlags().StringVar(&containerBackend, "container-backend", containerbackend.BackendAuto,
 		fmt.Sprintf("container engine backend to use, either \"%s\", \"%s\" or \"%s\"",
@@ -184,6 +200,10 @@ func newRunCmd() *cobra.Command {
 	cmd.PersistentFlags().BoolVar(&dockerfileImagePush, "dockerfile-image-push",
 		false, "whether to push whe image produced by the Dockerfile as CI environment feature")
 
+	// Tart-related flags
+	cmd.PersistentFlags().BoolVar(&tartLazyPull, "tart-lazy-pull", false,
+		"attempt to pull Tart VM images only if they are missing locally (helpful in case of registry rate limits)")
+
 	// Flags useful for debugging
 	cmd.PersistentFlags().BoolVar(&debugNoCleanup, "debug-no-cleanup", false,
 		"don't remove containers and volumes after execution")

internal/executor/executor.go

@@ -37,6 +37,7 @@ type Executor struct {
 	dirtyMode                bool
 	containerBackend         containerbackend.ContainerBackend
 	containerOptions         options.ContainerOptions
+	tartOptions              options.TartOptions
 }
 
 func New(projectDir string, tasks []*api.Task, opts ...Option) (*Executor, error) {
@@ -166,6 +167,7 @@ func (e *Executor) runSingleTask(ctx context.Context, task *build.Task) error {
 		TaskID:           task.ID,
 		DirtyMode:        e.dirtyMode,
 		ContainerOptions: e.containerOptions,
+		TartOptions:      e.tartOptions,
 	}
 
 	instanceRunOpts.SetLogger(taskLogger)

internal/executor/instance/persistentworker/isolation/tart/tart.go

@@ -53,7 +53,8 @@ func New(vmName string, sshUser string, sshPassword string, cpu uint32, memory u
 }
 
 func (tart *Tart) Run(ctx context.Context, config *runconfig.RunConfig) (err error) {
-	vm, err := NewVMClonedFrom(ctx, tart.vmName, tart.cpu, tart.memory, config.Logger())
+	vm, err := NewVMClonedFrom(ctx, tart.vmName, tart.cpu, tart.memory, config.TartOptions.LazyPull,
+		config.Logger())
 	if err != nil {
 		return fmt.Errorf("%w: failed to create VM cloned from %q: %v", ErrFailed, tart.vmName, err)
 	}

internal/executor/instance/persistentworker/isolation/tart/vm.go

@@ -18,7 +18,14 @@ type VM struct {
 	errChan      chan error
 }
 
-func NewVMClonedFrom(ctx context.Context, from string, cpu uint32, memory uint32, logger *echelon.Logger) (*VM, error) {
+func NewVMClonedFrom(
+	ctx context.Context,
+	from string,
+	cpu uint32,
+	memory uint32,
+	lazyPull bool,
+	logger *echelon.Logger,
+) (*VM, error) {
 	subCtx, subCtxCancel := context.WithCancel(ctx)
 
 	vm := &VM{
@@ -28,6 +35,20 @@ func NewVMClonedFrom(ctx context.Context, from string, cpu uint32, memory uint32
 		errChan:      make(chan error, 1),
 	}
 
+	pullLogger := logger.Scoped("pull virtual machine")
+	if !lazyPull {
+		pullLogger.Infof("Pulling virtual machine %s...", from)
+
+		if _, _, err := CmdWithLogger(ctx, pullLogger, "pull", from); err != nil {
+			pullLogger.FinishWithType(echelon.FinishTypeSucceeded)
+			return nil, err
+		}
+
+		pullLogger.FinishWithType(echelon.FinishTypeSucceeded)
+	} else {
+		pullLogger.FinishWithType(echelon.FinishTypeSkipped)
+	}
+
 	cloneLogger := logger.Scoped("clone virtual machine")
 	cloneLogger.Infof("Cloning virtual machine %s...", from)
 

internal/executor/instance/prebuilt.go

@@ -125,7 +125,7 @@ func (prebuilt *PrebuiltInstance) Run(ctx context.Context, config *runconfig.Run
 		Tags:       []string{prebuilt.Image},
 		Dockerfile: prebuilt.Dockerfile,
 		BuildArgs:  prebuilt.Arguments,
-		Pull:       config.ContainerOptions.EagerPull,
+		Pull:       !config.ContainerOptions.LazyPull,
 	})
 
 Outer:

internal/executor/instance/runconfig/runconfig.go

@@ -19,6 +19,7 @@ type RunConfig struct {
 	logger                     *echelon.Logger
 	DirtyMode                  bool
 	ContainerOptions           options.ContainerOptions
+	TartOptions                options.TartOptions
 	agentVersion               string
 }
 

internal/executor/options.go

@@ -50,3 +50,9 @@ func WithContainerBackend(containerBackend containerbackend.ContainerBackend) Op
 		e.containerBackend = containerBackend
 	}
 }
+
+func WithTartOptions(tartOptions options.TartOptions) Option {
+	return func(e *Executor) {
+		e.tartOptions = tartOptions
+	}
+}

internal/executor/options/container.go

@@ -7,7 +7,7 @@ import (
 )
 
 type ContainerOptions struct {
-	EagerPull    bool
+	LazyPull     bool
 	NoPullImages []string
 	NoCleanup    bool
 
@@ -26,7 +26,7 @@ func (copts ContainerOptions) ShouldPullImage(
 		}
 	}
 
-	if copts.EagerPull {
+	if !copts.LazyPull {
 		return true
 	}
 

internal/executor/options/container_test.go

@@ -11,7 +11,6 @@ import (
 
 func TestForcePull(t *testing.T) {
 	do := options.ContainerOptions{
-		EagerPull:    true,
 		NoPullImages: []string{"nonexistent.invalid/should/not/be:pulled"},
 	}
 
@@ -24,7 +23,7 @@ func TestForcePull(t *testing.T) {
 	// Shouldn't be pulled because it's blacklisted
 	assert.False(t, do.ShouldPullImage(ctx, backend, "nonexistent.invalid/should/not/be:pulled"))
 
-	// Should be pulled because EagerPull is set to true
+	// Should be pulled because lazy pull is disabled by default
 	image := canaryImage()
 
 	if err := backend.ImagePull(ctx, image); err != nil {

internal/executor/options/tart.go

@@ -0,0 +1,5 @@
+package options
+
+type TartOptions struct {
+	LazyPull bool
+}