fetch: check a cache directory for files matching expected SHAs

This adds a --cache-dir flag (default /var/cache/melange), which is
copied into the build environment at /var/cache/melange (constant), and
updates the fetch.yaml pipeline to look for files matching the
expected-sha{256,512} in that directory.

The intention is that builds could pre-fetch external dependencies into
the cache directory, then execute builds without hitting the network.

Files in --cache-dir that won't match any fetch are not copied into the
build environment.

Signed-off-by: Jason Hall <[email protected]>

README.md

@@ -52,7 +52,7 @@ Platform:      linux/amd64
 To use the examples, you'll generally want to mount your current directory into the container and provide elevated privileges e.g:
 
 ```shell
-docker run --privileged -v "$PWD":/work distroless.dev/melange build examples/gnu-hello.yaml
+docker run --privileged -v "$PWD":/work cgr.dev/chainguard/melange build examples/gnu-hello.yaml
 ```
 
 These examples require [Docker](https://docs.docker.com/get-docker/), but should also work with other runtimes such as [podman](https://podman.io/getting-started/installation).
@@ -190,8 +190,8 @@ melange build --template '{"Version": "1.22.0"}'
 
 ## Usage with apko
 
-To use a melange built APK in apko, either upload it to a package repository or use a "local" repository. Using a local repository allows a melange build and apko build to run in the same directory (or GitHub repo) without using external storage. 
-An example of this approach can be seen in the [nginx-image-demo repo](https://github.com/chainguard-dev/nginx-image-demo/). 
+To use a melange built APK in apko, either upload it to a package repository or use a "local" repository. Using a local repository allows a melange build and apko build to run in the same directory (or GitHub repo) without using external storage.
+An example of this approach can be seen in the [nginx-image-demo repo](https://github.com/chainguard-dev/nginx-image-demo/).
 
 ### Coming soon: Keyless signatures
 

pipelines/fetch.yaml

@@ -35,16 +35,35 @@ pipeline:
         exit 1
       fi
 
-      wget ${{inputs.uri}}
       bn=$(basename ${{inputs.uri}})
 
+      ls /var/cache/melange ## TODO(jason): Remove this; this just tests that the dir exists.
+      if [ ! "${{inputs.expected-sha256}}" == "" ]; then
+        fn="/var/cache/melange/sha256:${{inputs.expected-sha256}}"
+        printf "Looking for $fn in cache"
+        if [ -f $fn ]; then
+          printf "Found $fn in cache"
+          cp $fn $bn
+        fi
+      else
+        fn="/var/cache/melange/sha512:${{inputs.expected-sha512}}"
+        printf "Looking for $fn in cache"
+        if [ -f $fn ]; then
+          printf "Found $fn in cache"
+          cp $fn $bn
+        fi
+      fi
+
+      if [ ! -f $bn ]; then
+        wget ${{inputs.uri}}
+      fi
+
       if [ "${{inputs.expected-sha256}}" != "" ]; then
         printf "%s  %s\n" '${{inputs.expected-sha256}}' $bn | sha256sum -c
       else
         printf "%s  %s\n" '${{inputs.expected-sha512}}' $bn | sha512sum -c
       fi
 
       if [ "${{inputs.extract}}" = "true" ]; then
-        bn=$(basename ${{inputs.uri}})
         tar -x '--strip-components=${{inputs.strip-components}}' -f $bn
       fi

pkg/build/build.go

@@ -139,6 +139,7 @@ type Context struct {
 	DependencyLog     string
 	BinShOverlay      string
 	ignorePatterns    []*xignore.Pattern
+	CacheDir          string
 }
 
 type Dependencies struct {
@@ -152,6 +153,7 @@ func New(opts ...Option) (*Context, error) {
 		PipelineDir:     "/usr/share/melange/pipelines",
 		SourceDir:       ".",
 		OutDir:          ".",
+		CacheDir:        "/var/cache/melange",
 		Logger:          log.New(log.Writer(), "melange: ", log.LstdFlags|log.Lmsgprefix),
 		Arch:            apko_types.ParseArchitecture(runtime.GOARCH),
 	}
@@ -294,6 +296,14 @@ func WithSourceDir(sourceDir string) Option {
 	}
 }
 
+// WithCacheDir sets the cache directory to use.
+func WithCacheDir(cacheDir string) Option {
+	return func(ctx *Context) error {
+		ctx.CacheDir = cacheDir
+		return nil
+	}
+}
+
 // WithSigningKey sets the signing key path to use.
 func WithSigningKey(signingKey string) Option {
 	return func(ctx *Context) error {
@@ -605,6 +615,54 @@ func (ctx *Context) OverlayBinSh() error {
 	return nil
 }
 
+func (ctx *Context) PopulateCache() error {
+	ctx.Logger.Printf("populating cache from %s", ctx.CacheDir)
+
+	fsys := apkofs.DirFS(ctx.CacheDir)
+
+	// mkdir /var/cache/melange
+	if err := os.MkdirAll("/var/cache/melange", 0o755); err != nil {
+		return err
+	}
+
+	// --cache-dir doesn't exist, nothing to do.
+	if _, err := fs.Stat(fsys, "."); errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+
+	return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		fi, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		mode := fi.Mode()
+		if !mode.IsRegular() {
+			return nil
+		}
+
+		// Skip files in the cache that aren't named like sha256:... or sha512:...
+		// This is likely a bug, and won't be matched by any fetch.
+		base := filepath.Base(fi.Name())
+		if !strings.HasPrefix(base, "sha256:") &&
+			!strings.HasPrefix(base, "sha512:") {
+			return nil
+		}
+
+		ctx.Logger.Printf("  -> %s", path)
+
+		if err := copyFile(ctx.CacheDir, path, "/var/cache/melange", mode.Perm()); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func (ctx *Context) PopulateWorkspace() error {
 	if ctx.EmptyWorkspace {
 		ctx.Logger.Printf("empty workspace requested")
@@ -619,7 +677,7 @@ func (ctx *Context) PopulateWorkspace() error {
 
 	fsys := apkofs.DirFS(ctx.SourceDir)
 
-	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+	return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -645,11 +703,7 @@ func (ctx *Context) PopulateWorkspace() error {
 		}
 
 		return nil
-	}); err != nil {
-		return err
-	}
-
-	return nil
+	})
 }
 
 func (ctx *Context) BuildPackage() error {
@@ -681,6 +735,9 @@ func (ctx *Context) BuildPackage() error {
 		return fmt.Errorf("unable to install overlay /bin/sh: %w", err)
 	}
 
+	if err := ctx.PopulateCache(); err != nil {
+		return fmt.Errorf("unable to populate cache: %w", err)
+	}
 	if err := ctx.PopulateWorkspace(); err != nil {
 		return fmt.Errorf("unable to populate workspace: %w", err)
 	}

pkg/cli/build.go

@@ -32,6 +32,7 @@ func Build() *cobra.Command {
 	var workspaceDir string
 	var pipelineDir string
 	var sourceDir string
+	var cacheDir string
 	var signingKey string
 	var generateIndex bool
 	var useProot bool
@@ -56,6 +57,7 @@ func Build() *cobra.Command {
 				build.WithBuildDate(buildDate),
 				build.WithWorkspaceDir(workspaceDir),
 				build.WithPipelineDir(pipelineDir),
+				build.WithCacheDir(cacheDir),
 				build.WithSigningKey(signingKey),
 				build.WithGenerateIndex(generateIndex),
 				build.WithUseProot(useProot),
@@ -93,6 +95,7 @@ func Build() *cobra.Command {
 	cmd.Flags().StringVar(&workspaceDir, "workspace-dir", "", "directory used for the workspace at /home/build")
 	cmd.Flags().StringVar(&pipelineDir, "pipeline-dir", "/usr/share/melange/pipelines", "directory used to store defined pipelines")
 	cmd.Flags().StringVar(&sourceDir, "source-dir", "", "directory used for included sources")
+	cmd.Flags().StringVar(&cacheDir, "cache-dir", "/var/cache/melange", "directory used for cached inputs")
 	cmd.Flags().StringVar(&signingKey, "signing-key", "", "key to use for signing")
 	cmd.Flags().BoolVar(&generateIndex, "generate-index", true, "whether to generate APKINDEX.tar.gz")
 	cmd.Flags().BoolVar(&useProot, "use-proot", false, "whether to use proot for fakeroot")