Skip to content

Commit

Permalink
Introduce KOCACHE (ko-build#269)
Browse files Browse the repository at this point in the history
* Introduce KOCACHE

Cache binaries under $KOCACHE/<import path>/<platform>

Cache metdata mapping buildid to diffid and diffid to descriptor.

* Point TODO at issue

* Split layerCache into separate file

This makes things a little cleaner by having a single place that calls
buildLayer and passing a thunk down into the cache logic to call that on
a cache miss.

Also, remove the debug logging to make the code easier to follow (if you
need to recompile anyway, it's easy enough to add log lines).

* Move cached output to $KOCACHE/bin
  • Loading branch information
jonjohnsonjr authored Dec 8, 2021
1 parent 6ff3469 commit 54cddcc
Show file tree
Hide file tree
Showing 3 changed files with 337 additions and 23 deletions.
210 changes: 210 additions & 0 deletions pkg/build/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
// Copyright 2021 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"

v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
)

type diffIDToDescriptor map[v1.Hash]v1.Descriptor
type buildIDToDiffID map[string]v1.Hash

type layerCache struct {
buildToDiff map[string]buildIDToDiffID
diffToDesc map[string]diffIDToDescriptor
sync.Mutex
}

type layerFactory func() (v1.Layer, error)

func (c *layerCache) get(ctx context.Context, file string, miss layerFactory) (v1.Layer, error) {
if os.Getenv("KOCACHE") == "" {
return miss()
}

// Cache hit.
if diffid, desc, err := c.getMeta(ctx, file); err == nil {
return &lazyLayer{
diffid: *diffid,
desc: *desc,
buildLayer: miss,
}, nil
}

// Cache miss.
layer, err := miss()
if err != nil {
return nil, err
}
if err := c.put(ctx, file, layer); err != nil {
log.Printf("failed to cache metadata %s: %v", file, err)
}
return layer, nil
}

func (c *layerCache) getMeta(ctx context.Context, file string) (*v1.Hash, *v1.Descriptor, error) {
buildid, err := getBuildID(ctx, file)
if err != nil {
return nil, nil, err
}

if buildid == "" {
return nil, nil, fmt.Errorf("no buildid for %s", file)
}

btod, err := c.readBuildToDiff(file)
if err != nil {
return nil, nil, err
}
dtod, err := c.readDiffToDesc(file)
if err != nil {
return nil, nil, err
}

diffid, ok := btod[buildid]
if !ok {
return nil, nil, fmt.Errorf("no diffid for %q", buildid)
}

desc, ok := dtod[diffid]
if !ok {
return nil, nil, fmt.Errorf("no desc for %q", diffid)
}

return &diffid, &desc, nil
}

// Compute new layer metadata and cache it in-mem and on-disk.
func (c *layerCache) put(ctx context.Context, file string, layer v1.Layer) error {
buildid, err := getBuildID(ctx, file)
if err != nil {
return err
}

desc, err := partial.Descriptor(layer)
if err != nil {
return err
}

diffid, err := layer.DiffID()
if err != nil {
return err
}

btod, ok := c.buildToDiff[file]
if !ok {
btod = buildIDToDiffID{}
}
btod[buildid] = diffid

dtod, ok := c.diffToDesc[file]
if !ok {
dtod = diffIDToDescriptor{}
}
dtod[diffid] = *desc

// TODO: Implement better per-file locking.
c.Lock()
defer c.Unlock()

btodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "buildid-to-diffid"), os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return err
}
defer btodf.Close()

dtodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"), os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return err
}
defer dtodf.Close()

enc := json.NewEncoder(btodf)
enc.SetIndent("", " ")
if err := enc.Encode(&btod); err != nil {
return err
}

enc = json.NewEncoder(dtodf)
enc.SetIndent("", " ")
if err := enc.Encode(&dtod); err != nil {
return err
}

return nil
}

func (c *layerCache) readDiffToDesc(file string) (diffIDToDescriptor, error) {
if dtod, ok := c.diffToDesc[file]; ok {
return dtod, nil
}

dtodf, err := os.Open(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"))
if err != nil {
return nil, err
}
defer dtodf.Close()

var dtod diffIDToDescriptor
if err := json.NewDecoder(dtodf).Decode(&dtod); err != nil {
return nil, err
}
c.diffToDesc[file] = dtod
return dtod, nil
}

func (c *layerCache) readBuildToDiff(file string) (buildIDToDiffID, error) {
if btod, ok := c.buildToDiff[file]; ok {
return btod, nil
}

btodf, err := os.Open(filepath.Join(filepath.Dir(file), "buildid-to-diffid"))
if err != nil {
return nil, err
}
defer btodf.Close()

var btod buildIDToDiffID
if err := json.NewDecoder(btodf).Decode(&btod); err != nil {
return nil, err
}
c.buildToDiff[file] = btod
return btod, nil
}

func getBuildID(ctx context.Context, file string) (string, error) {
cmd := exec.CommandContext(ctx, "go", "tool", "buildid", file)
var output bytes.Buffer
cmd.Stderr = &output
cmd.Stdout = &output

if err := cmd.Run(); err != nil {
log.Printf("Unexpected error running \"go tool buildid %s\": %v\n%v", err, file, output.String())
return "", err
}
return strings.TrimSpace(output.String()), nil
}
75 changes: 52 additions & 23 deletions pkg/build/gobuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ type gobuild struct {
platformMatcher *platformMatcher
dir string
labels map[string]string

cache *layerCache
}

// Option is a functional option for NewGo.
Expand Down Expand Up @@ -117,6 +119,10 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
labels: gbo.labels,
dir: gbo.dir,
platformMatcher: matcher,
cache: &layerCache{
buildToDiff: map[string]buildIDToDiffID{},
diffToDesc: map[string]diffIDToDescriptor{},
},
}, nil
}

Expand Down Expand Up @@ -222,12 +228,6 @@ func platformToString(p v1.Platform) string {
}

func build(ctx context.Context, ip string, dir string, platform v1.Platform, config Config) (string, error) {
tmpDir, err := ioutil.TempDir("", "ko")
if err != nil {
return "", err
}
file := filepath.Join(tmpDir, "out")

buildArgs, err := createBuildArgs(config)
if err != nil {
return "", err
Expand All @@ -236,15 +236,31 @@ func build(ctx context.Context, ip string, dir string, platform v1.Platform, con
args := make([]string, 0, 4+len(buildArgs))
args = append(args, "build")
args = append(args, buildArgs...)
args = append(args, "-o", file)
args = append(args, ip)
cmd := exec.CommandContext(ctx, "go", args...)
cmd.Dir = dir

env, err := buildEnv(platform, os.Environ(), config.Env)
if err != nil {
return "", fmt.Errorf("could not create env for %s: %w", ip, err)
}

tmpDir, err := ioutil.TempDir("", "ko")
if err != nil {
return "", err
}

if dir := os.Getenv("KOCACHE"); dir != "" {
// TODO(#264): if KOCACHE is unset, default to filepath.Join(os.TempDir(), "ko").
tmpDir = filepath.Join(dir, "bin", ip, platformToString(platform))
if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
return "", err
}
}

file := filepath.Join(tmpDir, "out")

args = append(args, "-o", file)
args = append(args, ip)
cmd := exec.CommandContext(ctx, "go", args...)
cmd.Dir = dir
cmd.Env = env

var output bytes.Buffer
Expand All @@ -253,7 +269,9 @@ func build(ctx context.Context, ip string, dir string, platform v1.Platform, con

log.Printf("Building %s for %s", ip, platformToString(platform))
if err := cmd.Run(); err != nil {
os.RemoveAll(tmpDir)
if os.Getenv("KOCACHE") == "" {
os.RemoveAll(tmpDir)
}
log.Printf("Unexpected error running \"go build\": %v\n%v", err, output.String())
return "", err
}
Expand Down Expand Up @@ -643,7 +661,9 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
if err != nil {
return nil, err
}
defer os.RemoveAll(filepath.Dir(file))
if os.Getenv("KOCACHE") == "" {
defer os.RemoveAll(filepath.Dir(file))
}

var layers []mutate.Addendum

Expand Down Expand Up @@ -671,21 +691,15 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
appDir := "/ko-app"
appPath := path.Join(appDir, appFilename(ref.Path()))

// Construct a tarball with the binary and produce a layer.
binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{}, platform)
if err != nil {
return nil, err
miss := func() (v1.Layer, error) {
return buildLayer(appPath, file, platform)
}
binaryLayerBytes := binaryLayerBuf.Bytes()
binaryLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil
}, tarball.WithCompressedCaching, tarball.WithEstargzOptions(estargz.WithPrioritizedFiles([]string{
// When using estargz, prioritize downloading the binary entrypoint.
appPath,
})))

binaryLayer, err := g.cache.get(ctx, file, miss)
if err != nil {
return nil, err
}

layers = append(layers, mutate.Addendum{
Layer: binaryLayer,
History: v1.History{
Expand Down Expand Up @@ -759,6 +773,21 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
return si, nil
}

func buildLayer(appPath, file string, platform *v1.Platform) (v1.Layer, error) {
// Construct a tarball with the binary and produce a layer.
binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{}, platform)
if err != nil {
return nil, err
}
binaryLayerBytes := binaryLayerBuf.Bytes()
return tarball.LayerFromOpener(func() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil
}, tarball.WithCompressedCaching, tarball.WithEstargzOptions(estargz.WithPrioritizedFiles([]string{
// When using estargz, prioritize downloading the binary entrypoint.
appPath,
})))
}

// Append appPath to the PATH environment variable, if it exists. Otherwise,
// set the PATH environment variable to appPath.
func updatePath(cf *v1.ConfigFile, appPath string) {
Expand Down
Loading

0 comments on commit 54cddcc

Please sign in to comment.