[v0.11] cherry-picks

README.md

@@ -390,6 +390,7 @@ buildctl build ... \
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`
 * `compression-level=<value>`: choose compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcibly apply `compression` option to all layers
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=registry`
@@ -415,6 +416,7 @@ The directory layout conforms to OCI Image Spec v1.0.
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`.
 * `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcibly apply `compression` option to all layers
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=local`
@@ -449,6 +451,7 @@ in your workflow to expose the runtime.
   * `min`: only export layers for the resulting image
   * `max`: export all the layers of all intermediate steps
 * `scope=<scope>`: which scope cache object belongs to (default `buildkit`)
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=gha`
@@ -496,6 +499,7 @@ Others options are:
 * `prefix=<prefix>`: set global prefix to store / read files on s3 (default: empty)
 * `name=<manifest>`: specify name of the manifest to use (default `buildkit`)
   * Multiple manifest names can be specified at the same time, separated by `;`. The standard use case is to use the git sha1 as name, and the branch name as duplicate, and load both with 2 `import-cache` commands.
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=s3`
@@ -540,6 +544,7 @@ There are 2 options supported for Azure Blob Storage authentication:
 * `prefix=<prefix>`: set global prefix to store / read files on the Azure Blob Storage container (`<container>`) (default: empty)
 * `name=<manifest>`: specify name of the manifest to use (default: `buildkit`)
   * Multiple manifest names can be specified at the same time, separated by `;`. The standard use case is to use the git sha1 as name, and the branch name as duplicate, and load both with 2 `import-cache` commands.
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=azblob`

api/services/control/control.proto

@@ -204,8 +204,9 @@ message BuildHistoryRecord {
 	int32 Generation = 12;
 	Descriptor trace = 13;
 	bool pinned = 14;
+	int32 numCachedSteps = 15;
+	int32 numTotalSteps = 16;
 	// TODO: tags
-	// TODO: steps/cache summary
 	// TODO: unclipped logs
 }
 

client/client_test.go

@@ -168,6 +168,7 @@ func TestIntegration(t *testing.T) {
 		testBuildInfoInline,
 		testBuildInfoNoExport,
 		testZstdLocalCacheExport,
+		testCacheExportIgnoreError,
 		testZstdRegistryCacheImportExport,
 		testZstdLocalCacheImportExport,
 		testUncompressedLocalCacheImportExport,
@@ -4351,6 +4352,113 @@ func testZstdLocalCacheExport(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, dt[:4], []byte{0x28, 0xb5, 0x2f, 0xfd})
 }
 
+func testCacheExportIgnoreError(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureCacheExport)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	busybox := llb.Image("busybox:latest")
+	cmd := `sh -e -c "echo -n ignore-error > data"`
+
+	st := llb.Scratch()
+	st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	tests := map[string]struct {
+		Exports        []ExportEntry
+		CacheExports   []CacheOptionsEntry
+		expectedErrors []string
+	}{
+		"local-ignore-error": {
+			Exports: []ExportEntry{
+				{
+					Type:      ExporterLocal,
+					OutputDir: t.TempDir(),
+				},
+			},
+			CacheExports: []CacheOptionsEntry{
+				{
+					Type: "local",
+					Attrs: map[string]string{
+						"dest": "éèç",
+					},
+				},
+			},
+			expectedErrors: []string{"failed to solve", "contains value with non-printable ASCII characters"},
+		},
+		"registry-ignore-error": {
+			Exports: []ExportEntry{
+				{
+					Type: ExporterImage,
+					Attrs: map[string]string{
+						"name": "test-registry-ignore-error",
+						"push": "false",
+					},
+				},
+			},
+			CacheExports: []CacheOptionsEntry{
+				{
+					Type: "registry",
+					Attrs: map[string]string{
+						"ref": "fake-url:5000/myrepo:buildcache",
+					},
+				},
+			},
+			expectedErrors: []string{"failed to solve", "dial tcp: lookup fake-url", "no such host"},
+		},
+		"s3-ignore-error": {
+			Exports: []ExportEntry{
+				{
+					Type:      ExporterLocal,
+					OutputDir: t.TempDir(),
+				},
+			},
+			CacheExports: []CacheOptionsEntry{
+				{
+					Type: "s3",
+					Attrs: map[string]string{
+						"endpoint_url":      "http://fake-url:9000",
+						"bucket":            "my-bucket",
+						"region":            "us-east-1",
+						"access_key_id":     "minioadmin",
+						"secret_access_key": "minioadmin",
+						"use_path_style":    "true",
+					},
+				},
+			},
+			expectedErrors: []string{"failed to solve", "dial tcp: lookup fake-url", "no such host"},
+		},
+	}
+	ignoreErrorValues := []bool{true, false}
+	for _, ignoreError := range ignoreErrorValues {
+		ignoreErrStr := strconv.FormatBool(ignoreError)
+		for n, test := range tests {
+			require.Equal(t, 1, len(test.Exports))
+			require.Equal(t, 1, len(test.CacheExports))
+			require.NotEmpty(t, test.CacheExports[0].Attrs)
+			test.CacheExports[0].Attrs["ignore-error"] = ignoreErrStr
+			testName := fmt.Sprintf("%s-%s", n, ignoreErrStr)
+			t.Run(testName, func(t *testing.T) {
+				_, err = c.Solve(sb.Context(), def, SolveOpt{
+					Exports:      test.Exports,
+					CacheExports: test.CacheExports,
+				}, nil)
+				if ignoreError {
+					require.NoError(t, err)
+				} else {
+					require.Error(t, err)
+					for _, errStr := range test.expectedErrors {
+						require.Contains(t, err.Error(), errStr)
+					}
+				}
+			})
+		}
+	}
+}
+
 func testUncompressedLocalCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureCacheExport)
 	dir := t.TempDir()

client/ociindex/ociindex.go

@@ -4,76 +4,94 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	"path"
 
 	"github.com/gofrs/flock"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
 const (
-	// IndexJSONLockFileSuffix is the suffix of the lock file
-	IndexJSONLockFileSuffix = ".lock"
+	// indexFile is the name of the index file
+	indexFile = "index.json"
+
+	// lockFileSuffix is the suffix of the lock file
+	lockFileSuffix = ".lock"
 )
 
-// PutDescToIndex puts desc to index with tag.
-// Existing manifests with the same tag will be removed from the index.
-func PutDescToIndex(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
-	if index == nil {
-		index = &ocispecs.Index{}
+type StoreIndex struct {
+	indexPath string
+	lockPath  string
+}
+
+func NewStoreIndex(storePath string) StoreIndex {
+	indexPath := path.Join(storePath, indexFile)
+	return StoreIndex{
+		indexPath: indexPath,
+		lockPath:  indexPath + lockFileSuffix,
 	}
-	if index.SchemaVersion == 0 {
-		index.SchemaVersion = 2
+}
+
+func (s StoreIndex) Read() (*ocispecs.Index, error) {
+	lock := flock.New(s.lockPath)
+	locked, err := lock.TryRLock()
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not lock %s", s.lockPath)
 	}
-	if tag != "" {
-		if desc.Annotations == nil {
-			desc.Annotations = make(map[string]string)
-		}
-		desc.Annotations[ocispecs.AnnotationRefName] = tag
-		// remove existing manifests with the same tag
-		var manifests []ocispecs.Descriptor
-		for _, m := range index.Manifests {
-			if m.Annotations[ocispecs.AnnotationRefName] != tag {
-				manifests = append(manifests, m)
-			}
-		}
-		index.Manifests = manifests
+	if !locked {
+		return nil, errors.Errorf("could not lock %s", s.lockPath)
 	}
-	index.Manifests = append(index.Manifests, desc)
-	return nil
+	defer func() {
+		lock.Unlock()
+		os.RemoveAll(s.lockPath)
+	}()
+
+	b, err := os.ReadFile(s.indexPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not read %s", s.indexPath)
+	}
+	var idx ocispecs.Index
+	if err := json.Unmarshal(b, &idx); err != nil {
+		return nil, errors.Wrapf(err, "could not unmarshal %s (%q)", s.indexPath, string(b))
+	}
+	return &idx, nil
 }
 
-func PutDescToIndexJSONFileLocked(indexJSONPath string, desc ocispecs.Descriptor, tag string) error {
-	lockPath := indexJSONPath + IndexJSONLockFileSuffix
-	lock := flock.New(lockPath)
+func (s StoreIndex) Put(tag string, desc ocispecs.Descriptor) error {
+	lock := flock.New(s.lockPath)
 	locked, err := lock.TryLock()
 	if err != nil {
-		return errors.Wrapf(err, "could not lock %s", lockPath)
+		return errors.Wrapf(err, "could not lock %s", s.lockPath)
 	}
 	if !locked {
-		return errors.Errorf("could not lock %s", lockPath)
+		return errors.Errorf("could not lock %s", s.lockPath)
 	}
 	defer func() {
 		lock.Unlock()
-		os.RemoveAll(lockPath)
+		os.RemoveAll(s.lockPath)
 	}()
-	f, err := os.OpenFile(indexJSONPath, os.O_RDWR|os.O_CREATE, 0644)
+
+	f, err := os.OpenFile(s.indexPath, os.O_RDWR|os.O_CREATE, 0644)
 	if err != nil {
-		return errors.Wrapf(err, "could not open %s", indexJSONPath)
+		return errors.Wrapf(err, "could not open %s", s.indexPath)
 	}
 	defer f.Close()
+
 	var idx ocispecs.Index
 	b, err := io.ReadAll(f)
 	if err != nil {
-		return errors.Wrapf(err, "could not read %s", indexJSONPath)
+		return errors.Wrapf(err, "could not read %s", s.indexPath)
 	}
 	if len(b) > 0 {
 		if err := json.Unmarshal(b, &idx); err != nil {
-			return errors.Wrapf(err, "could not unmarshal %s (%q)", indexJSONPath, string(b))
+			return errors.Wrapf(err, "could not unmarshal %s (%q)", s.indexPath, string(b))
 		}
 	}
-	if err = PutDescToIndex(&idx, desc, tag); err != nil {
+
+	if err = insertDesc(&idx, desc, tag); err != nil {
 		return err
 	}
+
 	b, err = json.Marshal(idx)
 	if err != nil {
 		return err
@@ -87,27 +105,56 @@ func PutDescToIndexJSONFileLocked(indexJSONPath string, desc ocispecs.Descriptor
 	return nil
 }
 
-func ReadIndexJSONFileLocked(indexJSONPath string) (*ocispecs.Index, error) {
-	lockPath := indexJSONPath + IndexJSONLockFileSuffix
-	lock := flock.New(lockPath)
-	locked, err := lock.TryRLock()
+func (s StoreIndex) Get(tag string) (*ocispecs.Descriptor, error) {
+	idx, err := s.Read()
 	if err != nil {
-		return nil, errors.Wrapf(err, "could not lock %s", lockPath)
+		return nil, err
 	}
-	if !locked {
-		return nil, errors.Errorf("could not lock %s", lockPath)
+
+	for _, m := range idx.Manifests {
+		if t, ok := m.Annotations[ocispecs.AnnotationRefName]; ok && t == tag {
+			return &m, nil
+		}
 	}
-	defer func() {
-		lock.Unlock()
-		os.RemoveAll(lockPath)
-	}()
-	b, err := os.ReadFile(indexJSONPath)
+	return nil, nil
+}
+
+func (s StoreIndex) GetSingle() (*ocispecs.Descriptor, error) {
+	idx, err := s.Read()
 	if err != nil {
-		return nil, errors.Wrapf(err, "could not read %s", indexJSONPath)
+		return nil, err
 	}
-	var idx ocispecs.Index
-	if err := json.Unmarshal(b, &idx); err != nil {
-		return nil, errors.Wrapf(err, "could not unmarshal %s (%q)", indexJSONPath, string(b))
+
+	if len(idx.Manifests) == 1 {
+		return &idx.Manifests[0], nil
 	}
-	return &idx, nil
+	return nil, nil
+}
+
+// insertDesc puts desc to index with tag.
+// Existing manifests with the same tag will be removed from the index.
+func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
+	if index == nil {
+		return nil
+	}
+
+	if index.SchemaVersion == 0 {
+		index.SchemaVersion = 2
+	}
+	if tag != "" {
+		if desc.Annotations == nil {
+			desc.Annotations = make(map[string]string)
+		}
+		desc.Annotations[ocispecs.AnnotationRefName] = tag
+		// remove existing manifests with the same tag
+		var manifests []ocispecs.Descriptor
+		for _, m := range index.Manifests {
+			if m.Annotations[ocispecs.AnnotationRefName] != tag {
+				manifests = append(manifests, m)
+			}
+		}
+		index.Manifests = manifests
+	}
+	index.Manifests = append(index.Manifests, desc)
+	return nil
 }