Import zdfs package to support a tricky OCIv2 tgz format

Signed-off-by: Yifan Yuan <[email protected]>

.github/workflows/release.yml

@@ -39,7 +39,8 @@ jobs:
           RELEASE_VERSION=$(git tag --sort=v:refname -l v* | tail -1)
         fi
         echo "RELEASE_VERSION=${RELEASE_VERSION}"
-        docker buildx build --build-arg RELEASE_VERSION=${RELEASE_VERSION} --build-arg GO_VERSION=${{ env.GO_VERSION }} -f .github/workflows/release/Dockerfile --platform=linux/amd64,linux/arm64 -o releases/ .
+        RELEASE_NUM="$(date +%Y%m%d%H%M%S).$(git rev-parse --short HEAD)"
+        docker buildx build --build-arg RELEASE_NUM=${RELEASE_NUM} RELEASE_VERSION=${RELEASE_VERSION} --build-arg GO_VERSION=${{ env.GO_VERSION }} -f .github/workflows/release/Dockerfile --platform=linux/amd64,linux/arm64 -o releases/ .
         ls -l releases/*/
     - name: Upload
       uses: actions/upload-artifact@v3

.github/workflows/release/Dockerfile

@@ -28,6 +28,8 @@ ENV GOOS=${TARGETOS}
 ENV GOARCH=${TARGETARCH}
 ARG RELEASE_VERSION
 ENV SEMVER=${RELEASE_VERSION}
+ARG RELEASE_NUM
+ENV RELEASE=${RELEASE_NUM}
 RUN make && \
     nfpm pkg --packager deb --target /tmp/ && \
     nfpm pkg --packager rpm --target /tmp/

go.mod

@@ -4,14 +4,17 @@ go 1.19
 
 require (
 	github.com/containerd/containerd v1.6.26
-	github.com/containerd/continuity v0.3.0
-	github.com/containerd/go-cni v1.1.6
+	github.com/containerd/continuity v0.4.3
+	github.com/containerd/go-cni v1.1.9
+	github.com/containerd/log v0.1.0
+	github.com/data-accelerator/zdfs v0.1.3
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/jessevdk/go-flags v1.5.0
 	github.com/moby/locker v1.0.1
 	github.com/moby/sys/mountinfo v0.6.2
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
-	github.com/opencontainers/runtime-spec v1.1.0-rc.1
+	github.com/opencontainers/runtime-spec v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/sirupsen/logrus v1.9.3
@@ -24,25 +27,26 @@ require (
 )
 
 require (
-	github.com/Microsoft/go-winio v0.6.0 // indirect
-	github.com/Microsoft/hcsshim v0.9.10 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cilium/ebpf v0.7.0 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/cilium/ebpf v0.9.1 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/containerd/console v1.0.3 // indirect
-	github.com/containerd/fifo v1.0.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/containerd/go-runc v1.0.0 // indirect
-	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/ttrpc v1.1.2 // indirect
+	github.com/containerd/ttrpc v1.2.3 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
-	github.com/containernetworking/cni v1.1.1 // indirect
-	github.com/containernetworking/plugins v1.1.1 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/containernetworking/cni v1.1.2 // indirect
+	github.com/containernetworking/plugins v1.2.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/godbus/dbus/v5 v5.0.6 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -51,11 +55,11 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.3 // indirect
-	github.com/kr/pretty v0.3.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/moby/sys/signal v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.0 // indirect
 	github.com/moby/sys/symlink v0.2.0 // indirect
-	github.com/opencontainers/runc v1.1.12 // indirect
+	github.com/onsi/ginkgo/v2 v2.4.0 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
@@ -66,11 +70,11 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	golang.org/x/tools v0.10.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gotest.tools/v3 v3.5.0 // indirect
 )

nfpm.yaml

@@ -6,6 +6,7 @@ name: "overlaybd-snapshotter"
 arch: ${GOARCH}
 platform: ${GOOS}
 version: ${SEMVER}
+release: ${RELEASE}
 section: "default"
 priority: "extra"
 

pkg/snapshot/overlay.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/accelerated-container-image/pkg/metrics"
+	"github.com/data-accelerator/zdfs"
 	"github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd/errdefs"
@@ -716,6 +717,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount,
 			}
 			return o.basedOnBlockDeviceMount(ctx, s, RoDir)
 		}
+
 	}
 	return o.normalOverlayMount(s), nil
 }
@@ -777,6 +779,13 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 		}
 	}
 
+	if isOverlaybd, err := zdfs.PrepareOverlayBDSpec(ctx, key, id, o.snPath(id), oinfo, o.snPath); isOverlaybd {
+		log.G(ctx).Infof("sn: %s is an overlaybd layer", id)
+	} else if err != nil {
+		log.G(ctx).Errorf("prepare overlaybd spec failed(sn: %s): %s", id, err.Error())
+		return err
+	}
+
 	id, info, err := o.commit(ctx, name, key, opts...)
 	if err != nil {
 		return err
@@ -1170,6 +1179,18 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
 	if err != nil {
 		return "", snapshots.Info{}, errors.Wrap(err, "failed to get snapshot info")
 	}
+	img, ok := info.Labels[label.CRIImageRef]
+	if !ok {
+		img, ok = info.Labels[label.TargetImageRef]
+	}
+	if ok {
+		log.G(ctx).Infof("found imageRef: %s", img)
+		if err := os.WriteFile(filepath.Join(path, "image_ref"), []byte(img), 0644); err != nil {
+			log.G(ctx).Errorf("write imageRef '%s'. path: %s, err: %s", img, filepath.Join(path, "image_ref"), err.Error())
+		}
+	} else {
+		log.G(ctx).Warnf("imageRef meta not found.")
+	}
 	return id, info, nil
 }