Add support to overlaybd convert

Signed-off-by: HileQAQ <[email protected]>

Dockerfile

@@ -28,6 +28,8 @@ ARG BUILDKIT_VERSION=v0.10.6
 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0
 # Extra deps: Nydus Lazy-pulling
 ARG NYDUS_VERSION=v2.1.0
+# Extra deps: OverlayBD Lazy-pulling
+ARG OVERLAYBD_VERSION=0.5.2
 # Extra deps: Encryption
 ARG IMGCRYPT_VERSION=v1.1.7
 # Extra deps: Rootless
@@ -250,6 +252,10 @@ RUN perl -pi -e 's/multi-user.target/docker-entrypoint.target/g' /usr/local/lib/
   nerdctl completion bash >/etc/bash_completion.d/nerdctl
 COPY ./Dockerfile.d/etc_containerd_config.toml /etc/containerd/config.toml
 COPY ./Dockerfile.d/etc_buildkit_buildkitd.toml /etc/buildkit/buildkitd.toml
+# add overlaybd snapshotter plugin
+RUN echo '[proxy_plugins.overlaybd]\n \
+    type = "snapshot"\n \
+    address = "/run/overlaybd-snapshotter/overlaybd.sock"' >> /etc/containerd/config.toml
 VOLUME /var/lib/containerd
 VOLUME /var/lib/buildkit
 VOLUME /var/lib/containerd-stargz-grpc
@@ -293,6 +299,15 @@ RUN curl -L -o nydus-static.tgz "https://github.com/dragonflyoss/image-service/r
     tar xzf nydus-static.tgz && \
     mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
     rm nydus-static.tgz
+# install OverlayBD components
+ARG OVERLAYBD_VERSION
+RUN curl -L -o overlaybd-snapshotter.deb "https://github.com/containerd/accelerated-container-image/releases/download/v${OVERLAYBD_VERSION}/overlaybd-snapshotter_${OVERLAYBD_VERSION}_amd64.deb" && \
+    curl -L -o overlaybd-tcmu.deb "https://github.com/containerd/overlaybd/releases/download/v${OVERLAYBD_VERSION}/overlaybd-${OVERLAYBD_VERSION}-1.x86_64.deb" && \
+    apt install ./overlaybd-snapshotter.deb && \
+    apt install ./overlaybd-tcmu.deb 
+RUN systemctl enable --now /opt/overlaybd/snapshotter/overlaybd-snapshotter.service
+RUN systemctl enable --now /opt/overlaybd/overlaybd-tcmu.service
+RUN rm -f ./overlaybd-tcmu.deb ./overlaybd-snapshotter.deb
 CMD ["go", "test", "-v", "-timeout=20m", "./cmd/nerdctl/...", "-args", "-test.kill-daemon"]
 
 FROM test-integration AS test-integration-rootless

cmd/nerdctl/image_convert.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"strings"
 
+	overlaybdconvert "github.com/containerd/accelerated-container-image/pkg/convertor"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/converter"
@@ -95,6 +96,12 @@ func newImageConvertCommand() *cobra.Command {
 	imageConvertCommand.Flags().String("nydus-compressor", "lz4_block", "Nydus blob compression algorithm, possible values: `none`, `lz4_block`, `zstd`, default is `lz4_block`")
 	// #endregion
 
+	// #region overlaybd flags
+	imageConvertCommand.Flags().Bool("overlaybd", false, "Convert tar.gz layers to overlaybd layers")
+	imageConvertCommand.Flags().String("overlaybd-fs-type", "ext4", "Filesystem type for overlaybd")
+	imageConvertCommand.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
+	// #endregion
+
 	// #region generic flags
 	imageConvertCommand.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	imageConvertCommand.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -158,6 +165,10 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	overlaybd, err := cmd.Flags().GetBool("overlaybd")
+	if err != nil {
+		return err
+	}
 	oci, err := cmd.Flags().GetBool("oci")
 	if err != nil {
 		return err
@@ -167,10 +178,30 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	client, ctx, cancel, err := newClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer cancel()
+
 	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
-	if estargz || zstdchunked {
-		if estargz && zstdchunked {
-			return errors.New("option --estargz conflicts with --zstdchunked")
+	if estargz || zstdchunked || overlaybd || nydus {
+		convertCount := 0
+		if estargz {
+			convertCount++
+		}
+		if zstdchunked {
+			convertCount++
+		}
+		if overlaybd {
+			convertCount++
+		}
+		if nydus {
+			convertCount++
+		}
+
+		if convertCount > 1 {
+			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
 		}
 
 		var convertFunc converter.ConvertFunc
@@ -188,9 +219,44 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 				return err
 			}
 			convertType = "zstdchunked"
+		case overlaybd:
+			obdOpts, err := getOBDConvertOpts(cmd)
+			if err != nil {
+				return err
+			}
+			obdOpts = append(obdOpts, overlaybdconvert.WithClient(client))
+			obdOpts = append(obdOpts, overlaybdconvert.WithImageRef(srcRef))
+			convertFunc = overlaybdconvert.IndexConvertFunc(obdOpts...)
+			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(convertFunc))
+			convertType = "overlaybd"
+		case nydus:
+			nydusOpts, err := getNydusConvertOpts(cmd)
+			if err != nil {
+				return err
+			}
+			convertHooks := converter.ConvertHooks{
+				PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{
+					WorkDir:          nydusOpts.WorkDir,
+					BuilderPath:      nydusOpts.BuilderPath,
+					FsVersion:        nydusOpts.FsVersion,
+					ChunkDictPath:    nydusOpts.ChunkDictPath,
+					PrefetchPatterns: nydusOpts.PrefetchPatterns,
+				}),
+			}
+			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
+				converter.IndexConvertFuncWithHook(
+					nydusconvert.LayerConvertFunc(*nydusOpts),
+					true,
+					platMC,
+					convertHooks,
+				)),
+			)
+			convertType = "nydus"
 		}
-		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc))
 
+		if convertType != "overlaybd" {
+			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc))
+		}
 		if !oci {
 			logrus.Warnf("option --%s should be used in conjunction with --oci", convertType)
 		}
@@ -199,41 +265,6 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	if nydus {
-		if estargz {
-			return errors.New("option --nydus conflicts with --estargz")
-		}
-		if zstdchunked {
-			return errors.New("option --nydus conflicts with --zstdchunked")
-		}
-		if !oci {
-			logrus.Warnln("option --nydus should be used in conjunction with '--oci', forcibly enabling on oci mediatype for nydus conversion")
-		}
-
-		nydusOpts, err := getNydusConvertOpts(cmd)
-		if err != nil {
-			return err
-		}
-		convertFunc := nydusconvert.LayerConvertFunc(*nydusOpts)
-		convertHooks := converter.ConvertHooks{
-			PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{
-				WorkDir:          nydusOpts.WorkDir,
-				BuilderPath:      nydusOpts.BuilderPath,
-				FsVersion:        nydusOpts.FsVersion,
-				ChunkDictPath:    nydusOpts.ChunkDictPath,
-				PrefetchPatterns: nydusOpts.PrefetchPatterns,
-			}),
-		}
-		convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
-			converter.IndexConvertFuncWithHook(
-				convertFunc,
-				true,
-				platMC,
-				convertHooks,
-			)),
-		)
-	}
-
 	if uncompressValue {
 		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
 	}
@@ -242,12 +273,6 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
 	}
 
-	client, ctx, cancel, err := newClient(cmd)
-	if err != nil {
-		return err
-	}
-	defer cancel()
-
 	// converter.Convert() gains the lease by itself
 	newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 	if err != nil {
@@ -454,6 +479,23 @@ func getNydusConvertOpts(cmd *cobra.Command) (*nydusconvert.PackOption, error) {
 	}, nil
 }
 
+func getOBDConvertOpts(cmd *cobra.Command) ([]overlaybdconvert.Option, error) {
+	obdFsType, err := cmd.Flags().GetString("overlaybd-fs-type")
+	if err != nil {
+		return nil, err
+	}
+	obdDbstr, err := cmd.Flags().GetString("overlaybd-dbstr")
+	if err != nil {
+		return nil, err
+	}
+
+	obdOpts := []overlaybdconvert.Option{
+		overlaybdconvert.WithFsType(obdFsType),
+		overlaybdconvert.WithDbstr(obdDbstr),
+	}
+	return obdOpts, nil
+}
+
 func readPathsFromRecordFile(filename string) ([]string, error) {
 	r, err := os.Open(filename)
 	if err != nil {

cmd/nerdctl/image_convert_test.go

@@ -51,3 +51,18 @@ func TestImageConvertZstdChunked(t *testing.T) {
 	base.Cmd("image", "convert", "--zstdchunked", "--oci", "--zstdchunked-compression-level", "3",
 		testutil.CommonImage, convertedImage).AssertOK()
 }
+
+func TestImageConvertOverlayBD(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("no windows support yet")
+	}
+	testutil.DockerIncompatible(t)
+	t.Parallel()
+	base := testutil.NewBase(t)
+	convertedImage := testutil.Identifier(t) + ":overlaybd"
+	base.Cmd("rmi", convertedImage).Run()
+	defer base.Cmd("rmi", convertedImage).Run()
+	base.Cmd("pull", testutil.CommonImage).AssertOK()
+	base.Cmd("image", "convert", "--overlaybd", "--oci",
+		testutil.CommonImage, convertedImage).AssertOK()
+}

docs/overlaybd.md

@@ -26,4 +26,10 @@ See https://github.com/containerd/accelerated-container-image to learn further i
 nerdctl run --net host -it --rm --snapshotter=overlaybd registry.hub.docker.com/overlaybd/redis:6.2.1_obd
 ```
 
-For more details about how to build overlaybd image, please refer to [accelerated-container-image](https://github.com/containerd/accelerated-container-image/blob/main/docs/IMAGE_CONVERTOR.md) conversion tool.
+For more details about how to build overlaybd image, please refer to [accelerated-container-image](https://github.com/containerd/accelerated-container-image/blob/main/docs/IMAGE_CONVERTOR.md) conversion tool.
+
+## Build OverlayBD image using `nerdctl image convert`
+
+Nerdctl supports to convert an OCI image or docker format v2 image to OverlayBD image by using the `nerdctl image convert` command.
+
+Before the conversion, you should have the `overlaybd-snapshotter` binary installed, which build from [accelerated-container-image](https://github.com/containerd/accelerated-container-image). You can run the command like `nerdctl image convert --overlaybd --oci <source_image> <target_image>` to convert the `<source_image>` to a OverlayBD image whose tag is `<target_image>`.

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Microsoft/go-winio v0.6.0
 	github.com/compose-spec/compose-go v1.7.0
+	github.com/containerd/accelerated-container-image v0.5.2
 	github.com/containerd/cgroups v1.0.5-0.20220816231112-7083cd60b721
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.7.0-beta.0

go.sum

@@ -237,6 +237,8 @@ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u9
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/compose-spec/compose-go v1.7.0 h1:70HzJ/g81pdxF1ao9L7W2fgje/9FxNKH/davgvusEKc=
 github.com/compose-spec/compose-go v1.7.0/go.mod h1:Tb5Ae2PsYN3GTqYqzl2IRbTPiJtPZZjMw8UKUvmehFk=
+github.com/containerd/accelerated-container-image v0.5.2 h1:gnPqNqmynp/SjoAvFJpWTnZfaurtY9oop6etRSmFXPM=
+github.com/containerd/accelerated-container-image v0.5.2/go.mod h1:2gtaJ7535HlImo5DGYmUyRKSGOr9JqTqw1DX5FTpY3k=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=