Converter: support WithTar option for merge

Enable `WithTar` option to put bootstrap into a tar stream (no gzip),
so that we can make the bootstrap as an image layer.

Signed-off-by: Yan Song <[email protected]>

pkg/converter/convert.go

@@ -50,6 +50,8 @@ type MergeOption struct {
 	ChunkDictPath string
 	// PrefetchPatterns holds file path pattern list want to prefetch.
 	PrefetchPatterns string
+	// WithTar puts bootstrap into a tar stream (no gzip).
+	WithTar bool
 }
 
 func getBuilder() string {
@@ -251,16 +253,12 @@ func Convert(ctx context.Context, dest io.Writer, opt ConvertOption) (io.WriteCl
 }
 
 // Merge multiple nydus boostraps (from every layer of image) to a final boostrap.
-func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.ReadCloser, err error) {
+func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error {
 	workDir, err := ioutil.TempDir("", "nydus-converter-")
 	if err != nil {
-		return nil, errors.Wrap(err, "create work directory")
+		return errors.Wrap(err, "create work directory")
 	}
-	defer func() {
-		if err != nil {
-			os.RemoveAll(workDir)
-		}
-	}()
+	defer os.RemoveAll(workDir)
 
 	eg, ctx := errgroup.WithContext(ctx)
 	sourceBootstrapPaths := []string{}
@@ -293,7 +291,7 @@ func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.Read
 	}
 
 	if err := eg.Wait(); err != nil {
-		return nil, errors.Wrap(err, "unpack all bootstraps")
+		return errors.Wrap(err, "unpack all bootstraps")
 	}
 
 	targetBootstrapPath := filepath.Join(workDir, "bootstrap")
@@ -306,15 +304,27 @@ func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.Read
 		ChunkDictPath:        opt.ChunkDictPath,
 		PrefetchPatterns:     opt.PrefetchPatterns,
 	}); err != nil {
-		return nil, errors.Wrap(err, "merge bootstrap")
+		return errors.Wrap(err, "merge bootstrap")
 	}
 
-	reader, err = os.Open(targetBootstrapPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "open targe bootstrap")
+	var rc io.ReadCloser
+
+	if opt.WithTar {
+		rc, err = packToTar(targetBootstrapPath, bootstrapNameInTar, false)
+		if err != nil {
+			return errors.Wrap(err, "pack bootstrap to tar")
+		}
+	} else {
+		rc, err = os.Open(targetBootstrapPath)
+		if err != nil {
+			return errors.Wrap(err, "open targe bootstrap")
+		}
 	}
+	defer rc.Close()
 
-	return newReadCloser(reader, func() error {
-		return os.RemoveAll(workDir)
-	}), nil
+	if _, err = io.Copy(dest, rc); err != nil {
+		return errors.Wrap(err, "copy merged bootstrap")
+	}
+
+	return nil
 }

pkg/converter/convert_test.go

@@ -259,12 +259,13 @@ func buildChunkDict(t *testing.T, workDir string) (string, string) {
 		},
 	}
 
-	finalBootstrapReader, err := Merge(context.TODO(), layers, MergeOption{})
+	bootstrapPath := filepath.Join(workDir, "dict-bootstrap")
+	file, err := os.Create(bootstrapPath)
 	require.NoError(t, err)
-	defer finalBootstrapReader.Close()
+	defer file.Close()
 
-	bootstrapPath := filepath.Join(workDir, "dict-bootstrap")
-	writeToFile(t, finalBootstrapReader, bootstrapPath)
+	err = Merge(context.TODO(), layers, file, MergeOption{})
+	require.NoError(t, err)
 
 	dictBlobPath := ""
 	err = filepath.WalkDir(blobDir, func(path string, entry fs.DirEntry, err error) error {
@@ -325,14 +326,15 @@ func TestConverter(t *testing.T) {
 		},
 	}
 
-	finalBootstrapReader, err := Merge(context.TODO(), layers, MergeOption{
+	bootstrapPath := filepath.Join(workDir, "bootstrap")
+	file, err := os.Create(bootstrapPath)
+	require.NoError(t, err)
+	defer file.Close()
+
+	err = Merge(context.TODO(), layers, file, MergeOption{
 		ChunkDictPath: chunkDictBootstrapPath,
 	})
 	require.NoError(t, err)
-	defer finalBootstrapReader.Close()
-
-	bootstrapPath := filepath.Join(workDir, "bootstrap")
-	writeToFile(t, finalBootstrapReader, bootstrapPath)
 
 	verify(t, workDir)
 	dropCache(t)

pkg/converter/tool/builder.go

@@ -87,6 +87,8 @@ func Convert(option ConvertOption) error {
 func Merge(option MergeOption) error {
 	args := []string{
 		"merge",
+		"--log-level",
+		"warn",
 		"--prefetch-policy",
 		"fs",
 		"--bootstrap",

pkg/converter/utils.go

@@ -7,9 +7,13 @@
 package converter
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 )
 
 type readCloser struct {
@@ -94,3 +98,82 @@ func newCtxReader(ctx context.Context, reader io.Reader) io.Reader {
 		reader: reader,
 	}
 }
+
+// packToTar makes .tar(.gz) stream of file named `name` and return reader.
+func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
+	fi, err := os.Stat(src)
+	if err != nil {
+		return nil, err
+	}
+
+	dirHdr := &tar.Header{
+		Name:     filepath.Dir(name),
+		Mode:     0755,
+		Typeflag: tar.TypeDir,
+	}
+
+	hdr := &tar.Header{
+		Name: name,
+		Mode: 0444,
+		Size: fi.Size(),
+	}
+
+	reader, writer := io.Pipe()
+
+	go func() {
+		// Prepare targz writer
+		var tw *tar.Writer
+		var gw *gzip.Writer
+		var err error
+		var file *os.File
+
+		if compress {
+			gw = gzip.NewWriter(writer)
+			tw = tar.NewWriter(gw)
+		} else {
+			tw = tar.NewWriter(writer)
+		}
+
+		defer func() {
+			err1 := tw.Close()
+			var err2 error
+			if gw != nil {
+				err2 = gw.Close()
+			}
+
+			var finalErr error
+
+			// Return the first error encountered to the other end and ignore others.
+			if err != nil {
+				finalErr = err
+			} else if err1 != nil {
+				finalErr = err1
+			} else if err2 != nil {
+				finalErr = err2
+			}
+
+			writer.CloseWithError(finalErr)
+		}()
+
+		file, err = os.Open(src)
+		if err != nil {
+			return
+		}
+		defer file.Close()
+
+		// Write targz stream
+		if err = tw.WriteHeader(dirHdr); err != nil {
+			return
+		}
+
+		if err = tw.WriteHeader(hdr); err != nil {
+			return
+		}
+
+		if _, err = io.Copy(tw, file); err != nil {
+			return
+		}
+	}()
+
+	return reader, nil
+}