diff --git a/pkg/converter/convert.go b/pkg/converter/convert.go index 0abd344452..d9bf6a4535 100644 --- a/pkg/converter/convert.go +++ b/pkg/converter/convert.go @@ -50,6 +50,8 @@ type MergeOption struct { ChunkDictPath string // PrefetchPatterns holds file path pattern list want to prefetch. PrefetchPatterns string + // WithTar puts bootstrap into a tar stream (no gzip). + WithTar bool } func getBuilder() string { @@ -251,16 +253,12 @@ func Convert(ctx context.Context, dest io.Writer, opt ConvertOption) (io.WriteCl } // Merge multiple nydus boostraps (from every layer of image) to a final boostrap. -func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.ReadCloser, err error) { +func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error { workDir, err := ioutil.TempDir("", "nydus-converter-") if err != nil { - return nil, errors.Wrap(err, "create work directory") + return errors.Wrap(err, "create work directory") } - defer func() { - if err != nil { - os.RemoveAll(workDir) - } - }() + defer os.RemoveAll(workDir) eg, ctx := errgroup.WithContext(ctx) sourceBootstrapPaths := []string{} @@ -293,7 +291,7 @@ func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.Read } if err := eg.Wait(); err != nil { - return nil, errors.Wrap(err, "unpack all bootstraps") + return errors.Wrap(err, "unpack all bootstraps") } targetBootstrapPath := filepath.Join(workDir, "bootstrap") @@ -306,15 +304,27 @@ func Merge(ctx context.Context, layers []Layer, opt MergeOption) (reader io.Read ChunkDictPath: opt.ChunkDictPath, PrefetchPatterns: opt.PrefetchPatterns, }); err != nil { - return nil, errors.Wrap(err, "merge bootstrap") + return errors.Wrap(err, "merge bootstrap") } - reader, err = os.Open(targetBootstrapPath) - if err != nil { - return nil, errors.Wrap(err, "open targe bootstrap") + var rc io.ReadCloser + + if opt.WithTar { + rc, err = packToTar(targetBootstrapPath, bootstrapNameInTar, false) + if err != nil { + return errors.Wrap(err, "pack bootstrap to tar") + } + } else { + rc, err = os.Open(targetBootstrapPath) + if err != nil { + return errors.Wrap(err, "open targe bootstrap") + } } + defer rc.Close() - return newReadCloser(reader, func() error { - return os.RemoveAll(workDir) - }), nil + if _, err = io.Copy(dest, rc); err != nil { + return errors.Wrap(err, "copy merged bootstrap") + } + + return nil } diff --git a/pkg/converter/convert_test.go b/pkg/converter/convert_test.go index 82a183c11a..2782a697f1 100644 --- a/pkg/converter/convert_test.go +++ b/pkg/converter/convert_test.go @@ -259,12 +259,13 @@ func buildChunkDict(t *testing.T, workDir string) (string, string) { }, } - finalBootstrapReader, err := Merge(context.TODO(), layers, MergeOption{}) + bootstrapPath := filepath.Join(workDir, "dict-bootstrap") + file, err := os.Create(bootstrapPath) require.NoError(t, err) - defer finalBootstrapReader.Close() + defer file.Close() - bootstrapPath := filepath.Join(workDir, "dict-bootstrap") - writeToFile(t, finalBootstrapReader, bootstrapPath) + err = Merge(context.TODO(), layers, file, MergeOption{}) + require.NoError(t, err) dictBlobPath := "" err = filepath.WalkDir(blobDir, func(path string, entry fs.DirEntry, err error) error { @@ -325,14 +326,15 @@ func TestConverter(t *testing.T) { }, } - finalBootstrapReader, err := Merge(context.TODO(), layers, MergeOption{ + bootstrapPath := filepath.Join(workDir, "bootstrap") + file, err := os.Create(bootstrapPath) + require.NoError(t, err) + defer file.Close() + + err = Merge(context.TODO(), layers, file, MergeOption{ ChunkDictPath: chunkDictBootstrapPath, }) require.NoError(t, err) - defer finalBootstrapReader.Close() - - bootstrapPath := filepath.Join(workDir, "bootstrap") - writeToFile(t, finalBootstrapReader, bootstrapPath) verify(t, workDir) dropCache(t) diff --git a/pkg/converter/tool/builder.go b/pkg/converter/tool/builder.go index 86b0230f22..dfe654b8c7 100644 --- a/pkg/converter/tool/builder.go +++ b/pkg/converter/tool/builder.go @@ -87,6 +87,8 @@ func Convert(option ConvertOption) error { func Merge(option MergeOption) error { args := []string{ "merge", + "--log-level", + "warn", "--prefetch-policy", "fs", "--bootstrap", diff --git a/pkg/converter/utils.go b/pkg/converter/utils.go index 985347918b..8ee59279bc 100644 --- a/pkg/converter/utils.go +++ b/pkg/converter/utils.go @@ -7,9 +7,13 @@ package converter import ( + "archive/tar" + "compress/gzip" "context" "fmt" "io" + "os" + "path/filepath" ) type readCloser struct { @@ -94,3 +98,82 @@ func newCtxReader(ctx context.Context, reader io.Reader) io.Reader { reader: reader, } } + +// packToTar makes .tar(.gz) stream of file named `name` and return reader. +func packToTar(src string, name string, compress bool) (io.ReadCloser, error) { + fi, err := os.Stat(src) + if err != nil { + return nil, err + } + + dirHdr := &tar.Header{ + Name: filepath.Dir(name), + Mode: 0755, + Typeflag: tar.TypeDir, + } + + hdr := &tar.Header{ + Name: name, + Mode: 0444, + Size: fi.Size(), + } + + reader, writer := io.Pipe() + + go func() { + // Prepare targz writer + var tw *tar.Writer + var gw *gzip.Writer + var err error + var file *os.File + + if compress { + gw = gzip.NewWriter(writer) + tw = tar.NewWriter(gw) + } else { + tw = tar.NewWriter(writer) + } + + defer func() { + err1 := tw.Close() + var err2 error + if gw != nil { + err2 = gw.Close() + } + + var finalErr error + + // Return the first error encountered to the other end and ignore others. + if err != nil { + finalErr = err + } else if err1 != nil { + finalErr = err1 + } else if err2 != nil { + finalErr = err2 + } + + writer.CloseWithError(finalErr) + }() + + file, err = os.Open(src) + if err != nil { + return + } + defer file.Close() + + // Write targz stream + if err = tw.WriteHeader(dirHdr); err != nil { + return + } + + if err = tw.WriteHeader(hdr); err != nil { + return + } + + if _, err = io.Copy(tw, file); err != nil { + return + } + }() + + return reader, nil +}