Merge pull request #1107 from giuseppe/revert-zstd-encoder

revert usage of github.com/valyala/gozstd

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/google/go-intervals v0.0.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/json-iterator/go v1.1.12
+	github.com/klauspost/compress v1.13.6
 	github.com/klauspost/pgzip v1.2.5
 	github.com/mattn/go-shellwords v1.0.12
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible
@@ -26,7 +27,6 @@ require (
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
 	github.com/tchap/go-patricia v2.3.0+incompatible
 	github.com/ulikunitz/xz v0.5.10
-	github.com/valyala/gozstd v1.15.1
 	github.com/vbatts/tar-split v0.11.2
 	golang.org/x/net v0.0.0-20210825183410-e898025ed96a
 	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359

go.sum

@@ -635,8 +635,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/valyala/gozstd v1.15.1 h1:hpmJYWOpNs0rjDOMdbwWZplfNlfh1tOy0v7XiR9+iGQ=
-github.com/valyala/gozstd v1.15.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=

pkg/archive/archive_zstd_cgo.go → pkg/archive/archive_zstd.go

@@ -1,44 +1,41 @@
-// +build linux,cgo
-
 package archive
 
 import (
 	"io"
 
-	zstd "github.com/valyala/gozstd"
+	"github.com/klauspost/compress/zstd"
 )
 
 type wrapperZstdDecoder struct {
-	decoder *zstd.Reader
+	decoder *zstd.Decoder
 }
 
 func (w *wrapperZstdDecoder) Close() error {
-	w.decoder.Release()
+	w.decoder.Close()
 	return nil
 }
 
 func (w *wrapperZstdDecoder) DecodeAll(input, dst []byte) ([]byte, error) {
-	return zstd.Decompress(dst, input)
+	return w.decoder.DecodeAll(input, dst)
 }
 
 func (w *wrapperZstdDecoder) Read(p []byte) (int, error) {
 	return w.decoder.Read(p)
 }
 
 func (w *wrapperZstdDecoder) Reset(r io.Reader) error {
-	w.decoder.Reset(r, nil)
-	return nil
+	return w.decoder.Reset(r)
 }
 
 func (w *wrapperZstdDecoder) WriteTo(wr io.Writer) (int64, error) {
 	return w.decoder.WriteTo(wr)
 }
 
 func zstdReader(buf io.Reader) (io.ReadCloser, error) {
-	decoder := zstd.NewReader(buf)
-	return &wrapperZstdDecoder{decoder: decoder}, nil
+	decoder, err := zstd.NewReader(buf)
+	return &wrapperZstdDecoder{decoder: decoder}, err
 }
 
 func zstdWriter(dest io.Writer) (io.WriteCloser, error) {
-	return zstd.NewWriter(dest), nil
+	return zstd.NewWriter(dest)
 }

pkg/chunked/compression.go

@@ -11,10 +11,10 @@ import (
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containers/storage/pkg/chunked/compressor"
 	"github.com/containers/storage/pkg/chunked/internal"
+	"github.com/klauspost/compress/zstd"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
-	zstd "github.com/valyala/gozstd"
 	"github.com/vbatts/tar-split/archive/tar"
 )
 
@@ -256,8 +256,14 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
 		return nil, 0, errors.New("invalid manifest checksum")
 	}
 
+	decoder, err := zstd.NewReader(nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer decoder.Close()
+
 	b := make([]byte, 0, lengthUncompressed)
-	if decoded, err := zstd.Decompress(b, manifest); err == nil {
+	if decoded, err := decoder.DecodeAll(manifest, b); err == nil {
 		return decoded, int64(offset), nil
 	}
 

pkg/chunked/compressor/compressor.go

@@ -215,7 +215,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 	defer func() {
 		if zstdWriter != nil {
 			zstdWriter.Close()
-			zstdWriter.Release()
+			zstdWriter.Flush()
 		}
 	}()
 
@@ -225,8 +225,11 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r
 			if err := zstdWriter.Close(); err != nil {
 				return 0, err
 			}
+			if err := zstdWriter.Flush(); err != nil {
+				return 0, err
+			}
 			offset = dest.Count
-			zstdWriter.Reset(dest, nil, level)
+			zstdWriter.Reset(dest)
 		}
 		return offset, nil
 	}

pkg/chunked/internal/compression.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
-	zstd "github.com/valyala/gozstd"
 )
 
 type TOC struct {
@@ -178,6 +178,7 @@ func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, off
 	return appendZstdSkippableFrame(dest, manifestDataLE)
 }
 
-func ZstdWriterWithLevel(dest io.Writer, level int) (*zstd.Writer, error) {
-	return zstd.NewWriterLevel(dest, level), nil
+func ZstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) {
+	el := zstd.EncoderLevelFromZstd(level)
+	return zstd.NewWriter(dest, zstd.WithEncoderLevel(el))
 }

pkg/chunked/storage_linux.go

@@ -28,11 +28,11 @@ import (
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/types"
 	securejoin "github.com/cyphar/filepath-securejoin"
+	"github.com/klauspost/compress/zstd"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	zstd "github.com/valyala/gozstd"
 	"github.com/vbatts/tar-split/archive/tar"
 	"golang.org/x/sys/unix"
 )
@@ -63,8 +63,7 @@ type chunkedDiffer struct {
 	copyBuffer []byte
 
 	gzipReader *pgzip.Reader
-
-	zstdReader *zstd.Reader
+	zstdReader *zstd.Decoder
 	rawReader  io.Reader
 }
 
@@ -740,10 +739,15 @@ func (c *chunkedDiffer) prepareCompressedStreamToFile(partCompression compressed
 	case partCompression == fileTypeZstdChunked:
 		c.rawReader = io.LimitReader(from, mf.CompressedSize)
 		if c.zstdReader == nil {
-			r := zstd.NewReader(c.rawReader)
+			r, err := zstd.NewReader(c.rawReader)
+			if err != nil {
+				return partCompression, err
+			}
 			c.zstdReader = r
 		} else {
-			c.zstdReader.Reset(c.rawReader, nil)
+			if err := c.zstdReader.Reset(c.rawReader); err != nil {
+				return partCompression, err
+			}
 		}
 	case partCompression == fileTypeEstargz:
 		c.rawReader = io.LimitReader(from, mf.CompressedSize)
@@ -804,7 +808,7 @@ func appendHole(fd int, size int64) error {
 func (c *chunkedDiffer) appendCompressedStreamToFile(compression compressedFileType, destFile *destinationFile, size int64) error {
 	switch compression {
 	case fileTypeZstdChunked:
-		defer c.zstdReader.Reset(nil, nil)
+		defer c.zstdReader.Reset(nil)
 		if _, err := io.CopyBuffer(destFile.to, io.LimitReader(c.zstdReader, size), c.copyBuffer); err != nil {
 			return err
 		}
@@ -1341,7 +1345,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions) (gra
 	defer c.layersCache.release()
 	defer func() {
 		if c.zstdReader != nil {
-			c.zstdReader.Release()
+			c.zstdReader.Close()
 		}
 	}()