From 0fd62841d2fa41b716a898f27809d80dbaea439b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 18 Nov 2020 09:48:35 +0100 Subject: [PATCH] pkg/compression: new zstd variant zstd:chunked add a new custom variant of the zstd compression that permits to retrieve each file separately. The idea is based on CRFS and its stargz format for having seekable and indexable tarballs. One disadvantage of the stargz format is that a custom file is added to the tarball to store the layer metadata, and the metadata file is part of the image itself. Clients that are not aware of the stargz format will propagate the metadata file inside of the containers. The zstd compression supports embeddeding additional data as part of the stream that the zstd decompressor will ignore (skippable frame), so the issue above with CRFS can be solved directly within the zstd compression format. Beside this minor advantage, zstd is much faster and compresses better than gzip, so take this opportunity to push the zstd format further. The zstd compression is supported by the OCI image specs since August 2019: https://github.com/opencontainers/image-spec/pull/788 and has been supported by containers/image since then. Clients that are not aware of the zstd:chunked format, won't notice any difference when handling a blob that uses the variant. Signed-off-by: Giuseppe Scrivano --- pkg/compression/compression.go | 11 +++++++---- pkg/compression/compression_linux.go | 11 +++++++++++ pkg/compression/compression_unsupported.go | 11 +++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 pkg/compression/compression_linux.go create mode 100644 pkg/compression/compression_unsupported.go diff --git a/pkg/compression/compression.go b/pkg/compression/compression.go index 8672b243dd..bc075df326 100644 --- a/pkg/compression/compression.go +++ b/pkg/compression/compression.go @@ -27,12 +27,15 @@ var ( Xz = internal.NewAlgorithm("Xz", "xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor) // Zstd compression. Zstd = internal.NewAlgorithm("zstd", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor) + // Zstd:chunked compression. + ZstdChunked = internal.NewAlgorithm("zstd:chunked", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, getZstdCompressor) compressionAlgorithms = map[string]Algorithm{ - Gzip.Name(): Gzip, - Bzip2.Name(): Bzip2, - Xz.Name(): Xz, - Zstd.Name(): Zstd, + Gzip.Name(): Gzip, + Bzip2.Name(): Bzip2, + Xz.Name(): Xz, + Zstd.Name(): Zstd, + ZstdChunked.Name(): ZstdChunked, } ) diff --git a/pkg/compression/compression_linux.go b/pkg/compression/compression_linux.go new file mode 100644 index 0000000000..2e9be2959f --- /dev/null +++ b/pkg/compression/compression_linux.go @@ -0,0 +1,11 @@ +package compression + +import( + "io" + + "github.com/containers/storage/pkg/chunked" +) + +func getZstdCompressor(r io.Writer, metadata map[string]string, level *int) (CompressorFunc, error) { + return chunked.ZstdCompressor +} diff --git a/pkg/compression/compression_unsupported.go b/pkg/compression/compression_unsupported.go new file mode 100644 index 0000000000..7844636085 --- /dev/null +++ b/pkg/compression/compression_unsupported.go @@ -0,0 +1,11 @@ +// +build !linux + +package compression + +import ( + "fmt" +) + +func getZstdCompressor(r io.Writer, metadata map[string]string, level *int) (CompressorFunc, error) { + return nil, fmt.Errorf("compressor not supported on this arch") +}