From 9c2758905f669797b67289c586147d71f9df5482 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Wed, 21 Apr 2021 21:16:23 -0600 Subject: [PATCH] zstd compression: add more options For why, see https://github.com/tailscale/tailscale/blob/a16eb6ac415138e83ea380f4170649c9c55314e4/smallzstd/zstd.go We go with a slightly larger window size (64K vs 8K) to allow for slightly large compression while still having a much much smaller window than default (8M). --- pkg/kgo/compression.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/kgo/compression.go b/pkg/kgo/compression.go index 8420a639..e7296220 100644 --- a/pkg/kgo/compression.go +++ b/pkg/kgo/compression.go @@ -135,7 +135,10 @@ out: New: func() interface{} { zstdEnc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(level), - zstd.WithEncoderConcurrency(1)) + zstd.WithWindowSize(64<<10), + zstd.WithEncoderConcurrency(1), + zstd.WithZeroFrames(true), + ) if err != nil { zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) @@ -230,7 +233,10 @@ func newDecompressor() *decompressor { }, unzstdPool: sync.Pool{ New: func() interface{} { - zstdDec, _ := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + zstdDec, _ := zstd.NewReader(nil, + zstd.WithDecoderLowmem(true), + zstd.WithDecoderConcurrency(1), + ) r := &zstdDecoder{zstdDec} runtime.SetFinalizer(r, func(r *zstdDecoder) { r.inner.Close()