Skip to content

Commit

Permalink
zstd compression: add more options
Browse files Browse the repository at this point in the history
For why, see https://github.com/tailscale/tailscale/blob/a16eb6ac415138e83ea380f4170649c9c55314e4/smallzstd/zstd.go

We go with a slightly larger window size (64K vs 8K) to allow for
slightly large compression while still having a much much smaller window
than default (8M).
  • Loading branch information
twmb committed Apr 22, 2021
1 parent 0554ad5 commit 9c27589
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions pkg/kgo/compression.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,10 @@ out:
New: func() interface{} {
zstdEnc, err := zstd.NewWriter(nil,
zstd.WithEncoderLevel(level),
zstd.WithEncoderConcurrency(1))
zstd.WithWindowSize(64<<10),
zstd.WithEncoderConcurrency(1),
zstd.WithZeroFrames(true),
)
if err != nil {
zstdEnc, _ = zstd.NewWriter(nil,
zstd.WithEncoderConcurrency(1))
Expand Down Expand Up @@ -230,7 +233,10 @@ func newDecompressor() *decompressor {
},
unzstdPool: sync.Pool{
New: func() interface{} {
zstdDec, _ := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
zstdDec, _ := zstd.NewReader(nil,
zstd.WithDecoderLowmem(true),
zstd.WithDecoderConcurrency(1),
)
r := &zstdDecoder{zstdDec}
runtime.SetFinalizer(r, func(r *zstdDecoder) {
r.inner.Close()
Expand Down

0 comments on commit 9c27589

Please sign in to comment.