Skip to content

Commit

Permalink
re-allocate dst buffer only if it has more than 4096 free bytes of ca…
Browse files Browse the repository at this point in the history
…pacity

This should reduce the number of allocations during compression/decompression
  • Loading branch information
valyala committed Oct 16, 2021
1 parent 8dfd7aa commit ab202a0
Showing 1 changed file with 13 additions and 6 deletions.
19 changes: 13 additions & 6 deletions gozstd.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ func compress(cctx, cctxDict *cctxWrapper, dst, src []byte, cd *CDict, compressi
// All OK.
return dst[:dstLen+compressedSize]
}

if C.ZSTD_getErrorCode(result) != C.ZSTD_error_dstSize_tooSmall {
// Unexpected error.
panic(fmt.Errorf("BUG: unexpected error during compression with cd=%p: %s", cd, errStr(result)))
Expand All @@ -139,8 +138,12 @@ func compress(cctx, cctxDict *cctxWrapper, dst, src []byte, cd *CDict, compressi

result := compressInternal(cctx, cctxDict, dst[dstLen:dstLen+compressBound], src, cd, compressionLevel, true)
compressedSize := int(result)
// Re-allocate dst in order to remove superflouos capacity and reduce memory usage.
return append([]byte{}, dst[:dstLen+compressedSize]...)
dst = dst[:dstLen+compressedSize]
if cap(dst)-len(dst) > 4096 {
// Re-allocate dst in order to remove superflouos capacity and reduce memory usage.
dst = append([]byte{}, dst...)
}
return dst
}

func compressInternal(cctx, cctxDict *cctxWrapper, dst, src []byte, cd *CDict, compressionLevel int, mustSucceed bool) C.size_t {
Expand Down Expand Up @@ -235,7 +238,7 @@ func decompress(dctx, dctxDict *dctxWrapper, dst, src []byte, dd *DDict) ([]byte
}

dstLen := len(dst)
if cap(dst)-dstLen >= len(src) {
if cap(dst) > dstLen {
// Fast path - try decompressing without dst resize.
result := decompressInternal(dctx, dctxDict, dst[dstLen:cap(dst)], src, dd)
decompressedSize := int(result)
Expand Down Expand Up @@ -271,8 +274,12 @@ func decompress(dctx, dctxDict *dctxWrapper, dst, src []byte, dd *DDict) ([]byte
result := decompressInternal(dctx, dctxDict, dst[dstLen:dstLen+decompressBound], src, dd)
decompressedSize := int(result)
if decompressedSize >= 0 {
// Re-allocate dst in order to remove superflouos capacity and reduce memory usage.
return append([]byte{}, dst[:dstLen+decompressedSize]...), nil
dst = dst[:dstLen+decompressedSize]
if cap(dst)-len(dst) > 4096 {
// Re-allocate dst in order to remove superflouos capacity and reduce memory usage.
dst = append([]byte{}, dst...)
}
return dst, nil
}

// Error during decompression.
Expand Down

0 comments on commit ab202a0

Please sign in to comment.