Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace ZSTD_findDecompressedSize with ZSTD_getFrameContentSize #62

Closed
wants to merge 1 commit into from
Closed

Replace ZSTD_findDecompressedSize with ZSTD_getFrameContentSize #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/decompression.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,11 @@ function TranscodingStreams.process(codec::ZstdDecompressor, input::Memory, outp
end

function TranscodingStreams.expectedsize(codec::ZstdDecompressor, input::Memory)
ret = find_decompressed_size(input.ptr, input.size)
ret = LibZstd.ZSTD_getFrameContentSize(input.ptr, input.size)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not a proper replacement. getFrameContentSize only reads the size of a single frame.

Note 5: ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to read each contained frame header. This is fast as most of the data is skipped, however it does mean that all frame data must be present and valid.

if ret == ZSTD_CONTENTSIZE_ERROR
# something is bad, but we ignore it here
return Int(input.size)
# but no reason to allocate a ton of extra space if the data isn't valid zstd
return 8
elseif ret == ZSTD_CONTENTSIZE_UNKNOWN
# random guess
return Int(input.size * 2)
Expand Down
4 changes: 0 additions & 4 deletions src/libzstd.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,3 @@ end

const ZSTD_CONTENTSIZE_UNKNOWN = Culonglong(0) - 1
const ZSTD_CONTENTSIZE_ERROR = Culonglong(0) - 2

function find_decompressed_size(src::Ptr, size::Integer)
return LibZstd.ZSTD_findDecompressedSize(src, size)
end
8 changes: 4 additions & 4 deletions test/compress_endOp.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ using Test
GC.@preserve data begin
# default endOp
@test CodecZstd.compress!(cstream; endOp=:continue) == 0
@test CodecZstd.find_decompressed_size(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
@test CodecZstd.LibZstd.ZSTD_getFrameContentSize(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
end
finally
Base.Libc.free(cstream.obuffer.dst)
Expand All @@ -33,7 +33,7 @@ end
try
GC.@preserve data begin
@test CodecZstd.compress!(cstream; endOp=:flush) == 0
@test CodecZstd.find_decompressed_size(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
@test CodecZstd.LibZstd.ZSTD_getFrameContentSize(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
end
finally
Base.Libc.free(cstream.obuffer.dst)
Expand All @@ -53,7 +53,7 @@ end
GC.@preserve data begin
# The frame should contain the decompressed size
@test CodecZstd.compress!(cstream; endOp=:end) == 0
@test CodecZstd.find_decompressed_size(cstream.obuffer.dst, cstream.obuffer.pos) == sizeof(data)
@test CodecZstd.LibZstd.ZSTD_getFrameContentSize(cstream.obuffer.dst, cstream.obuffer.pos) == sizeof(data)
end
finally
Base.Libc.free(cstream.obuffer.dst)
Expand All @@ -64,7 +64,7 @@ end
data = rand(1:100, 1024*1024)
compressed = transcode(ZstdFrameCompressor, copy(reinterpret(UInt8, data)))
GC.@preserve compressed begin
@test CodecZstd.find_decompressed_size(pointer(compressed), sizeof(compressed)) == sizeof(data)
@test CodecZstd.LibZstd.ZSTD_getFrameContentSize(pointer(compressed), sizeof(compressed)) == sizeof(data)
end
@test reinterpret(Int, transcode(ZstdDecompressor, compressed)) == data
iob = IOBuffer()
Expand Down
Loading