From 5a7395707481900a11aae277b1ef403abeaad777 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Wed, 30 Jun 2021 03:58:40 +0100 Subject: [PATCH 01/16] show(io::IO, int) optimization --- base/Base.jl | 2 + base/intfuncs.jl | 171 +++++++++++++++++++++++++++++++++++++++++------ base/show.jl | 8 +-- test/intfuncs.jl | 26 +++++++ 4 files changed, 182 insertions(+), 25 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index 42a506479326b..d07452fd4efb0 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -488,6 +488,8 @@ function __init__() if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES") MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"]) end + + init_print_int_scratch() nothing end diff --git a/base/intfuncs.jl b/base/intfuncs.jl index e312000ac39f1..37aa5b945348d 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -639,10 +639,25 @@ ndigits(x::Integer; base::Integer=10, pad::Integer=1) = max(pad, ndigits0z(x, ba ## integer to string functions ## -function bin(x::Unsigned, pad::Int, neg::Bool) - m = 8 * sizeof(x) - leading_zeros(x) - n = neg + max(pad, m) - a = StringVector(n) +const PRINT_INT_SCRATCH_LENGTH = 256 +const PRINT_INT_SCRATCH = Vector{UInt8}[StringVector(PRINT_INT_SCRATCH_LENGTH)] + +@inline function print_int_scratch(n) + # FIXME: scratch = @inbounds print_int_scratch_[Threads.threadid()] + scratch = @inbounds PRINT_INT_SCRATCH[1] + resize!(scratch, n) + scratch +end + +function init_print_int_scratch() + # FIXME: resize_nthreads!(PRINT_INT_SCRATCH, print_int_scratch_[1]) + # print_int_scratch() also needs to use Threads.threadid(), but it's + # not clear how to replace that as @eval into base is prohibited + + nothing +end + +@inline function bin_impl_(a, x::Unsigned, n::Int, neg::Bool) # for i in 0x0:UInt(n-1) # automatic vectorization produces redundant codes # @inbounds a[n - i] = 0x30 + (((x >> i) % UInt8)::UInt8 & 0x1) # end @@ -663,13 +678,29 @@ function bin(x::Unsigned, pad::Int, neg::Bool) i -= 1 end if neg; @inbounds a[1]=0x2d; end + + nothing +end + +function bin(x::Unsigned, pad::Int, neg::Bool) + m = 8 * sizeof(x) - Base.leading_zeros(x) + n = neg + max(pad, m) + a = Base.StringVector(n) + + bin_impl_(a, x, n, neg) String(a) end -function oct(x::Unsigned, pad::Int, neg::Bool) - m = div(8 * sizeof(x) - leading_zeros(x) + 2, 3) +function bin_io(io::IO, x::Unsigned, pad::Int, neg::Bool) + m = 8 * sizeof(x) - Base.leading_zeros(x) n = neg + max(pad, m) - a = StringVector(n) + scratch = print_int_scratch(n) + + bin_impl_(scratch, x, n, neg) + write(io, scratch) +end + +@inline function oct_impl_(a, x::Unsigned, n::Int, neg::Bool) i = n while i > neg @inbounds a[i] = 0x30 + ((x % UInt8)::UInt8 & 0x7) @@ -677,15 +708,31 @@ function oct(x::Unsigned, pad::Int, neg::Bool) i -= 1 end if neg; @inbounds a[1]=0x2d; end + + nothing +end + +function oct(x::Unsigned, pad::Int, neg::Bool) + m = div(8 * sizeof(x) - Base.leading_zeros(x) + 2, 3) + n = neg + max(pad, m) + a = Base.StringVector(n) + + oct_impl_(a, x, n, neg) String(a) end +function oct_io(io::IO, x::Unsigned, pad::Int, neg::Bool) + m = div(8 * sizeof(x) - leading_zeros(x) + 2, 3) + n = neg + max(pad, m) + scratch = print_int_scratch(n) + oct_impl_(scratch, x, n, neg) + write(io, scratch) +end + # 2-digit decimal characters ("00":"99") const _dec_d100 = UInt16[(0x30 + i % 10) << 0x8 + (0x30 + i ÷ 10) for i = 0:99] -function dec(x::Unsigned, pad::Int, neg::Bool) - n = neg + ndigits(x, pad=pad) - a = StringVector(n) +@inline function dec_impl_(a, x::Unsigned, n::Int, neg::Bool) i = n @inbounds while i >= 2 d, r = divrem(x, 0x64) @@ -699,13 +746,25 @@ function dec(x::Unsigned, pad::Int, neg::Bool) @inbounds a[i] = 0x30 + (rem(x, 0xa) % UInt8)::UInt8 end if neg; @inbounds a[1]=0x2d; end - String(a) + + nothing end -function hex(x::Unsigned, pad::Int, neg::Bool) - m = 2 * sizeof(x) - (leading_zeros(x) >> 2) - n = neg + max(pad, m) - a = StringVector(n) +function dec(x::Unsigned, pad::Int, neg::Bool) + n = neg + ndigits(x, pad=pad) + scratch = Base.StringVector(n) + dec_impl_(scratch, x, n, neg) + String(scratch) +end + +function dec_io(io::IO, x::Unsigned, pad::Int, neg::Bool) + n = neg + ndigits(x, pad=pad) + scratch = print_int_scratch(n) + dec_impl_(scratch, x, n, neg) + write(io, scratch) +end + +@inline function hex_impl_(a, x::Unsigned, n::Int, neg::Bool) i = n while i >= 2 b = (x % UInt8)::UInt8 @@ -720,19 +779,32 @@ function hex(x::Unsigned, pad::Int, neg::Bool) @inbounds a[i] = d + ifelse(d > 0x9, 0x57, 0x30) end if neg; @inbounds a[1]=0x2d; end + + nothing +end + +function hex(x::Unsigned, pad::Int, neg::Bool) + m = 2 * sizeof(x) - (Base.leading_zeros(x) >> 2) + n = neg + max(pad, m) + a = Base.StringVector(n) + + hex_impl_(a, x, n, neg) String(a) end +function hex_io(io::IO, x::Unsigned, pad::Int, neg::Bool) + m = 2 * sizeof(x) - (leading_zeros(x) >> 2) + n = neg + max(pad, m) + scratch = print_int_scratch(n) + hex_impl_(scratch, x, n, neg) + write(io, scratch) +end + const base36digits = UInt8['0':'9';'a':'z'] const base62digits = UInt8['0':'9';'A':'Z';'a':'z'] -function _base(base::Integer, x::Integer, pad::Int, neg::Bool) - (x >= 0) | (base < 0) || throw(DomainError(x, "For negative `x`, `base` must be negative.")) - 2 <= abs(base) <= 62 || throw(DomainError(base, "base must satisfy 2 ≤ abs(base) ≤ 62")) - b = (base % Int)::Int +@inline function _base_impl_(a, b::Integer, x::Integer, n::Int, neg::Bool) digits = abs(b) <= 36 ? base36digits : base62digits - n = neg + ndigits(x, base=b, pad=pad) - a = StringVector(n) i = n @inbounds while i > neg if b > 0 @@ -745,9 +817,33 @@ function _base(base::Integer, x::Integer, pad::Int, neg::Bool) i -= 1 end if neg; @inbounds a[1]=0x2d; end + + nothing +end + +function _base(base::Integer, x::Integer, pad::Int, neg::Bool) + (x >= 0) | (base < 0) || throw(DomainError(x, "For negative `x`, `base` must be negative.")) + 2 <= abs(base) <= 62 || throw(DomainError(base, "base must satisfy 2 ≤ abs(base) ≤ 62")) + b = (base % Int)::Int + n = neg + ndigits(x, base=b, pad=pad) + a = StringVector(n) + + _base_impl_(a, b, x, n, neg) String(a) end +function _base_io(io::IO, base::Integer, x::Integer, pad::Int, neg::Bool) + (x >= 0) | (base < 0) || throw(DomainError(x, "For negative `x`, `base` must be negative.")) + 2 <= abs(base) <= 62 || throw(DomainError(base, "base must satisfy 2 ≤ abs(base) ≤ 62")) + b = (base % Int)::Int + n = neg + ndigits(x, base=b, pad=pad) + + scratch = print_int_scratch(n) + _base_impl_(scratch, b, x, n, neg) + + write(io, scratch) +end + split_sign(n::Integer) = unsigned(abs(n)), n < 0 split_sign(n::Unsigned) = n, false @@ -787,6 +883,39 @@ function string(n::Integer; base::Integer = 10, pad::Integer = 1) end end +function print_int(io::IO, n::Integer; base::Integer = 10, pad::Integer = 1) + pad = (min(max(pad, typemin(Int)), typemax(Int)) % Int)::Int + if base == 2 + (n_positive, neg) = split_sign(n) + bin_io(io, n_positive, pad, neg) + elseif base == 8 + (n_positive, neg) = split_sign(n) + oct_io(io, n_positive, pad, neg) + elseif base == 10 + (n_positive, neg) = split_sign(n) + dec_io(io, n_positive, pad, neg) + elseif base == 16 + (n_positive, neg) = split_sign(n) + hex_io(io, n_positive, pad, neg) + else + _base_io(io, base, base > 0 ? unsigned(abs(n)) : convert(Signed, n), pad, (base>0) & (n<0)) + end + + return nothing +end + +function print_int(io::IO, prefix::Tuple, n::Integer; base::Integer = 10, pad::Integer = 1) + lock(io) + try + print(io, prefix...) + print_int(io, n; base = base, pad = pad) + finally + unlock(io) + end + + return nothing +end + string(b::Bool) = b ? "true" : "false" """ diff --git a/base/show.jl b/base/show.jl index 7835ab8b94db0..aab274b172e38 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1051,11 +1051,11 @@ end show(io::IO, ::Nothing) = print(io, "nothing") show(io::IO, b::Bool) = print(io, get(io, :typeinfo, Any) === Bool ? (b ? "1" : "0") : (b ? "true" : "false")) -show(io::IO, n::Signed) = (write(io, string(n)); nothing) -show(io::IO, n::Unsigned) = print(io, "0x", string(n, pad = sizeof(n)<<1, base = 16)) -print(io::IO, n::Unsigned) = print(io, string(n)) +show(io::IO, n::Signed) = print_int(io, n) +show(io::IO, n::Unsigned) = print_int(io, ("0x", ), n; pad = sizeof(n)<<1, base = 16) +print(io::IO, n::Unsigned) = print_int(io, n) -show(io::IO, p::Ptr) = print(io, typeof(p), " @0x$(string(UInt(p), base = 16, pad = Sys.WORD_SIZE>>2))") +show(io::IO, p::Ptr) = print_int(io, (typeof(p), " @0x"), UInt(p); base = 16, pad = Sys.WORD_SIZE>>2) has_tight_type(p::Pair) = typeof(p.first) == typeof(p).parameters[1] && diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 8a8f046310622..e3d371bc5c30e 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -356,6 +356,32 @@ end @test bitstring(1035) == (Int == Int32 ? "00000000000000000000010000001011" : "0000000000000000000000000000000000000000000000000000010000001011") @test bitstring(Int128(3)) == "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011" + + + @test sprint(print_int, UInt32('3'), base = 2) == "110011" + @test sprint(print_int, UInt32('3'), pad = 7, base = 2) == "0110011" + @test sprint(print_int, 3, base = 2) == "11" + @test sprint(print_int, 3, pad = 2, base = 2) == "11" + @test sprint(print_int, 3, pad = Int32(2), base = Int32(2)) == "11" + @test sprint(print_int, 3, pad = typemin(Int128) + 3, base = 0x2) == "11" + @test sprint(print_int, 3, pad = 3, base = 2) == "011" + @test sprint(print_int, -3, base = 2) == "-11" + @test sprint(print_int, -3, pad = 3, base = 2) == "-011" + + @test sprint(print_int, 9, base = 8) == "11" + @test sprint(print_int, -9, base = 8) == "-11" + @test sprint(print_int, -9, base = 8, pad = 5) == "-00011" + @test sprint(print_int, -9, base = 8, pad = Int32(5)) == "-00011" + + @test sprint(print_int, 121, base = 10) == "121" + @test sprint(print_int, 121, base = 10, pad = 5) == "00121" + @test sprint(print_int, 121, base = 10, pad = 5) == "00121" + + @test sprint(print_int, 12, base = 16) == "c" + @test sprint(print_int, -12, pad = 3, base = 16) == "-00c" + @test sprint(print_int, -12, pad = Int32(3), base = Int32(16)) == "-00c" + + @test sprint(print_int, 5, pad = 7, base = 2) == "0000101" end @testset "digits/base" begin From 6061ab4f64d6ace012a88426af477f6004999458 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Wed, 30 Jun 2021 07:00:32 +0100 Subject: [PATCH 02/16] Fix print(signed) --- base/show.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/show.jl b/base/show.jl index aab274b172e38..3aeb9e3f77eb3 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1053,6 +1053,7 @@ show(io::IO, ::Nothing) = print(io, "nothing") show(io::IO, b::Bool) = print(io, get(io, :typeinfo, Any) === Bool ? (b ? "1" : "0") : (b ? "true" : "false")) show(io::IO, n::Signed) = print_int(io, n) show(io::IO, n::Unsigned) = print_int(io, ("0x", ), n; pad = sizeof(n)<<1, base = 16) +print(io::IO, n::Signed) = print_int(io, n) print(io::IO, n::Unsigned) = print_int(io, n) show(io::IO, p::Ptr) = print_int(io, (typeof(p), " @0x"), UInt(p); base = 16, pad = Sys.WORD_SIZE>>2) From 7a9b43f87a8051f6f93ab91ef553ff335ee8b519 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Wed, 30 Jun 2021 09:09:20 +0100 Subject: [PATCH 03/16] Fix intfuncs tests --- test/intfuncs.jl | 53 ++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/test/intfuncs.jl b/test/intfuncs.jl index e3d371bc5c30e..2f667090bfd27 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -357,31 +357,36 @@ end "0000000000000000000000000000000000000000000000000000010000001011") @test bitstring(Int128(3)) == "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011" + test_print_int = (args...; kwargs...) -> begin + iobuf = IOBuffer() + Base.print_int(iobuf, args...; kwargs...) + String(take!(iobuf)) + end - @test sprint(print_int, UInt32('3'), base = 2) == "110011" - @test sprint(print_int, UInt32('3'), pad = 7, base = 2) == "0110011" - @test sprint(print_int, 3, base = 2) == "11" - @test sprint(print_int, 3, pad = 2, base = 2) == "11" - @test sprint(print_int, 3, pad = Int32(2), base = Int32(2)) == "11" - @test sprint(print_int, 3, pad = typemin(Int128) + 3, base = 0x2) == "11" - @test sprint(print_int, 3, pad = 3, base = 2) == "011" - @test sprint(print_int, -3, base = 2) == "-11" - @test sprint(print_int, -3, pad = 3, base = 2) == "-011" - - @test sprint(print_int, 9, base = 8) == "11" - @test sprint(print_int, -9, base = 8) == "-11" - @test sprint(print_int, -9, base = 8, pad = 5) == "-00011" - @test sprint(print_int, -9, base = 8, pad = Int32(5)) == "-00011" - - @test sprint(print_int, 121, base = 10) == "121" - @test sprint(print_int, 121, base = 10, pad = 5) == "00121" - @test sprint(print_int, 121, base = 10, pad = 5) == "00121" - - @test sprint(print_int, 12, base = 16) == "c" - @test sprint(print_int, -12, pad = 3, base = 16) == "-00c" - @test sprint(print_int, -12, pad = Int32(3), base = Int32(16)) == "-00c" - - @test sprint(print_int, 5, pad = 7, base = 2) == "0000101" + @test test_print_int(UInt32('3'), base = 2) == "110011" + @test test_print_int(UInt32('3'), pad = 7, base = 2) == "0110011" + @test test_print_int(3, base = 2) == "11" + @test test_print_int(3, pad = 2, base = 2) == "11" + @test test_print_int(3, pad = Int32(2), base = Int32(2)) == "11" + @test test_print_int(3, pad = typemin(Int128) + 3, base = 0x2) == "11" + @test test_print_int(3, pad = 3, base = 2) == "011" + @test test_print_int(-3, base = 2) == "-11" + @test test_print_int(-3, pad = 3, base = 2) == "-011" + + @test test_print_int(9, base = 8) == "11" + @test test_print_int(-9, base = 8) == "-11" + @test test_print_int(-9, base = 8, pad = 5) == "-00011" + @test test_print_int(-9, base = 8, pad = Int32(5)) == "-00011" + + @test test_print_int(121, base = 10) == "121" + @test test_print_int(121, base = 10, pad = 5) == "00121" + @test test_print_int(121, base = 10, pad = 5) == "00121" + + @test test_print_int(12, base = 16) == "c" + @test test_print_int(-12, pad = 3, base = 16) == "-00c" + @test test_print_int(-12, pad = Int32(3), base = Int32(16)) == "-00c" + + @test test_print_int(5, pad = 7, base = 2) == "0000101" end @testset "digits/base" begin From 154e6d41825a82aac422bf524f3441fca5755010 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Wed, 30 Jun 2021 09:20:00 +0100 Subject: [PATCH 04/16] Fix show tests --- base/show.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index 3aeb9e3f77eb3..b4139ac7858e2 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1053,7 +1053,7 @@ show(io::IO, ::Nothing) = print(io, "nothing") show(io::IO, b::Bool) = print(io, get(io, :typeinfo, Any) === Bool ? (b ? "1" : "0") : (b ? "true" : "false")) show(io::IO, n::Signed) = print_int(io, n) show(io::IO, n::Unsigned) = print_int(io, ("0x", ), n; pad = sizeof(n)<<1, base = 16) -print(io::IO, n::Signed) = print_int(io, n) +print(io::IO, n::Signed) = show(io, n) print(io::IO, n::Unsigned) = print_int(io, n) show(io::IO, p::Ptr) = print_int(io, (typeof(p), " @0x"), UInt(p); base = 16, pad = Sys.WORD_SIZE>>2) From 59c65763540edd2fe11439c77d5b8fabef4596a8 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Thu, 1 Jul 2021 06:46:31 +0100 Subject: [PATCH 05/16] Stack-allocate scratch space --- base/Base.jl | 2 -- base/intfuncs.jl | 73 +++++++++++++++++++++++++++++------------------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index d07452fd4efb0..42a506479326b 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -488,8 +488,6 @@ function __init__() if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES") MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"]) end - - init_print_int_scratch() nothing end diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 37aa5b945348d..6336951c1aaca 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -639,24 +639,32 @@ ndigits(x::Integer; base::Integer=10, pad::Integer=1) = max(pad, ndigits0z(x, ba ## integer to string functions ## -const PRINT_INT_SCRATCH_LENGTH = 256 -const PRINT_INT_SCRATCH = Vector{UInt8}[StringVector(PRINT_INT_SCRATCH_LENGTH)] - -@inline function print_int_scratch(n) - # FIXME: scratch = @inbounds print_int_scratch_[Threads.threadid()] - scratch = @inbounds PRINT_INT_SCRATCH[1] - resize!(scratch, n) - scratch +struct Scratch + p::Ptr{UInt8} + length::UInt64 end -function init_print_int_scratch() - # FIXME: resize_nthreads!(PRINT_INT_SCRATCH, print_int_scratch_[1]) - # print_int_scratch() also needs to use Threads.threadid(), but it's - # not clear how to replace that as @eval into base is prohibited +@inline function with_scratch(f, n::Int64) + scratch = Scratch( + llvmcall( + """%2 = alloca i8, i64 %0 + %3 = ptrtoint i8* %2 to i64 + ret i64 %3""", + Ptr{UInt8}, Tuple{Int64}, n ), + unsafe_trunc(UInt64, n) + ) + + f(scratch) +end - nothing +@propagate_inbounds function setindex!(a::Scratch, v, i) + @boundscheck (i <= a.length || throw(BoundsError(a, i))) + unsafe_store!(a.p, convert(UInt8, v), i) + a end +write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) + @inline function bin_impl_(a, x::Unsigned, n::Int, neg::Bool) # for i in 0x0:UInt(n-1) # automatic vectorization produces redundant codes # @inbounds a[n - i] = 0x30 + (((x >> i) % UInt8)::UInt8 & 0x1) @@ -694,10 +702,11 @@ end function bin_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = 8 * sizeof(x) - Base.leading_zeros(x) n = neg + max(pad, m) - scratch = print_int_scratch(n) - bin_impl_(scratch, x, n, neg) - write(io, scratch) + with_scratch(n) do scratch + bin_impl_(scratch, x, n, neg) + write(io, scratch) + end end @inline function oct_impl_(a, x::Unsigned, n::Int, neg::Bool) @@ -724,9 +733,11 @@ end function oct_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = div(8 * sizeof(x) - leading_zeros(x) + 2, 3) n = neg + max(pad, m) - scratch = print_int_scratch(n) - oct_impl_(scratch, x, n, neg) - write(io, scratch) + + with_scratch(n) do scratch + oct_impl_(scratch, x, n, neg) + write(io, scratch) + end end # 2-digit decimal characters ("00":"99") @@ -759,9 +770,11 @@ end function dec_io(io::IO, x::Unsigned, pad::Int, neg::Bool) n = neg + ndigits(x, pad=pad) - scratch = print_int_scratch(n) - dec_impl_(scratch, x, n, neg) - write(io, scratch) + + with_scratch(n) do scratch + dec_impl_(scratch, x, n, neg) + write(io, scratch) + end end @inline function hex_impl_(a, x::Unsigned, n::Int, neg::Bool) @@ -795,9 +808,11 @@ end function hex_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = 2 * sizeof(x) - (leading_zeros(x) >> 2) n = neg + max(pad, m) - scratch = print_int_scratch(n) - hex_impl_(scratch, x, n, neg) - write(io, scratch) + + with_scratch(n) do scratch + hex_impl_(scratch, x, n, neg) + write(io, scratch) + end end const base36digits = UInt8['0':'9';'a':'z'] @@ -838,10 +853,10 @@ function _base_io(io::IO, base::Integer, x::Integer, pad::Int, neg::Bool) b = (base % Int)::Int n = neg + ndigits(x, base=b, pad=pad) - scratch = print_int_scratch(n) - _base_impl_(scratch, b, x, n, neg) - - write(io, scratch) + with_scratch(n) do scratch + _base_impl_(scratch, b, x, n, neg) + write(io, scratch) + end end split_sign(n::Integer) = unsigned(abs(n)), n < 0 From 62765fc0924d2221f47dc3259fe38dc25442b8d5 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 01:54:15 +0100 Subject: [PATCH 06/16] alloca() in disguise --- base/intfuncs.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 6336951c1aaca..cbcab2018bc50 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -639,22 +639,21 @@ ndigits(x::Integer; base::Integer=10, pad::Integer=1) = max(pad, ndigits0z(x, ba ## integer to string functions ## +const PRINT_SCRATCH_LENGTH = 256 +struct Buf + data::NTuple{PRINT_SCRATCH_LENGTH,UInt8} + + Buf(::UndefInitializer) = new() +end + struct Scratch p::Ptr{UInt8} length::UInt64 end -@inline function with_scratch(f, n::Int64) - scratch = Scratch( - llvmcall( - """%2 = alloca i8, i64 %0 - %3 = ptrtoint i8* %2 to i64 - ret i64 %3""", - Ptr{UInt8}, Tuple{Int64}, n ), - unsafe_trunc(UInt64, n) - ) - - f(scratch) +function with_scratch(f, n) + buf = Ref(Buf(undef)) + GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) end @propagate_inbounds function setindex!(a::Scratch, v, i) From 69d08c8d20e46301c74b79dc44419b53c829dec8 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 01:54:35 +0100 Subject: [PATCH 07/16] StringVector -> stack buffer --- base/intfuncs.jl | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index cbcab2018bc50..ae90d4e0119e3 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -664,6 +664,8 @@ end write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) +String(s::Scratch) = unsafe_string(s.p, s.length) + @inline function bin_impl_(a, x::Unsigned, n::Int, neg::Bool) # for i in 0x0:UInt(n-1) # automatic vectorization produces redundant codes # @inbounds a[n - i] = 0x30 + (((x >> i) % UInt8)::UInt8 & 0x1) @@ -692,10 +694,12 @@ end function bin(x::Unsigned, pad::Int, neg::Bool) m = 8 * sizeof(x) - Base.leading_zeros(x) n = neg + max(pad, m) - a = Base.StringVector(n) - bin_impl_(a, x, n, neg) - String(a) + + with_scratch(n) do a + bin_impl_(a, x, n, neg) + String(a) + end end function bin_io(io::IO, x::Unsigned, pad::Int, neg::Bool) @@ -723,10 +727,11 @@ end function oct(x::Unsigned, pad::Int, neg::Bool) m = div(8 * sizeof(x) - Base.leading_zeros(x) + 2, 3) n = neg + max(pad, m) - a = Base.StringVector(n) - oct_impl_(a, x, n, neg) - String(a) + with_scratch(n) do a + oct_impl_(a, x, n, neg) + String(a) + end end function oct_io(io::IO, x::Unsigned, pad::Int, neg::Bool) @@ -762,9 +767,11 @@ end function dec(x::Unsigned, pad::Int, neg::Bool) n = neg + ndigits(x, pad=pad) - scratch = Base.StringVector(n) - dec_impl_(scratch, x, n, neg) - String(scratch) + + with_scratch(n) do scratch + dec_impl_(scratch, x, n, neg) + String(scratch) + end end function dec_io(io::IO, x::Unsigned, pad::Int, neg::Bool) @@ -798,10 +805,11 @@ end function hex(x::Unsigned, pad::Int, neg::Bool) m = 2 * sizeof(x) - (Base.leading_zeros(x) >> 2) n = neg + max(pad, m) - a = Base.StringVector(n) - hex_impl_(a, x, n, neg) - String(a) + with_scratch(n) do a + hex_impl_(a, x, n, neg) + String(a) + end end function hex_io(io::IO, x::Unsigned, pad::Int, neg::Bool) @@ -840,10 +848,11 @@ function _base(base::Integer, x::Integer, pad::Int, neg::Bool) 2 <= abs(base) <= 62 || throw(DomainError(base, "base must satisfy 2 ≤ abs(base) ≤ 62")) b = (base % Int)::Int n = neg + ndigits(x, base=b, pad=pad) - a = StringVector(n) - _base_impl_(a, b, x, n, neg) - String(a) + with_scratch(n) do a + _base_impl_(a, b, x, n, neg) + String(a) + end end function _base_io(io::IO, base::Integer, x::Integer, pad::Int, neg::Bool) From 4e6eb22ac058f9e85aa6c47e7e25f49e11b5ecb0 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 03:02:25 +0100 Subject: [PATCH 08/16] Move scratch to a separate file --- base/Base.jl | 1 + base/intfuncs.jl | 27 --------------------------- base/scratch.jl | 26 ++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 base/scratch.jl diff --git a/base/Base.jl b/base/Base.jl index 42a506479326b..4c90d3e458bd6 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -228,6 +228,7 @@ include("io.jl") include("iobuffer.jl") # strings & printing +include("scratch.jl") include("intfuncs.jl") include("strings/strings.jl") include("regex.jl") diff --git a/base/intfuncs.jl b/base/intfuncs.jl index ae90d4e0119e3..3a81a265f43e5 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -639,33 +639,6 @@ ndigits(x::Integer; base::Integer=10, pad::Integer=1) = max(pad, ndigits0z(x, ba ## integer to string functions ## -const PRINT_SCRATCH_LENGTH = 256 -struct Buf - data::NTuple{PRINT_SCRATCH_LENGTH,UInt8} - - Buf(::UndefInitializer) = new() -end - -struct Scratch - p::Ptr{UInt8} - length::UInt64 -end - -function with_scratch(f, n) - buf = Ref(Buf(undef)) - GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) -end - -@propagate_inbounds function setindex!(a::Scratch, v, i) - @boundscheck (i <= a.length || throw(BoundsError(a, i))) - unsafe_store!(a.p, convert(UInt8, v), i) - a -end - -write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) - -String(s::Scratch) = unsafe_string(s.p, s.length) - @inline function bin_impl_(a, x::Unsigned, n::Int, neg::Bool) # for i in 0x0:UInt(n-1) # automatic vectorization produces redundant codes # @inbounds a[n - i] = 0x30 + (((x >> i) % UInt8)::UInt8 & 0x1) diff --git a/base/scratch.jl b/base/scratch.jl new file mode 100644 index 0000000000000..f9fc44cb5ce55 --- /dev/null +++ b/base/scratch.jl @@ -0,0 +1,26 @@ +const PRINT_SCRATCH_LENGTH = 256 +struct ScratchBuf + data::NTuple{PRINT_SCRATCH_LENGTH,UInt8} + + ScratchBuf(::UndefInitializer) = new() +end + +struct Scratch + p::Ptr{UInt8} + length::UInt64 +end + +function with_scratch(f, n) + buf = Ref(ScratchBuf(undef)) + GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) +end + +@propagate_inbounds function setindex!(a::Scratch, v, i) + @boundscheck (i <= a.length || throw(BoundsError(a, i))) + unsafe_store!(a.p, convert(UInt8, v), i) + a +end + +write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) + +String(s::Scratch) = unsafe_string(s.p, s.length) From 9dbc3f94144fc72514940fe41dc27f55dfe33d77 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 04:03:08 +0100 Subject: [PATCH 09/16] show(io, float) optimization --- base/ryu/Ryu.jl | 44 +++++++++++++++++++++++++------------------- base/ryu/shortest.jl | 2 +- base/scratch.jl | 38 ++++++++++++++++++++++++++++++++++---- 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/base/ryu/Ryu.jl b/base/ryu/Ryu.jl index 81d1c41f4c19f..4b53bc80e3ed9 100644 --- a/base/ryu/Ryu.jl +++ b/base/ryu/Ryu.jl @@ -1,6 +1,7 @@ module Ryu import .Base: significand_bits, significand_mask, exponent_bits, exponent_mask, exponent_bias, exponent_max, uinttype +import .Base: with_scratch include("utils.jl") include("shortest.jl") @@ -45,9 +46,10 @@ function writeshortest(x::T, decchar::UInt8=UInt8('.'), typed::Bool=false, compact::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(neededdigits(T)) - pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) - return String(resize!(buf, pos - 1)) + with_scratch(neededdigits(T)) do buf + pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) + return String(@inbounds view(buf, 1:pos - 1)) + end end """ @@ -73,9 +75,10 @@ function writefixed(x::T, hash::Bool=false, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(precision + neededdigits(T)) - pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) - return String(resize!(buf, pos - 1)) + with_scratch(precision + neededdigits(T)) do buf + pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) + return String(@inbounds view(buf, 1:pos - 1)) + end end """ @@ -103,26 +106,29 @@ function writeexp(x::T, expchar::UInt8=UInt8('e'), decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(precision + neededdigits(T)) - pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) - return String(resize!(buf, pos - 1)) + with_scratch(precision + neededdigits(T)) do buf + pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) + return String(@inbounds view(buf, 1:pos - 1)) + end end function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false) where {T <: Base.IEEEFloat} compact = get(io, :compact, false)::Bool - buf = Base.StringVector(neededdigits(T)) - typed = !forceuntyped && !compact && get(io, :typeinfo, Any) != typeof(x) - pos = writeshortest(buf, 1, x, false, false, true, -1, - (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) - write(io, resize!(buf, pos - 1)) - return + with_scratch(neededdigits(T)) do buf + typed = !forceuntyped && !compact && get(io, :typeinfo, Any) != typeof(x) + pos = writeshortest(buf, 1, x, false, false, true, -1, + (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) + write(io, @inbounds view(buf, 1:pos - 1)) + return + end end function Base.string(x::T) where {T <: Base.IEEEFloat} - buf = Base.StringVector(neededdigits(T)) - pos = writeshortest(buf, 1, x, false, false, true, -1, - UInt8('e'), false, UInt8('.'), false, false) - return String(resize!(buf, pos - 1)) + with_scratch(neededdigits(T)) do buf + pos = writeshortest(buf, 1, x, false, false, true, -1, + UInt8('e'), false, UInt8('.'), false, false) + return String(@inbounds view(buf, 1:pos - 1)) + end end Base.print(io::IO, x::Union{Float16, Float32}) = show(io, x, true, true) diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index f95c09d235e6d..401de99227ca5 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -224,7 +224,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. return b, e10 end -function writeshortest(buf::Vector{UInt8}, pos, x::T, +function writeshortest(buf, pos, x::T, plus=false, space=false, hash=true, precision=-1, expchar=UInt8('e'), padexp=false, decchar=UInt8('.'), typed=false, compact=false) where {T} diff --git a/base/scratch.jl b/base/scratch.jl index f9fc44cb5ce55..c3389953d9af4 100644 --- a/base/scratch.jl +++ b/base/scratch.jl @@ -1,4 +1,5 @@ -const PRINT_SCRATCH_LENGTH = 256 +# NOTE: ryu/Ryu.jl: neededdigits(::Type{Float64}) = 309 + 17 +const PRINT_SCRATCH_LENGTH = 309 + 17 struct ScratchBuf data::NTuple{PRINT_SCRATCH_LENGTH,UInt8} @@ -11,16 +12,45 @@ struct Scratch end function with_scratch(f, n) - buf = Ref(ScratchBuf(undef)) - GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) + if n <= PRINT_SCRATCH_LENGTH + buf = Ref(ScratchBuf(undef)) + GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) + else + tls = task_local_storage() + buf = get!(tls, :PRINT_SCRATCH) do + Vector{UInt8}(undef, n) + end::Vector{UInt8} + resize!(buf, n) + GC.@preserve buf f(Scratch(pointer(buf), length(buf))) + end end @propagate_inbounds function setindex!(a::Scratch, v, i) - @boundscheck (i <= a.length || throw(BoundsError(a, i))) + @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) unsafe_store!(a.p, convert(UInt8, v), i) a end +@propagate_inbounds function getindex(a::Scratch, i) + @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) + unsafe_load(a.p, i) +end + +@propagate_inbounds function view(a::Scratch, index::UnitRange) + @boundscheck ((index.start > 0 && index.stop <= a.length) || throw(BoundsError)) + index.start > index.stop && return Scratch(Ptr{UInt8}(), 0) + Scratch(a.p + index.start - 1, unsafe_trunc(UInt64, index.stop - index.start + 1)) +end + +pointer(a::Scratch) = a.p +pointer(a::Scratch, pos) = a.p + pos - 1 +length(a::Scratch) = a.length +@inline function unsafe_copyto!(dst::Scratch, dst_pos, src::Vector{UInt8}, src_pos, N) + GC.@preserve dst, src, begin + unsafe_copyto!(pointer(dst, dst_pos), pointer(src, src_pos), N) + end +end + write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) String(s::Scratch) = unsafe_string(s.p, s.length) From f864790de43c4b51ae513b634b69e392bb02138b Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 04:04:59 +0100 Subject: [PATCH 10/16] scatter inbounds --- base/ryu/Ryu.jl | 12 ++++++------ base/ryu/exp.jl | 4 ++-- base/ryu/fixed.jl | 4 ++-- base/ryu/shortest.jl | 4 ++-- base/ryu/utils.jl | 12 ++++++------ 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/base/ryu/Ryu.jl b/base/ryu/Ryu.jl index 4b53bc80e3ed9..2f10730f88f07 100644 --- a/base/ryu/Ryu.jl +++ b/base/ryu/Ryu.jl @@ -1,7 +1,7 @@ module Ryu import .Base: significand_bits, significand_mask, exponent_bits, exponent_mask, exponent_bias, exponent_max, uinttype -import .Base: with_scratch +import .Base: with_scratch, @propagate_inbounds include("utils.jl") include("shortest.jl") @@ -47,7 +47,7 @@ function writeshortest(x::T, typed::Bool=false, compact::Bool=false) where {T <: Base.IEEEFloat} with_scratch(neededdigits(T)) do buf - pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) + pos = @inbounds writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -76,7 +76,7 @@ function writefixed(x::T, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} with_scratch(precision + neededdigits(T)) do buf - pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) + pos = @inbounds writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -107,7 +107,7 @@ function writeexp(x::T, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} with_scratch(precision + neededdigits(T)) do buf - pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) + pos = @inbounds writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -116,7 +116,7 @@ function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false compact = get(io, :compact, false)::Bool with_scratch(neededdigits(T)) do buf typed = !forceuntyped && !compact && get(io, :typeinfo, Any) != typeof(x) - pos = writeshortest(buf, 1, x, false, false, true, -1, + pos = @inbounds writeshortest(buf, 1, x, false, false, true, -1, (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) write(io, @inbounds view(buf, 1:pos - 1)) return @@ -125,7 +125,7 @@ end function Base.string(x::T) where {T <: Base.IEEEFloat} with_scratch(neededdigits(T)) do buf - pos = writeshortest(buf, 1, x, false, false, true, -1, + pos = @inbounds writeshortest(buf, 1, x, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, false) return String(@inbounds view(buf, 1:pos - 1)) end diff --git a/base/ryu/exp.jl b/base/ryu/exp.jl index 30291212d014d..de87eb2958859 100644 --- a/base/ryu/exp.jl +++ b/base/ryu/exp.jl @@ -1,7 +1,7 @@ -function writeexp(buf, pos, v::T, +@propagate_inbounds function writeexp(buf, pos, v::T, precision=-1, plus=false, space=false, hash=false, expchar=UInt8('e'), decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @assert 0 < pos <= length(buf) + @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) diff --git a/base/ryu/fixed.jl b/base/ryu/fixed.jl index e0085f5c66dab..65a6084c4dcef 100644 --- a/base/ryu/fixed.jl +++ b/base/ryu/fixed.jl @@ -1,7 +1,7 @@ -function writefixed(buf, pos, v::T, +@propagate_inbounds function writefixed(buf, pos, v::T, precision=-1, plus=false, space=false, hash=false, decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @assert 0 < pos <= length(buf) + @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index 401de99227ca5..e0bb5293f9aed 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -224,11 +224,11 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. return b, e10 end -function writeshortest(buf, pos, x::T, +@propagate_inbounds function writeshortest(buf, pos, x::T, plus=false, space=false, hash=true, precision=-1, expchar=UInt8('e'), padexp=false, decchar=UInt8('.'), typed=false, compact=false) where {T} - @assert 0 < pos <= length(buf) + @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) # special cases if x == 0 if typed && x isa Float16 diff --git a/base/ryu/utils.jl b/base/ryu/utils.jl index 352f8f19cb9be..062e9ed22915e 100644 --- a/base/ryu/utils.jl +++ b/base/ryu/utils.jl @@ -44,7 +44,7 @@ Computes `e == 0 ? 1 : ceil(log2(5^e))`. This is valid for `e < 3529` (if perfor """ pow5bits(e) = ((e * 1217359) >> 19) + 1 -"""" +""" Ryu.mulshift(m::U, mula, j) where {U<:Unsigned} Compute `(m * mul) >> j`, where `j >= 8*sizeof(U)`. The type of the results is the larger of `U` or `UInt32`. @@ -195,7 +195,7 @@ Compute `(m * mul) >> j % 10^9` where `mul = mula + mulb<<64 + mulc<<128`, and ` return (v % UInt32) - UInt32(1000000000) * shifted end -@inline function append_sign(x, plus, space, buf, pos) +@propagate_inbounds function append_sign(x, plus, space, buf, pos) if signbit(x) && !isnan(x) # suppress minus sign for signaling NaNs buf[pos] = UInt8('-') pos += 1 @@ -209,7 +209,7 @@ end return pos end -@inline function append_n_digits(olength, digits, buf, pos) +@propagate_inbounds function append_n_digits(olength, digits, buf, pos) i = 0 while digits >= 10000 c = digits % 10000 @@ -237,7 +237,7 @@ end return pos + i end -@inline function append_d_digits(olength, digits, buf, pos, decchar) +@propagate_inbounds function append_d_digits(olength, digits, buf, pos, decchar) i = 0 while digits >= 10000 c = digits % 10000 @@ -268,7 +268,7 @@ end return pos + i end -@inline function append_c_digits(count, digits, buf, pos) +@propagate_inbounds function append_c_digits(count, digits, buf, pos) i = 0 while i < count - 1 c = (digits % 100) << 1 @@ -283,7 +283,7 @@ end return pos + i end -@inline function append_nine_digits(digits, buf, pos) +@propagate_inbounds function append_nine_digits(digits, buf, pos) if digits == 0 for _ = 1:9 buf[pos] = UInt8('0') From 39d1eb282e3712a0072090c760b283c3d3c48345 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 07:23:17 +0100 Subject: [PATCH 11/16] Revert "scatter inbounds" This reverts commit f864790de43c4b51ae513b634b69e392bb02138b. --- base/ryu/Ryu.jl | 12 ++++++------ base/ryu/exp.jl | 4 ++-- base/ryu/fixed.jl | 4 ++-- base/ryu/shortest.jl | 4 ++-- base/ryu/utils.jl | 12 ++++++------ 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/base/ryu/Ryu.jl b/base/ryu/Ryu.jl index 2f10730f88f07..4b53bc80e3ed9 100644 --- a/base/ryu/Ryu.jl +++ b/base/ryu/Ryu.jl @@ -1,7 +1,7 @@ module Ryu import .Base: significand_bits, significand_mask, exponent_bits, exponent_mask, exponent_bias, exponent_max, uinttype -import .Base: with_scratch, @propagate_inbounds +import .Base: with_scratch include("utils.jl") include("shortest.jl") @@ -47,7 +47,7 @@ function writeshortest(x::T, typed::Bool=false, compact::Bool=false) where {T <: Base.IEEEFloat} with_scratch(neededdigits(T)) do buf - pos = @inbounds writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) + pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -76,7 +76,7 @@ function writefixed(x::T, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} with_scratch(precision + neededdigits(T)) do buf - pos = @inbounds writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) + pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -107,7 +107,7 @@ function writeexp(x::T, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} with_scratch(precision + neededdigits(T)) do buf - pos = @inbounds writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) + pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end end @@ -116,7 +116,7 @@ function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false compact = get(io, :compact, false)::Bool with_scratch(neededdigits(T)) do buf typed = !forceuntyped && !compact && get(io, :typeinfo, Any) != typeof(x) - pos = @inbounds writeshortest(buf, 1, x, false, false, true, -1, + pos = writeshortest(buf, 1, x, false, false, true, -1, (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) write(io, @inbounds view(buf, 1:pos - 1)) return @@ -125,7 +125,7 @@ end function Base.string(x::T) where {T <: Base.IEEEFloat} with_scratch(neededdigits(T)) do buf - pos = @inbounds writeshortest(buf, 1, x, false, false, true, -1, + pos = writeshortest(buf, 1, x, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, false) return String(@inbounds view(buf, 1:pos - 1)) end diff --git a/base/ryu/exp.jl b/base/ryu/exp.jl index de87eb2958859..30291212d014d 100644 --- a/base/ryu/exp.jl +++ b/base/ryu/exp.jl @@ -1,7 +1,7 @@ -@propagate_inbounds function writeexp(buf, pos, v::T, +function writeexp(buf, pos, v::T, precision=-1, plus=false, space=false, hash=false, expchar=UInt8('e'), decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) + @assert 0 < pos <= length(buf) startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) diff --git a/base/ryu/fixed.jl b/base/ryu/fixed.jl index 65a6084c4dcef..e0085f5c66dab 100644 --- a/base/ryu/fixed.jl +++ b/base/ryu/fixed.jl @@ -1,7 +1,7 @@ -@propagate_inbounds function writefixed(buf, pos, v::T, +function writefixed(buf, pos, v::T, precision=-1, plus=false, space=false, hash=false, decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) + @assert 0 < pos <= length(buf) startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index e0bb5293f9aed..401de99227ca5 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -224,11 +224,11 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. return b, e10 end -@propagate_inbounds function writeshortest(buf, pos, x::T, +function writeshortest(buf, pos, x::T, plus=false, space=false, hash=true, precision=-1, expchar=UInt8('e'), padexp=false, decchar=UInt8('.'), typed=false, compact=false) where {T} - @boundscheck (0 < pos <= length(buf) || throw(BoundsError())) + @assert 0 < pos <= length(buf) # special cases if x == 0 if typed && x isa Float16 diff --git a/base/ryu/utils.jl b/base/ryu/utils.jl index 062e9ed22915e..352f8f19cb9be 100644 --- a/base/ryu/utils.jl +++ b/base/ryu/utils.jl @@ -44,7 +44,7 @@ Computes `e == 0 ? 1 : ceil(log2(5^e))`. This is valid for `e < 3529` (if perfor """ pow5bits(e) = ((e * 1217359) >> 19) + 1 -""" +"""" Ryu.mulshift(m::U, mula, j) where {U<:Unsigned} Compute `(m * mul) >> j`, where `j >= 8*sizeof(U)`. The type of the results is the larger of `U` or `UInt32`. @@ -195,7 +195,7 @@ Compute `(m * mul) >> j % 10^9` where `mul = mula + mulb<<64 + mulc<<128`, and ` return (v % UInt32) - UInt32(1000000000) * shifted end -@propagate_inbounds function append_sign(x, plus, space, buf, pos) +@inline function append_sign(x, plus, space, buf, pos) if signbit(x) && !isnan(x) # suppress minus sign for signaling NaNs buf[pos] = UInt8('-') pos += 1 @@ -209,7 +209,7 @@ end return pos end -@propagate_inbounds function append_n_digits(olength, digits, buf, pos) +@inline function append_n_digits(olength, digits, buf, pos) i = 0 while digits >= 10000 c = digits % 10000 @@ -237,7 +237,7 @@ end return pos + i end -@propagate_inbounds function append_d_digits(olength, digits, buf, pos, decchar) +@inline function append_d_digits(olength, digits, buf, pos, decchar) i = 0 while digits >= 10000 c = digits % 10000 @@ -268,7 +268,7 @@ end return pos + i end -@propagate_inbounds function append_c_digits(count, digits, buf, pos) +@inline function append_c_digits(count, digits, buf, pos) i = 0 while i < count - 1 c = (digits % 100) << 1 @@ -283,7 +283,7 @@ end return pos + i end -@propagate_inbounds function append_nine_digits(digits, buf, pos) +@inline function append_nine_digits(digits, buf, pos) if digits == 0 for _ = 1:9 buf[pos] = UInt8('0') From e65d424889973048e2c16fb65ed1e289722528d8 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Fri, 2 Jul 2021 07:28:38 +0100 Subject: [PATCH 12/16] Fixup --- base/scratch.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/scratch.jl b/base/scratch.jl index c3389953d9af4..a45ea90298081 100644 --- a/base/scratch.jl +++ b/base/scratch.jl @@ -37,7 +37,7 @@ end end @propagate_inbounds function view(a::Scratch, index::UnitRange) - @boundscheck ((index.start > 0 && index.stop <= a.length) || throw(BoundsError)) + @boundscheck ((index.start > 0 && index.stop <= a.length) || throw(BoundsError(a, index))) index.start > index.stop && return Scratch(Ptr{UInt8}(), 0) Scratch(a.p + index.start - 1, unsafe_trunc(UInt64, index.stop - index.start + 1)) end From 8e82b6dedcdf397cf959c162cc0bb5274487bc45 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Mon, 26 Jul 2021 07:34:42 +0100 Subject: [PATCH 13/16] Rename Scratch -> ScratchBuffer --- base/Base.jl | 2 +- base/intfuncs.jl | 20 +++++++-------- base/ryu/Ryu.jl | 12 ++++----- base/scratch.jl | 56 ------------------------------------------ base/scratch_buffer.jl | 56 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 73 deletions(-) delete mode 100644 base/scratch.jl create mode 100644 base/scratch_buffer.jl diff --git a/base/Base.jl b/base/Base.jl index 4c90d3e458bd6..383d8ac3023aa 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -228,7 +228,7 @@ include("io.jl") include("iobuffer.jl") # strings & printing -include("scratch.jl") +include("scratch_buffer.jl") include("intfuncs.jl") include("strings/strings.jl") include("regex.jl") diff --git a/base/intfuncs.jl b/base/intfuncs.jl index bc3358c5c1f97..be61c6ab7cbf6 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -669,7 +669,7 @@ function bin(x::Unsigned, pad::Int, neg::Bool) n = neg + max(pad, m) - with_scratch(n) do a + with_scratch_buffer(n) do a bin_impl_(a, x, n, neg) String(a) end @@ -679,7 +679,7 @@ function bin_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = 8 * sizeof(x) - Base.leading_zeros(x) n = neg + max(pad, m) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch bin_impl_(scratch, x, n, neg) write(io, scratch) end @@ -701,7 +701,7 @@ function oct(x::Unsigned, pad::Int, neg::Bool) m = div(8 * sizeof(x) - Base.leading_zeros(x) + 2, 3) n = neg + max(pad, m) - with_scratch(n) do a + with_scratch_buffer(n) do a oct_impl_(a, x, n, neg) String(a) end @@ -711,7 +711,7 @@ function oct_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = div(8 * sizeof(x) - leading_zeros(x) + 2, 3) n = neg + max(pad, m) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch oct_impl_(scratch, x, n, neg) write(io, scratch) end @@ -741,7 +741,7 @@ end function dec(x::Unsigned, pad::Int, neg::Bool) n = neg + ndigits(x, pad=pad) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch dec_impl_(scratch, x, n, neg) String(scratch) end @@ -750,7 +750,7 @@ end function dec_io(io::IO, x::Unsigned, pad::Int, neg::Bool) n = neg + ndigits(x, pad=pad) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch dec_impl_(scratch, x, n, neg) write(io, scratch) end @@ -779,7 +779,7 @@ function hex(x::Unsigned, pad::Int, neg::Bool) m = 2 * sizeof(x) - (Base.leading_zeros(x) >> 2) n = neg + max(pad, m) - with_scratch(n) do a + with_scratch_buffer(n) do a hex_impl_(a, x, n, neg) String(a) end @@ -789,7 +789,7 @@ function hex_io(io::IO, x::Unsigned, pad::Int, neg::Bool) m = 2 * sizeof(x) - (leading_zeros(x) >> 2) n = neg + max(pad, m) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch hex_impl_(scratch, x, n, neg) write(io, scratch) end @@ -822,7 +822,7 @@ function _base(base::Integer, x::Integer, pad::Int, neg::Bool) b = (base % Int)::Int n = neg + ndigits(x, base=b, pad=pad) - with_scratch(n) do a + with_scratch_buffer(n) do a _base_impl_(a, b, x, n, neg) String(a) end @@ -834,7 +834,7 @@ function _base_io(io::IO, base::Integer, x::Integer, pad::Int, neg::Bool) b = (base % Int)::Int n = neg + ndigits(x, base=b, pad=pad) - with_scratch(n) do scratch + with_scratch_buffer(n) do scratch _base_impl_(scratch, b, x, n, neg) write(io, scratch) end diff --git a/base/ryu/Ryu.jl b/base/ryu/Ryu.jl index 4b53bc80e3ed9..4bfc472341767 100644 --- a/base/ryu/Ryu.jl +++ b/base/ryu/Ryu.jl @@ -1,7 +1,7 @@ module Ryu import .Base: significand_bits, significand_mask, exponent_bits, exponent_mask, exponent_bias, exponent_max, uinttype -import .Base: with_scratch +import .Base: with_scratch_buffer include("utils.jl") include("shortest.jl") @@ -46,7 +46,7 @@ function writeshortest(x::T, decchar::UInt8=UInt8('.'), typed::Bool=false, compact::Bool=false) where {T <: Base.IEEEFloat} - with_scratch(neededdigits(T)) do buf + with_scratch_buffer(neededdigits(T)) do buf pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) return String(@inbounds view(buf, 1:pos - 1)) end @@ -75,7 +75,7 @@ function writefixed(x::T, hash::Bool=false, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - with_scratch(precision + neededdigits(T)) do buf + with_scratch_buffer(precision + neededdigits(T)) do buf pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end @@ -106,7 +106,7 @@ function writeexp(x::T, expchar::UInt8=UInt8('e'), decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - with_scratch(precision + neededdigits(T)) do buf + with_scratch_buffer(precision + neededdigits(T)) do buf pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) return String(@inbounds view(buf, 1:pos - 1)) end @@ -114,7 +114,7 @@ end function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false) where {T <: Base.IEEEFloat} compact = get(io, :compact, false)::Bool - with_scratch(neededdigits(T)) do buf + with_scratch_buffer(neededdigits(T)) do buf typed = !forceuntyped && !compact && get(io, :typeinfo, Any) != typeof(x) pos = writeshortest(buf, 1, x, false, false, true, -1, (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) @@ -124,7 +124,7 @@ function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false end function Base.string(x::T) where {T <: Base.IEEEFloat} - with_scratch(neededdigits(T)) do buf + with_scratch_buffer(neededdigits(T)) do buf pos = writeshortest(buf, 1, x, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, false) return String(@inbounds view(buf, 1:pos - 1)) diff --git a/base/scratch.jl b/base/scratch.jl deleted file mode 100644 index a45ea90298081..0000000000000 --- a/base/scratch.jl +++ /dev/null @@ -1,56 +0,0 @@ -# NOTE: ryu/Ryu.jl: neededdigits(::Type{Float64}) = 309 + 17 -const PRINT_SCRATCH_LENGTH = 309 + 17 -struct ScratchBuf - data::NTuple{PRINT_SCRATCH_LENGTH,UInt8} - - ScratchBuf(::UndefInitializer) = new() -end - -struct Scratch - p::Ptr{UInt8} - length::UInt64 -end - -function with_scratch(f, n) - if n <= PRINT_SCRATCH_LENGTH - buf = Ref(ScratchBuf(undef)) - GC.@preserve buf f(Scratch(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) - else - tls = task_local_storage() - buf = get!(tls, :PRINT_SCRATCH) do - Vector{UInt8}(undef, n) - end::Vector{UInt8} - resize!(buf, n) - GC.@preserve buf f(Scratch(pointer(buf), length(buf))) - end -end - -@propagate_inbounds function setindex!(a::Scratch, v, i) - @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) - unsafe_store!(a.p, convert(UInt8, v), i) - a -end - -@propagate_inbounds function getindex(a::Scratch, i) - @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) - unsafe_load(a.p, i) -end - -@propagate_inbounds function view(a::Scratch, index::UnitRange) - @boundscheck ((index.start > 0 && index.stop <= a.length) || throw(BoundsError(a, index))) - index.start > index.stop && return Scratch(Ptr{UInt8}(), 0) - Scratch(a.p + index.start - 1, unsafe_trunc(UInt64, index.stop - index.start + 1)) -end - -pointer(a::Scratch) = a.p -pointer(a::Scratch, pos) = a.p + pos - 1 -length(a::Scratch) = a.length -@inline function unsafe_copyto!(dst::Scratch, dst_pos, src::Vector{UInt8}, src_pos, N) - GC.@preserve dst, src, begin - unsafe_copyto!(pointer(dst, dst_pos), pointer(src, src_pos), N) - end -end - -write(io::IO, a::Scratch) = unsafe_write(io, a.p, a.length) - -String(s::Scratch) = unsafe_string(s.p, s.length) diff --git a/base/scratch_buffer.jl b/base/scratch_buffer.jl new file mode 100644 index 0000000000000..e2ddaa9b3431a --- /dev/null +++ b/base/scratch_buffer.jl @@ -0,0 +1,56 @@ +# NOTE: ryu/Ryu.jl: neededdigits(::Type{Float64}) = 309 + 17 +const SCRATCH_BUFFER_INLINE_LENGTH = 309 + 17 +struct ScratchBufferInline + data::NTuple{SCRATCH_BUFFER_INLINE_LENGTH,UInt8} + + ScratchBufferInline(::UndefInitializer) = new() +end + +struct ScratchBuffer + p::Ptr{UInt8} + length::UInt64 +end + +function with_scratch_buffer(f, n) + if n <= SCRATCH_BUFFER_INLINE_LENGTH + buf = Ref(ScratchBufferInline(undef)) + GC.@preserve buf f(ScratchBuffer(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) + else + tls = task_local_storage() + buf = get!(tls, :SCRATCH_BUFFER) do + Vector{UInt8}(undef, n) + end::Vector{UInt8} + resize!(buf, n) + GC.@preserve buf f(ScratchBuffer(pointer(buf), length(buf))) + end +end + +@propagate_inbounds function setindex!(a::ScratchBuffer, v, i) + @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) + unsafe_store!(a.p, convert(UInt8, v), i) + a +end + +@propagate_inbounds function getindex(a::ScratchBuffer, i) + @boundscheck (1 <= i <= a.length || throw(BoundsError(a, i))) + unsafe_load(a.p, i) +end + +@propagate_inbounds function view(a::ScratchBuffer, index::UnitRange) + @boundscheck ((index.start > 0 && index.stop <= a.length) || throw(BoundsError(a, index))) + index.start > index.stop && return ScratchBuffer(Ptr{UInt8}(), 0) + ScratchBuffer(a.p + index.start - 1, unsafe_trunc(UInt64, index.stop - index.start + 1)) +end + +pointer(a::ScratchBuffer) = a.p +pointer(a::ScratchBuffer, pos) = a.p + pos - 1 +length(a::ScratchBuffer) = a.length +@inline function unsafe_copyto!(dst::ScratchBuffer, dst_pos, src::Vector{UInt8}, src_pos, N) + GC.@preserve dst, src, begin + unsafe_copyto!(pointer(dst, dst_pos), pointer(src, src_pos), N) + end +end + +write(io::IO, a::ScratchBuffer) = unsafe_write(io, a.p, a.length) + +String(s::ScratchBuffer) = unsafe_string(s.p, s.length) From 8c06a0049205eefb1099fa3de84ff05687185b50 Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Mon, 26 Jul 2021 09:59:16 +0100 Subject: [PATCH 14/16] Remove UndefInitializer argument --- base/scratch_buffer.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/scratch_buffer.jl b/base/scratch_buffer.jl index e2ddaa9b3431a..9b46647605504 100644 --- a/base/scratch_buffer.jl +++ b/base/scratch_buffer.jl @@ -3,7 +3,7 @@ const SCRATCH_BUFFER_INLINE_LENGTH = 309 + 17 struct ScratchBufferInline data::NTuple{SCRATCH_BUFFER_INLINE_LENGTH,UInt8} - ScratchBufferInline(::UndefInitializer) = new() + ScratchBufferInline() = new() end struct ScratchBuffer @@ -13,7 +13,7 @@ end function with_scratch_buffer(f, n) if n <= SCRATCH_BUFFER_INLINE_LENGTH - buf = Ref(ScratchBufferInline(undef)) + buf = Ref(ScratchBufferInline()) GC.@preserve buf f(ScratchBuffer(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) else tls = task_local_storage() From e5069a15eb52e0a18f24bd1532290e4017501edd Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Mon, 25 Apr 2022 11:46:59 +0200 Subject: [PATCH 15/16] Dynamic-allocated ScratchBuffer space --- base/scratch_buffer.jl | 45 +++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/base/scratch_buffer.jl b/base/scratch_buffer.jl index 9b46647605504..9dd235d887f79 100644 --- a/base/scratch_buffer.jl +++ b/base/scratch_buffer.jl @@ -1,28 +1,41 @@ # NOTE: ryu/Ryu.jl: neededdigits(::Type{Float64}) = 309 + 17 -const SCRATCH_BUFFER_INLINE_LENGTH = 309 + 17 -struct ScratchBufferInline - data::NTuple{SCRATCH_BUFFER_INLINE_LENGTH,UInt8} - - ScratchBufferInline() = new() -end +const SCRATCH_BUFFER_MIN_LENGTH = 309 + 17 struct ScratchBuffer p::Ptr{UInt8} length::UInt64 end -function with_scratch_buffer(f, n) - if n <= SCRATCH_BUFFER_INLINE_LENGTH - buf = Ref(ScratchBufferInline()) - GC.@preserve buf f(ScratchBuffer(pointer_from_objref(buf), unsafe_trunc(UInt64, n))) +# NOTE: similar to pcre.jl, Threads module isn't available at the time of loading +_tid() = Int(ccall(:jl_threadid, Int16, ())) + 1 +_nth() = Int(unsafe_load(cglobal(:jl_n_threads, Cint))) + +const SCRATCH_BUFFERS = map(1:_nth()) do _ + Vector{UInt8}(undef, SCRATCH_BUFFER_MIN_LENGTH), false +end + +function with_scratch_buffer(f, n::Int) + buf, buf_inuse = @inbounds SCRATCH_BUFFERS[_tid()] + if buf_inuse + buf = Vector{UInt8}(undef, max(SCRATCH_BUFFER_MIN_LENGTH, n)) else - tls = task_local_storage() - buf = get!(tls, :SCRATCH_BUFFER) do - Vector{UInt8}(undef, n) - end::Vector{UInt8} - resize!(buf, n) - GC.@preserve buf f(ScratchBuffer(pointer(buf), length(buf))) + @inbounds SCRATCH_BUFFERS[_tid()] = (buf, true) end + + resize!(buf, n) + # NOTE: we could just `f(buf)`, but String(buf) doesn't play well and causes allocation on the next `resize!(buf)` + res = GC.@preserve buf f(ScratchBuffer(pointer(buf), length(buf))) + + # The intent is to reuse buffers when it's cheap and reallocate when synchronization/exception handling is required. + # We'll "waste" a buffer when: + # * task is migrated while `f()` is running + # * exception is thrown from `f()` + # * multiple tasks are using `with_scratch_buffer()` from the same thread (e.g. f() is blocking) + # Those shouldn't happen often. + # NOTE: _tid() could be different here if task migrated. + @inbounds SCRATCH_BUFFERS[_tid()] = (buf, false) + + res end @propagate_inbounds function setindex!(a::ScratchBuffer, v, i) From 0d1be8f2488c23c18e8a3d7a28c0b4cec01ae98e Mon Sep 17 00:00:00 2001 From: Sergey Fokin Date: Tue, 26 Apr 2022 14:03:53 +0200 Subject: [PATCH 16/16] Initialize thread-local buffers --- base/Base.jl | 1 + base/scratch_buffer.jl | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/base/Base.jl b/base/Base.jl index deb0a24c4b8a6..f8fa2581dfcb8 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -493,6 +493,7 @@ function __init__() # Base library init reinit_stdio() Multimedia.reinit_displays() # since Multimedia.displays uses stdout as fallback + init_scratch_buffers() # initialize loading init_depot_path() init_load_path() diff --git a/base/scratch_buffer.jl b/base/scratch_buffer.jl index 9dd235d887f79..4115ef4646820 100644 --- a/base/scratch_buffer.jl +++ b/base/scratch_buffer.jl @@ -14,6 +14,13 @@ const SCRATCH_BUFFERS = map(1:_nth()) do _ Vector{UInt8}(undef, SCRATCH_BUFFER_MIN_LENGTH), false end +function init_scratch_buffers() + while length(SCRATCH_BUFFERS) < _nth() + push!(SCRATCH_BUFFERS, (Vector{UInt8}(undef, SCRATCH_BUFFER_MIN_LENGTH), false)) + end + nothing +end + function with_scratch_buffer(f, n::Int) buf, buf_inuse = @inbounds SCRATCH_BUFFERS[_tid()] if buf_inuse