From 237c8d342ea00a74abf61100fadc73de05ce649e Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 10 Aug 2022 09:41:08 -0400 Subject: [PATCH 1/3] Reintroduce SparseArrays in the system image (#46278) --- base/sysimg.jl | 1 + test/precompile.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index f5a7fb22bf2bdd..79f01a0e9a1d25 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -50,6 +50,7 @@ let :InteractiveUtils, :LibGit2, :Profile, + :SparseArrays, :UUIDs, # 3-depth packages diff --git a/test/precompile.jl b/test/precompile.jl index c23fffb6630888..ac2c63ff7af081 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -364,7 +364,7 @@ precompile_test_harness(false) do dir :LazyArtifacts, :LibCURL, :LibCURL_jll, :LibGit2, :Libdl, :LinearAlgebra, :Logging, :Markdown, :Mmap, :MozillaCACerts_jll, :NetworkOptions, :OpenBLAS_jll, :Pkg, :Printf, :Profile, :p7zip_jll, :REPL, :Random, :SHA, :Serialization, :SharedArrays, :Sockets, - :TOML, :Tar, :Test, :UUIDs, :Unicode, + :SparseArrays, :TOML, :Tar, :Test, :UUIDs, :Unicode, :nghttp2_jll] ), ) From f3459234837b13e26abebc22c21d897398151588 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Thu, 11 Aug 2022 01:24:52 +0800 Subject: [PATCH 2/3] improve type-based offset axes check (#45260) * Follow up to #45236 (make `length(::StepRange{Int8,Int128})` type-stable) * Fully drop `_tuple_any` (unneeded now) * Make sure `has_offset_axes(::StepRange)` could be const folded. And define some "cheap" `firstindex` * Do offset axes check on `A`'s parent rather than itself. This avoid some unneeded `axes` call, thus more possible be folded by the compiler. Co-authored-by: Jameson Nash --- base/abstractarray.jl | 6 +++-- base/multidimensional.jl | 1 + base/permuteddimsarray.jl | 1 + base/range.jl | 47 ++++++++++++++++++++++----------------- base/reinterpretarray.jl | 2 ++ base/subarray.jl | 2 ++ base/tuple.jl | 9 -------- test/abstractarray.jl | 33 +++++++++++++++++++++++---- test/compiler/inline.jl | 7 ------ test/ranges.jl | 23 +++++++++++++++++-- 10 files changed, 86 insertions(+), 45 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e9b3c6667dc22a..1690aa4a9e56fe 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -104,9 +104,11 @@ If multiple arguments are passed, equivalent to `has_offset_axes(A) | has_offset See also [`require_one_based_indexing`](@ref). """ -has_offset_axes(A) = _tuple_any(x->Int(first(x))::Int != 1, axes(A)) +has_offset_axes(A) = _any_tuple(x->Int(first(x))::Int != 1, false, axes(A)...) has_offset_axes(A::AbstractVector) = Int(firstindex(A))::Int != 1 # improve performance of a common case (ranges) -has_offset_axes(A...) = _tuple_any(has_offset_axes, A) +# Use `_any_tuple` to avoid unneeded invoke. +# note: this could call `any` directly if the compiler can infer it +has_offset_axes(As...) = _any_tuple(has_offset_axes, false, As...) has_offset_axes(::Colon) = false """ diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 3eecdf17e53181..487be1d9b42a00 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -335,6 +335,7 @@ module IteratorsMD # AbstractArray implementation Base.axes(iter::CartesianIndices{N,R}) where {N,R} = map(Base.axes1, iter.indices) Base.IndexStyle(::Type{CartesianIndices{N,R}}) where {N,R} = IndexCartesian() + Base.has_offset_axes(iter::CartesianIndices) = Base.has_offset_axes(iter.indices...) # getindex for a 0D CartesianIndices is necessary for disambiguation @propagate_inbounds function Base.getindex(iter::CartesianIndices{0,R}) where {R} CartesianIndex() diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index dae288584aa89f..ea1863de8b7084 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -48,6 +48,7 @@ end Base.parent(A::PermutedDimsArray) = A.parent Base.size(A::PermutedDimsArray{T,N,perm}) where {T,N,perm} = genperm(size(parent(A)), perm) Base.axes(A::PermutedDimsArray{T,N,perm}) where {T,N,perm} = genperm(axes(parent(A)), perm) +Base.has_offset_axes(A::PermutedDimsArray) = Base.has_offset_axes(A.parent) Base.similar(A::PermutedDimsArray, T::Type, dims::Base.Dims) = similar(parent(A), T, dims) diff --git a/base/range.jl b/base/range.jl index be4d46f796ac2d..4b2bf18e9f6344 100644 --- a/base/range.jl +++ b/base/range.jl @@ -689,6 +689,9 @@ step_hp(r::AbstractRange) = step(r) axes(r::AbstractRange) = (oneto(length(r)),) +# Needed to ensure `has_offset_axes` can constant-fold. +has_offset_axes(::StepRange) = false + # n.b. checked_length for these is defined iff checked_add and checked_sub are # defined between the relevant types function checked_length(r::OrdinalRange{T}) where T @@ -750,64 +753,66 @@ length(r::OneTo) = Integer(r.stop - zero(r.stop)) length(r::StepRangeLen) = r.len length(r::LinRange) = r.len -let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128} - global length, checked_length +let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, + smallints = (Int === Int64 ? + Union{Int8, UInt8, Int16, UInt16, Int32, UInt32} : + Union{Int8, UInt8, Int16, UInt16}), + bitints = Union{bigints, smallints} + global length, checked_length, firstindex # compile optimization for which promote_type(T, Int) == T length(r::OneTo{T}) where {T<:bigints} = r.stop # slightly more accurate length and checked_length in extreme cases # (near typemax) for types with known `unsigned` functions function length(r::OrdinalRange{T}) where T<:bigints s = step(r) - isempty(r) && return zero(T) diff = last(r) - first(r) + isempty(r) && return zero(diff) # if |s| > 1, diff might have overflowed, but unsigned(diff)÷s should # therefore still be valid (if the result is representable at all) # n.b. !(s isa T) if s isa Unsigned || -1 <= s <= 1 || s == -s - a = div(diff, s) % T + a = div(diff, s) % typeof(diff) elseif s < 0 - a = div(unsigned(-diff), -s) % T + a = div(unsigned(-diff), -s) % typeof(diff) else - a = div(unsigned(diff), s) % T + a = div(unsigned(diff), s) % typeof(diff) end - return a + oneunit(T) + return a + oneunit(a) end function checked_length(r::OrdinalRange{T}) where T<:bigints s = step(r) - isempty(r) && return zero(T) stop, start = last(r), first(r) + ET = promote_type(typeof(stop), typeof(start)) + isempty(r) && return zero(ET) # n.b. !(s isa T) if s > 1 diff = stop - start - a = convert(T, div(unsigned(diff), s)) + a = convert(ET, div(unsigned(diff), s)) elseif s < -1 diff = start - stop - a = convert(T, div(unsigned(diff), -s)) + a = convert(ET, div(unsigned(diff), -s)) elseif s > 0 - a = div(checked_sub(stop, start), s) + a = convert(ET, div(checked_sub(stop, start), s)) else - a = div(checked_sub(start, stop), -s) + a = convert(ET, div(checked_sub(start, stop), -s)) end - return checked_add(convert(T, a), oneunit(T)) + return checked_add(a, oneunit(a)) end -end + firstindex(r::StepRange{<:bigints,<:bitints}) = one(last(r)-first(r)) -# some special cases to favor default Int type -let smallints = (Int === Int64 ? - Union{Int8, UInt8, Int16, UInt16, Int32, UInt32} : - Union{Int8, UInt8, Int16, UInt16}) - global length, checked_length - # n.b. !(step isa T) + # some special cases to favor default Int type function length(r::OrdinalRange{<:smallints}) s = step(r) isempty(r) && return 0 - return div(Int(last(r)) - Int(first(r)), s) + 1 + # n.b. !(step isa T) + return Int(div(Int(last(r)) - Int(first(r)), s)) + 1 end length(r::AbstractUnitRange{<:smallints}) = Int(last(r)) - Int(first(r)) + 1 length(r::OneTo{<:smallints}) = Int(r.stop) checked_length(r::OrdinalRange{<:smallints}) = length(r) checked_length(r::AbstractUnitRange{<:smallints}) = length(r) checked_length(r::OneTo{<:smallints}) = length(r) + firstindex(::StepRange{<:smallints,<:bitints}) = 1 end first(r::OrdinalRange{T}) where {T} = convert(T, r.start) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index d1287781931676..f34c295918f6ab 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -325,6 +325,8 @@ function axes(a::ReshapedReinterpretArray{T,N,S} where {N}) where {T,S} end axes(a::NonReshapedReinterpretArray{T,0}) where {T} = () +has_offset_axes(a::ReinterpretArray) = has_offset_axes(a.parent) + elsize(::Type{<:ReinterpretArray{T}}) where {T} = sizeof(T) unsafe_convert(::Type{Ptr{T}}, a::ReinterpretArray{T,N,S} where N) where {T,S} = Ptr{T}(unsafe_convert(Ptr{S},a.parent)) diff --git a/base/subarray.jl b/base/subarray.jl index 17bd6450f0d79d..214a2f98afe313 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -459,3 +459,5 @@ function _indices_sub(i1::AbstractArray, I...) @inline (axes(i1)..., _indices_sub(I...)...) end + +has_offset_axes(S::SubArray) = has_offset_axes(S.indices...) diff --git a/base/tuple.jl b/base/tuple.jl index 6711318bba78ea..875d0173c60596 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -581,15 +581,6 @@ any(x::Tuple{Bool}) = x[1] any(x::Tuple{Bool, Bool}) = x[1]|x[2] any(x::Tuple{Bool, Bool, Bool}) = x[1]|x[2]|x[3] -# equivalent to any(f, t), to be used only in bootstrap -_tuple_any(f::Function, t::Tuple) = _tuple_any(f, false, t...) -function _tuple_any(f::Function, tf::Bool, a, b...) - @inline - _tuple_any(f, tf | f(a), b...) -end -_tuple_any(f::Function, tf::Bool) = tf - - # a version of `in` esp. for NamedTuple, to make it pure, and not compiled for each tuple length function sym_in(x::Symbol, @nospecialize itr::Tuple{Vararg{Symbol}}) @_total_meta diff --git a/test/abstractarray.jl b/test/abstractarray.jl index ad05af606140f5..1ca91cad77d61d 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1583,11 +1583,11 @@ end @test length(rr) == length(r) end -struct FakeZeroDimArray <: AbstractArray{Int, 0} end -Base.strides(::FakeZeroDimArray) = () -Base.size(::FakeZeroDimArray) = () +module IRUtils + include("compiler/irutils.jl") +end + @testset "strides for ReshapedArray" begin - # Type-based contiguous check is tested in test/compiler/inline.jl function check_strides(A::AbstractArray) # Make sure stride(A, i) is equivalent with strides(A)[i] (if 1 <= i <= ndims(A)) dims = ntuple(identity, ndims(A)) @@ -1598,6 +1598,10 @@ Base.size(::FakeZeroDimArray) = () end return true end + # Type-based contiguous Check + a = vec(reinterpret(reshape, Int16, reshape(view(reinterpret(Int32, randn(10)), 2:11), 5, :))) + f(a) = only(strides(a)); + @test IRUtils.fully_eliminated(f, Base.typesof(a)) && f(a) == 1 # General contiguous check a = view(rand(10,10), 1:10, 1:10) @test check_strides(vec(a)) @@ -1629,6 +1633,9 @@ Base.size(::FakeZeroDimArray) = () @test_throws "Input is not strided." strides(reshape(a,3,5,3,2)) @test_throws "Input is not strided." strides(reshape(a,5,3,3,2)) # Zero dimensional parent + struct FakeZeroDimArray <: AbstractArray{Int, 0} end + Base.strides(::FakeZeroDimArray) = () + Base.size(::FakeZeroDimArray) = () a = reshape(FakeZeroDimArray(),1,1,1) @test @inferred(strides(a)) == (1, 1, 1) # Dense parent (but not StridedArray) @@ -1660,3 +1667,21 @@ end @test (@inferred A[i,i,i]) === A[1] @test (@inferred to_indices([], (1, CIdx(1, 1), 1, CIdx(1, 1), 1, CIdx(1, 1), 1))) == ntuple(Returns(1), 10) end + +@testset "type-based offset axes check" begin + a = randn(ComplexF64, 10) + ta = reinterpret(Float64, a) + tb = reinterpret(Float64, view(a, 1:2:10)) + tc = reinterpret(Float64, reshape(view(a, 1:3:10), 2, 2, 1)) + # Issue #44040 + @test IRUtils.fully_eliminated(Base.require_one_based_indexing, Base.typesof(ta, tc)) + @test IRUtils.fully_eliminated(Base.require_one_based_indexing, Base.typesof(tc, tc)) + @test IRUtils.fully_eliminated(Base.require_one_based_indexing, Base.typesof(ta, tc, tb)) + # Ranges && CartesianIndices + @test IRUtils.fully_eliminated(Base.require_one_based_indexing, Base.typesof(1:10, Base.OneTo(10), 1.0:2.0, LinRange(1.0, 2.0, 2), 1:2:10, CartesianIndices((1:2:10, 1:2:10)))) + # Remind us to call `any` in `Base.has_offset_axes` once our compiler is ready. + @inline _has_offset_axes(A) = @inline any(x -> Int(first(x))::Int != 1, axes(A)) + @inline _has_offset_axes(As...) = @inline any(_has_offset_axes, As) + a, b = zeros(2, 2, 2), zeros(2, 2) + @test_broken IRUtils.fully_eliminated(_has_offset_axes, Base.typesof(a, a, b, b)) +end diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 60de41402cceab..0a60aa7635e889 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -988,13 +988,6 @@ end @invoke conditional_escape!(false::Any, x::Any) end -@testset "strides for ReshapedArray (PR#44027)" begin - # Type-based contiguous check - a = vec(reinterpret(reshape,Int16,reshape(view(reinterpret(Int32,randn(10)),2:11),5,:))) - f(a) = only(strides(a)); - @test fully_eliminated(f, Tuple{typeof(a)}) && f(a) == 1 -end - @testset "elimination of `get_binding_type`" begin m = Module() @eval m begin diff --git a/test/ranges.jl b/test/ranges.jl index 6d038747e706d3..404a908adf6b36 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -2031,8 +2031,17 @@ end end @testset "length(StepRange()) type stability" begin - typeof(length(StepRange(1,Int128(1),1))) == typeof(length(StepRange(1,Int128(1),0))) - typeof(checked_length(StepRange(1,Int128(1),1))) == typeof(checked_length(StepRange(1,Int128(1),0))) + for SR in (StepRange{Int,Int128}, StepRange{Int8,Int128}) + r1, r2 = SR(1, 1, 1), SR(1, 1, 0) + @test typeof(length(r1)) == typeof(checked_length(r1)) == + typeof(length(r2)) == typeof(checked_length(r2)) + end + SR = StepRange{Union{Int64,Int128},Int} + test_length(r, l) = length(r) === checked_length(r) === l + @test test_length(SR(Int64(1), 1, Int128(1)), Int128(1)) + @test test_length(SR(Int64(1), 1, Int128(0)), Int128(0)) + @test test_length(SR(Int64(1), 1, Int64(1)), Int64(1)) + @test test_length(SR(Int64(1), 1, Int64(0)), Int64(0)) end @testset "LinRange eltype for element types that wrap integers" begin @@ -2346,3 +2355,13 @@ end @test isempty(range(typemax(Int), length=0, step=UInt(2))) @test length(range(1, length=typemax(Int128))) === typemax(Int128) + +@testset "firstindex(::StepRange{<:Base.BitInteger})" begin + test_firstindex(x) = firstindex(x) === first(Base.axes1(x)) + for T in Base.BitInteger_types, S in Base.BitInteger_types + @test test_firstindex(StepRange{T,S}(1, 1, 1)) + @test test_firstindex(StepRange{T,S}(1, 1, 0)) + end + @test test_firstindex(StepRange{Union{Int64,Int128},Int}(Int64(1), 1, Int128(1))) + @test test_firstindex(StepRange{Union{Int64,Int128},Int}(Int64(1), 1, Int128(0))) +end From d94ed88696824c82cdb6a4c4f1195c8cf831e278 Mon Sep 17 00:00:00 2001 From: pchintalapudi <34727397+pchintalapudi@users.noreply.github.com> Date: Wed, 10 Aug 2022 19:57:28 -0700 Subject: [PATCH 3/3] Mark code instance fields as const or atomic (#46203) This coopts half of #44968 to mark jl_code_instance_t fields as either atomic or const. Co-authored-by: Valentin Churavy --- base/compiler/abstractinterpretation.jl | 12 +++-- base/compiler/ssair/inlining.jl | 2 +- base/compiler/ssair/passes.jl | 2 +- base/compiler/typeinfer.jl | 5 +- src/aotcompile.cpp | 10 ++-- src/codegen.cpp | 69 ++++++++++++++----------- src/dump.c | 11 ++-- src/gf.c | 6 +-- src/jitlayers.cpp | 8 +-- src/jltypes.c | 5 +- src/julia.h | 4 +- src/precompile.c | 8 +-- src/staticdata.c | 11 ++-- test/compiler/inline.jl | 2 +- test/core.jl | 2 +- 15 files changed, 92 insertions(+), 65 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 8fdc66be14bded..31e34506a90467 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1107,9 +1107,15 @@ function const_prop_methodinstance_heuristic( return false else code = get(code_cache(interp), mi, nothing) - if isdefined(code, :inferred) && inlining_policy( - interp, code.inferred, IR_FLAG_NULL, mi, argtypes) !== nothing - return true + if isdefined(code, :inferred) + if isa(code, CodeInstance) + inferred = @atomic :monotonic code.inferred + else + inferred = code.inferred + end + if inlining_policy(interp, inferred, IR_FLAG_NULL, mi, argtypes) !== nothing + return true + end end end end diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index f2b6acf41c2b5d..44d9067679d52a 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -832,7 +832,7 @@ function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) et !== nothing && push!(et, mi) return ConstantCase(quoted(code.rettype_const)) else - src = code.inferred + src = @atomic :monotonic code.inferred end effects = decode_effects(code.ipo_purity_bits) else # fallback pass for external AbstractInterpreter cache diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 58626f5279dde2..594b77b38654a5 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -956,7 +956,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, mi:: et !== nothing && push!(et, mi) return true end - src = code.inferred + src = @atomic :monotonic code.inferred else src = code end diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index a61d407bd7239b..2c66083b9024bc 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -880,7 +880,8 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize mi = specialize_method(method, atype, sparams)::MethodInstance code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # return existing rettype if the code is already inferred - if code.inferred === nothing && is_stmt_inline(get_curr_ssaflag(caller)) + inferred = @atomic :monotonic code.inferred + if inferred === nothing && is_stmt_inline(get_curr_ssaflag(caller)) # we already inferred this edge before and decided to discard the inferred code, # nevertheless we re-infer it here again and keep it around in the local cache # since the inliner will request to use it later @@ -1010,7 +1011,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance) code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # see if this code already exists in the cache - inf = code.inferred + inf = @atomic :monotonic code.inferred if use_const_api(code) i == 2 && ccall(:jl_typeinf_end, Cvoid, ()) tree = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ()) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 1a43fc450db6fc..282024f3874b8f 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -219,7 +219,7 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance jl_code_instance_t *codeinst = NULL; if (ci != jl_nothing) { codeinst = (jl_code_instance_t*)ci; - *src_out = (jl_code_info_t*)codeinst->inferred; + *src_out = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); jl_method_t *def = codeinst->def->def.method; if ((jl_value_t*)*src_out == jl_nothing) *src_out = NULL; @@ -234,8 +234,10 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance *src_out = jl_type_infer(mi, world, 0); if (*src_out) { codeinst = jl_get_method_inferred(mi, (*src_out)->rettype, (*src_out)->min_world, (*src_out)->max_world); - if ((*src_out)->inferred && !codeinst->inferred) - codeinst->inferred = jl_nothing; + if ((*src_out)->inferred) { + jl_value_t *null = nullptr; + jl_atomic_cmpswap_relaxed(&codeinst->inferred, &null, jl_nothing); + } } } } @@ -1007,7 +1009,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz jl_value_t *ci = jl_rettype_inferred(mi, world, world); if (ci != jl_nothing) { jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; - src = (jl_code_info_t*)codeinst->inferred; + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); if ((jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method)) src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src); jlrettype = codeinst->rettype; diff --git a/src/codegen.cpp b/src/codegen.cpp index 2f3974c3a51104..e54dc61fcb6e78 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4839,13 +4839,18 @@ static std::pair get_oc_function(jl_codectx_t &ctx, jl_met jl_method_instance_t *mi = jl_specializations_get_linfo(closure_method, sigtype, jl_emptysvec); jl_code_instance_t *ci = (jl_code_instance_t*)jl_rettype_inferred(mi, ctx.world, ctx.world); - if (ci == NULL || (jl_value_t*)ci == jl_nothing || ci->inferred == NULL || ci->inferred == jl_nothing) { + if (ci == NULL || (jl_value_t*)ci == jl_nothing) { + JL_GC_POP(); + return std::make_pair((Function*)NULL, (Function*)NULL); + } + auto inferred = jl_atomic_load_relaxed(&ci->inferred); + if (!inferred || inferred == jl_nothing) { JL_GC_POP(); return std::make_pair((Function*)NULL, (Function*)NULL); } ++EmittedOpaqueClosureFunctions; - ir = jl_uncompress_ir(closure_method, ci, (jl_array_t*)ci->inferred); + ir = jl_uncompress_ir(closure_method, ci, (jl_array_t*)inferred); // TODO: Emit this inline and outline it late using LLVM's coroutine support. orc::ThreadSafeModule closure_m = jl_create_llvm_module( @@ -8223,7 +8228,7 @@ jl_llvm_functions_t jl_emit_codeinst( JL_TIMING(CODEGEN); JL_GC_PUSH1(&src); if (!src) { - src = (jl_code_info_t*)codeinst->inferred; + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); jl_method_t *def = codeinst->def->def.method; if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def)) src = jl_uncompress_ir(def, codeinst, (jl_array_t*)src); @@ -8254,36 +8259,38 @@ jl_llvm_functions_t jl_emit_codeinst( jl_add_code_in_flight(f, codeinst, DL); } - if (// don't alter `inferred` when the code is not directly being used - params.world && + if (params.world) {// don't alter `inferred` when the code is not directly being used + auto inferred = jl_atomic_load_relaxed(&codeinst->inferred); // don't change inferred state - codeinst->inferred) { - jl_method_t *def = codeinst->def->def.method; - if (// keep code when keeping everything - !(JL_DELETE_NON_INLINEABLE) || - // aggressively keep code when debugging level >= 2 - jl_options.debug_level > 1) { - // update the stored code - if (codeinst->inferred != (jl_value_t*)src) { - if (jl_is_method(def)) { - src = (jl_code_info_t*)jl_compress_ir(def, src); - assert(jl_typeis(src, jl_array_uint8_type)); - codeinst->relocatability = ((uint8_t*)jl_array_data(src))[jl_array_len(src)-1]; + if (inferred) { + jl_method_t *def = codeinst->def->def.method; + if (// keep code when keeping everything + !(JL_DELETE_NON_INLINEABLE) || + // aggressively keep code when debugging level >= 2 + jl_options.debug_level > 1) { + // update the stored code + if (inferred != (jl_value_t*)src) { + if (jl_is_method(def)) { + src = (jl_code_info_t*)jl_compress_ir(def, src); + assert(jl_typeis(src, jl_array_uint8_type)); + codeinst->relocatability = ((uint8_t*)jl_array_data(src))[jl_array_len(src)-1]; + } + jl_atomic_store_release(&codeinst->inferred, (jl_value_t*)src); + jl_gc_wb(codeinst, src); + } + } + else if (jl_is_method(def)) {// don't delete toplevel code + if (// and there is something to delete (test this before calling jl_ir_inlining_cost) + inferred != jl_nothing && + // don't delete inlineable code, unless it is constant + (codeinst->invoke == jl_fptr_const_return_addr || + (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) && + // don't delete code when generating a precompile file + !(params.imaging || jl_options.incremental)) { + // if not inlineable, code won't be needed again + jl_atomic_store_release(&codeinst->inferred, jl_nothing); } - codeinst->inferred = (jl_value_t*)src; - jl_gc_wb(codeinst, src); } - } - else if (// don't delete toplevel code - jl_is_method(def) && - // and there is something to delete (test this before calling jl_ir_inlining_cost) - codeinst->inferred != jl_nothing && - // don't delete inlineable code, unless it is constant - (codeinst->invoke == jl_fptr_const_return_addr || (jl_ir_inlining_cost((jl_array_t*)codeinst->inferred) == UINT16_MAX)) && - // don't delete code when generating a precompile file - !(params.imaging || jl_options.incremental)) { - // if not inlineable, code won't be needed again - codeinst->inferred = jl_nothing; } } } @@ -8336,7 +8343,7 @@ void jl_compile_workqueue( // Reinfer the function. The JIT came along and removed the inferred // method body. See #34993 if (policy != CompilationPolicy::Default && - codeinst->inferred && codeinst->inferred == jl_nothing) { + jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { src = jl_type_infer(codeinst->def, jl_atomic_load_acquire(&jl_world_counter), 0); if (src) { orc::ThreadSafeModule result_m = diff --git a/src/dump.c b/src/dump.c index cdf6dedb83e590..37aa011e1deae9 100644 --- a/src/dump.c +++ b/src/dump.c @@ -689,10 +689,10 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ write_uint8(s->s, TAG_CODE_INSTANCE); write_uint8(s->s, flags); write_uint32(s->s, codeinst->ipo_purity_bits); - write_uint32(s->s, codeinst->purity_bits); + write_uint32(s->s, jl_atomic_load_relaxed(&codeinst->purity_bits)); jl_serialize_value(s, (jl_value_t*)codeinst->def); if (write_ret_type) { - jl_serialize_value(s, codeinst->inferred); + jl_serialize_value(s, jl_atomic_load_relaxed(&codeinst->inferred)); jl_serialize_value(s, codeinst->rettype_const); jl_serialize_value(s, codeinst->rettype); jl_serialize_value(s, codeinst->argescapes); @@ -1907,11 +1907,12 @@ static jl_value_t *jl_deserialize_value_code_instance(jl_serializer_state *s, jl int validate = (flags >> 0) & 3; int constret = (flags >> 2) & 1; codeinst->ipo_purity_bits = read_uint32(s->s); - codeinst->purity_bits = read_uint32(s->s); + jl_atomic_store_relaxed(&codeinst->purity_bits, read_uint32(s->s)); codeinst->def = (jl_method_instance_t*)jl_deserialize_value(s, (jl_value_t**)&codeinst->def); jl_gc_wb(codeinst, codeinst->def); - codeinst->inferred = jl_deserialize_value(s, &codeinst->inferred); - jl_gc_wb(codeinst, codeinst->inferred); + jl_value_t *inferred = jl_deserialize_value(s, NULL); + jl_atomic_store_release(&codeinst->inferred, inferred); + jl_gc_wb(codeinst, inferred); codeinst->rettype_const = jl_deserialize_value(s, &codeinst->rettype_const); if (codeinst->rettype_const) jl_gc_wb(codeinst, codeinst->rettype_const); diff --git a/src/gf.c b/src/gf.c index 1d36589a082f5a..c9d91ec836a0b4 100644 --- a/src/gf.c +++ b/src/gf.c @@ -363,7 +363,7 @@ JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *mi, size_t mi jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); while (codeinst) { if (codeinst->min_world <= min_world && max_world <= codeinst->max_world) { - jl_value_t *code = codeinst->inferred; + jl_value_t *code = jl_atomic_load_relaxed(&codeinst->inferred); if (code && (code == jl_nothing || jl_ir_flag_inferred((jl_array_t*)code))) return (jl_value_t*)codeinst; } @@ -409,7 +409,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( codeinst->min_world = min_world; codeinst->max_world = max_world; codeinst->rettype = rettype; - codeinst->inferred = inferred; + jl_atomic_store_release(&codeinst->inferred, inferred); //codeinst->edges = NULL; if ((const_flags & 2) == 0) inferred_const = NULL; @@ -424,7 +424,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( jl_atomic_store_relaxed(&codeinst->precompile, 0); jl_atomic_store_relaxed(&codeinst->next, NULL); codeinst->ipo_purity_bits = ipo_effects; - codeinst->purity_bits = effects; + jl_atomic_store_relaxed(&codeinst->purity_bits, effects); codeinst->argescapes = argescapes; codeinst->relocatability = relocatability; return codeinst; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 6e5253a952fa32..ad91d10124b5a3 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -326,7 +326,7 @@ jl_code_instance_t *jl_generate_fptr_impl(jl_method_instance_t *mi JL_PROPAGATES jl_value_t *ci = jl_rettype_inferred(mi, world, world); jl_code_instance_t *codeinst = (ci == jl_nothing ? NULL : (jl_code_instance_t*)ci); if (codeinst) { - src = (jl_code_info_t*)codeinst->inferred; + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); if ((jl_value_t*)src == jl_nothing) src = NULL; else if (jl_is_method(mi->def.method)) @@ -352,8 +352,10 @@ jl_code_instance_t *jl_generate_fptr_impl(jl_method_instance_t *mi JL_PROPAGATES else if (src && jl_is_code_info(src)) { if (!codeinst) { codeinst = jl_get_method_inferred(mi, src->rettype, src->min_world, src->max_world); - if (src->inferred && !codeinst->inferred) - codeinst->inferred = jl_nothing; + if (src->inferred) { + jl_value_t *null = nullptr; + jl_atomic_cmpswap_relaxed(&codeinst->inferred, &null, jl_nothing); + } } _jl_compile_codeinst(codeinst, src, world, context); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) diff --git a/src/jltypes.c b/src/jltypes.c index a6b63863d8e7c0..8eb43076e46a57 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2578,8 +2578,11 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, 0, 1, 1); jl_svecset(jl_code_instance_type->types, 1, jl_code_instance_type); - const static uint32_t code_instance_constfields[1] = { 0x00000001 }; // (1<<1); + const static uint32_t code_instance_constfields[1] = { 0b000001010111101 }; // Set fields 1, 3-6, 8, 10 as const + const static uint32_t code_instance_atomicfields[1] = { 0b011100101000010 }; // Set fields 2, 7, 9, 12-14 as atomic + //Fields 11 and 15 must be protected by locks, and thus all operations on jl_code_instance_t are threadsafe jl_code_instance_type->name->constfields = code_instance_constfields; + jl_code_instance_type->name->atomicfields = code_instance_atomicfields; jl_const_type = jl_new_datatype(jl_symbol("Const"), core, jl_any_type, jl_emptysvec, jl_perm_symsvec(1, "val"), diff --git a/src/julia.h b/src/julia.h index 09cd7db196197e..1980d22eeaf38b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -392,7 +392,7 @@ typedef struct _jl_code_instance_t { // inference state cache jl_value_t *rettype; // return type for fptr jl_value_t *rettype_const; // inferred constant return value, or null - jl_value_t *inferred; // inferred jl_code_info_t, or jl_nothing, or null + _Atomic(jl_value_t *) inferred; // inferred jl_code_info_t, or jl_nothing, or null //TODO: jl_array_t *edges; // stored information about edges from this object //TODO: uint8_t absolute_max; // whether true max world is unknown @@ -425,7 +425,7 @@ typedef struct _jl_code_instance_t { }; #else uint32_t ipo_purity_bits; - uint32_t purity_bits; + _Atomic(uint32_t) purity_bits; #endif jl_value_t *argescapes; // escape information of call arguments diff --git a/src/precompile.c b/src/precompile.c index bc0239ccf3cc50..8e1fed06837ed3 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -336,9 +336,11 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur while (codeinst) { int do_compile = 0; if (jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return) { - if (codeinst->inferred && codeinst->inferred != jl_nothing && - jl_ir_flag_inferred((jl_array_t*)codeinst->inferred) && - (jl_ir_inlining_cost((jl_array_t*)codeinst->inferred) == UINT16_MAX)) { + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); + if (inferred && + inferred != jl_nothing && + jl_ir_flag_inferred((jl_array_t*)inferred) && + (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { do_compile = 1; } else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { diff --git a/src/staticdata.c b/src/staticdata.c index 8a3d4132c42f5d..9f5c1e64ba9281 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1707,13 +1707,16 @@ static void strip_specializations_(jl_method_instance_t *mi) assert(jl_is_method_instance(mi)); jl_code_instance_t *codeinst = mi->cache; while (codeinst) { - if (codeinst->inferred && codeinst->inferred != jl_nothing) { + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); + if (inferred && inferred != jl_nothing) { if (jl_options.strip_ir) { - record_field_change(&codeinst->inferred, jl_nothing); + record_field_change(&inferred, jl_nothing); } else if (jl_options.strip_metadata) { - codeinst->inferred = strip_codeinfo_meta(mi->def.method, codeinst->inferred, 0); - jl_gc_wb(codeinst, codeinst->inferred); + jl_value_t *stripped = strip_codeinfo_meta(mi->def.method, inferred, 0); + if (jl_atomic_cmpswap_relaxed(&codeinst->inferred, &inferred, stripped)) { + jl_gc_wb(codeinst, stripped); + } } } codeinst = jl_atomic_load_relaxed(&codeinst->next); diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 0a60aa7635e889..c6a3e95c9de135 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -625,7 +625,7 @@ let specs = collect(only(methods(f42078)).specializations) mi = specs[findfirst(!isnothing, specs)]::Core.MethodInstance codeinf = getcache(mi)::Core.CodeInstance - codeinf.inferred = nothing + @atomic codeinf.inferred = nothing end let # inference should re-infer `f42078(::Int)` and we should get the same code diff --git a/test/core.jl b/test/core.jl index f6c7f3a8133027..a0bd78df8e1ce0 100644 --- a/test/core.jl +++ b/test/core.jl @@ -14,7 +14,7 @@ include("testenv.jl") # sanity tests that our built-in types are marked correctly for const fields for (T, c) in ( (Core.CodeInfo, []), - (Core.CodeInstance, [:def]), + (Core.CodeInstance, [:def, :min_world, :max_world, :rettype, :rettype_const, :ipo_purity_bits, :argescapes]), (Core.Method, [#=:name, :module, :file, :line, :primary_world, :sig, :slot_syms, :external_mt, :nargs, :called, :nospecialize, :nkw, :isva, :pure, :is_for_opaque_closure, :constprop=#]), (Core.MethodInstance, [#=:def, :specTypes, :sparam_vals]=#]), (Core.MethodTable, [:module]),