Backports for 1.11.2 (JuliaLang#56228)

Backported PRs:
- [x] JuliaLang#55886 <!-- irrationals: restrict assume effects annotations to
known types -->
- [x] JuliaLang#55867 <!-- update `hash` doc string: `widen` not required any
more -->
- [x] JuliaLang#56084 <!-- slightly improve inference in precompilation code -->
- [x] JuliaLang#56088 <!-- make `Base.ANSIIterator` have a concrete field -->
- [x] JuliaLang#54093 <!-- Fix `JULIA_CPU_TARGET` being propagated to workers
precompiling stdlib pkgimages -->
- [x] JuliaLang#56165 <!-- Fix markdown list in installation.md -->
- [x] JuliaLang#56148 <!-- Make loading work when stdlib deps are missing in the
manifest -->
- [x] JuliaLang#56174 <!-- Fix implicit `convert(String, ...)` in several places
-->
- [x] JuliaLang#56159 <!-- Add invalidation barriers for `displaysize` and
`implicit_typeinfo` -->
- [x] JuliaLang#56089 <!-- Call `MulAddMul` instead of multiplication in
_generic_matmatmul! -->
- [x] JuliaLang#56195 <!-- Include default user depot when JULIA_DEPOT_PATH has
leading empty entry -->
- [x] JuliaLang#56215 <!-- [REPL] fix lock ordering mistake in load_pkg -->
- [x] JuliaLang#56251 <!-- REPL: run repl hint generation for modeswitch chars
when not switching -->
- [x] JuliaLang#56092 <!-- stream: fix reading LibuvStream into array -->
- [x] JuliaLang#55870 <!-- fix infinite recursion in `promote_type` for
`Irrational` -->
- [x] JuliaLang#56227 <!-- Do not call `rand` during sysimage precompilation -->
- [x] JuliaLang#55741 <!-- Change annotations to use a NamedTuple -->
- [x] JuliaLang#56149 <!-- Specialize adding/subtracting mixed
Upper/LowerTriangular -->
- [x] JuliaLang#56214 <!-- fix precompile process flags -->
- [x] JuliaLang#54471
- [x] JuliaLang#55622
- [x] JuliaLang#55704
- [x] JuliaLang#55764

NEWS.md

@@ -37,6 +37,10 @@ Language changes
   omit the default user depot ([#51448]).
 * Precompilation cache files are now relocatable and their validity is now verified through
   a content hash of their source files instead of their `mtime` ([#49866]).
+* Extensions may now depend on other extensions, if their triggers include all triggers of any
+  extension they wish to depend upon (+ at least one other trigger). Ext-to-ext dependencies
+  that don't meet this requirement are now blocked from using `Base.get_extension` during pre-
+  compilation, to prevent extension cycles [#55557].
 
 Compiler/Runtime improvements
 -----------------------------
@@ -139,6 +143,17 @@ Standard library changes
   `length(::Stateful)` method. The last type parameter of `Stateful` is gone, too. Issue: ([#47790]),
   PR: ([#51747]).
 
+#### Package Manager
+
+* It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml.
+  This can be used to add non-registered normal or test dependencies.
+* Pkg now obeys `[compat]` bounds for `julia` and raises an error if the version of the running Julia binary is incompatible with the bounds in `Project.toml`.
+  Pkg has always obeyed this compat when working with Registry packages. This change affects mostly local packages
+* `pkg> add` and `Pkg.add` will now add compat entries for new direct dependencies if the active environment is a
+  package (has a `name` and `uuid` entry).
+* Dependencies can now be directly added as weak deps or extras via the `pkg> add --weak/extra Foo` or
+  `Pkg.add("Foo", target=:weakdeps/:extras)` forms.
+
 #### StyledStrings
 
 * A new standard library for handling styling in a more comprehensive and structured way ([#49586]).

base/Base.jl

@@ -508,6 +508,7 @@ include("deepcopy.jl")
 include("download.jl")
 include("summarysize.jl")
 include("errorshow.jl")
+include("util.jl")
 
 include("initdefs.jl")
 Filesystem.__postinit__()
@@ -524,7 +525,6 @@ include("loading.jl")
 
 # misc useful functions & macros
 include("timing.jl")
-include("util.jl")
 include("client.jl")
 include("asyncmap.jl")
 
@@ -605,7 +605,7 @@ function __init__()
     empty!(explicit_loaded_modules)
     empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
     for (mod, key) in module_keys
-        loaded_precompiles[key => module_build_id(mod)] = mod
+        push!(get!(Vector{Module}, loaded_precompiles, key), mod)
     end
     if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
         MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])

base/abstractarray.jl

@@ -1634,10 +1634,10 @@ typed_vcat(::Type{T}) where {T} = Vector{T}()
 typed_hcat(::Type{T}) where {T} = Vector{T}()
 
 ## cat: special cases
-vcat(X::T...) where {T}         = T[ X[i] for i=1:length(X) ]
-vcat(X::T...) where {T<:Number} = T[ X[i] for i=1:length(X) ]
-hcat(X::T...) where {T}         = T[ X[j] for i=1:1, j=1:length(X) ]
-hcat(X::T...) where {T<:Number} = T[ X[j] for i=1:1, j=1:length(X) ]
+vcat(X::T...) where {T}         = T[ X[i] for i=eachindex(X) ]
+vcat(X::T...) where {T<:Number} = T[ X[i] for i=eachindex(X) ]
+hcat(X::T...) where {T}         = T[ X[j] for i=1:1, j=eachindex(X) ]
+hcat(X::T...) where {T<:Number} = T[ X[j] for i=1:1, j=eachindex(X) ]
 
 vcat(X::Number...) = hvcat_fill!(Vector{promote_typeof(X...)}(undef, length(X)), X)
 hcat(X::Number...) = hvcat_fill!(Matrix{promote_typeof(X...)}(undef, 1,length(X)), X)

base/array.jl

@@ -351,6 +351,17 @@ copy
     return $(Expr(:new, :(typeof(a)), :(memoryref(newmem)), :(a.size)))
 end
 
+# a mutating version of copyto! that results in dst aliasing src afterwards
+function _take!(dst::Array{T,N}, src::Array{T,N}) where {T,N}
+    if getfield(dst, :ref) !== getfield(src, :ref)
+        setfield!(dst, :ref, getfield(src, :ref))
+    end
+    if getfield(dst, :size) !== getfield(src, :size)
+        setfield!(dst, :size, getfield(src, :size))
+    end
+    return dst
+end
+
 ## Constructors ##
 
 similar(a::Array{T,1}) where {T}                    = Vector{T}(undef, size(a,1))

base/arrayshow.jl

@@ -545,6 +545,12 @@ typeinfo_eltype(typeinfo::Type{<:AbstractArray{T}}) where {T} = eltype(typeinfo)
 typeinfo_eltype(typeinfo::Type{<:AbstractDict{K,V}}) where {K,V} = eltype(typeinfo)
 typeinfo_eltype(typeinfo::Type{<:AbstractSet{T}}) where {T} = eltype(typeinfo)
 
+# This is a fancy way to make de-specialize a call to `typeinfo_implicit(T)`
+# which is unfortunately invalidated by Dates
+#  (https://github.com/JuliaLang/julia/issues/56080)
+#
+# This makes the call less efficient, but avoids being invalidated by Dates.
+_typeinfo_implicit(@nospecialize(T)) = Base.invoke_in_world(Base.tls_world_age(), typeinfo_implicit, T)::Bool
 
 # types that can be parsed back accurately from their un-decorated representations
 function typeinfo_implicit(@nospecialize(T))
@@ -553,9 +559,9 @@ function typeinfo_implicit(@nospecialize(T))
         return true
     end
     return isconcretetype(T) &&
-        ((T <: Array && typeinfo_implicit(eltype(T))) ||
-         ((T <: Tuple || T <: Pair) && all(typeinfo_implicit, fieldtypes(T))) ||
-         (T <: AbstractDict && typeinfo_implicit(keytype(T)) && typeinfo_implicit(valtype(T))))
+        ((T <: Array && _typeinfo_implicit(eltype(T))) ||
+         ((T <: Tuple || T <: Pair) && all(_typeinfo_implicit, fieldtypes(T))) ||
+         (T <: AbstractDict && _typeinfo_implicit(keytype(T)) && _typeinfo_implicit(valtype(T))))
 end
 
 # X not constrained, can be any iterable (cf. show_vector)
@@ -573,7 +579,7 @@ function typeinfo_prefix(io::IO, X)
     if X isa AbstractDict
         if eltype_X == eltype_ctx
             sprint(show_type_name, typeof(X).name; context=io), false
-        elseif !isempty(X) && typeinfo_implicit(keytype(X)) && typeinfo_implicit(valtype(X))
+        elseif !isempty(X) && _typeinfo_implicit(keytype(X)) && _typeinfo_implicit(valtype(X))
             sprint(show_type_name, typeof(X).name; context=io), true
         else
             sprint(print, typeof(X); context=io), false
@@ -582,7 +588,7 @@ function typeinfo_prefix(io::IO, X)
         # Types hard-coded here are those which are created by default for a given syntax
         if eltype_X == eltype_ctx
             "", false
-        elseif !isempty(X) && typeinfo_implicit(eltype_X)
+        elseif !isempty(X) && _typeinfo_implicit(eltype_X)
             "", true
         elseif print_without_params(eltype_X)
             sprint(show_type_name, unwrap_unionall(eltype_X).name; context=io), false # Print "Array" rather than "Array{T,N}"

base/docs/basedocs.jl

@@ -37,6 +37,14 @@ kw"help", kw"Julia", kw"julia", kw""
 available for direct use. Names can also be used via dot syntax (e.g. `Foo.foo` to access
 the name `foo`), whether they are `export`ed or not.
 See the [manual section about modules](@ref modules) for details.
+
+!!! note
+    When two or more packages/modules export a name and that name does not refer to the
+    same thing in each of the packages, and the packages are loaded via `using` without
+    an explicit list of names, it is an error to reference that name without qualification.
+    It is thus recommended that code intended to be forward-compatible with future versions
+    of its dependencies and of Julia, e.g., code in released packages, list the names it
+    uses from each loaded package, e.g., `using Foo: Foo, f` rather than `using Foo`.
 """
 kw"using"
 

base/errorshow.jl

@@ -196,7 +196,7 @@ function showerror(io::IO, ex::CanonicalIndexError)
     print(io, "CanonicalIndexError: ", ex.func, " not defined for ", ex.type)
 end
 
-typesof(@nospecialize args...) = Tuple{Any[ Core.Typeof(args[i]) for i in 1:length(args) ]...}
+typesof(@nospecialize args...) = Tuple{Any[Core.Typeof(arg) for arg in args]...}
 
 function print_with_compare(io::IO, @nospecialize(a::DataType), @nospecialize(b::DataType), color::Symbol)
     if a.name === b.name
@@ -273,7 +273,7 @@ function showerror(io::IO, ex::MethodError)
         arg_types_param = arg_types_param[3:end]
         san_arg_types_param = san_arg_types_param[3:end]
         keys = kwt.parameters[1]::Tuple
-        kwargs = Any[(keys[i], fieldtype(kwt, i)) for i in 1:length(keys)]
+        kwargs = Any[(keys[i], fieldtype(kwt, i)) for i in eachindex(keys)]
         arg_types = rewrap_unionall(Tuple{arg_types_param...}, arg_types)
     end
     if f === Base.convert && length(arg_types_param) == 2 && !is_arg_types
@@ -687,7 +687,7 @@ function show_reduced_backtrace(io::IO, t::Vector)
 
     push!(repeated_cycle, (0,0,0)) # repeated_cycle is never empty
     frame_counter = 1
-    for i in 1:length(displayed_stackframes)
+    for i in eachindex(displayed_stackframes)
         (frame, n) = displayed_stackframes[i]
 
         print_stackframe(io, frame_counter, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS)
@@ -864,7 +864,7 @@ end
 function _collapse_repeated_frames(trace)
     kept_frames = trues(length(trace))
     last_frame = nothing
-    for i in 1:length(trace)
+    for i in eachindex(trace)
         frame::StackFrame, _ = trace[i]
         if last_frame !== nothing && frame.file == last_frame.file && frame.line == last_frame.line
             #=
@@ -909,7 +909,7 @@ function _collapse_repeated_frames(trace)
                 end
                 if length(last_params) > length(params)
                     issame = true
-                    for i = 1:length(params)
+                    for i = eachindex(params)
                         issame &= params[i] == last_params[i]
                     end
                     if issame

base/expr.jl

@@ -68,7 +68,7 @@ function copy_exprs(@nospecialize(x))
     end
     return x
 end
-copy_exprargs(x::Array{Any,1}) = Any[copy_exprs(@inbounds x[i]) for i in 1:length(x)]
+copy_exprargs(x::Array{Any,1}) = Any[copy_exprs(@inbounds x[i]) for i in eachindex(x)]
 
 @eval exprarray(head::Symbol, arg::Array{Any,1}) = $(Expr(:new, :Expr, :head, :arg))
 

base/genericmemory.jl

@@ -80,17 +80,16 @@ function _unsetindex!(A::MemoryRef{T}) where T
     MemT = typeof(mem)
     arrayelem = datatype_arrayelem(MemT)
     elsz = datatype_layoutsize(MemT)
-    isboxed = 1; isunion = 2
+    isbits = 0; isboxed = 1; isunion = 2
+    arrayelem == isbits && datatype_pointerfree(T::DataType) && return A
     t = @_gc_preserve_begin mem
     p = Ptr{Ptr{Cvoid}}(@inbounds pointer(A))
     if arrayelem == isboxed
         Intrinsics.atomic_pointerset(p, C_NULL, :monotonic)
     elseif arrayelem != isunion
-        if !datatype_pointerfree(T::DataType)
-            for j = 1:Core.sizeof(Ptr{Cvoid}):elsz
-                # XXX: this violates memory ordering, since it writes more than one C_NULL to each
-                Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic)
-            end
+        for j = 1:Core.sizeof(Ptr{Cvoid}):elsz
+            # XXX: this violates memory ordering, since it writes more than one C_NULL to each
+            Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic)
         end
     end
     @_gc_preserve_end t
@@ -114,7 +113,17 @@ function unsafe_copyto!(dest::MemoryRef{T}, src::MemoryRef{T}, n) where {T}
     @_terminates_globally_notaskstate_meta
     n == 0 && return dest
     @boundscheck memoryref(dest, n), memoryref(src, n)
-    ccall(:jl_genericmemory_copyto, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int), dest.mem, dest.ptr_or_offset, src.mem, src.ptr_or_offset, Int(n))
+    if isbitstype(T)
+        tdest = @_gc_preserve_begin dest
+        tsrc = @_gc_preserve_begin src
+        pdest = unsafe_convert(Ptr{Cvoid}, dest)
+        psrc = unsafe_convert(Ptr{Cvoid}, src)
+        memmove(pdest, psrc, aligned_sizeof(T) * n)
+        @_gc_preserve_end tdest
+        @_gc_preserve_end tsrc
+    else
+        ccall(:jl_genericmemory_copyto, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int), dest.mem, dest.ptr_or_offset, src.mem, src.ptr_or_offset, Int(n))
+    end
     return dest
 end
 

base/hashing.jl

@@ -11,9 +11,7 @@ optional second argument `h` is another hash code to be mixed with the result.
 New types should implement the 2-argument form, typically by calling the 2-argument `hash`
 method recursively in order to mix hashes of the contents with each other (and with `h`).
 Typically, any type that implements `hash` should also implement its own [`==`](@ref) (hence
-[`isequal`](@ref)) to guarantee the property mentioned above. Types supporting subtraction
-(operator `-`) should also implement [`widen`](@ref), which is required to hash
-values inside heterogeneous arrays.
+[`isequal`](@ref)) to guarantee the property mentioned above.
 
 The hash value may change when a new Julia process is started.
 

base/initdefs.jl

@@ -112,20 +112,23 @@ function init_depot_path()
 
         # otherwise, populate the depot path with the entries in JULIA_DEPOT_PATH,
         # expanding empty strings to the bundled depot
-        populated = false
-        for path in eachsplit(str, Sys.iswindows() ? ';' : ':')
+        pushfirst_default = true
+        for (i, path) in enumerate(eachsplit(str, Sys.iswindows() ? ';' : ':'))
             if isempty(path)
                 append_bundled_depot_path!(DEPOT_PATH)
             else
                 path = expanduser(path)
                 path in DEPOT_PATH || push!(DEPOT_PATH, path)
-                populated = true
+                if i == 1
+                    # if a first entry is given, don't add the default depot at the start
+                    pushfirst_default = false
+                end
             end
         end
 
         # backwards compatibility: if JULIA_DEPOT_PATH only contains empty entries
         # (e.g., JULIA_DEPOT_PATH=':'), make sure to use the default depot
-        if !populated
+        if pushfirst_default
             pushfirst!(DEPOT_PATH, joinpath(homedir(), ".julia"))
         end
     else

base/intfuncs.jl

@@ -976,7 +976,7 @@ end
 
 Return an array with element type `T` (default `Int`) of the digits of `n` in the given
 base, optionally padded with zeros to a specified size. More significant digits are at
-higher indices, such that `n == sum(digits[k]*base^(k-1) for k=1:length(digits))`.
+higher indices, such that `n == sum(digits[k]*base^(k-1) for k in eachindex(digits))`.
 
 See also [`ndigits`](@ref), [`digits!`](@ref),
 and for base 2 also [`bitstring`](@ref), [`count_ones`](@ref).

base/irrationals.jl

@@ -45,14 +45,22 @@ promote_rule(::Type{<:AbstractIrrational}, ::Type{Float16}) = Float16
 promote_rule(::Type{<:AbstractIrrational}, ::Type{Float32}) = Float32
 promote_rule(::Type{<:AbstractIrrational}, ::Type{<:AbstractIrrational}) = Float64
 promote_rule(::Type{<:AbstractIrrational}, ::Type{T}) where {T<:Real} = promote_type(Float64, T)
-promote_rule(::Type{S}, ::Type{T}) where {S<:AbstractIrrational,T<:Number} = promote_type(promote_type(S, real(T)), T)
+
+function promote_rule(::Type{S}, ::Type{T}) where {S<:AbstractIrrational,T<:Number}
+    U = promote_type(S, real(T))
+    if S <: U
+        # prevent infinite recursion
+        promote_type(Float64, T)
+    else
+        promote_type(U, T)
+    end
+end
 
 AbstractFloat(x::AbstractIrrational) = Float64(x)::Float64
 Float16(x::AbstractIrrational) = Float16(Float32(x)::Float32)
 Complex{T}(x::AbstractIrrational) where {T<:Real} = Complex{T}(T(x))
 
-# XXX this may change `DEFAULT_PRECISION`, thus not effect free
-@assume_effects :total function Rational{T}(x::AbstractIrrational) where T<:Integer
+function _irrational_to_rational(::Type{T}, x::AbstractIrrational) where T<:Integer
     o = precision(BigFloat)
     p = 256
     while true
@@ -66,13 +74,16 @@ Complex{T}(x::AbstractIrrational) where {T<:Real} = Complex{T}(T(x))
         p += 32
     end
 end
-Rational{BigInt}(x::AbstractIrrational) = throw(ArgumentError("Cannot convert an AbstractIrrational to a Rational{BigInt}: use rationalize(BigInt, x) instead"))
+Rational{T}(x::AbstractIrrational) where {T<:Integer} = _irrational_to_rational(T, x)
+_throw_argument_error_irrational_to_rational_bigint() = throw(ArgumentError("Cannot convert an AbstractIrrational to a Rational{BigInt}: use rationalize(BigInt, x) instead"))
+Rational{BigInt}(::AbstractIrrational) = _throw_argument_error_irrational_to_rational_bigint()
 
-@assume_effects :total function (t::Type{T})(x::AbstractIrrational, r::RoundingMode) where T<:Union{Float32,Float64}
+function _irrational_to_float(::Type{T}, x::AbstractIrrational, r::RoundingMode) where T<:Union{Float32,Float64}
     setprecision(BigFloat, 256) do
         T(BigFloat(x)::BigFloat, r)
     end
 end
+(::Type{T})(x::AbstractIrrational, r::RoundingMode) where {T<:Union{Float32,Float64}} = _irrational_to_float(T, x, r)
 
 float(::Type{<:AbstractIrrational}) = Float64
 
@@ -110,14 +121,18 @@ end
 <=(x::AbstractFloat, y::AbstractIrrational) = x < y
 
 # Irrational vs Rational
-@assume_effects :total function rationalize(::Type{T}, x::AbstractIrrational; tol::Real=0) where T
+function _rationalize_irrational(::Type{T}, x::AbstractIrrational, tol::Real) where {T<:Integer}
     return rationalize(T, big(x), tol=tol)
 end
-@assume_effects :total function lessrational(rx::Rational{<:Integer}, x::AbstractIrrational)
-    # an @assume_effects :total version of `<` for determining if the rationalization of
-    # an irrational number required rounding up or down
+function rationalize(::Type{T}, x::AbstractIrrational; tol::Real=0) where {T<:Integer}
+    return _rationalize_irrational(T, x, tol)
+end
+function _lessrational(rx::Rational, x::AbstractIrrational)
     return rx < big(x)
 end
+function lessrational(rx::Rational, x::AbstractIrrational)
+    return _lessrational(rx, x)
+end
 function <(x::AbstractIrrational, y::Rational{T}) where T
     T <: Unsigned && x < 0.0 && return true
     rx = rationalize(T, x)