Merge branch 'master' into subtype_fix

NEWS.md

@@ -158,6 +158,8 @@ Standard library changes
 
 #### DelimitedFiles
 
+* DelimitedFiles has been promoted from being a standard library to a separate package. It now has to be explicitly installed to be used.
+
 
 Deprecated or removed
 ---------------------

base/compiler/optimize.jl

@@ -509,8 +509,8 @@ macro pass(name, expr)
     end
 end
 
-matchpass(optimize_until::Int, stage, _name) = optimize_until < stage
-matchpass(optimize_until::String, _stage, name) = optimize_until == name
+matchpass(optimize_until::Int, stage, _) = optimize_until == stage
+matchpass(optimize_until::String, _, name) = optimize_until == name
 matchpass(::Nothing, _, _) = false
 
 function run_passes(
@@ -519,7 +519,7 @@ function run_passes(
     caller::InferenceResult,
     optimize_until = nothing,  # run all passes by default
 )
-    __stage__ = 1  # used by @pass
+    __stage__ = 0  # used by @pass
     # NOTE: The pass name MUST be unique for `optimize_until::AbstractString` to work
     @pass "convert"   ir = convert_to_ircode(ci, sv)
     @pass "slot2reg"  ir = slot2reg(ir, ci, sv)

base/compiler/typeutils.jl

@@ -4,8 +4,6 @@
 # lattice utilities #
 #####################
 
-isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME
-
 # true if Type{T} is inlineable as constant T
 # requires that T is a singleton, s.t. T == S implies T === S
 isconstType(@nospecialize t) = isType(t) && hasuniquerep(t.parameters[1])

base/deprecated.jl

@@ -159,7 +159,7 @@ function firstcaller(bt::Vector, funcsyms)
                 li = lkup.linfo
                 if li isa Core.MethodInstance
                     ft = ccall(:jl_first_argument_datatype, Any, (Any,), (li.def::Method).sig)
-                    if isa(ft, DataType) && ft.name === Type.body.name
+                    if isType(ft)
                         ft = unwrap_unionall(ft.parameters[1])
                         found = (isa(ft, DataType) && ft.name.name in funcsyms)
                     end

base/errorshow.jl

@@ -400,7 +400,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=()
     # pool MethodErrors for these two functions.
     if f === convert && !isempty(arg_types_param)
         at1 = arg_types_param[1]
-        if isa(at1,DataType) && (at1::DataType).name === Type.body.name && !Core.Compiler.has_free_typevars(at1)
+        if isType(at1) && !Core.Compiler.has_free_typevars(at1)
             push!(funcs, (at1.parameters[1], arg_types_param[2:end]))
         end
     end

base/loading.jl

@@ -1312,8 +1312,11 @@ end
 
 # get a top-level Module from the given key
 root_module(key::PkgId) = @lock require_lock loaded_modules[key]
-root_module(where::Module, name::Symbol) =
-    root_module(identify_package(where, String(name)))
+function root_module(where::Module, name::Symbol)
+    key = identify_package(where, String(name))
+    key isa PkgId || throw(KeyError(name))
+    return root_module(key)
+end
 maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing)
 
 root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key)

base/reflection.jl

@@ -605,12 +605,14 @@ has_free_typevars(@nospecialize(t)) = ccall(:jl_has_free_typevars, Cint, (Any,),
 
 # equivalent to isa(v, Type) && isdispatchtuple(Tuple{v}) || v === Union{}
 # and is thus perhaps most similar to the old (pre-1.0) `isleaftype` query
-const _TYPE_NAME = Type.body.name
 function isdispatchelem(@nospecialize v)
     return (v === Bottom) || (v === typeof(Bottom)) || isconcretedispatch(v) ||
-        (isa(v, DataType) && v.name === _TYPE_NAME && !has_free_typevars(v)) # isType(v)
+        (isType(v) && !has_free_typevars(v))
 end
 
+const _TYPE_NAME = Type.body.name
+isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME
+
 """
     isconcretetype(T)
 

base/show.jl

@@ -2392,8 +2392,7 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg
         end
         s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io)
         print_within_stacktrace(io, s, bold=true)
-    elseif isa(ft, DataType) && ft.name === Type.body.name &&
-        (f = ft.parameters[1]; !isa(f, TypeVar))
+    elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar))
         uwf = unwrap_unionall(f)
         parens = isa(f, UnionAll) && !(isa(uwf, DataType) && f === uwf.name.wrapper)
         parens && print(io, "(")

base/sort.jl

@@ -1464,10 +1464,15 @@ import Core.Intrinsics: slt_int
 import ..Sort: sort!, UIntMappable, uint_map, uint_unmap
 import ...Order: lt, DirectOrdering
 
-const Floats = Union{Float32,Float64}
-const FPSortable = Union{ # Mixed Float32 and Float64 are not allowed.
+# IEEEFloat is not available in Core.Compiler
+const Floats = Union{Float16, Float32, Float64}
+# fpsort is not safe for vectors of mixed bitwidth such as Vector{Union{Float32, Float64}}.
+# This type allows us to dispatch only when it is safe to do so. See #42739 for more info.
+const FPSortable = Union{
+    AbstractVector{Union{Float16, Missing}},
     AbstractVector{Union{Float32, Missing}},
     AbstractVector{Union{Float64, Missing}},
+    AbstractVector{Float16},
     AbstractVector{Float32},
     AbstractVector{Float64},
     AbstractVector{Missing}}
@@ -1484,6 +1489,12 @@ right(o::Perm) = Perm(right(o.order), o.data)
 lt(::Left, x::T, y::T) where {T<:Floats} = slt_int(y, x)
 lt(::Right, x::T, y::T) where {T<:Floats} = slt_int(x, y)
 
+uint_map(x::Float16, ::Left) = ~reinterpret(UInt16, x)
+uint_unmap(::Type{Float16}, u::UInt16, ::Left) = reinterpret(Float16, ~u)
+uint_map(x::Float16, ::Right) = reinterpret(UInt16, x)
+uint_unmap(::Type{Float16}, u::UInt16, ::Right) = reinterpret(Float16, u)
+UIntMappable(::Type{Float16}, ::Union{Left, Right}) = UInt16
+
 uint_map(x::Float32, ::Left) = ~reinterpret(UInt32, x)
 uint_unmap(::Type{Float32}, u::UInt32, ::Left) = reinterpret(Float32, ~u)
 uint_map(x::Float32, ::Right) = reinterpret(UInt32, x)

base/sysimg.jl

@@ -71,7 +71,7 @@ let
         # 7-depth packages
         :LazyArtifacts,
     ]
-    maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)
+    maxlen = maximum(textwidth.(string.(stdlibs)))
 
     tot_time_stdlib = 0.0
     # use a temp module to avoid leaving the type of this closure in Main

deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5

@@ -0,0 +1 @@
+9e337aa579ec47017d7b5ef2df3bcd02

deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512

@@ -0,0 +1 @@
+99e67b045691f8d9c16680e77014a33a507250377c037a7cf23e5dc2f0294c771a834e6d14f6a3a19a8def0ebb61a6fde17a4f3c1284cab0f4d7b5ea8c128d5d

julia.spdx.json

@@ -146,18 +146,6 @@
             "copyrightText": "Copyright (c) 2014: Elliot Saba",
             "summary": "A performant, 100% native-julia SHA1, SHA2, and SHA3 implementation"
         },
-        {
-            "name": "DelimitedFiles.jl",
-            "SPDXID": "SPDXRef-JuliaDelimitedFiles",
-            "downloadLocation": "git+https://github.com/JuliaData/DelimitedFiles.jl.git",
-            "filesAnalyzed": false,
-            "homepage": "https://julialang.org",
-            "sourceInfo": "The git hash of the version in use can be found in the file stdlib/DelimitedFiles.version",
-            "licenseConcluded": "MIT",
-            "licenseDeclared": "MIT",
-            "copyrightText": "Copyright (c) 2012-2022 The Julia Programming Language",
-            "summary": "A package for reading and writing files with delimited values."
-        },
         {
             "name": "dSFMT",
             "SPDXID": "SPDXRef-dSFMT",
@@ -503,11 +491,6 @@
             "relationshipType": "BUILD_DEPENDENCY_OF",
             "relatedSpdxElement": "SPDXRef-JuliaMain"
         },
-        {
-            "spdxElementId": "SPDXRef-JuliaDelimitedFiles",
-            "relationshipType": "BUILD_DEPENDENCY_OF",
-            "relatedSpdxElement": "SPDXRef-JuliaMain"
-        },
         {
             "spdxElementId": "SPDXRef-dSFMT",
             "relationshipType": "BUILD_DEPENDENCY_OF",

src/ccall.cpp

@@ -760,6 +760,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
     ir = static_eval(ctx, ir_arg);
     if (!ir) {
         emit_error(ctx, "error statically evaluating llvm IR argument");
+        JL_GC_POP();
         return jl_cgval_t();
     }
     if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) {
@@ -771,6 +772,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
         rt = static_eval(ctx, args[2]);
         if (!rt) {
             emit_error(ctx, "error statically evaluating llvmcall return type");
+            JL_GC_POP();
             return jl_cgval_t();
         }
     }
@@ -783,30 +785,35 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
         at = static_eval(ctx, args[3]);
         if (!at) {
             emit_error(ctx, "error statically evaluating llvmcall argument tuple");
+            JL_GC_POP();
             return jl_cgval_t();
         }
     }
     if (jl_is_tuple(ir)) {
         // if the IR is a tuple, we expect (mod, fn)
         if (jl_nfields(ir) != 2) {
             emit_error(ctx, "Tuple as first argument to llvmcall must have exactly two children");
+            JL_GC_POP();
             return jl_cgval_t();
         }
         entry = jl_fieldref(ir, 1);
         if (!jl_is_string(entry)) {
             emit_error(ctx, "Function name passed to llvmcall must be a string");
+            JL_GC_POP();
             return jl_cgval_t();
         }
         ir = jl_fieldref(ir, 0);
 
         if (!jl_is_string(ir) && !jl_typeis(ir, jl_array_uint8_type)) {
             emit_error(ctx, "Module IR passed to llvmcall must be a string or an array of bytes");
+            JL_GC_POP();
             return jl_cgval_t();
         }
     }
     else {
         if (!jl_is_string(ir)) {
             emit_error(ctx, "Function IR passed to llvmcall must be a string");
+            JL_GC_POP();
             return jl_cgval_t();
         }
     }
@@ -835,6 +842,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
         argtypes.push_back(t);
         if (4 + i > nargs) {
             emit_error(ctx, "Missing arguments to llvmcall!");
+            JL_GC_POP();
             return jl_cgval_t();
         }
         jl_value_t *argi = args[4 + i];
@@ -889,6 +897,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
             raw_string_ostream stream(message);
             Err.print("", stream, true);
             emit_error(ctx, stream.str());
+            JL_GC_POP();
             return jl_cgval_t();
         }
 
@@ -906,6 +915,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
                 raw_string_ostream stream(message);
                 Err.print("", stream, true);
                 emit_error(ctx, stream.str());
+                JL_GC_POP();
                 return jl_cgval_t();
             }
         }
@@ -923,6 +933,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
                 raw_string_ostream stream(message);
                 stream << Message;
                 emit_error(ctx, stream.str());
+                JL_GC_POP();
                 return jl_cgval_t();
             }
             Mod = std::move(ModuleOrErr.get());
@@ -931,6 +942,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
         Function *f = Mod->getFunction(jl_string_data(entry));
         if (!f) {
             emit_error(ctx, "Module IR does not contain specified entry function");
+            JL_GC_POP();
             return jl_cgval_t();
         }
         f->setName(ir_name);
@@ -959,6 +971,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
     raw_string_ostream stream(message);
     if (verifyFunction(*def, &stream)) {
         emit_error(ctx, stream.str());
+        JL_GC_POP();
         return jl_cgval_t();
     }
     def->setLinkage(GlobalVariable::LinkOnceODRLinkage);

src/dlload.c

@@ -4,6 +4,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
+#ifdef __GLIBC__
+#include <link.h>
+#endif
 
 #include "platform.h"
 #include "julia.h"
@@ -97,9 +100,62 @@ static void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOI
 }
 #endif
 
+#if defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)
+struct link_map;
+typedef void* (dlopen_prototype)(const char* filename, int flags);
+
+/* This function is copied from the memory sanitizer runtime.
+   Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+   See https://llvm.org/LICENSE.txt for license information.
+   SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+*/
+static inline uintptr_t RoundUpTo(uintptr_t size, uintptr_t boundary) {
+  return (size + boundary - 1) & ~(boundary - 1);
+}
+static inline uintptr_t RoundDownTo(uintptr_t x, uintptr_t boundary) {
+  return x & ~(boundary - 1);
+}
+void ForEachMappedRegion(struct link_map *map, void (*cb)(const void *, uintptr_t)) {
+#if !defined(_OS_FREEBSD_)
+  typedef ElfW(Phdr) Elf_Phdr;
+  typedef ElfW(Ehdr) Elf_Ehdr;
+#endif
+  char *base = (char *)map->l_addr;
+  Elf_Ehdr *ehdr = (Elf_Ehdr *)base;
+  char *phdrs = base + ehdr->e_phoff;
+  char *phdrs_end = phdrs + ehdr->e_phnum * ehdr->e_phentsize;
+
+  // Find the segment with the minimum base so we can "relocate" the p_vaddr
+  // fields.  Typically ET_DYN objects (DSOs) have base of zero and ET_EXEC
+  // objects have a non-zero base.
+  uintptr_t preferred_base = (uintptr_t)-1;
+  for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) {
+    Elf_Phdr *phdr = (Elf_Phdr *)iter;
+    if (phdr->p_type == PT_LOAD && preferred_base > (uintptr_t)phdr->p_vaddr)
+      preferred_base = (uintptr_t)phdr->p_vaddr;
+  }
+
+  // Compute the delta from the real base to get a relocation delta.
+  intptr_t delta = (uintptr_t)base - preferred_base;
+  // Now we can figure out what the loader really mapped.
+  for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) {
+    Elf_Phdr *phdr = (Elf_Phdr *)iter;
+    if (phdr->p_type == PT_LOAD) {
+      uintptr_t seg_start = phdr->p_vaddr + delta;
+      uintptr_t seg_end = seg_start + phdr->p_memsz;
+      // None of these values are aligned.  We consider the ragged edges of the
+      // load command as defined, since they are mapped from the file.
+      seg_start = RoundDownTo(seg_start, jl_page_size);
+      seg_end = RoundUpTo(seg_end, jl_page_size);
+      cb((void *)seg_start, seg_end - seg_start);
+    }
+  }
+}
+#endif
+
+#if defined(_OS_WINDOWS_)
 JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT
 {
-#if defined(_OS_WINDOWS_)
     size_t len = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0);
     if (!len) return NULL;
     WCHAR *wfilename = (WCHAR*)alloca(len * sizeof(WCHAR));
@@ -108,8 +164,32 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI
     if (lib)
         needsSymRefreshModuleList = 1;
     return lib;
+}
 #else
-    return dlopen(filename,
+JL_DLLEXPORT JL_NO_SANITIZE void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT
+{
+    /* The sanitizers break RUNPATH use in dlopen for annoying reasons that are
+       are hard to fix. Specifically, libc will use the return address of the
+       caller to determine certain paths and flags that affect .so location lookup.
+       To work around this, we need to avoid using the sanitizer's dlopen interposition,
+       instead using the real dlopen directly from the current shared library.
+       Of course, this does mean that we need to manually perform the work that
+       the sanitizers would otherwise do. */
+#if (defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)) && __GLIBC__
+    static dlopen_prototype *dlopen = NULL;
+    if (!dlopen) {
+        dlopen = (dlopen_prototype*)dlsym(RTLD_NEXT, "dlopen");
+        if (!dlopen)
+            return NULL;
+        void *libdl_handle = dlopen("libdl.so", RTLD_NOW | RTLD_NOLOAD);
+        dlopen = (dlopen_prototype*)dlsym(libdl_handle, "dlopen");
+        dlclose(libdl_handle);
+        assert(dlopen);
+    }
+    // The real interceptors check the validty of the string here, but let's
+    // just skip that for the time being.
+#endif
+    void *hnd = dlopen(filename,
                   (flags & JL_RTLD_NOW ? RTLD_NOW : RTLD_LAZY)
                   | JL_RTLD(flags, LOCAL)
                   | JL_RTLD(flags, GLOBAL)
@@ -126,8 +206,15 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI
                   | JL_RTLD(flags, FIRST)
 #endif
                   );
+#if defined(_COMPILER_MSAN_ENABLED_) && defined(__GLIBC__)
+    link_map *map = (link_map*)handle;
+    if (filename && map)
+      ForEachMappedRegion(map, __msan_unpoison);
 #endif
+    return hnd;
 }
+#endif
+
 
 JL_DLLEXPORT int jl_dlclose(void *handle) JL_NOTSAFEPOINT
 {