diff --git a/Makefile b/Makefile index 6c6d39709f9aed..3b1c6f0d0bc7bd 100644 --- a/Makefile +++ b/Makefile @@ -169,14 +169,17 @@ $(build_datarootdir)/julia/base.cache: $(JULIA_SYSIMG) | $(DIRS) $(build_dataroo $(call cygpath_w,$@)) # public libraries, that are installed in $(prefix)/lib +ifeq ($(JULIA_BUILD_MODE),release) JL_TARGETS := julia -ifeq ($(BUNDLE_DEBUG_LIBS),1) -JL_TARGETS += julia-debug +else ifeq ($(JULIA_BUILD_MODE),debug) +JL_TARGETS := julia-debug endif # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest libjulia-internal libjulia-codegen -ifeq ($(BUNDLE_DEBUG_LIBS),1) +JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest +ifeq ($(JULIA_BUILD_MODE),release) +JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen +else ifeq ($(JULIA_BUILD_MODE),debug) JL_PRIVATE_LIBS-0 += libjulia-internal-debug libjulia-codegen-debug endif ifeq ($(USE_GPL_LIBS), 1) @@ -237,38 +240,32 @@ endef install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html -ifeq ($(BUNDLE_DEBUG_LIBS),1) - @$(MAKE) $(QUIET_MAKE) all -else - @$(MAKE) $(QUIET_MAKE) release -endif + @$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE) @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \ mkdir -p $(DESTDIR)$$subdir; \ done - $(INSTALL_M) $(build_bindir)/julia $(DESTDIR)$(bindir)/ -ifeq ($(BUNDLE_DEBUG_LIBS),1) - $(INSTALL_M) $(build_bindir)/julia-debug $(DESTDIR)$(bindir)/ -endif + $(INSTALL_M) $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) $(DESTDIR)$(bindir)/ ifeq ($(OS),WINNT) - -$(INSTALL_M) $(filter-out $(build_bindir)/libjulia-debug.dll,$(wildcard $(build_bindir)/*.dll)) $(DESTDIR)$(bindir)/ + -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ +ifeq ($(JULIA_BUILD_MODE),release) -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ +else ifeq ($(JULIA_BUILD_MODE),debug) + -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ +endif # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it. -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/ -ifeq ($(BUNDLE_DEBUG_LIBS),1) - -$(INSTALL_M) $(build_bindir)/libjulia-debug.dll $(DESTDIR)$(bindir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ -endif -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(OS),Darwin) +ifeq ($(JULIA_BUILD_MODE),release) -cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) -cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) -cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) -cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) endif @@ -283,10 +280,11 @@ endif done else # libjulia in Darwin framework has special location and name +ifeq ($(JULIA_BUILD_MODE),release) $(INSTALL_M) $(build_libdir)/libjulia.$(SOMAJOR).$(SOMINOR).dylib $(DESTDIR)$(prefix)/$(framework_dylib) @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/$(FRAMEWORK_NAME).dSYM $(DESTDIR)$(prefix)/$(framework_dylib) @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/sys.dylib.dSYM $(build_private_libdir)/sys.dylib -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_libdir)/libjulia-debug.$(SOMAJOR).$(SOMINOR).dylib $(DESTDIR)$(prefix)/$(framework_dylib)_debug @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/$(FRAMEWORK_NAME)_debug.dSYM $(DESTDIR)$(prefix)/$(framework_dylib)_debug @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/sys-debug.dylib.dSYM $(build_private_libdir)/sys-debug.dylib @@ -314,8 +312,9 @@ endif # Copy public headers cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia # Copy system image +ifeq ($(JULIA_BUILD_MODE),release) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif @@ -364,16 +363,18 @@ endif RELEASE_TARGET=$(DESTDIR)$(prefix)/$(framework_dylib); \ DEBUG_TARGET=$(DESTDIR)$(prefix)/$(framework_dylib)_debug; \ fi; \ - $(call stringreplace,$${RELEASE_TARGET},sys.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys.$(SHLIB_EXT)); \ - if [ "$(BUNDLE_DEBUG_LIBS)" = "1" ]; then \ + if [ "$(JULIA_BUILD_MODE)" = "release" ]; then \ + $(call stringreplace,$${RELEASE_TARGET},sys.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys.$(SHLIB_EXT)); \ + elif [ "$(JULIA_BUILD_MODE)" = "debug" ]; then \ $(call stringreplace,$${DEBUG_TARGET},sys-debug.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys-debug.$(SHLIB_EXT)); \ fi; endif # Set rpath for libjulia-internal, which is moving from `../lib` to `../lib/julia`. We only need to do this for Linux/FreeBSD ifneq (,$(findstring $(OS),Linux FreeBSD)) +ifeq ($(JULIA_BUILD_MODE),release) $(PATCHELF) --set-rpath '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(PATCHELF) --set-rpath '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) endif endif @@ -381,16 +382,16 @@ endif ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS)) # Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS) +ifeq ($(JULIA_BUILD_MODE),release) $(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT),$(LOADER_BUILD_DEP_LIBS)$$,$(LOADER_INSTALL_DEP_LIBS)) -ifeq ($(OS),Darwin) - # Codesign the libjulia we just modified - $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT)" -endif - -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT),$(LOADER_DEBUG_BUILD_DEP_LIBS)$$,$(LOADER_DEBUG_INSTALL_DEP_LIBS)) +endif ifeq ($(OS),Darwin) # Codesign the libjulia we just modified +ifeq ($(JULIA_BUILD_MODE),release) + $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT)" +else ifeq ($(JULIA_BUILD_MODE),debug) $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT)" endif endif diff --git a/NEWS.md b/NEWS.md index 886e24b706f1a9..f871424e47337b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -158,6 +158,8 @@ Standard library changes #### DelimitedFiles +* DelimitedFiles has been promoted from being a standard library to a separate package. It now has to be explicitly installed to be used. + Deprecated or removed --------------------- diff --git a/base/compiler/typeutils.jl b/base/compiler/typeutils.jl index c70394e891c9fb..0e59fc9daa8aeb 100644 --- a/base/compiler/typeutils.jl +++ b/base/compiler/typeutils.jl @@ -4,8 +4,6 @@ # lattice utilities # ##################### -isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME - # true if Type{T} is inlineable as constant T # requires that T is a singleton, s.t. T == S implies T === S isconstType(@nospecialize t) = isType(t) && hasuniquerep(t.parameters[1]) diff --git a/base/deprecated.jl b/base/deprecated.jl index 83ea85cddff7a9..61ba282f390dea 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -159,7 +159,7 @@ function firstcaller(bt::Vector, funcsyms) li = lkup.linfo if li isa Core.MethodInstance ft = ccall(:jl_first_argument_datatype, Any, (Any,), (li.def::Method).sig) - if isa(ft, DataType) && ft.name === Type.body.name + if isType(ft) ft = unwrap_unionall(ft.parameters[1]) found = (isa(ft, DataType) && ft.name.name in funcsyms) end diff --git a/base/errorshow.jl b/base/errorshow.jl index 1b00419cd85108..9218abe02a1879 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -400,7 +400,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=() # pool MethodErrors for these two functions. if f === convert && !isempty(arg_types_param) at1 = arg_types_param[1] - if isa(at1,DataType) && (at1::DataType).name === Type.body.name && !Core.Compiler.has_free_typevars(at1) + if isType(at1) && !Core.Compiler.has_free_typevars(at1) push!(funcs, (at1.parameters[1], arg_types_param[2:end])) end end diff --git a/base/loading.jl b/base/loading.jl index b3c0ee639a206e..79bc66fe456c3a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1312,8 +1312,11 @@ end # get a top-level Module from the given key root_module(key::PkgId) = @lock require_lock loaded_modules[key] -root_module(where::Module, name::Symbol) = - root_module(identify_package(where, String(name))) +function root_module(where::Module, name::Symbol) + key = identify_package(where, String(name)) + key isa PkgId || throw(KeyError(name)) + return root_module(key) +end maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing) root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key) diff --git a/base/namedtuple.jl b/base/namedtuple.jl index b2ebb3f9d0d7e3..70ebba34abe87c 100644 --- a/base/namedtuple.jl +++ b/base/namedtuple.jl @@ -320,6 +320,7 @@ get(nt::NamedTuple, key::Union{Integer, Symbol}, default) = isdefined(nt, key) ? get(f::Callable, nt::NamedTuple, key::Union{Integer, Symbol}) = isdefined(nt, key) ? getfield(nt, key) : f() tail(t::NamedTuple{names}) where names = NamedTuple{tail(names)}(t) front(t::NamedTuple{names}) where names = NamedTuple{front(names)}(t) +reverse(nt::NamedTuple) = NamedTuple{reverse(keys(nt))}(reverse(values(nt))) @assume_effects :total function diff_names(an::Tuple{Vararg{Symbol}}, bn::Tuple{Vararg{Symbol}}) @nospecialize an bn diff --git a/base/reflection.jl b/base/reflection.jl index 94c103fb5c8de7..f377e7b16b5711 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -588,7 +588,7 @@ isbitstype(@nospecialize t) = (@_total_meta; isa(t, DataType) && (t.flags & 0x8) Return `true` if `x` is an instance of an [`isbitstype`](@ref) type. """ -isbits(@nospecialize x) = (@_total_meta; typeof(x).flags & 0x8 == 0x8) +isbits(@nospecialize x) = isbitstype(typeof(x)) """ isdispatchtuple(T) @@ -605,12 +605,14 @@ has_free_typevars(@nospecialize(t)) = ccall(:jl_has_free_typevars, Cint, (Any,), # equivalent to isa(v, Type) && isdispatchtuple(Tuple{v}) || v === Union{} # and is thus perhaps most similar to the old (pre-1.0) `isleaftype` query -const _TYPE_NAME = Type.body.name function isdispatchelem(@nospecialize v) return (v === Bottom) || (v === typeof(Bottom)) || isconcretedispatch(v) || - (isa(v, DataType) && v.name === _TYPE_NAME && !has_free_typevars(v)) # isType(v) + (isType(v) && !has_free_typevars(v)) end +const _TYPE_NAME = Type.body.name +isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME + """ isconcretetype(T) diff --git a/base/show.jl b/base/show.jl index a9bbb27df6790e..6f17fe6ee2ef66 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2392,8 +2392,7 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg end s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io) print_within_stacktrace(io, s, bold=true) - elseif isa(ft, DataType) && ft.name === Type.body.name && - (f = ft.parameters[1]; !isa(f, TypeVar)) + elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar)) uwf = unwrap_unionall(f) parens = isa(f, UnionAll) && !(isa(uwf, DataType) && f === uwf.name.wrapper) parens && print(io, "(") diff --git a/base/sort.jl b/base/sort.jl index 59fa14aa99f658..8a3d2e2871cfe6 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -1464,10 +1464,15 @@ import Core.Intrinsics: slt_int import ..Sort: sort!, UIntMappable, uint_map, uint_unmap import ...Order: lt, DirectOrdering -const Floats = Union{Float32,Float64} -const FPSortable = Union{ # Mixed Float32 and Float64 are not allowed. +# IEEEFloat is not available in Core.Compiler +const Floats = Union{Float16, Float32, Float64} +# fpsort is not safe for vectors of mixed bitwidth such as Vector{Union{Float32, Float64}}. +# This type allows us to dispatch only when it is safe to do so. See #42739 for more info. +const FPSortable = Union{ + AbstractVector{Union{Float16, Missing}}, AbstractVector{Union{Float32, Missing}}, AbstractVector{Union{Float64, Missing}}, + AbstractVector{Float16}, AbstractVector{Float32}, AbstractVector{Float64}, AbstractVector{Missing}} @@ -1484,6 +1489,12 @@ right(o::Perm) = Perm(right(o.order), o.data) lt(::Left, x::T, y::T) where {T<:Floats} = slt_int(y, x) lt(::Right, x::T, y::T) where {T<:Floats} = slt_int(x, y) +uint_map(x::Float16, ::Left) = ~reinterpret(UInt16, x) +uint_unmap(::Type{Float16}, u::UInt16, ::Left) = reinterpret(Float16, ~u) +uint_map(x::Float16, ::Right) = reinterpret(UInt16, x) +uint_unmap(::Type{Float16}, u::UInt16, ::Right) = reinterpret(Float16, u) +UIntMappable(::Type{Float16}, ::Union{Left, Right}) = UInt16 + uint_map(x::Float32, ::Left) = ~reinterpret(UInt32, x) uint_unmap(::Type{Float32}, u::UInt32, ::Left) = reinterpret(Float32, ~u) uint_map(x::Float32, ::Right) = reinterpret(UInt32, x) diff --git a/base/sysimg.jl b/base/sysimg.jl index 79f01a0e9a1d25..183934d77fb6bc 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -71,7 +71,7 @@ let # 7-depth packages :LazyArtifacts, ] - maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0) + maxlen = maximum(textwidth.(string.(stdlibs))) tot_time_stdlib = 0.0 # use a temp module to avoid leaving the type of this closure in Main diff --git a/contrib/fixup-libstdc++.sh b/contrib/fixup-libstdc++.sh index 1c19d98a54b1e4..7442d995448a14 100755 --- a/contrib/fixup-libstdc++.sh +++ b/contrib/fixup-libstdc++.sh @@ -11,7 +11,8 @@ fi libdir="$1" private_libdir="$2" -if [ ! -f "$private_libdir/libjulia-internal.so" ]; then +if [ ! -f "$private_libdir/libjulia-internal.so" ] && \ + [ ! -f "$private_libdir/libjulia-internal-debug.so" ]; then echo "ERROR: Could not open $private_libdir/libjulia-internal.so" >&2 exit 2 fi @@ -24,7 +25,11 @@ find_shlib () } # Discover libstdc++ location and name -LIBSTD=$(find_shlib "$private_libdir/libjulia-internal.so" "libstdc++.so") +if [ -f "$private_libdir/libjulia-internal.so" ]; then + LIBSTD=$(find_shlib "$private_libdir/libjulia-internal.so" "libstdc++.so") +elif [ -f "$private_libdir/libjulia-internal-debug.so" ]; then + LIBSTD=$(find_shlib "$private_libdir/libjulia-internal-debug.so" "libstdc++.so") +fi LIBSTD_NAME=$(basename $LIBSTD) LIBSTD_DIR=$(dirname $LIBSTD) diff --git a/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 new file mode 100644 index 00000000000000..ea7ad31ce70351 --- /dev/null +++ b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 @@ -0,0 +1 @@ +9e337aa579ec47017d7b5ef2df3bcd02 diff --git a/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 new file mode 100644 index 00000000000000..3ef1b9730fa601 --- /dev/null +++ b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 @@ -0,0 +1 @@ +99e67b045691f8d9c16680e77014a33a507250377c037a7cf23e5dc2f0294c771a834e6d14f6a3a19a8def0ebb61a6fde17a4f3c1284cab0f4d7b5ea8c128d5d diff --git a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 b/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 deleted file mode 100644 index 8a7bea807777b7..00000000000000 --- a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ee424db01b8b85de4f927dbe5a294f6a diff --git a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 b/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 deleted file mode 100644 index cb3e080eac9433..00000000000000 --- a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -aba17eb434325938daa43ae1fe518fc72eefc4d0a8cd1f8cfb0411361cf43a5e9b14ceeb145ab26af58336b6f99dd169668a40091a6156777068ea0fc5daad8a diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index c3012efa1d8b1d..f39fb7669ba0dd 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -1097,7 +1097,7 @@ they are equivalent to `broadcast` calls and are fused with other nested "dot" c You can also combine dot operations with function chaining using [`|>`](@ref), as in this example: ```jldoctest -julia> [1:5;] .|> [x->x^2, inv, x->2*x, -, isodd] +julia> 1:5 .|> [x->x^2, inv, x->2*x, -, isodd] 5-element Vector{Real}: 1 0.5 diff --git a/julia.spdx.json b/julia.spdx.json index 2e7e368a49c0d4..9c1911958096e6 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -146,18 +146,6 @@ "copyrightText": "Copyright (c) 2014: Elliot Saba", "summary": "A performant, 100% native-julia SHA1, SHA2, and SHA3 implementation" }, - { - "name": "DelimitedFiles.jl", - "SPDXID": "SPDXRef-JuliaDelimitedFiles", - "downloadLocation": "git+https://github.com/JuliaData/DelimitedFiles.jl.git", - "filesAnalyzed": false, - "homepage": "https://julialang.org", - "sourceInfo": "The git hash of the version in use can be found in the file stdlib/DelimitedFiles.version", - "licenseConcluded": "MIT", - "licenseDeclared": "MIT", - "copyrightText": "Copyright (c) 2012-2022 The Julia Programming Language", - "summary": "A package for reading and writing files with delimited values." - }, { "name": "dSFMT", "SPDXID": "SPDXRef-dSFMT", @@ -503,11 +491,6 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, - { - "spdxElementId": "SPDXRef-JuliaDelimitedFiles", - "relationshipType": "BUILD_DEPENDENCY_OF", - "relatedSpdxElement": "SPDXRef-JuliaMain" - }, { "spdxElementId": "SPDXRef-dSFMT", "relationshipType": "BUILD_DEPENDENCY_OF", diff --git a/src/ccall.cpp b/src/ccall.cpp index 88c80b333b0272..0fddf5425acc03 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -760,6 +760,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar ir = static_eval(ctx, ir_arg); if (!ir) { emit_error(ctx, "error statically evaluating llvm IR argument"); + JL_GC_POP(); return jl_cgval_t(); } if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) { @@ -771,6 +772,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar rt = static_eval(ctx, args[2]); if (!rt) { emit_error(ctx, "error statically evaluating llvmcall return type"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -783,6 +785,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar at = static_eval(ctx, args[3]); if (!at) { emit_error(ctx, "error statically evaluating llvmcall argument tuple"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -790,23 +793,27 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar // if the IR is a tuple, we expect (mod, fn) if (jl_nfields(ir) != 2) { emit_error(ctx, "Tuple as first argument to llvmcall must have exactly two children"); + JL_GC_POP(); return jl_cgval_t(); } entry = jl_fieldref(ir, 1); if (!jl_is_string(entry)) { emit_error(ctx, "Function name passed to llvmcall must be a string"); + JL_GC_POP(); return jl_cgval_t(); } ir = jl_fieldref(ir, 0); if (!jl_is_string(ir) && !jl_typeis(ir, jl_array_uint8_type)) { emit_error(ctx, "Module IR passed to llvmcall must be a string or an array of bytes"); + JL_GC_POP(); return jl_cgval_t(); } } else { if (!jl_is_string(ir)) { emit_error(ctx, "Function IR passed to llvmcall must be a string"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -835,6 +842,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar argtypes.push_back(t); if (4 + i > nargs) { emit_error(ctx, "Missing arguments to llvmcall!"); + JL_GC_POP(); return jl_cgval_t(); } jl_value_t *argi = args[4 + i]; @@ -889,6 +897,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); Err.print("", stream, true); emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } @@ -906,6 +915,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); Err.print("", stream, true); emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } } @@ -923,6 +933,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); stream << Message; emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } Mod = std::move(ModuleOrErr.get()); @@ -931,6 +942,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar Function *f = Mod->getFunction(jl_string_data(entry)); if (!f) { emit_error(ctx, "Module IR does not contain specified entry function"); + JL_GC_POP(); return jl_cgval_t(); } f->setName(ir_name); @@ -959,6 +971,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); if (verifyFunction(*def, &stream)) { emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } def->setLinkage(GlobalVariable::LinkOnceODRLinkage); diff --git a/src/dlload.c b/src/dlload.c index 230a31ed3d695b..308c8317cd5d68 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -4,6 +4,9 @@ #include #include #include +#ifdef __GLIBC__ +#include +#endif #include "platform.h" #include "julia.h" @@ -97,9 +100,62 @@ static void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOI } #endif +#if defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_) +struct link_map; +typedef void* (dlopen_prototype)(const char* filename, int flags); + +/* This function is copied from the memory sanitizer runtime. + Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + See https://llvm.org/LICENSE.txt for license information. + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ +static inline uintptr_t RoundUpTo(uintptr_t size, uintptr_t boundary) { + return (size + boundary - 1) & ~(boundary - 1); +} +static inline uintptr_t RoundDownTo(uintptr_t x, uintptr_t boundary) { + return x & ~(boundary - 1); +} +void ForEachMappedRegion(struct link_map *map, void (*cb)(const void *, uintptr_t)) { +#if !defined(_OS_FREEBSD_) + typedef ElfW(Phdr) Elf_Phdr; + typedef ElfW(Ehdr) Elf_Ehdr; +#endif + char *base = (char *)map->l_addr; + Elf_Ehdr *ehdr = (Elf_Ehdr *)base; + char *phdrs = base + ehdr->e_phoff; + char *phdrs_end = phdrs + ehdr->e_phnum * ehdr->e_phentsize; + + // Find the segment with the minimum base so we can "relocate" the p_vaddr + // fields. Typically ET_DYN objects (DSOs) have base of zero and ET_EXEC + // objects have a non-zero base. + uintptr_t preferred_base = (uintptr_t)-1; + for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) { + Elf_Phdr *phdr = (Elf_Phdr *)iter; + if (phdr->p_type == PT_LOAD && preferred_base > (uintptr_t)phdr->p_vaddr) + preferred_base = (uintptr_t)phdr->p_vaddr; + } + + // Compute the delta from the real base to get a relocation delta. + intptr_t delta = (uintptr_t)base - preferred_base; + // Now we can figure out what the loader really mapped. + for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) { + Elf_Phdr *phdr = (Elf_Phdr *)iter; + if (phdr->p_type == PT_LOAD) { + uintptr_t seg_start = phdr->p_vaddr + delta; + uintptr_t seg_end = seg_start + phdr->p_memsz; + // None of these values are aligned. We consider the ragged edges of the + // load command as defined, since they are mapped from the file. + seg_start = RoundDownTo(seg_start, jl_page_size); + seg_end = RoundUpTo(seg_end, jl_page_size); + cb((void *)seg_start, seg_end - seg_start); + } + } +} +#endif + +#if defined(_OS_WINDOWS_) JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT { -#if defined(_OS_WINDOWS_) size_t len = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0); if (!len) return NULL; WCHAR *wfilename = (WCHAR*)alloca(len * sizeof(WCHAR)); @@ -108,8 +164,32 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI if (lib) needsSymRefreshModuleList = 1; return lib; +} #else - return dlopen(filename, +JL_DLLEXPORT JL_NO_SANITIZE void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT +{ + /* The sanitizers break RUNPATH use in dlopen for annoying reasons that are + are hard to fix. Specifically, libc will use the return address of the + caller to determine certain paths and flags that affect .so location lookup. + To work around this, we need to avoid using the sanitizer's dlopen interposition, + instead using the real dlopen directly from the current shared library. + Of course, this does mean that we need to manually perform the work that + the sanitizers would otherwise do. */ +#if (defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)) && __GLIBC__ + static dlopen_prototype *dlopen = NULL; + if (!dlopen) { + dlopen = (dlopen_prototype*)dlsym(RTLD_NEXT, "dlopen"); + if (!dlopen) + return NULL; + void *libdl_handle = dlopen("libdl.so", RTLD_NOW | RTLD_NOLOAD); + dlopen = (dlopen_prototype*)dlsym(libdl_handle, "dlopen"); + dlclose(libdl_handle); + assert(dlopen); + } + // The real interceptors check the validty of the string here, but let's + // just skip that for the time being. +#endif + void *hnd = dlopen(filename, (flags & JL_RTLD_NOW ? RTLD_NOW : RTLD_LAZY) | JL_RTLD(flags, LOCAL) | JL_RTLD(flags, GLOBAL) @@ -126,8 +206,15 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI | JL_RTLD(flags, FIRST) #endif ); +#if defined(_COMPILER_MSAN_ENABLED_) && defined(__GLIBC__) + link_map *map = (link_map*)handle; + if (filename && map) + ForEachMappedRegion(map, __msan_unpoison); #endif + return hnd; } +#endif + JL_DLLEXPORT int jl_dlclose(void *handle) JL_NOTSAFEPOINT { diff --git a/src/flisp/cvalues.c b/src/flisp/cvalues.c index 071a0b16429717..a5635c238ba3c3 100644 --- a/src/flisp/cvalues.c +++ b/src/flisp/cvalues.c @@ -108,7 +108,7 @@ static value_t cprim(fl_context_t *fl_ctx, fltype_t *type, size_t sz) return tagptr(pcp, TAG_CPRIM); } -value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +static value_t _cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz, int may_finalize) { cvalue_t *pcv; int str=0; @@ -127,7 +127,7 @@ value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) pcv = (cvalue_t*)alloc_words(fl_ctx, nw); pcv->type = type; pcv->data = &pcv->_space[0]; - if (type->vtable != NULL && type->vtable->finalize != NULL) + if (may_finalize && type->vtable != NULL && type->vtable->finalize != NULL) add_finalizer(fl_ctx, pcv); } else { @@ -148,6 +148,16 @@ value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) return tagptr(pcv, TAG_CVALUE); } +value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +{ + return _cvalue(fl_ctx, type, sz, 1); +} + +value_t cvalue_no_finalizer(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +{ + return _cvalue(fl_ctx, type, sz, 0); +} + value_t cvalue_from_data(fl_context_t *fl_ctx, fltype_t *type, void *data, size_t sz) { value_t cv; diff --git a/src/flisp/flisp.h b/src/flisp/flisp.h index e77904a32d1f20..b031e456cd3fe3 100644 --- a/src/flisp/flisp.h +++ b/src/flisp/flisp.h @@ -328,6 +328,7 @@ typedef float fl_float_t; typedef value_t (*builtin_t)(fl_context_t*, value_t*, uint32_t); value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) JL_NOTSAFEPOINT; +value_t cvalue_no_finalizer(fl_context_t *fl_ctx, fltype_t *type, size_t sz) JL_NOTSAFEPOINT; void add_finalizer(fl_context_t *fl_ctx, cvalue_t *cv); void cv_autorelease(fl_context_t *fl_ctx, cvalue_t *cv); void cv_pin(fl_context_t *fl_ctx, cvalue_t *cv); diff --git a/src/flisp/table.c b/src/flisp/table.c index a24cdf3bc06e8b..1d8aed358e88dd 100644 --- a/src/flisp/table.c +++ b/src/flisp/table.c @@ -87,9 +87,7 @@ value_t fl_table(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) value_t nt; // prevent small tables from being added to finalizer list if (cnt <= HT_N_INLINE) { - fl_ctx->table_vtable.finalize = NULL; - nt = cvalue(fl_ctx, fl_ctx->tabletype, sizeof(htable_t)); - fl_ctx->table_vtable.finalize = free_htable; + nt = cvalue_no_finalizer(fl_ctx, fl_ctx->tabletype, sizeof(htable_t)); } else { nt = cvalue(fl_ctx, fl_ctx->tabletype, 2*sizeof(void*)); @@ -104,6 +102,12 @@ value_t fl_table(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) else k = arg; } + if (h->table != &h->_space[0]) { + // We expected to use the inline table, but we ended up outgrowing it. + // Make sure to register the finalizer. + add_finalizer(fl_ctx, (cvalue_t*)ptr(nt)); + ((cvalue_t*)ptr(nt))->len = 2*sizeof(void*); + } return nt; } diff --git a/src/support/platform.h b/src/support/platform.h index cf65fa01423feb..1afe0148be045f 100644 --- a/src/support/platform.h +++ b/src/support/platform.h @@ -43,24 +43,37 @@ #error Unsupported compiler #endif + +#define JL_NO_ASAN +#define JL_NO_MSAN +#define JL_NO_TSAN #if defined(__has_feature) // Clang flavor #if __has_feature(address_sanitizer) #define _COMPILER_ASAN_ENABLED_ +#undef JL_NO_ASAN +#define JL_NO_ASAN __attribute__((no_sanitize("address"))) #endif #if __has_feature(memory_sanitizer) #define _COMPILER_MSAN_ENABLED_ +#undef JL_NO_MSAN +#define JL_NO_MSAN __attribute__((no_sanitize("mempry"))) #endif #if __has_feature(thread_sanitizer) #if __clang_major__ < 11 #error Thread sanitizer runtime libraries in clang < 11 leak memory and cannot be used #endif #define _COMPILER_TSAN_ENABLED_ +#undef JL_NO_TSAN +#define JL_NO_TSAN __attribute__((no_sanitize("thread"))) #endif #else // GCC flavor #if defined(__SANITIZE_ADDRESS__) #define _COMPILER_ASAN_ENABLED_ +#undef JL_NO_ASAN +#define JL_NO_ASAN __attribute__((no_sanitize("address"))) #endif #endif // __has_feature +#define JL_NO_SANITIZE JL_NO_ASAN JL_NO_MSAN JL_NO_TSAN /******************************************************************************* * OS * diff --git a/src/typemap.c b/src/typemap.c index dfa8ac67f6abcf..cbabbe361daa5a 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -349,14 +349,13 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * goto exit; JL_GC_POP(); return 1; +exit: + JL_GC_POP(); + return 0; } else { return jl_typemap_node_visitor((jl_typemap_entry_t*)cache, fptr, closure); } - -exit: - JL_GC_POP(); - return 0; } static unsigned jl_supertype_height(jl_datatype_t *dt) diff --git a/stdlib/.gitignore b/stdlib/.gitignore index 038b2d9602b2a3..ffbc2f12f52da2 100644 --- a/stdlib/.gitignore +++ b/stdlib/.gitignore @@ -5,8 +5,6 @@ /Statistics /LibCURL-* /LibCURL -/DelimitedFiles-* -/DelimitedFiles /Downloads-* /Downloads /ArgTools-* diff --git a/stdlib/DelimitedFiles.version b/stdlib/DelimitedFiles.version deleted file mode 100644 index 220a1482822e0e..00000000000000 --- a/stdlib/DelimitedFiles.version +++ /dev/null @@ -1,4 +0,0 @@ -DELIMITEDFILES_BRANCH = main -DELIMITEDFILES_SHA1 = 495ebc81ef7fe2c083ca4c1d7e43a22cc1e49490 -DELIMITEDFILES_GIT_URL := https://github.com/JuliaData/DelimitedFiles.jl.git -DELIMITEDFILES_TAR_URL = https://api.github.com/repos/JuliaData/DelimitedFiles.jl/tarball/$1 diff --git a/stdlib/Makefile b/stdlib/Makefile index 5a72290ce96f00..d45be468626cd5 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -41,13 +41,13 @@ endef $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll)))) -STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ +STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \ Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization SHA \ SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \ $(JLL_NAMES) -STDLIBS_EXT = Pkg Statistics LibCURL DelimitedFiles Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA +STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z)))) diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 815ca8ca323a9a..49cb1c091ea5f6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = cebcbc0a4af16fb7933fe101e8ef89a368231e60 +PKG_SHA1 = 8d77a6cac48113d143e028209aa28f10d5473032 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 3c16c804ccebcc..7658a227b51cdc 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -198,7 +198,7 @@ function complete_symbol(sym::String, @nospecialize(ffunc), context_module::Modu # Looking for a member of a type if t isa DataType && t != Any # Check for cases like Type{typeof(+)} - if t isa DataType && t.name === Base._TYPE_NAME + if Base.isType(t) t = typeof(t.parameters[1]) end # Only look for fields if this is a concrete type diff --git a/stdlib/Random/docs/src/index.md b/stdlib/Random/docs/src/index.md index 0f7636cf2444f5..e344e479474402 100644 --- a/stdlib/Random/docs/src/index.md +++ b/stdlib/Random/docs/src/index.md @@ -33,6 +33,8 @@ unbounded integers, the interval must be specified (e.g. `rand(big.(1:6))`). Additionally, normal and exponential distributions are implemented for some `AbstractFloat` and `Complex` types, see [`randn`](@ref) and [`randexp`](@ref) for details. +To generate random numbers from other distributions, see the [Distributions.jl](https://juliastats.org/Distributions.jl/stable/) package. + !!! warning Because the precise way in which random numbers are generated is considered an implementation detail, bug fixes and speed improvements may change the stream of numbers that are generated after a version change. Relying on a specific seed or generated stream of numbers during unit testing is thus discouraged - consider testing properties of the methods in question instead. diff --git a/test/loading.jl b/test/loading.jl index 98f20bdebf4ae7..16690c6e046136 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -730,6 +730,14 @@ end end end +@testset "Issue #25719" begin + empty!(LOAD_PATH) + @test Base.root_module(Core, :Core) == Core + push!(LOAD_PATH, "@stdlib") + @test Base.root_module(Base, :Test) == Test + @test_throws KeyError(:SomeNonExistentPackage) Base.root_module(Base, :SomeNonExistentPackage) +end + ## cleanup after tests ## for env in keys(envs) diff --git a/test/namedtuple.jl b/test/namedtuple.jl index 94eb14008dc523..6c41ab788b7331 100644 --- a/test/namedtuple.jl +++ b/test/namedtuple.jl @@ -147,6 +147,8 @@ end @test Base.front((a = 1, )) ≡ NamedTuple() @test_throws ArgumentError Base.tail(NamedTuple()) @test_throws ArgumentError Base.front(NamedTuple()) +@test @inferred(reverse((a=1,))) === (a=1,) +@test @inferred(reverse((a=1, b=:c))) === (b=:c, a=1) # syntax errors diff --git a/test/reflection.jl b/test/reflection.jl index 5cb20256538e07..e28e92142e5f67 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -2,6 +2,8 @@ using Test +include("compiler/irutils.jl") + # code_native / code_llvm (issue #8239) # It's hard to really test these, but just running them should be # sufficient to catch segfault bugs. @@ -66,6 +68,7 @@ end # module ReflectionTest @test isbits((1,2)) @test !isbits([1]) @test isbits(nothing) +@test fully_eliminated(isbits, (Int,)) # issue #16670 @test isconcretetype(Int) diff --git a/test/sorting.jl b/test/sorting.jl index b2f663932c0ed7..9766ee99ce7514 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -652,8 +652,7 @@ end T = eltype(x) U = UIntN(Val(sizeof(T))) for order in [Forward, Reverse, Base.Sort.Float.Left(), Base.Sort.Float.Right(), By(Forward, identity)] - if order isa Base.Order.By || T === Float16 || - ((T <: AbstractFloat) == (order isa DirectOrdering)) + if order isa Base.Order.By || ((T <: AbstractFloat) == (order isa DirectOrdering)) @test Base.Sort.UIntMappable(T, order) === nothing continue end