From 61f58be14eaf9fd7a8def5f18a94d2fe1da42ac8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 11 Aug 2022 12:12:35 -0400 Subject: [PATCH 01/14] Improve error message from root_module (#46306) Fix: #25719 Co-authored-by: Jameson Nash --- base/loading.jl | 7 +++++-- test/loading.jl | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b3c0ee639a206e..79bc66fe456c3a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1312,8 +1312,11 @@ end # get a top-level Module from the given key root_module(key::PkgId) = @lock require_lock loaded_modules[key] -root_module(where::Module, name::Symbol) = - root_module(identify_package(where, String(name))) +function root_module(where::Module, name::Symbol) + key = identify_package(where, String(name)) + key isa PkgId || throw(KeyError(name)) + return root_module(key) +end maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing) root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key) diff --git a/test/loading.jl b/test/loading.jl index 98f20bdebf4ae7..16690c6e046136 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -730,6 +730,14 @@ end end end +@testset "Issue #25719" begin + empty!(LOAD_PATH) + @test Base.root_module(Core, :Core) == Core + push!(LOAD_PATH, "@stdlib") + @test Base.root_module(Base, :Test) == Test + @test_throws KeyError(:SomeNonExistentPackage) Base.root_module(Base, :SomeNonExistentPackage) +end + ## cleanup after tests ## for env in keys(envs) From 75e2402f3353cfaa5fa71336c7350714cb61c538 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 11 Aug 2022 22:40:45 +0200 Subject: [PATCH 02/14] remove DelimitedFiles from being an stdlib (#45540) --- NEWS.md | 2 ++ .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - julia.spdx.json | 17 ----------------- stdlib/.gitignore | 2 -- stdlib/DelimitedFiles.version | 4 ---- stdlib/Makefile | 4 ++-- stdlib/Pkg.version | 2 +- 10 files changed, 7 insertions(+), 28 deletions(-) create mode 100644 deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 create mode 100644 deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 delete mode 100644 stdlib/DelimitedFiles.version diff --git a/NEWS.md b/NEWS.md index 886e24b706f1a9..f871424e47337b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -158,6 +158,8 @@ Standard library changes #### DelimitedFiles +* DelimitedFiles has been promoted from being a standard library to a separate package. It now has to be explicitly installed to be used. + Deprecated or removed --------------------- diff --git a/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 new file mode 100644 index 00000000000000..ea7ad31ce70351 --- /dev/null +++ b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/md5 @@ -0,0 +1 @@ +9e337aa579ec47017d7b5ef2df3bcd02 diff --git a/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 new file mode 100644 index 00000000000000..3ef1b9730fa601 --- /dev/null +++ b/deps/checksums/Pkg-8d77a6cac48113d143e028209aa28f10d5473032.tar.gz/sha512 @@ -0,0 +1 @@ +99e67b045691f8d9c16680e77014a33a507250377c037a7cf23e5dc2f0294c771a834e6d14f6a3a19a8def0ebb61a6fde17a4f3c1284cab0f4d7b5ea8c128d5d diff --git a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 b/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 deleted file mode 100644 index 8a7bea807777b7..00000000000000 --- a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ee424db01b8b85de4f927dbe5a294f6a diff --git a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 b/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 deleted file mode 100644 index cb3e080eac9433..00000000000000 --- a/deps/checksums/Pkg-cebcbc0a4af16fb7933fe101e8ef89a368231e60.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -aba17eb434325938daa43ae1fe518fc72eefc4d0a8cd1f8cfb0411361cf43a5e9b14ceeb145ab26af58336b6f99dd169668a40091a6156777068ea0fc5daad8a diff --git a/julia.spdx.json b/julia.spdx.json index 2e7e368a49c0d4..9c1911958096e6 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -146,18 +146,6 @@ "copyrightText": "Copyright (c) 2014: Elliot Saba", "summary": "A performant, 100% native-julia SHA1, SHA2, and SHA3 implementation" }, - { - "name": "DelimitedFiles.jl", - "SPDXID": "SPDXRef-JuliaDelimitedFiles", - "downloadLocation": "git+https://github.com/JuliaData/DelimitedFiles.jl.git", - "filesAnalyzed": false, - "homepage": "https://julialang.org", - "sourceInfo": "The git hash of the version in use can be found in the file stdlib/DelimitedFiles.version", - "licenseConcluded": "MIT", - "licenseDeclared": "MIT", - "copyrightText": "Copyright (c) 2012-2022 The Julia Programming Language", - "summary": "A package for reading and writing files with delimited values." - }, { "name": "dSFMT", "SPDXID": "SPDXRef-dSFMT", @@ -503,11 +491,6 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, - { - "spdxElementId": "SPDXRef-JuliaDelimitedFiles", - "relationshipType": "BUILD_DEPENDENCY_OF", - "relatedSpdxElement": "SPDXRef-JuliaMain" - }, { "spdxElementId": "SPDXRef-dSFMT", "relationshipType": "BUILD_DEPENDENCY_OF", diff --git a/stdlib/.gitignore b/stdlib/.gitignore index 038b2d9602b2a3..ffbc2f12f52da2 100644 --- a/stdlib/.gitignore +++ b/stdlib/.gitignore @@ -5,8 +5,6 @@ /Statistics /LibCURL-* /LibCURL -/DelimitedFiles-* -/DelimitedFiles /Downloads-* /Downloads /ArgTools-* diff --git a/stdlib/DelimitedFiles.version b/stdlib/DelimitedFiles.version deleted file mode 100644 index 220a1482822e0e..00000000000000 --- a/stdlib/DelimitedFiles.version +++ /dev/null @@ -1,4 +0,0 @@ -DELIMITEDFILES_BRANCH = main -DELIMITEDFILES_SHA1 = 495ebc81ef7fe2c083ca4c1d7e43a22cc1e49490 -DELIMITEDFILES_GIT_URL := https://github.com/JuliaData/DelimitedFiles.jl.git -DELIMITEDFILES_TAR_URL = https://api.github.com/repos/JuliaData/DelimitedFiles.jl/tarball/$1 diff --git a/stdlib/Makefile b/stdlib/Makefile index 5a72290ce96f00..d45be468626cd5 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -41,13 +41,13 @@ endef $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll)))) -STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ +STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \ Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization SHA \ SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \ $(JLL_NAMES) -STDLIBS_EXT = Pkg Statistics LibCURL DelimitedFiles Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA +STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z)))) diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 815ca8ca323a9a..49cb1c091ea5f6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = cebcbc0a4af16fb7933fe101e8ef89a368231e60 +PKG_SHA1 = 8d77a6cac48113d143e028209aa28f10d5473032 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 95cbb9f618401a1d5e5487467edad33f0e010950 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Thu, 11 Aug 2022 19:44:46 -0500 Subject: [PATCH 03/14] Extend sort's floating point optimizations to Float16 (#46149) --- base/sort.jl | 15 +++++++++++++-- test/sorting.jl | 3 +-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/base/sort.jl b/base/sort.jl index 59fa14aa99f658..8a3d2e2871cfe6 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -1464,10 +1464,15 @@ import Core.Intrinsics: slt_int import ..Sort: sort!, UIntMappable, uint_map, uint_unmap import ...Order: lt, DirectOrdering -const Floats = Union{Float32,Float64} -const FPSortable = Union{ # Mixed Float32 and Float64 are not allowed. +# IEEEFloat is not available in Core.Compiler +const Floats = Union{Float16, Float32, Float64} +# fpsort is not safe for vectors of mixed bitwidth such as Vector{Union{Float32, Float64}}. +# This type allows us to dispatch only when it is safe to do so. See #42739 for more info. +const FPSortable = Union{ + AbstractVector{Union{Float16, Missing}}, AbstractVector{Union{Float32, Missing}}, AbstractVector{Union{Float64, Missing}}, + AbstractVector{Float16}, AbstractVector{Float32}, AbstractVector{Float64}, AbstractVector{Missing}} @@ -1484,6 +1489,12 @@ right(o::Perm) = Perm(right(o.order), o.data) lt(::Left, x::T, y::T) where {T<:Floats} = slt_int(y, x) lt(::Right, x::T, y::T) where {T<:Floats} = slt_int(x, y) +uint_map(x::Float16, ::Left) = ~reinterpret(UInt16, x) +uint_unmap(::Type{Float16}, u::UInt16, ::Left) = reinterpret(Float16, ~u) +uint_map(x::Float16, ::Right) = reinterpret(UInt16, x) +uint_unmap(::Type{Float16}, u::UInt16, ::Right) = reinterpret(Float16, u) +UIntMappable(::Type{Float16}, ::Union{Left, Right}) = UInt16 + uint_map(x::Float32, ::Left) = ~reinterpret(UInt32, x) uint_unmap(::Type{Float32}, u::UInt32, ::Left) = reinterpret(Float32, ~u) uint_map(x::Float32, ::Right) = reinterpret(UInt32, x) diff --git a/test/sorting.jl b/test/sorting.jl index b2f663932c0ed7..9766ee99ce7514 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -652,8 +652,7 @@ end T = eltype(x) U = UIntN(Val(sizeof(T))) for order in [Forward, Reverse, Base.Sort.Float.Left(), Base.Sort.Float.Right(), By(Forward, identity)] - if order isa Base.Order.By || T === Float16 || - ((T <: AbstractFloat) == (order isa DirectOrdering)) + if order isa Base.Order.By || ((T <: AbstractFloat) == (order isa DirectOrdering)) @test Base.Sort.UIntMappable(T, order) === nothing continue end From e4a3106affaeb06bf5c6d1db646597598d3c0784 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 12 Aug 2022 07:43:50 -0400 Subject: [PATCH 04/14] Fix stack-use-after-scope in typemap (#46319) The value `a` is referenced in a GC frame past its liftetime. In practice this probably causes few issues, but in theory, the compiler is allowed to start scribbling garbage into the stack slot as soon as it goes out of scope. Asan complains about this (correctly). --- src/typemap.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index dfa8ac67f6abcf..cbabbe361daa5a 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -349,14 +349,13 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * goto exit; JL_GC_POP(); return 1; +exit: + JL_GC_POP(); + return 0; } else { return jl_typemap_node_visitor((jl_typemap_entry_t*)cache, fptr, closure); } - -exit: - JL_GC_POP(); - return 0; } static unsigned jl_supertype_height(jl_datatype_t *dt) From 5fc4ba91931526a08fc1bf8d3937aac731ee2cc6 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 12 Aug 2022 10:00:30 -0400 Subject: [PATCH 05/14] flisp: Fix memory leaks (#46249) There's two independent issues here: 1. The table allocator assumes that small tables will be stored inline and do not need a finalizer. This is mostly true, except that hash collisions can cause premature growing of the inline table, so even for relatively small tables, we need to validate that the storage was not allocated out-of-line. 2. It is unsafe to clear the vtable finalizer pointer during the table allocation to supress the `add_finalizer` call. This is because the allocation of the table object itself may trigger GC of a different table, and without the finalizer set in the vtable, freeing of that table's memory space would get skipped. Co-authored-by: Jeff Bezanson --- src/flisp/cvalues.c | 14 ++++++++++++-- src/flisp/flisp.h | 1 + src/flisp/table.c | 10 +++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/flisp/cvalues.c b/src/flisp/cvalues.c index 071a0b16429717..a5635c238ba3c3 100644 --- a/src/flisp/cvalues.c +++ b/src/flisp/cvalues.c @@ -108,7 +108,7 @@ static value_t cprim(fl_context_t *fl_ctx, fltype_t *type, size_t sz) return tagptr(pcp, TAG_CPRIM); } -value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +static value_t _cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz, int may_finalize) { cvalue_t *pcv; int str=0; @@ -127,7 +127,7 @@ value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) pcv = (cvalue_t*)alloc_words(fl_ctx, nw); pcv->type = type; pcv->data = &pcv->_space[0]; - if (type->vtable != NULL && type->vtable->finalize != NULL) + if (may_finalize && type->vtable != NULL && type->vtable->finalize != NULL) add_finalizer(fl_ctx, pcv); } else { @@ -148,6 +148,16 @@ value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) return tagptr(pcv, TAG_CVALUE); } +value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +{ + return _cvalue(fl_ctx, type, sz, 1); +} + +value_t cvalue_no_finalizer(fl_context_t *fl_ctx, fltype_t *type, size_t sz) +{ + return _cvalue(fl_ctx, type, sz, 0); +} + value_t cvalue_from_data(fl_context_t *fl_ctx, fltype_t *type, void *data, size_t sz) { value_t cv; diff --git a/src/flisp/flisp.h b/src/flisp/flisp.h index e77904a32d1f20..b031e456cd3fe3 100644 --- a/src/flisp/flisp.h +++ b/src/flisp/flisp.h @@ -328,6 +328,7 @@ typedef float fl_float_t; typedef value_t (*builtin_t)(fl_context_t*, value_t*, uint32_t); value_t cvalue(fl_context_t *fl_ctx, fltype_t *type, size_t sz) JL_NOTSAFEPOINT; +value_t cvalue_no_finalizer(fl_context_t *fl_ctx, fltype_t *type, size_t sz) JL_NOTSAFEPOINT; void add_finalizer(fl_context_t *fl_ctx, cvalue_t *cv); void cv_autorelease(fl_context_t *fl_ctx, cvalue_t *cv); void cv_pin(fl_context_t *fl_ctx, cvalue_t *cv); diff --git a/src/flisp/table.c b/src/flisp/table.c index a24cdf3bc06e8b..1d8aed358e88dd 100644 --- a/src/flisp/table.c +++ b/src/flisp/table.c @@ -87,9 +87,7 @@ value_t fl_table(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) value_t nt; // prevent small tables from being added to finalizer list if (cnt <= HT_N_INLINE) { - fl_ctx->table_vtable.finalize = NULL; - nt = cvalue(fl_ctx, fl_ctx->tabletype, sizeof(htable_t)); - fl_ctx->table_vtable.finalize = free_htable; + nt = cvalue_no_finalizer(fl_ctx, fl_ctx->tabletype, sizeof(htable_t)); } else { nt = cvalue(fl_ctx, fl_ctx->tabletype, 2*sizeof(void*)); @@ -104,6 +102,12 @@ value_t fl_table(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) else k = arg; } + if (h->table != &h->_space[0]) { + // We expected to use the inline table, but we ended up outgrowing it. + // Make sure to register the finalizer. + add_finalizer(fl_ctx, (cvalue_t*)ptr(nt)); + ((cvalue_t*)ptr(nt))->len = 2*sizeof(void*); + } return nt; } From 0e3e00dcb3512413ec35403d1d9135183293f99e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 12 Aug 2022 18:18:36 -0400 Subject: [PATCH 06/14] Work around dlopen not working properly under sanitizer instrumentation (#46255) `dlopen` has a mis-feature where it looks at the return address to determine the calling object to look at it's RUNPATH. Because asan intercepts `dlopen`, the calling object check finds asan rather than julia, causing an incorrect RUNPATH (and other flags to be used). Arguably, this is mostly a libc problem, because there isn't really a way to directly specify the resolution scope. I have sent a proposal to libc-coord [1] to fix this, but of course, we can't wait for that to percolate down to us. Instead, this takes advantage of the fact that almost all of our dlopen calls go through a single entrypoint in jl_dlopen, so we can insert additional logic here to make this work. This doesn't catch uses of `dlopen` in jlls (which is a problem for things like plugin loading in various jlls), but it at least makes base julia work. We can punt the jll question to another day - either with a patched libc in PkgEval or by patching the jll source with an analogous patch. Regardless, with this, Julia bootstraps properly under asan, without any special LD_LIBRARY_PATH hacks. [1] https://www.openwall.com/lists/libc-coord/2022/08/04/1 --- src/dlload.c | 91 +++++++++++++++++++++++++++++++++++++++++- src/support/platform.h | 13 ++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/src/dlload.c b/src/dlload.c index 230a31ed3d695b..308c8317cd5d68 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -4,6 +4,9 @@ #include #include #include +#ifdef __GLIBC__ +#include +#endif #include "platform.h" #include "julia.h" @@ -97,9 +100,62 @@ static void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOI } #endif +#if defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_) +struct link_map; +typedef void* (dlopen_prototype)(const char* filename, int flags); + +/* This function is copied from the memory sanitizer runtime. + Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + See https://llvm.org/LICENSE.txt for license information. + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ +static inline uintptr_t RoundUpTo(uintptr_t size, uintptr_t boundary) { + return (size + boundary - 1) & ~(boundary - 1); +} +static inline uintptr_t RoundDownTo(uintptr_t x, uintptr_t boundary) { + return x & ~(boundary - 1); +} +void ForEachMappedRegion(struct link_map *map, void (*cb)(const void *, uintptr_t)) { +#if !defined(_OS_FREEBSD_) + typedef ElfW(Phdr) Elf_Phdr; + typedef ElfW(Ehdr) Elf_Ehdr; +#endif + char *base = (char *)map->l_addr; + Elf_Ehdr *ehdr = (Elf_Ehdr *)base; + char *phdrs = base + ehdr->e_phoff; + char *phdrs_end = phdrs + ehdr->e_phnum * ehdr->e_phentsize; + + // Find the segment with the minimum base so we can "relocate" the p_vaddr + // fields. Typically ET_DYN objects (DSOs) have base of zero and ET_EXEC + // objects have a non-zero base. + uintptr_t preferred_base = (uintptr_t)-1; + for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) { + Elf_Phdr *phdr = (Elf_Phdr *)iter; + if (phdr->p_type == PT_LOAD && preferred_base > (uintptr_t)phdr->p_vaddr) + preferred_base = (uintptr_t)phdr->p_vaddr; + } + + // Compute the delta from the real base to get a relocation delta. + intptr_t delta = (uintptr_t)base - preferred_base; + // Now we can figure out what the loader really mapped. + for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) { + Elf_Phdr *phdr = (Elf_Phdr *)iter; + if (phdr->p_type == PT_LOAD) { + uintptr_t seg_start = phdr->p_vaddr + delta; + uintptr_t seg_end = seg_start + phdr->p_memsz; + // None of these values are aligned. We consider the ragged edges of the + // load command as defined, since they are mapped from the file. + seg_start = RoundDownTo(seg_start, jl_page_size); + seg_end = RoundUpTo(seg_end, jl_page_size); + cb((void *)seg_start, seg_end - seg_start); + } + } +} +#endif + +#if defined(_OS_WINDOWS_) JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT { -#if defined(_OS_WINDOWS_) size_t len = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0); if (!len) return NULL; WCHAR *wfilename = (WCHAR*)alloca(len * sizeof(WCHAR)); @@ -108,8 +164,32 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI if (lib) needsSymRefreshModuleList = 1; return lib; +} #else - return dlopen(filename, +JL_DLLEXPORT JL_NO_SANITIZE void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT +{ + /* The sanitizers break RUNPATH use in dlopen for annoying reasons that are + are hard to fix. Specifically, libc will use the return address of the + caller to determine certain paths and flags that affect .so location lookup. + To work around this, we need to avoid using the sanitizer's dlopen interposition, + instead using the real dlopen directly from the current shared library. + Of course, this does mean that we need to manually perform the work that + the sanitizers would otherwise do. */ +#if (defined(_COMPILER_MSAN_ENABLED_) || defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)) && __GLIBC__ + static dlopen_prototype *dlopen = NULL; + if (!dlopen) { + dlopen = (dlopen_prototype*)dlsym(RTLD_NEXT, "dlopen"); + if (!dlopen) + return NULL; + void *libdl_handle = dlopen("libdl.so", RTLD_NOW | RTLD_NOLOAD); + dlopen = (dlopen_prototype*)dlsym(libdl_handle, "dlopen"); + dlclose(libdl_handle); + assert(dlopen); + } + // The real interceptors check the validty of the string here, but let's + // just skip that for the time being. +#endif + void *hnd = dlopen(filename, (flags & JL_RTLD_NOW ? RTLD_NOW : RTLD_LAZY) | JL_RTLD(flags, LOCAL) | JL_RTLD(flags, GLOBAL) @@ -126,8 +206,15 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOI | JL_RTLD(flags, FIRST) #endif ); +#if defined(_COMPILER_MSAN_ENABLED_) && defined(__GLIBC__) + link_map *map = (link_map*)handle; + if (filename && map) + ForEachMappedRegion(map, __msan_unpoison); #endif + return hnd; } +#endif + JL_DLLEXPORT int jl_dlclose(void *handle) JL_NOTSAFEPOINT { diff --git a/src/support/platform.h b/src/support/platform.h index cf65fa01423feb..1afe0148be045f 100644 --- a/src/support/platform.h +++ b/src/support/platform.h @@ -43,24 +43,37 @@ #error Unsupported compiler #endif + +#define JL_NO_ASAN +#define JL_NO_MSAN +#define JL_NO_TSAN #if defined(__has_feature) // Clang flavor #if __has_feature(address_sanitizer) #define _COMPILER_ASAN_ENABLED_ +#undef JL_NO_ASAN +#define JL_NO_ASAN __attribute__((no_sanitize("address"))) #endif #if __has_feature(memory_sanitizer) #define _COMPILER_MSAN_ENABLED_ +#undef JL_NO_MSAN +#define JL_NO_MSAN __attribute__((no_sanitize("mempry"))) #endif #if __has_feature(thread_sanitizer) #if __clang_major__ < 11 #error Thread sanitizer runtime libraries in clang < 11 leak memory and cannot be used #endif #define _COMPILER_TSAN_ENABLED_ +#undef JL_NO_TSAN +#define JL_NO_TSAN __attribute__((no_sanitize("thread"))) #endif #else // GCC flavor #if defined(__SANITIZE_ADDRESS__) #define _COMPILER_ASAN_ENABLED_ +#undef JL_NO_ASAN +#define JL_NO_ASAN __attribute__((no_sanitize("address"))) #endif #endif // __has_feature +#define JL_NO_SANITIZE JL_NO_ASAN JL_NO_MSAN JL_NO_TSAN /******************************************************************************* * OS * From d2aedf48ca799cbe11c6db9188e8af9ee5340e5f Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 13 Aug 2022 10:01:23 +0900 Subject: [PATCH 07/14] define `isType` in `Base` (#46326) --- base/compiler/typeutils.jl | 2 -- base/deprecated.jl | 2 +- base/errorshow.jl | 2 +- base/reflection.jl | 6 ++++-- base/show.jl | 3 +-- stdlib/REPL/src/REPLCompletions.jl | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/base/compiler/typeutils.jl b/base/compiler/typeutils.jl index c70394e891c9fb..0e59fc9daa8aeb 100644 --- a/base/compiler/typeutils.jl +++ b/base/compiler/typeutils.jl @@ -4,8 +4,6 @@ # lattice utilities # ##################### -isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME - # true if Type{T} is inlineable as constant T # requires that T is a singleton, s.t. T == S implies T === S isconstType(@nospecialize t) = isType(t) && hasuniquerep(t.parameters[1]) diff --git a/base/deprecated.jl b/base/deprecated.jl index 83ea85cddff7a9..61ba282f390dea 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -159,7 +159,7 @@ function firstcaller(bt::Vector, funcsyms) li = lkup.linfo if li isa Core.MethodInstance ft = ccall(:jl_first_argument_datatype, Any, (Any,), (li.def::Method).sig) - if isa(ft, DataType) && ft.name === Type.body.name + if isType(ft) ft = unwrap_unionall(ft.parameters[1]) found = (isa(ft, DataType) && ft.name.name in funcsyms) end diff --git a/base/errorshow.jl b/base/errorshow.jl index 1b00419cd85108..9218abe02a1879 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -400,7 +400,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=() # pool MethodErrors for these two functions. if f === convert && !isempty(arg_types_param) at1 = arg_types_param[1] - if isa(at1,DataType) && (at1::DataType).name === Type.body.name && !Core.Compiler.has_free_typevars(at1) + if isType(at1) && !Core.Compiler.has_free_typevars(at1) push!(funcs, (at1.parameters[1], arg_types_param[2:end])) end end diff --git a/base/reflection.jl b/base/reflection.jl index 94c103fb5c8de7..00b8d974d4b357 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -605,12 +605,14 @@ has_free_typevars(@nospecialize(t)) = ccall(:jl_has_free_typevars, Cint, (Any,), # equivalent to isa(v, Type) && isdispatchtuple(Tuple{v}) || v === Union{} # and is thus perhaps most similar to the old (pre-1.0) `isleaftype` query -const _TYPE_NAME = Type.body.name function isdispatchelem(@nospecialize v) return (v === Bottom) || (v === typeof(Bottom)) || isconcretedispatch(v) || - (isa(v, DataType) && v.name === _TYPE_NAME && !has_free_typevars(v)) # isType(v) + (isType(v) && !has_free_typevars(v)) end +const _TYPE_NAME = Type.body.name +isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME + """ isconcretetype(T) diff --git a/base/show.jl b/base/show.jl index a9bbb27df6790e..6f17fe6ee2ef66 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2392,8 +2392,7 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg end s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io) print_within_stacktrace(io, s, bold=true) - elseif isa(ft, DataType) && ft.name === Type.body.name && - (f = ft.parameters[1]; !isa(f, TypeVar)) + elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar)) uwf = unwrap_unionall(f) parens = isa(f, UnionAll) && !(isa(uwf, DataType) && f === uwf.name.wrapper) parens && print(io, "(") diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 3c16c804ccebcc..7658a227b51cdc 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -198,7 +198,7 @@ function complete_symbol(sym::String, @nospecialize(ffunc), context_module::Modu # Looking for a member of a type if t isa DataType && t != Any # Check for cases like Type{typeof(+)} - if t isa DataType && t.name === Base._TYPE_NAME + if Base.isType(t) t = typeof(t.parameters[1]) end # Only look for fields if this is a concrete type From e1fa6a51e4142fbf25019b1c95ebc3b5b7f4e8a1 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 12 Aug 2022 21:32:16 -0400 Subject: [PATCH 08/14] Fix mismatched GC frame mgmt in llvmcall codegen (#46335) Unfortunately clang-sa is still disabled on .cpp files, otherwise it would have caught this. Found by asan. --- src/ccall.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ccall.cpp b/src/ccall.cpp index 88c80b333b0272..0fddf5425acc03 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -760,6 +760,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar ir = static_eval(ctx, ir_arg); if (!ir) { emit_error(ctx, "error statically evaluating llvm IR argument"); + JL_GC_POP(); return jl_cgval_t(); } if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) { @@ -771,6 +772,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar rt = static_eval(ctx, args[2]); if (!rt) { emit_error(ctx, "error statically evaluating llvmcall return type"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -783,6 +785,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar at = static_eval(ctx, args[3]); if (!at) { emit_error(ctx, "error statically evaluating llvmcall argument tuple"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -790,23 +793,27 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar // if the IR is a tuple, we expect (mod, fn) if (jl_nfields(ir) != 2) { emit_error(ctx, "Tuple as first argument to llvmcall must have exactly two children"); + JL_GC_POP(); return jl_cgval_t(); } entry = jl_fieldref(ir, 1); if (!jl_is_string(entry)) { emit_error(ctx, "Function name passed to llvmcall must be a string"); + JL_GC_POP(); return jl_cgval_t(); } ir = jl_fieldref(ir, 0); if (!jl_is_string(ir) && !jl_typeis(ir, jl_array_uint8_type)) { emit_error(ctx, "Module IR passed to llvmcall must be a string or an array of bytes"); + JL_GC_POP(); return jl_cgval_t(); } } else { if (!jl_is_string(ir)) { emit_error(ctx, "Function IR passed to llvmcall must be a string"); + JL_GC_POP(); return jl_cgval_t(); } } @@ -835,6 +842,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar argtypes.push_back(t); if (4 + i > nargs) { emit_error(ctx, "Missing arguments to llvmcall!"); + JL_GC_POP(); return jl_cgval_t(); } jl_value_t *argi = args[4 + i]; @@ -889,6 +897,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); Err.print("", stream, true); emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } @@ -906,6 +915,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); Err.print("", stream, true); emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } } @@ -923,6 +933,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); stream << Message; emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } Mod = std::move(ModuleOrErr.get()); @@ -931,6 +942,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar Function *f = Mod->getFunction(jl_string_data(entry)); if (!f) { emit_error(ctx, "Module IR does not contain specified entry function"); + JL_GC_POP(); return jl_cgval_t(); } f->setName(ir_name); @@ -959,6 +971,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar raw_string_ostream stream(message); if (verifyFunction(*def, &stream)) { emit_error(ctx, stream.str()); + JL_GC_POP(); return jl_cgval_t(); } def->setLinkage(GlobalVariable::LinkOnceODRLinkage); From f4cee905e258967948dc7a9ec5ed7f62b95e3410 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Sun, 14 Aug 2022 13:36:32 -0500 Subject: [PATCH 09/14] Replace `reduce(max` with `maximum` in sysimg.jl (#46343) --- base/sysimg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index 79f01a0e9a1d25..183934d77fb6bc 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -71,7 +71,7 @@ let # 7-depth packages :LazyArtifacts, ] - maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0) + maxlen = maximum(textwidth.(string.(stdlibs))) tot_time_stdlib = 0.0 # use a temp module to avoid leaving the type of this closure in Main From fde3a68005720ed74f802d9aad5e039e59cfd52b Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Mon, 15 Aug 2022 10:05:28 +0200 Subject: [PATCH 10/14] Allow elimination of `isbits` for non-const argument (#46330) It should be sufficient for the _type_ of `x` to be known at compile time to fully eliminate `isbits(x)`. However, the pevious code relied on `isbits` being called on its argument during inference (by declaring it `:total`) for eliminating it. But if the value of `x` was not known, only its type, this would not work. The new implementation has the added benefit of being simpler and DRYer (wrt. `isbitstype`). --- base/reflection.jl | 2 +- test/reflection.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/reflection.jl b/base/reflection.jl index 00b8d974d4b357..f377e7b16b5711 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -588,7 +588,7 @@ isbitstype(@nospecialize t) = (@_total_meta; isa(t, DataType) && (t.flags & 0x8) Return `true` if `x` is an instance of an [`isbitstype`](@ref) type. """ -isbits(@nospecialize x) = (@_total_meta; typeof(x).flags & 0x8 == 0x8) +isbits(@nospecialize x) = isbitstype(typeof(x)) """ isdispatchtuple(T) diff --git a/test/reflection.jl b/test/reflection.jl index 5cb20256538e07..e28e92142e5f67 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -2,6 +2,8 @@ using Test +include("compiler/irutils.jl") + # code_native / code_llvm (issue #8239) # It's hard to really test these, but just running them should be # sufficient to catch segfault bugs. @@ -66,6 +68,7 @@ end # module ReflectionTest @test isbits((1,2)) @test !isbits([1]) @test isbits(nothing) +@test fully_eliminated(isbits, (Int,)) # issue #16670 @test isconcretetype(Int) From 99f3722e5c6cb8b285227b1d57d987d997221c3a Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Mon, 15 Aug 2022 15:40:36 +0300 Subject: [PATCH 11/14] Add `Base.reverse(::NamedTuple)` (#45488) --- base/namedtuple.jl | 1 + test/namedtuple.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/namedtuple.jl b/base/namedtuple.jl index b2ebb3f9d0d7e3..70ebba34abe87c 100644 --- a/base/namedtuple.jl +++ b/base/namedtuple.jl @@ -320,6 +320,7 @@ get(nt::NamedTuple, key::Union{Integer, Symbol}, default) = isdefined(nt, key) ? get(f::Callable, nt::NamedTuple, key::Union{Integer, Symbol}) = isdefined(nt, key) ? getfield(nt, key) : f() tail(t::NamedTuple{names}) where names = NamedTuple{tail(names)}(t) front(t::NamedTuple{names}) where names = NamedTuple{front(names)}(t) +reverse(nt::NamedTuple) = NamedTuple{reverse(keys(nt))}(reverse(values(nt))) @assume_effects :total function diff_names(an::Tuple{Vararg{Symbol}}, bn::Tuple{Vararg{Symbol}}) @nospecialize an bn diff --git a/test/namedtuple.jl b/test/namedtuple.jl index 94eb14008dc523..6c41ab788b7331 100644 --- a/test/namedtuple.jl +++ b/test/namedtuple.jl @@ -147,6 +147,8 @@ end @test Base.front((a = 1, )) ≡ NamedTuple() @test_throws ArgumentError Base.tail(NamedTuple()) @test_throws ArgumentError Base.front(NamedTuple()) +@test @inferred(reverse((a=1,))) === (a=1,) +@test @inferred(reverse((a=1, b=:c))) === (b=:c, a=1) # syntax errors From 240f9acf3ed3b03a13ebaf94d0768fbbe8421072 Mon Sep 17 00:00:00 2001 From: Arno Strouwen Date: Mon, 15 Aug 2022 14:53:16 +0200 Subject: [PATCH 12/14] Remove superfluous vcat in dot syntax docs (#46334) --- doc/src/manual/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index c3012efa1d8b1d..f39fb7669ba0dd 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -1097,7 +1097,7 @@ they are equivalent to `broadcast` calls and are fused with other nested "dot" c You can also combine dot operations with function chaining using [`|>`](@ref), as in this example: ```jldoctest -julia> [1:5;] .|> [x->x^2, inv, x->2*x, -, isodd] +julia> 1:5 .|> [x->x^2, inv, x->2*x, -, isodd] 5-element Vector{Real}: 1 0.5 From c80316e125c201ad247e77ac0e945092b866d3c7 Mon Sep 17 00:00:00 2001 From: Michael Creel Date: Mon, 15 Aug 2022 06:05:20 -0700 Subject: [PATCH 13/14] Update index.md (#46342) Add sentence mentioning that random numbers from other distributions can be obtained using the Distributions.jl package, with a link to the documentation. --- stdlib/Random/docs/src/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/Random/docs/src/index.md b/stdlib/Random/docs/src/index.md index 0f7636cf2444f5..e344e479474402 100644 --- a/stdlib/Random/docs/src/index.md +++ b/stdlib/Random/docs/src/index.md @@ -33,6 +33,8 @@ unbounded integers, the interval must be specified (e.g. `rand(big.(1:6))`). Additionally, normal and exponential distributions are implemented for some `AbstractFloat` and `Complex` types, see [`randn`](@ref) and [`randexp`](@ref) for details. +To generate random numbers from other distributions, see the [Distributions.jl](https://juliastats.org/Distributions.jl/stable/) package. + !!! warning Because the precise way in which random numbers are generated is considered an implementation detail, bug fixes and speed improvements may change the stream of numbers that are generated after a version change. Relying on a specific seed or generated stream of numbers during unit testing is thus discouraged - consider testing properties of the methods in question instead. From ec98087cbf19bac26f6be05df9282746b0cffe78 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Aug 2022 09:03:23 +0200 Subject: [PATCH 14/14] Make JULIA_BUILD_MODE=debug work with make install. (#46320) Replaces BUNDLE_DEBUG_LIBS, and fixes fixup-libstdc++.sh. --- Makefile | 61 +++++++++++++++++++------------------- contrib/fixup-libstdc++.sh | 9 ++++-- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 6c6d39709f9aed..3b1c6f0d0bc7bd 100644 --- a/Makefile +++ b/Makefile @@ -169,14 +169,17 @@ $(build_datarootdir)/julia/base.cache: $(JULIA_SYSIMG) | $(DIRS) $(build_dataroo $(call cygpath_w,$@)) # public libraries, that are installed in $(prefix)/lib +ifeq ($(JULIA_BUILD_MODE),release) JL_TARGETS := julia -ifeq ($(BUNDLE_DEBUG_LIBS),1) -JL_TARGETS += julia-debug +else ifeq ($(JULIA_BUILD_MODE),debug) +JL_TARGETS := julia-debug endif # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest libjulia-internal libjulia-codegen -ifeq ($(BUNDLE_DEBUG_LIBS),1) +JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest +ifeq ($(JULIA_BUILD_MODE),release) +JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen +else ifeq ($(JULIA_BUILD_MODE),debug) JL_PRIVATE_LIBS-0 += libjulia-internal-debug libjulia-codegen-debug endif ifeq ($(USE_GPL_LIBS), 1) @@ -237,38 +240,32 @@ endef install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html -ifeq ($(BUNDLE_DEBUG_LIBS),1) - @$(MAKE) $(QUIET_MAKE) all -else - @$(MAKE) $(QUIET_MAKE) release -endif + @$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE) @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \ mkdir -p $(DESTDIR)$$subdir; \ done - $(INSTALL_M) $(build_bindir)/julia $(DESTDIR)$(bindir)/ -ifeq ($(BUNDLE_DEBUG_LIBS),1) - $(INSTALL_M) $(build_bindir)/julia-debug $(DESTDIR)$(bindir)/ -endif + $(INSTALL_M) $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) $(DESTDIR)$(bindir)/ ifeq ($(OS),WINNT) - -$(INSTALL_M) $(filter-out $(build_bindir)/libjulia-debug.dll,$(wildcard $(build_bindir)/*.dll)) $(DESTDIR)$(bindir)/ + -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ +ifeq ($(JULIA_BUILD_MODE),release) -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ +else ifeq ($(JULIA_BUILD_MODE),debug) + -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ +endif # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it. -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/ -ifeq ($(BUNDLE_DEBUG_LIBS),1) - -$(INSTALL_M) $(build_bindir)/libjulia-debug.dll $(DESTDIR)$(bindir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ -endif -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(OS),Darwin) +ifeq ($(JULIA_BUILD_MODE),release) -cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) -cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) -cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) -cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) endif @@ -283,10 +280,11 @@ endif done else # libjulia in Darwin framework has special location and name +ifeq ($(JULIA_BUILD_MODE),release) $(INSTALL_M) $(build_libdir)/libjulia.$(SOMAJOR).$(SOMINOR).dylib $(DESTDIR)$(prefix)/$(framework_dylib) @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/$(FRAMEWORK_NAME).dSYM $(DESTDIR)$(prefix)/$(framework_dylib) @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/sys.dylib.dSYM $(build_private_libdir)/sys.dylib -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_libdir)/libjulia-debug.$(SOMAJOR).$(SOMINOR).dylib $(DESTDIR)$(prefix)/$(framework_dylib)_debug @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/$(FRAMEWORK_NAME)_debug.dSYM $(DESTDIR)$(prefix)/$(framework_dylib)_debug @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/sys-debug.dylib.dSYM $(build_private_libdir)/sys-debug.dylib @@ -314,8 +312,9 @@ endif # Copy public headers cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia # Copy system image +ifeq ($(JULIA_BUILD_MODE),release) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif @@ -364,16 +363,18 @@ endif RELEASE_TARGET=$(DESTDIR)$(prefix)/$(framework_dylib); \ DEBUG_TARGET=$(DESTDIR)$(prefix)/$(framework_dylib)_debug; \ fi; \ - $(call stringreplace,$${RELEASE_TARGET},sys.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys.$(SHLIB_EXT)); \ - if [ "$(BUNDLE_DEBUG_LIBS)" = "1" ]; then \ + if [ "$(JULIA_BUILD_MODE)" = "release" ]; then \ + $(call stringreplace,$${RELEASE_TARGET},sys.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys.$(SHLIB_EXT)); \ + elif [ "$(JULIA_BUILD_MODE)" = "debug" ]; then \ $(call stringreplace,$${DEBUG_TARGET},sys-debug.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys-debug.$(SHLIB_EXT)); \ fi; endif # Set rpath for libjulia-internal, which is moving from `../lib` to `../lib/julia`. We only need to do this for Linux/FreeBSD ifneq (,$(findstring $(OS),Linux FreeBSD)) +ifeq ($(JULIA_BUILD_MODE),release) $(PATCHELF) --set-rpath '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(PATCHELF) --set-rpath '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) endif endif @@ -381,16 +382,16 @@ endif ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS)) # Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS) +ifeq ($(JULIA_BUILD_MODE),release) $(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT),$(LOADER_BUILD_DEP_LIBS)$$,$(LOADER_INSTALL_DEP_LIBS)) -ifeq ($(OS),Darwin) - # Codesign the libjulia we just modified - $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT)" -endif - -ifeq ($(BUNDLE_DEBUG_LIBS),1) +else ifeq ($(JULIA_BUILD_MODE),debug) $(call stringreplace,$(DESTDIR)$(shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT),$(LOADER_DEBUG_BUILD_DEP_LIBS)$$,$(LOADER_DEBUG_INSTALL_DEP_LIBS)) +endif ifeq ($(OS),Darwin) # Codesign the libjulia we just modified +ifeq ($(JULIA_BUILD_MODE),release) + $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT)" +else ifeq ($(JULIA_BUILD_MODE),debug) $(JULIAHOME)/contrib/codesign.sh "$(MACOS_CODESIGN_IDENTITY)" "$(DESTDIR)$(shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT)" endif endif diff --git a/contrib/fixup-libstdc++.sh b/contrib/fixup-libstdc++.sh index 1c19d98a54b1e4..7442d995448a14 100755 --- a/contrib/fixup-libstdc++.sh +++ b/contrib/fixup-libstdc++.sh @@ -11,7 +11,8 @@ fi libdir="$1" private_libdir="$2" -if [ ! -f "$private_libdir/libjulia-internal.so" ]; then +if [ ! -f "$private_libdir/libjulia-internal.so" ] && \ + [ ! -f "$private_libdir/libjulia-internal-debug.so" ]; then echo "ERROR: Could not open $private_libdir/libjulia-internal.so" >&2 exit 2 fi @@ -24,7 +25,11 @@ find_shlib () } # Discover libstdc++ location and name -LIBSTD=$(find_shlib "$private_libdir/libjulia-internal.so" "libstdc++.so") +if [ -f "$private_libdir/libjulia-internal.so" ]; then + LIBSTD=$(find_shlib "$private_libdir/libjulia-internal.so" "libstdc++.so") +elif [ -f "$private_libdir/libjulia-internal-debug.so" ]; then + LIBSTD=$(find_shlib "$private_libdir/libjulia-internal-debug.so" "libstdc++.so") +fi LIBSTD_NAME=$(basename $LIBSTD) LIBSTD_DIR=$(dirname $LIBSTD)