From cbdc2242d9118d883d3c4904f83893eb8b65f450 Mon Sep 17 00:00:00 2001 From: Ronan Arraes Jardim Chagas Date: Fri, 3 Nov 2017 12:39:22 -0200 Subject: [PATCH 01/44] Clarify precedence of numeric literal coefs over parenthesis (#21800) (cherry picked from commit ae51693ad4ce45837e3fea96ccfe26051a2ab2ac) --- doc/src/manual/integers-and-floating-point-numbers.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/src/manual/integers-and-floating-point-numbers.md b/doc/src/manual/integers-and-floating-point-numbers.md index 1b819938cc625..f19b08443848b 100644 --- a/doc/src/manual/integers-and-floating-point-numbers.md +++ b/doc/src/manual/integers-and-floating-point-numbers.md @@ -611,6 +611,11 @@ Numeric literals also work as coefficients to parenthesized expressions: julia> 2(x-1)^2 - 3(x-1) + 1 3 ``` +!!! note + The precedence of numeric literal coefficients used for implicit + multiplication is higher than other binary operators such as multiplication + (`*`), and division (`/`, `\`, and `//`). This means, for example, that + `1 / 2im` equals `-0.5im` and `6 // 2(2 + 1)` equals `1 // 1`. Additionally, parenthesized expressions can be used as coefficients to variables, implying multiplication of the expression by the variable: From b63c98c844a1242f3020eeaa02c49d4ccd770b6e Mon Sep 17 00:00:00 2001 From: "Todd A. Anderson" Date: Sun, 25 Jun 2017 09:16:46 -0700 Subject: [PATCH 02/44] Preclude toplevel thunks from being printed when dump_compiles_stream is non NULL to prevent non-escaping double-quotes and also because it probably doesn't help the users of the dump_compiles_stream feature. Ref #22538 (cherry picked from commit 5b9b8ce5b80a8ff7d6d63a3716ea6856160341ff) --- src/codegen.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 1e08cfce6b336..1847c063595d5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1325,7 +1325,11 @@ jl_llvm_functions_t jl_compile_linfo(jl_method_instance_t **pli, jl_code_info_t JL_UNLOCK(&codegen_lock); // Might GC - if (dump_compiles_stream != NULL) { + // If logging of the compilation stream is enabled then dump the function to the stream + // ... unless li->def isn't defined here meaning the function is a toplevel thunk and + // would have its CodeInfo printed in the stream, which might contain double-quotes that + // would not be properly escaped given the double-quotes added to the stream below. + if (dump_compiles_stream != NULL && li->def) { uint64_t this_time = jl_hrtime(); jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", this_time - last_time); jl_static_show(dump_compiles_stream, (jl_value_t*)li); From 0379efaa35d8c2fa5b8efddee081caa440993a78 Mon Sep 17 00:00:00 2001 From: "Todd A. Anderson" Date: Mon, 26 Jun 2017 18:36:15 -0700 Subject: [PATCH 03/44] Add tests of jl_dump_compiles functionality. One for regular functions and one to make sure that toplevel thunks aren't printed. Ref #22538 (cherry picked from commit dd5f4231de26bc604406b24cb4bc4466ad4059ab) --- test/codegen.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/codegen.jl b/test/codegen.jl index 6925ef1266ff8..a5009875ef290 100644 --- a/test/codegen.jl +++ b/test/codegen.jl @@ -51,9 +51,13 @@ end function test_jl_dump_compiles() tfile = tempname() io = open(tfile, "w") - @eval(test_jl_dump_compiles_internal(x) = x) ccall(:jl_dump_compiles, Void, (Ptr{Void},), io.handle) - @eval test_jl_dump_compiles_internal(1) + eval(@noinline function test_jl_dump_compiles_internal(x) + if x > 0 + test_jl_dump_compiles_internal(x-1) + end + end) + test_jl_dump_compiles_internal(1) ccall(:jl_dump_compiles, Void, (Ptr{Void},), C_NULL) close(io) tstats = stat(tfile) @@ -67,9 +71,8 @@ end function test_jl_dump_compiles_toplevel_thunks() tfile = tempname() io = open(tfile, "w") - topthunk = expand(Main, :(for i in 1:10; end)) ccall(:jl_dump_compiles, Void, (Ptr{Void},), io.handle) - Core.eval(Main, topthunk) + eval(expand(Main, :(for i in 1:10 end))) ccall(:jl_dump_compiles, Void, (Ptr{Void},), C_NULL) close(io) tstats = stat(tfile) From 67ea744c717c3e1f9c10bf32caf2602a733de643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bogumi=C5=82=20Kami=C5=84ski?= Date: Wed, 20 Sep 2017 16:28:36 +0200 Subject: [PATCH 04/44] iostream: Fix \r\n handling in reading buffers (#22621) (cherry picked from commit 1c0573e61953577945474962e9a48f787f21df28) --- src/flisp/iostream.c | 4 ++-- src/support/ios.c | 10 +++------- src/support/ios.h | 2 +- src/sys.c | 14 +++++++++++--- test/iobuffer.jl | 19 +++++++++++++++++++ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index a8de001ad8532..197e60be731d5 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -333,7 +333,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_setbuf(&dest, data, 80, 0); char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil"); ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil"); - size_t n = ios_copyuntil(&dest, src, delim, 0); + size_t n = ios_copyuntil(&dest, src, delim); cv->len = n; if (dest.buf != data) { // outgrew initial space @@ -352,7 +352,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil"); ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil"); char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil"); - return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 0)); + return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim)); } value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) diff --git a/src/support/ios.c b/src/support/ios.c index 70af4078f6e9e..22bd6c156b0f0 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -803,7 +803,7 @@ size_t ios_copyall(ios_t *to, ios_t *from) #define LINE_CHUNK_SIZE 160 -size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) { size_t total = 0, avail = (size_t)(from->size - from->bpos); while (!ios_eof(from)) { @@ -822,11 +822,7 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) } else { size_t ntowrite = pd - (from->buf+from->bpos) + 1; - size_t nchomp = 0; - if (chomp) { - nchomp = ios_nchomp(from, ntowrite); - } - written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); + written = ios_write(to, from->buf+from->bpos, ntowrite); from->bpos += ntowrite; total += written; return total; @@ -1153,7 +1149,7 @@ char *ios_readline(ios_t *s) { ios_t dest; ios_mem(&dest, 0); - ios_copyuntil(&dest, s, '\n', 0); + ios_copyuntil(&dest, s, '\n'); size_t n; return ios_take_buffer(&dest, &n); } diff --git a/src/support/ios.h b/src/support/ios.h index 66003f81d82d2..be1470125d907 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -94,7 +94,7 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s); JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); -JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp); +JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim); JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index a1cdc280cc7fb..ba3a44a352717 100644 --- a/src/sys.c +++ b/src/sys.c @@ -254,9 +254,9 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint { jl_array_t *a; // manually inlined common case - char *pd = (char*)memchr(s->buf+s->bpos, delim, (size_t)(s->size - s->bpos)); + char *pd = (char*)memchr(s->buf + s->bpos, delim, (size_t)(s->size - s->bpos)); if (pd) { - size_t n = pd-(s->buf+s->bpos)+1; + size_t n = pd - (s->buf + s->bpos) + 1; if (str) { size_t nchomp = 0; if (chomp) { @@ -275,7 +275,15 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint ios_t dest; ios_mem(&dest, 0); ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyuntil(&dest, s, delim, chomp); + size_t n = ios_copyuntil(&dest, s, delim); + if (chomp && n > 0 && dest.buf[n - 1] == '\n') { + n--; + if (n > 0 && dest.buf[n - 1] == '\r') { + n--; + } + int truncret = ios_trunc(&dest, n); // it should always be possible to truncate dest + assert(truncret == 0); + } if (dest.buf != a->data) { a = jl_take_buffer(&dest); } diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 5921b3714b4be..a7de17b61e50a 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -88,6 +88,25 @@ write(io,"\n\r\n\n\r \n") > 0 @test readlines(IOBuffer(""), chomp=true) == [] @test readlines(IOBuffer("first\nsecond"), chomp=false) == String["first\n", "second"] @test readlines(IOBuffer("first\nsecond"), chomp=true) == String["first", "second"] + +let fname = tempname() + for dochomp in [true, false], + endline in ["\n", "\r\n"], + i in -5:5 + + ref = ("1"^(2^17 - i)) * endline + open(fname, "w") do io + write(io, ref) + end + x = readlines(fname, chomp = dochomp) + if dochomp + ref = chomp(ref) + end + @test ref == x[1] + end + rm(fname) +end + Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 From 614547f02fe8ca1814d5ebf29eab440c8d064e04 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Wed, 20 Sep 2017 01:16:01 -0700 Subject: [PATCH 05/44] Fix getindex on LQPackedQ. (#23782) (cherry picked from commit 25622635747c57edc8493312b393d7ce271f6cb6) --- base/linalg/lq.jl | 9 ++------- test/linalg/lq.jl | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index a647550bcf9a4..fd4f95fbe2593 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -69,13 +69,8 @@ function getindex(A::LQ, d::Symbol) end end -function getindex(A::LQPackedQ, i::Integer, j::Integer) - x = zeros(eltype(A), size(A, 1)) - x[i] = 1 - y = zeros(eltype(A), size(A, 2)) - y[j] = 1 - return dot(x, A*y) -end +getindex(A::LQPackedQ, i::Integer, j::Integer) = + A_mul_B!(A, setindex!(zeros(eltype(A), size(A, 2)), 1, j))[i] getq(A::LQ) = LQPackedQ(A.factors, A.τ) diff --git a/test/linalg/lq.jl b/test/linalg/lq.jl index a9cfcdd54b34f..30b55ba0c9573 100644 --- a/test/linalg/lq.jl +++ b/test/linalg/lq.jl @@ -103,3 +103,34 @@ bimg = randn(n,2)/2 end end end + +@testset "getindex on LQPackedQ (#23733)" begin + function getqs(F::Base.LinAlg.LQ) + implicitQ = F[:Q] + explicitQ = A_mul_B!(implicitQ, eye(eltype(implicitQ), size(implicitQ.factors, 2))) + return implicitQ, explicitQ + end + + m, n = 3, 3 # thin Q 3-by-3, square Q 3-by-3 + implicitQ, explicitQ = getqs(lqfact(randn(m, n))) + @test implicitQ[1, 1] == explicitQ[1, 1] + @test implicitQ[m, 1] == explicitQ[m, 1] + @test implicitQ[1, n] == explicitQ[1, n] + @test implicitQ[m, n] == explicitQ[m, n] + + m, n = 3, 4 # thin Q 3-by-4, square Q 4-by-4 + implicitQ, explicitQ = getqs(lqfact(randn(m, n))) + @test implicitQ[1, 1] == explicitQ[1, 1] + @test implicitQ[m, 1] == explicitQ[m, 1] + @test implicitQ[1, n] == explicitQ[1, n] + @test implicitQ[m, n] == explicitQ[m, n] + @test implicitQ[m+1, 1] == explicitQ[m+1, 1] + @test implicitQ[m+1, n] == explicitQ[m+1, n] + + m, n = 4, 3 # thin Q 3-by-3, square Q 3-by-3 + implicitQ, explicitQ = getqs(lqfact(randn(m, n))) + @test implicitQ[1, 1] == explicitQ[1, 1] + @test implicitQ[n, 1] == explicitQ[n, 1] + @test implicitQ[1, n] == explicitQ[1, n] + @test implicitQ[n, n] == explicitQ[n, n] +end From b1f399cd7eebbaafcb9b1135e6641e6b262e1c2f Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Wed, 20 Sep 2017 16:52:39 +0530 Subject: [PATCH 06/44] Explicitly import warn_once, shell_escape, uv_error into Distributed Ref #23791 (cherry picked from commit e5acd8ddd968cba27b939e5326bfc7bc0f27b7c7) --- base/distributed/Distributed.jl | 3 ++- base/distributed/managers.jl | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/base/distributed/Distributed.jl b/base/distributed/Distributed.jl index 73270b0672b3b..06e2bd9b06a6c 100644 --- a/base/distributed/Distributed.jl +++ b/base/distributed/Distributed.jl @@ -10,7 +10,8 @@ import Base: getindex, wait, put!, take!, fetch, isready, push!, length, using Base: Process, Semaphore, JLOptions, AnyDict, buffer_writes, wait_connected, VERSION_STRING, sync_begin, sync_add, sync_end, async_run_thunk, binding_module, notify_error, atexit, julia_exename, julia_cmd, - AsyncGenerator, display_error, acquire, release, invokelatest + AsyncGenerator, display_error, acquire, release, invokelatest, warn_once, + shell_escape, uv_error # NOTE: clusterserialize.jl imports additional symbols from Base.Serializer for use diff --git a/base/distributed/managers.jl b/base/distributed/managers.jl index 66753ee09be6d..169214287cd04 100644 --- a/base/distributed/managers.jl +++ b/base/distributed/managers.jl @@ -184,7 +184,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, cmd = `cd $dir '&&' $tval $exename $exeflags` # shell login (-l) with string command (-c) to launch julia process - cmd = `sh -l -c $(Base.shell_escape(cmd))` + cmd = `sh -l -c $(shell_escape(cmd))` # remote launch with ssh with given ssh flags / host / port information # -T → disable pseudo-terminal allocation @@ -194,7 +194,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, # forwarded connections are causing collisions # -n → Redirects stdin from /dev/null (actually, prevents reading from stdin). # Used when running ssh in the background. - cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(Base.shell_escape(cmd))` + cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(shell_escape(cmd))` # launch the remote Julia process @@ -377,7 +377,7 @@ connection to worker with id `pid`, specified by `config` and return a pair of ` objects. Messages from `pid` to current process will be read off `instrm`, while messages to be sent to `pid` will be written to `outstrm`. The custom transport implementation must ensure that messages are delivered and received completely and in order. -`Base.connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between +`connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between workers. """ function connect(manager::ClusterManager, pid::Int, config::WorkerConfig) @@ -481,7 +481,7 @@ end function bind_client_port(s) err = ccall(:jl_tcp_bind, Int32, (Ptr{Void}, UInt16, UInt32, Cuint), s.handle, hton(client_port[]), hton(UInt32(0)), 0) - Base.uv_error("bind() failed", err) + uv_error("bind() failed", err) _addr, port = Base._sockname(s, true) client_port[] = port @@ -516,7 +516,7 @@ end Implemented by cluster managers. It is called on the master process, by [`rmprocs`](@ref). It should cause the remote worker specified by `pid` to exit. -`Base.kill(manager::ClusterManager.....)` executes a remote `exit()` +`kill(manager::ClusterManager.....)` executes a remote `exit()` on `pid`. """ function kill(manager::ClusterManager, pid::Int, config::WorkerConfig) From 2f409e6d6c29cb30061814619ca52f96413913e3 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 23 Sep 2017 00:15:02 -0400 Subject: [PATCH 07/44] fix #22842, dispatch confusion with `Type{T}` vs. `typeof(T)` (#23831) (cherry picked from commit f4619d01ec65790942a92cb6ef10643ddc57c709) --- src/jltypes.c | 9 +++++++-- test/core.jl | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index ea7418dfb19a9..0722339f522af 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -588,8 +588,13 @@ static int typekey_eq(jl_datatype_t *tt, jl_value_t **key, size_t n) } for(j=0; j < n; j++) { jl_value_t *kj = key[j], *tj = jl_svecref(tt->parameters,j); - if (tj != kj && !jl_types_equal(tj, kj)) - return 0; + if (tj != kj) { + // require exact same Type{T}. see e.g. issue #22842 + if (jl_is_type_type(tj) || jl_is_type_type(kj)) + return 0; + if (!jl_types_equal(tj, kj)) + return 0; + } } return 1; } diff --git a/test/core.jl b/test/core.jl index d3d7eafa26abd..616efb9f5bd67 100644 --- a/test/core.jl +++ b/test/core.jl @@ -907,6 +907,12 @@ let @test_throws MethodError foor(StridedArray) end +# issue #22842 +f22842(x::UnionAll) = UnionAll +f22842(x::DataType) = length(x.parameters) +@test f22842(Tuple{Vararg{Int64,N} where N}) == 1 +@test f22842(Tuple{Vararg{Int64,N}} where N) === UnionAll + # issue #1153 mutable struct SI{m, s, kg} value::AbstractFloat From 4ba27b924ee83dcd1a29f5385c8c432fc4a68dd3 Mon Sep 17 00:00:00 2001 From: Pablo Zubieta Date: Tue, 7 Nov 2017 14:30:08 -0600 Subject: [PATCH 08/44] Fix length calculation of broadcast over tuples (#23887) (cherry picked from commit 698ef27a22a1ccb464cfbbc496fb5e6fa5b7c369) --- base/broadcast.jl | 27 ++++++++++++++++++++++++--- test/broadcast.jl | 16 ++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 48de50d54d5c4..53b7bc0fc7dd9 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -128,6 +128,8 @@ end end Base.@propagate_inbounds _broadcast_getindex(A, I) = _broadcast_getindex(containertype(A), A, I) +# `(x,)`, where `x` is a scalar, broadcasts the same way as `[x]` or `x` +Base.@propagate_inbounds _broadcast_getindex(::Type{Tuple}, A::Tuple{Any}, I) = A[1] Base.@propagate_inbounds _broadcast_getindex(::Type{Array}, A::Ref, I) = A[] Base.@propagate_inbounds _broadcast_getindex(::ScalarType, A, I) = A Base.@propagate_inbounds _broadcast_getindex(::Any, A, I) = A[I] @@ -334,13 +336,32 @@ end end @inline broadcast_c(f, ::Type{Any}, a...) = f(a...) @inline broadcast_c(f, ::Type{Tuple}, A, Bs...) = - tuplebroadcast(f, first_tuple(A, Bs...), A, Bs...) + tuplebroadcast(f, tuplebroadcast_maxtuple(A, Bs...), A, Bs...) @inline tuplebroadcast(f, ::NTuple{N,Any}, As...) where {N} = ntuple(k -> f(tuplebroadcast_getargs(As, k)...), Val{N}) @inline tuplebroadcast(f, ::NTuple{N,Any}, ::Type{T}, As...) where {N,T} = ntuple(k -> f(T, tuplebroadcast_getargs(As, k)...), Val{N}) -first_tuple(A::Tuple, Bs...) = A -@inline first_tuple(A, Bs...) = first_tuple(Bs...) +# When the result of broadcast is a tuple it can only come from mixing n-tuples +# of the same length with scalars and 1-tuples. So, in order to have a +# type-stable broadcast, we need to find a tuple of maximum length (except when +# there are only scalars, empty tuples and 1-tuples, in which case the +# returned value will be an empty tuple). +# The following methods compare broadcast arguments pairwise to determine the +# length of the final tuple. +tuplebroadcast_maxtuple(A, B) = + _tuplebroadcast_maxtuple(containertype(A), containertype(B), A, B) +@inline tuplebroadcast_maxtuple(A, Bs...) = + tuplebroadcast_maxtuple(A, tuplebroadcast_maxtuple(Bs...)) +tuplebroadcast_maxtuple(A::NTuple{N,Any}, ::NTuple{N,Any}...) where {N} = A +# Here we use the containertype trait to easier disambiguate between methods +_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Any}, A, B) = (nothing,) +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Any}, A, B) = A +_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Tuple}, A, B) = B +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B::Tuple{Any}) = A +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, B) = B +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, ::Tuple{Any}) = A +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B) = + throw(DimensionMismatch("tuples could not be broadcast to a common size")) tuplebroadcast_getargs(::Tuple{}, k) = () @inline tuplebroadcast_getargs(As, k) = (_broadcast_getindex(first(As), k), tuplebroadcast_getargs(tail(As), k)...) diff --git a/test/broadcast.jl b/test/broadcast.jl index ccb54599d6dbf..fed20c55d314f 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -515,8 +515,16 @@ end Nullable("hello")) end -# Issue #21291 -let t = (0, 1, 2) - o = 1 - @test @inferred(broadcast(+, t, o)) == (1, 2, 3) +@testset "broadcast resulting in tuples" begin + # Issue #21291 + let t = (0, 1, 2) + o = 1 + @test @inferred(broadcast(+, t, o)) == (1, 2, 3) + end + + # Issue #23647 + @test (1, 2, 3) .+ (1,) == (1,) .+ (1, 2, 3) == (2, 3, 4) + @test (1,) .+ () == () .+ (1,) == () .+ () == () + @test (1, 2) .+ (1, 2) == (2, 4) + @test_throws DimensionMismatch (1, 2) .+ (1, 2, 3) end From fab29e9170cf1e5cb8dd6ba716355271b55b6ec4 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 28 Sep 2017 09:17:08 +0200 Subject: [PATCH 09/44] Fix return value in asyncmap! (#23906) (cherry picked from commit 4d2e1d7cdf4cbce4650cbe6b6c5ebde5b86cca10) --- base/asyncmap.jl | 2 +- test/abstractarray.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/asyncmap.jl b/base/asyncmap.jl index 19846ab7a0ced..3931cc9743749 100644 --- a/base/asyncmap.jl +++ b/base/asyncmap.jl @@ -420,5 +420,5 @@ returning a collection. """ function asyncmap!(f, r, c1, c...; ntasks=0, batch_size=nothing) foreach(identity, AsyncCollector(f, r, c1, c...; ntasks=ntasks, batch_size=batch_size)) - c + r end diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 24a181a58b76b..8bad4bb84a6b2 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -683,8 +683,9 @@ function test_map(::Type{TestAbstractArray}) # Map to destination collection map!((x,y,z)->x*y*z, A, Float64[1:10...], Float64[1:10...], Float64[1:10...]) @test A == map(x->x*x*x, Float64[1:10...]) - Base.asyncmap!((x,y,z)->x*y*z, B, Float64[1:10...], Float64[1:10...], Float64[1:10...]) + C = Base.asyncmap!((x,y,z)->x*y*z, B, Float64[1:10...], Float64[1:10...], Float64[1:10...]) @test A == B + @test B === C end @testset "issue #15689, mapping an abstract type" begin From d8ae4be0fce730aeae2b8ff5653bb377e236b554 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 10 Oct 2017 01:01:38 -0400 Subject: [PATCH 10/44] fix #23996, bug in local function capturing in lambda-optimize-vars (#24004) (cherry picked from commit c61aa276a3e89f4cb0ccbe797ed48f49c4e5ae20) --- src/julia-syntax.scm | 24 +++++++++++++++++++----- test/core.jl | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 18fab57cf4bc5..307a5d0a0666b 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2955,14 +2955,29 @@ f(x) = yt(x) (take body) (reverse! acc))) +;; find all methods for the same function as `ex` in `body` +(define (all-methods-for ex body) + (let ((mname (method-expr-name ex))) + (expr-find-all (lambda (s) + (and (length> s 2) (eq? (car s) 'method) + (eq? (method-expr-name s) mname))) + body + identity + (lambda (x) (and (pair? x) (not (eq? (car x) 'lambda))))))) + ;; clear capture bit for vars assigned once at the top, to avoid allocating ;; some unnecessary Boxes. (define (lambda-optimize-vars! lam) - (define (expr-uses-var ex v) + (define (expr-uses-var ex v stmts) (cond ((assignment? ex) (expr-contains-eq v (caddr ex))) ((eq? (car ex) 'method) (and (length> ex 2) - (assq v (cadr (lam:vinfo (cadddr ex)))))) + ;; a method expression captures a variable if any methods for the + ;; same function do. + (let ((all-methods (all-methods-for ex (cons 'body stmts)))) + (any (lambda (ex) + (assq v (cadr (lam:vinfo (cadddr ex))))) + all-methods)))) (else (expr-contains-eq v ex)))) (assert (eq? (car lam) 'lambda)) (let ((vi (car (lam:vinfo lam)))) @@ -2987,7 +3002,7 @@ f(x) = yt(x) ;; TODO: reorder leading statements to put assignments where the RHS is ;; `simple-atom?` at the top. (for-each (lambda (e) - (set! unused (filter (lambda (v) (not (expr-uses-var e v))) + (set! unused (filter (lambda (v) (not (expr-uses-var e v leading))) unused)) (if (and (memq (car e) '(method =)) (memq (cadr e) unused)) (let ((v (assq (cadr e) vi))) @@ -3137,9 +3152,8 @@ f(x) = yt(x) (and name (symbol (string "#" name "#" (current-julia-module-counter)))))) (alldefs (expr-find-all - (lambda (ex) (and (eq? (car ex) 'method) + (lambda (ex) (and (length> ex 2) (eq? (car ex) 'method) (not (eq? ex e)) - (length> ex 2) (eq? (method-expr-name ex) name))) (lam:body lam) identity diff --git a/test/core.jl b/test/core.jl index 616efb9f5bd67..c98cbe952015e 100644 --- a/test/core.jl +++ b/test/core.jl @@ -350,6 +350,20 @@ c23558(n,k) = end @test c23558(10, 5) == 252 +# issue #23996 +function foo23996(xs...) + rets = [] + bar(::Int) = push!(rets, 1) + foobar() = push!(rets, 3) + bar(::AbstractFloat) = push!(rets, 2) + bar(::Bool) = foobar() + for x in xs + bar(x) + end + rets +end +@test foo23996(1,2.0,false) == [1,2,3] + # variable scope, globals glob_x = 23 function glotest() From c7a63d971370dbada16d6e004ed51c343f2f9966 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 5 Oct 2017 18:26:18 -0400 Subject: [PATCH 11/44] fix #24002, lowering error in comprehensions at top level (#24005) the temp variables were not declared local (cherry picked from commit 7504a381631328bd8e6411a179e12977b0d0a41c) --- src/julia-syntax.scm | 14 ++++++-------- test/arrayops.jl | 8 ++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 307a5d0a0666b..bb2cf33723849 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2393,11 +2393,10 @@ (oneresult (make-ssavalue)) (lengths (map (lambda (x) (make-ssavalue)) ranges)) (states (map (lambda (x) (gensy)) ranges)) - (is (map (lambda (x) (gensy)) ranges)) (rv (map (lambda (x) (make-ssavalue)) ranges))) ;; construct loops to cycle over all dimensions of an n-d comprehension - (define (construct-loops ranges rv is states lengths) + (define (construct-loops ranges rv states lengths) (if (null? ranges) `(block (= ,oneresult ,expr) (inbounds true) @@ -2406,28 +2405,27 @@ (= ,ri (call (top +) ,ri 1))) `(block (= ,(car states) (call (top start) ,(car rv))) - (local (:: ,(car is) (call (core typeof) ,(car lengths)))) - (= ,(car is) 0) - (while (call (top !=) ,(car is) ,(car lengths)) + (while (call (top !) (call (top done) ,(car rv) ,(car states))) (scope-block (block - (= ,(car is) (call (top +) ,(car is) 1)) (= (tuple ,(cadr (car ranges)) ,(car states)) (call (top next) ,(car rv) ,(car states))) ;; *** either this or force all for loop vars local ,.(map (lambda (r) `(local ,r)) (lhs-vars (cadr (car ranges)))) - ,(construct-loops (cdr ranges) (cdr rv) (cdr is) (cdr states) (cdr lengths)))))))) + ,(construct-loops (cdr ranges) (cdr rv) (cdr states) (cdr lengths)))))))) ;; Evaluate the comprehension `(block + ,.(map (lambda (v) `(local ,v)) states) + (local ,ri) ,.(map (lambda (v r) `(= ,v ,(caddr r))) rv ranges) ,.(map (lambda (v r) `(= ,v (call (top length) ,r))) lengths rv) (scope-block (block (= ,result (call (curly Array ,atype ,(length lengths)) ,@lengths)) (= ,ri 1) - ,(construct-loops (reverse ranges) (reverse rv) is states (reverse lengths)) + ,(construct-loops (reverse ranges) (reverse rv) states (reverse lengths)) ,result))))) (define (lhs-vars e) diff --git a/test/arrayops.jl b/test/arrayops.jl index e1a0a2497edac..fb417597f8221 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1170,6 +1170,14 @@ end end end +# issue #24002 +module I24002 +s1() = 1 +y = Int[i for i in 1:10] +end +@test I24002.y == [1:10;] +@test I24002.s1() == 1 + @testset "eachindexvalue" begin A14 = [11 13; 12 14] R = CartesianRange(indices(A14)) From 83234e8b50cf375a5183efd85c3dd8f3b933c8e8 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 6 Oct 2017 21:25:15 -0400 Subject: [PATCH 12/44] Fix cgmemmgr for high address allocations This can certainly happen on 32bit with enough allocation. It's not entirely clear if it can happen on any 64bit linux but a solution seems possible so it's also included. Ref #24033 (cherry picked from commit 9c3cafb8ffdbac6e96c1d28e0bda65b8d0625f93) --- Make.inc | 7 +++++++ src/cgmemmgr.cpp | 29 +++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index acb5e2f737810..abf180b4a6788 100644 --- a/Make.inc +++ b/Make.inc @@ -709,6 +709,13 @@ endif JULIA_CPU_TARGET ?= native +ifneq ($(OS),WINNT) +# Windows headers with this configuration conflicts with LLVM +# (Symbol renames are done with macros) +# We mainly need this on linux for cgmemmgr so don't worry about windows for now... +JCXXFLAGS += -D_FILE_OFFSET_BITS=64 +endif + # Set some ARCH-specific flags ifneq ($(USEICC),1) ifeq ($(ISX86),1) diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index 85316d96676d3..95c491c496278 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -264,6 +264,31 @@ static void *alloc_shared_page(size_t size, size_t *id, bool exec) #ifdef _OS_LINUX_ // Using `/proc/self/mem`, A.K.A. Keno's remote memory manager. +ssize_t pwrite_addr(int fd, const void *buf, size_t nbyte, uintptr_t addr) +{ + static_assert(sizeof(off_t) >= 8, "off_t is smaller than 64bits"); +#ifdef _P64 + const uintptr_t sign_bit = uintptr_t(1) << 63; + if (__unlikely(sign_bit & addr)) { + // This case should not happen with default kernel on 64bit since the address belongs + // to kernel space (linear mapping). + // However, it seems possible to change this at kernel compile time. + + // pwrite doesn't support offset with sign bit set but lseek does. + // This is obviously not thread safe but none of the mem manager does anyway... + // From the kernel code, `lseek` with `SEEK_SET` can't fail. + // However, this can possibly confuse the glibc wrapper to think that + // we have invalid input value. Use syscall directly to be sure. + syscall(SYS_lseek, (long)fd, addr, (long)SEEK_SET); + // The return value can be -1 when the glibc syscall function + // think we have an error return with and `addr` that's too large. + // Ignore the return value for now. + return write(fd, buf, nbyte); + } +#endif + return pwrite(fd, buf, nbyte, (off_t)addr); +} + // Do not call this directly. // Use `get_self_mem_fd` which has a guard to call this only once. static int _init_self_mem() @@ -296,7 +321,7 @@ static int _init_self_mem() assert(test_pg != MAP_FAILED && "Cannot allocate executable memory"); const uint64_t v = 0xffff000012345678u; - int ret = pwrite(fd, (const void*)&v, sizeof(uint64_t), (uintptr_t)test_pg); + int ret = pwrite_addr(fd, (const void*)&v, sizeof(uint64_t), (uintptr_t)test_pg); if (ret != sizeof(uint64_t) || *(volatile uint64_t*)test_pg != v) { munmap(test_pg, jl_page_size); close(fd); @@ -315,7 +340,7 @@ static int get_self_mem_fd() static void write_self_mem(void *dest, void *ptr, size_t size) { while (size > 0) { - ssize_t ret = pwrite(get_self_mem_fd(), ptr, size, (uintptr_t)dest); + ssize_t ret = pwrite_addr(get_self_mem_fd(), ptr, size, (uintptr_t)dest); if ((size_t)ret == size) return; if (ret == -1 && (errno == EAGAIN || errno == EINTR)) From 10392dc8a37c078111c76ce298b58c3d45f08675 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 10 Oct 2017 00:04:26 -0400 Subject: [PATCH 13/44] init task system first (#24066) This has no dependencies on the rest of julia_init, but later functionality (like __init__) may want to be able to use Tasks (cherry picked from commit 3fa9b9c81f321dee95f0a9a150bcb086cf6dba56) --- src/task.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/task.c b/src/task.c index c4e0a465bcfac..053795004676c 100644 --- a/src/task.c +++ b/src/task.c @@ -294,11 +294,11 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) { // keep this function small, since we want to keep the stack frame // leading up to this also quite small - _julia_init(rel); #ifdef COPY_STACKS char __stk; jl_set_base_ctx(&__stk); // separate function, to record the size of a stack frame #endif + _julia_init(rel); } static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) From cbfa82eddbffe1cc07b5dc7f8860536850d0dbf4 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Tue, 10 Oct 2017 18:47:26 -0400 Subject: [PATCH 14/44] Fix the randmatstat python microbenchmark (#24089) Fix #23741 (cherry picked from commit 8afbe45b5b1f3e8bd2010f2fa85c5854e9c4fa0a) --- test/perf/micro/perf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/perf/micro/perf.py b/test/perf/micro/perf.py index b1bd40f150be9..5718d31a711e1 100644 --- a/test/perf/micro/perf.py +++ b/test/perf/micro/perf.py @@ -43,13 +43,13 @@ def randmatstat(t): n = 5 v = zeros(t) w = zeros(t) - for i in range(1,t): + for i in range(t): a = randn(n, n) b = randn(n, n) c = randn(n, n) d = randn(n, n) - P = concatenate((a, b, c, d)) - Q = concatenate((concatenate((a, b)), concatenate((c, d))),axis=1) + P = concatenate((a, b, c, d), axis=1) + Q = concatenate((concatenate((a, b), axis=1), concatenate((c, d), axis=1)), axis=0) v[i] = trace(matrix_power(dot(P.T,P), 4)) w[i] = trace(matrix_power(dot(Q.T,Q), 4)) return (std(v)/mean(v), std(w)/mean(w)) From 0b2c53bc81584ae3d471576148bf8f2bf69c4e80 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 12 Oct 2017 12:59:57 -0400 Subject: [PATCH 15/44] Do not save and restore signal mask when initializing tasks (#24090) This is consistent with what we do with `ASM_COPY_STACK` and it causes new tasks to be initialized to the wrong signal masks now that we initialize tasks before signal handlers. This cause ARM core test to fail after #24066 since the task running the test have SIGINT masked off. (cherry picked from commit 56040a706c76851770c96b5a216d925ca3d28ae9) --- src/task.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/task.c b/src/task.c index 053795004676c..ec01ca5efd369 100644 --- a/src/task.c +++ b/src/task.c @@ -283,7 +283,7 @@ void NOINLINE jl_set_base_ctx(char *__stk) jl_ptls_t ptls = jl_get_ptls_states(); ptls->stackbase = (char*)(((uintptr_t)__stk + sizeof(*__stk))&-16); // also ensures stackbase is 16-byte aligned #ifndef ASM_COPY_STACKS - if (jl_setjmp(ptls->base_ctx, 1)) { + if (jl_setjmp(ptls->base_ctx, 0)) { start_task(); } #endif From 2500378cf47ea152dea59993b1b8bc82830a4e8f Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Thu, 12 Oct 2017 18:48:22 -0400 Subject: [PATCH 16/44] fixes enum capitalization (#24105) * fixes enum capitalization This brings the example in the `instances` docstring in line with the naming convention used elsewhere in the enum docs. See #19506 * more enum consistency (cherry picked from commit 05045841189269dbf5aead770c1152207ea279fd) --- base/Enums.jl | 4 ++-- base/reflection.jl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index 22f72ca714ba7..7519b5a8cc4c5 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -29,10 +29,10 @@ end @noinline enum_argument_error(typename, x) = throw(ArgumentError(string("invalid value for Enum $(typename): $x"))) """ - @enum EnumName[::BaseType] EnumValue1[=x] EnumValue2[=y] + @enum EnumName[::BaseType] value1[=x] value2[=y] Create an `Enum{BaseType}` subtype with name `EnumName` and enum member values of -`EnumValue1` and `EnumValue2` with optional assigned values of `x` and `y`, respectively. +`value1` and `value2` with optional assigned values of `x` and `y`, respectively. `EnumName` can be used just like other types and enum member values as regular values, such as ```jldoctest diff --git a/base/reflection.jl b/base/reflection.jl index 1417e0da6a824..c9a4230c44fa4 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -390,10 +390,10 @@ Return a collection of all instances of the given type, if applicable. Mostly us enumerated types (see `@enum`). ```jldoctest -julia> @enum Colors Red Blue Green +julia> @enum Color red blue green -julia> instances(Colors) -(Red::Colors = 0, Blue::Colors = 1, Green::Colors = 2) +julia> instances(Color) +(red::Color = 0, blue::Color = 1, green::Color = 2) ``` """ function instances end From 5a3cbdc069a974f6cba4a520c6ca25b10e645ed8 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sun, 15 Oct 2017 20:01:05 +0200 Subject: [PATCH 17/44] Don't use copy_oftype in \(QRX, B) to make sure that solution array is mutable (#24110) * Don't use copy_oftype in \(QRX, B) to make sure that solution array is mutable. Fixes #24107 * Always return B in A_ldiv_B!(QRPivoted, B) (cherry picked from commit f9c662dbbb2aa2469aa85786146a445705d0c776) --- base/linalg/qr.jl | 34 +++++++++++++--------------------- test/linalg/qr.jl | 5 +++++ 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 1e5941b60a1d6..4425c677d3883 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -695,11 +695,12 @@ function A_ldiv_B!(A::QRPivoted{T}, B::StridedMatrix{T}, rcond::Real) where T<:B nr = min(mA,nA) nrhs = size(B, 2) if nr == 0 - return zeros(T, 0, nrhs), 0 + return B, 0 end ar = abs(A.factors[1]) if ar == 0 - return zeros(T, nA, nrhs), 0 + B[1:nA, :] = 0 + return B, 0 end rnk = 1 xmin = ones(T, 1) @@ -791,23 +792,8 @@ _cut_B(x::AbstractVector, r::UnitRange) = length(x) > length(r) ? x[r] : x _cut_B(X::AbstractMatrix, r::UnitRange) = size(X, 1) > length(r) ? X[r,:] : X ## append right hand side with zeros if necessary -function _append_zeros(b::AbstractVector, T::Type, n) - if n > length(b) - x = zeros(T, n) - return copy!(x, b) - else - return copy_oftype(b, T) - end -end -function _append_zeros(B::AbstractMatrix, T::Type, n) - if n > size(B, 1) - X = zeros(T, (n, size(B, 2))) - X[1:size(B,1), :] = B - return X - else - return copy_oftype(B, T) - end -end +_zeros(::Type{T}, b::AbstractVector, n::Integer) where {T} = zeros(T, max(length(b), n)) +_zeros(::Type{T}, B::AbstractMatrix, n::Integer) where {T} = zeros(T, max(size(B, 1), n), size(B, 2)) function (\)(A::Union{QR{TA},QRCompactWY{TA},QRPivoted{TA}}, B::AbstractVecOrMat{TB}) where {TA,TB} S = promote_type(TA,TB) @@ -816,7 +802,10 @@ function (\)(A::Union{QR{TA},QRCompactWY{TA},QRPivoted{TA}}, B::AbstractVecOrMat AA = convert(Factorization{S}, A) - X = A_ldiv_B!(AA, _append_zeros(B, S, n)) + X = _zeros(S, B, n) + X[1:size(B, 1), :] = B + + A_ldiv_B!(AA, X) return _cut_B(X, 1:n) end @@ -838,7 +827,10 @@ function (\)(A::Union{QR{T},QRCompactWY{T},QRPivoted{T}}, BIn::VecOrMat{Complex{ # |x4|y4| B = reshape(transpose(reinterpret(T, BIn, (2, length(BIn)))), size(BIn, 1), 2*size(BIn, 2)) - X = A_ldiv_B!(A, _append_zeros(B, T, n)) + X = _zeros(T, B, n) + X[1:size(B, 1), :] = B + + A_ldiv_B!(A, X) # |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| # |z2|z4| <- |y1|y2|y3|y4| <- |x2|y2| <- |x2|y2|x4|y4| diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index d7525732c1462..b80f3e867071e 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -191,3 +191,8 @@ let @test A \ B == zeros(2, 1) @test qrfact(A, Val{true}) \ B == zeros(2, 1) end + +@testset "Issue 24107" begin + A = rand(200,2) + @test A \ linspace(0,1,200) == A \ collect(linspace(0,1,200)) +end From f2841b5dd93ead15637cc1445549ac206a40864e Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 18 Oct 2017 17:23:54 -0400 Subject: [PATCH 18/44] bugfix in collect(A) for zero-dimensional array Ref #24187 (cherry picked from commit c59b8b184e10805a23ebc1e0702e0199147afbb1) --- base/array.jl | 2 +- test/abstractarray.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 633dfed5c1469..36625e29f6d20 100644 --- a/base/array.jl +++ b/base/array.jl @@ -445,7 +445,7 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) return a end -_collect_indices(::Tuple{}, A) = copy!(Vector{eltype(A)}(), A) +_collect_indices(::Tuple{}, A) = copy!(Array{eltype(A)}(), A) _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) = copy!(Array{eltype(A)}(length.(indsA)), A) function _collect_indices(indsA, A) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 8bad4bb84a6b2..94d301eb58873 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -841,3 +841,8 @@ end @testset "checkbounds_indices method ambiguities #20989" begin @test Base.checkbounds_indices(Bool, (1:1,), ([CartesianIndex(1)],)) end + +@testset "zero-dimensional copy" begin + Z = Array{Int}(); Z[] = 17 + @test Z == collect(Z) == copy(Z) +end From ee355384a0b7bdf326e33efb6b126438ba97cfff Mon Sep 17 00:00:00 2001 From: Timo Kluck Date: Mon, 23 Oct 2017 08:13:45 +0200 Subject: [PATCH 19/44] Simple documentation update: Remove obsolete comment about vector transpose (#24267) * Remove obsolete comment about vector transpose This is not true anymore since the introduction of `RowVector`: julia> typeof([1,2]') RowVector{Int64,Array{Int64,1}} julia> [1,2]' * [3,4] 11 (cherry picked from commit bb703387aa59452983cc09aa1744139f835d895b) --- doc/src/manual/noteworthy-differences.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/src/manual/noteworthy-differences.md b/doc/src/manual/noteworthy-differences.md index ba2e1cae9cb34..f9a01b1022c43 100644 --- a/doc/src/manual/noteworthy-differences.md +++ b/doc/src/manual/noteworthy-differences.md @@ -136,9 +136,8 @@ For users coming to Julia from R, these are some noteworthy differences: * Julia does not provide `nrow` and `ncol`. Instead, use `size(M, 1)` for `nrow(M)` and `size(M, 2)` for `ncol(M)`. * Julia is careful to distinguish scalars, vectors and matrices. In R, `1` and `c(1)` are the same. - In Julia, they can not be used interchangeably. One potentially confusing result of this is that - `x' * y` for vectors `x` and `y` is a 1-element vector, not a scalar. To get a scalar, use [`dot(x, y)`](@ref). - * Julia's [`diag()`](@ref) and [`diagm()`](@ref) are not like R's. + In Julia, they cannot be used interchangeably. + * Julia's [`diag`](@ref) and [`diagm`](@ref) are not like R's. * Julia cannot assign to the results of function calls on the left hand side of an assignment operation: you cannot write `diag(M) = ones(n)`. * Julia discourages populating the main namespace with functions. Most statistical functionality From c26b271be0c8b63e992494e1bb5ca827008a97cc Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 26 Oct 2017 11:35:30 +0200 Subject: [PATCH 20/44] Print some useful info when hitting some cases of invalid UTF-8 (#24311) (cherry picked from commit 29fcb3714ccba4df16bcce043b6e1e125e738776) --- base/strings/errors.jl | 2 +- test/strings/basic.jl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/base/strings/errors.jl b/base/strings/errors.jl index 2d3aa42455286..308bfd157e5b7 100644 --- a/base/strings/errors.jl +++ b/base/strings/errors.jl @@ -3,7 +3,7 @@ ## Error messages for Unicode / UTF support const UTF_ERR_SHORT = "invalid UTF-8 sequence starting at index <<1>> (0x<<2>> missing one or more continuation bytes)" -const UTF_ERR_INVALID_INDEX = "invalid character index" +const UTF_ERR_INVALID_INDEX = "invalid character index <<1>> (0x<<2>> is a continuation byte)" mutable struct UnicodeError <: Exception errmsg::AbstractString ##< A UTF_ERR_ message diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 36737bb027499..db322257b3a18 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -468,3 +468,17 @@ Base.endof(x::CharStr) = endof(x.chars) # issue #12495: check that logical indexing attempt raises ArgumentError @test_throws ArgumentError "abc"[[true, false, true]] @test_throws ArgumentError "abc"[BitArray([true, false, true])] + +@testset "invalid code point" begin + s = String([0x61, 0xba, 0x41]) + @test !isvalid(s) + @test_throws UnicodeError s[2] + e = try + s[2] + catch e + e + end + b = IOBuffer() + show(b, e) + @test String(take!(b)) == "UnicodeError: invalid character index 2 (0xba is a continuation byte)" +end From a10d6f7ff2f0fdc51711c603d7b6257e44f7a36c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 2 Nov 2017 17:03:40 -0400 Subject: [PATCH 21/44] fix #24305, stack overflow when intersecting sequence of big unions (#24314) (cherry picked from commit a547d7331555b26c933d35d07c81f2ec1a7bb635) --- src/subtype.c | 17 +++++++++++------ test/subtype.jl | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index e7c040699a33d..41e270fee3e1c 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2034,11 +2034,15 @@ static jl_value_t *intersect_all(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) e->Runions.depth = 0; e->Runions.more = 0; memset(e->Runions.stack, 0, sizeof(e->Runions.stack)); - int lastset = 0, niter = 0; + jl_value_t **is; + JL_GC_PUSHARGS(is, 2); + int lastset = 0, niter = 0, total_iter = 0; jl_value_t *ii = intersect(x, y, e, 0); while (e->Runions.more) { - if (e->emptiness_only && ii != jl_bottom_type) + if (e->emptiness_only && ii != jl_bottom_type) { + JL_GC_POP(); return ii; + } e->Runions.depth = 0; int set = e->Runions.more - 1; e->Runions.more = 0; @@ -2047,8 +2051,6 @@ static jl_value_t *intersect_all(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) statestack_set(&e->Runions, i, 0); lastset = set; - jl_value_t **is; - JL_GC_PUSHARGS(is, 2); is[0] = ii; is[1] = intersect(x, y, e, 0); if (is[0] == jl_bottom_type) @@ -2060,10 +2062,13 @@ static jl_value_t *intersect_all(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) ii = jl_type_union(is, 2); niter++; } - JL_GC_POP(); - if (niter > 3) + total_iter++; + if (niter > 3 || total_iter > 400000) { + JL_GC_POP(); return y; + } } + JL_GC_POP(); return ii; } diff --git a/test/subtype.jl b/test/subtype.jl index b34e9a2a7d7a1..2756241a04343 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -894,6 +894,12 @@ function test_intersection() A = Tuple{Ref, Vararg{Any}} B = Tuple{Vararg{Union{Z,Ref,Void}}} where Z<:Union{Ref,Void} @test B <: _type_intersect(A, B) + # TODO: this would be a better version of that test: + #let T = _type_intersect(A, B) + # @test T <: A + # @test T <: B + # @test Tuple{Ref, Vararg{Union{Ref,Void}}} <: T + #end @testintersect(Tuple{Int,Any,Vararg{A}} where A>:Integer, Tuple{Any,Int,Vararg{A}} where A>:Integer, Tuple{Int,Int,Vararg{A}} where A>:Integer) @@ -1154,3 +1160,22 @@ end let (t, e) = intersection_env(Tuple{Union{Int,Int8}}, Tuple{T} where T) @test e[1] isa TypeVar end + +# issue #24305 +f24305(x) = [g24305(x) g24305(x) g24305(x) g24305(x); g24305(x) g24305(x) 0 0]; +@test_throws UndefVarError f24305(1) + +f1_24305(x,y,z) = x*y-z^2-1 +f2_24305(x,y,z) = x*y*z+y^2-x^2-2 +f3_24305(x,y,z) = exp(x)+z-exp(y)-3 +Fun_24305(x) = [ f1_24305(x[1],x[2],x[3]); f2_24305(x[1],x[2],x[3]); f3_24305(x[1],x[2],x[3]) ] +Jac_24305(x) = [ x[2] x[1] -2*x[3] ; x[2]*x[3]-2x[1] x[1]*x[3]+2x[2] x[1]*x[2] ; exp(x[1]) -exp(x[2]) 1 ] + +x_24305 = ones(3) + +for it = 1:5 + h = - \(Jac_24305(x_24305), Fun_24305(x_24305)) + global x_24305 = x_24305 + h +end + +@test round.(x_24305, 2) == [1.78, 1.42, 1.24] From 5cd144ffa328fdd4cd9e983b616c7205f9bb4f51 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Sun, 29 Oct 2017 20:26:13 +0100 Subject: [PATCH 22/44] size -> indices to fix show of OffsetArrays (#24393) (cherry picked from commit 12517bd8ca20f574b65426ada6a28e69963b2bce) --- base/show.jl | 2 +- test/offsetarray.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index b62e0d9acfc1f..0d09148fac157 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1589,7 +1589,7 @@ function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices) if length(ind) > 10 if ii == ind[4] && all(d->idxs[d]==first(tailinds[d]),1:i-1) for j=i+1:nd - szj = size(a,j+2) + szj = length(indices(a, j+2)) indj = tailinds[j] if szj>10 && first(indj)+2 < idxs[j] <= last(indj)-3 @goto skip diff --git a/test/offsetarray.jl b/test/offsetarray.jl index bb755579d63a2..0f2d7aad45beb 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -158,6 +158,7 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,5), (10,-9))) # rows&c cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,5), (10,-9))) # columns fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,10^3), (10,-9))) # rows fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neither fits +cmp_showf(Base.show, io, OffsetArray(rand(1,1,10^3,1), (1,2,3,4))) # issue in #24393 targets1 = ["0-dimensional $OAs_name.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", "$OAs_name.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", "$OAs_name.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", From d05e152b18628184823bd07497a9f53464333d74 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Mon, 6 Nov 2017 19:26:37 +0100 Subject: [PATCH 23/44] Exclude version_git.jl from list of dependencies for HTML docs This file is modified by the light-source-dist target when preparing tarballs. Depending on it means that the docs will be rebuilt when calling 'make install' from the tarballs. This requires downloading files, which is problematic for replicable builds and may not be possible on build servers. Ref #24496 (cherry picked from commit bd019a4037f59c0ab1db8a08c0d4a4aa23f52ad5) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9dfbf522a1d21..e00c597f83ea1 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ julia_flisp.boot.inc.phony: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony # Build the HTML docs (skipped if already exists, notably in tarballs) -$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl \) -prune -o -type f -print) +$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print) @$(MAKE) docs # doc needs to live under $(build_docdir), not under $(build_datarootdir)/julia/ From b13ae0fe1c9399a506a19da334fe5db6cbefd431 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Mon, 6 Nov 2017 19:33:59 +0100 Subject: [PATCH 24/44] Add instruction to release-candidate target about testing builds without Internet It is generally a good idea to test that final tarballs build and pass the tests. Disabling Internet access allows ensuring that builds are reproducible, and is necessary for some build servers. Ref #24496 (cherry picked from commit 8cb370152f57b0c2578dffdde48da3925e08551a) --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e00c597f83ea1..6fcea3cfc4aa4 100644 --- a/Makefile +++ b/Makefile @@ -154,11 +154,12 @@ release-candidate: release testall @echo 5. Create tag, push to github "\(git tag v\`cat VERSION\` && git push --tags\)" #"` # These comments deal with incompetent syntax highlighting rules @echo 6. Clean out old .tar.gz files living in deps/, "\`git clean -fdx\`" seems to work #"` @echo 7. Replace github release tarball with tarballs created from make light-source-dist and make full-source-dist - @echo 8. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms - @echo 9. Upload to AWS, update https://julialang.org/downloads and http://status.julialang.org/stable links - @echo 10. Update checksums on AWS for tarball and packaged binaries - @echo 11. Announce on mailing lists - @echo 12. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20 + @echo 8. Check that 'make && make install && make test' succeed with unpacked tarballs even without Internet access. + @echo 9. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms + @echo 10. Upload to AWS, update https://julialang.org/downloads and http://status.julialang.org/stable links + @echo 11. Update checksums on AWS for tarball and packaged binaries + @echo 12. Announce on mailing lists + @echo 13. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20 @echo $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir) From 30b3625fa9733a59e0fc886bb0f397fd024908b5 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 13 Sep 2017 02:01:26 -0500 Subject: [PATCH 25/44] Fix IndexLinear views with a 1-dimensional parent having offset indices Fixes https://github.com/JuliaArrays/OffsetArrays.jl/issues/27 Ref #23686 (cherry picked from commit afd56a6cec47fbb52777cecaf897fd80739631fb) --- base/subarray.jl | 3 +++ test/offsetarray.jl | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/base/subarray.jl b/base/subarray.jl index 9c2e31aae95e9..d4348b845c334 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -260,6 +260,9 @@ end # Computing the first index simply steps through the indices, accumulating the # sum of index each multiplied by the parent's stride. # The running sum is `f`; the cumulative stride product is `s`. +# If the parent is a vector, then we offset the parent's own indices with parameters of I +compute_offset1(parent::AbstractVector, stride1::Integer, I::Tuple{Range}) = + (@_inline_meta; first(I[1]) - first(indices1(I[1]))*stride1) # If the result is one-dimensional and it's a Colon, then linear # indexing uses the indices along the given dimension. Otherwise # linear indexing always starts with 1. diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 0f2d7aad45beb..88cebeb7d9098 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -115,6 +115,17 @@ S = view(A, :, :) @test S[1,4] == S[4] == 4 @test_throws BoundsError S[1,1] @test indices(S) === (0:1, 3:4) +# https://github.com/JuliaArrays/OffsetArrays.jl/issues/27 +g = OffsetArray(collect(-2:3), (-3,)) +gv = view(g, -1:2) +@test indices(gv, 1) === Base.OneTo(4) +@test collect(gv) == collect(-1:2) +gv = view(g, OffsetArray(-1:2, (-2,))) +@test indices(gv, 1) === -1:2 +@test collect(gv) == collect(-1:2) +gv = view(g, OffsetArray(-1:2, (-1,))) +@test indices(gv, 1) === 0:3 +@test collect(gv) == collect(-1:2) # iteration for (a,d) in zip(A, A0) From d4683016c1222a92f80df28329c66e77369712dc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 13 Sep 2017 11:40:31 -0400 Subject: [PATCH 26/44] CI: switch travis to sudo-required (#22986) (cherry picked from commit 0189574cd36e8db7be0e0fe3a6f0956f440562b8) --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66afbebdd48c8..137f4c7d7afa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: cpp -sudo: false +sudo: required +dist: trusty matrix: include: - os: linux @@ -71,8 +72,11 @@ before_install: ln -s /usr/bin/g++-5 $HOME/bin/x86_64-linux-gnu-g++; gcc --version; BAR="bar -i 30"; - BUILDOPTS="-j3 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1"; + BUILDOPTS="-j5 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1"; echo "override ARCH=$ARCH" >> Make.user; + sudo sh -c "echo 0 > /proc/sys/net/ipv6/conf/lo/disable_ipv6"; + export JULIA_CPU_CORES=4; + export JULIA_TEST_MAXRSS_MB=1200; TESTSTORUN="all"; elif [ `uname` = "Darwin" ]; then brew update; @@ -94,6 +98,8 @@ before_install: export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$DYLD_FALLBACK_LIBRARY_PATH\" \$1"; export BUILDOPTS="$BUILDOPTS spawn=\$(JULIA_MACOS_SPAWN)"; make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make; + export JULIA_CPU_CORES=2; + export JULIA_TEST_MAXRSS_MB=600; TESTSTORUN="all --skip linalg/triangular subarray"; fi # TODO: re enable these if possible without timing out - git clone -q git://git.kitenet.net/moreutils script: @@ -132,9 +138,8 @@ script: /tmp/julia/bin/julia-debug --precompiled=no -e 'true' - /tmp/julia/bin/julia -e 'versioninfo()' - pushd /tmp/julia/share/julia/test - - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && - /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && - /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg + - /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && + /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg - popd # test that the embedding code works on our installation - mkdir /tmp/embedding-test && From 298421fa0a8fb37fdeb32239ed504cf13aa5e015 Mon Sep 17 00:00:00 2001 From: Eduardo Pinho Date: Sun, 29 Oct 2017 15:08:02 +0000 Subject: [PATCH 27/44] Fix JavaScript randmatstat matrix calculations (#24385) (cherry picked from commit 268f878b33bbcf69b5c60d2409f973f0278d07f5) --- test/perf/micro/perf.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/perf/micro/perf.js b/test/perf/micro/perf.js index 3ae22fd886b62..5ee8ec8f14a30 100644 --- a/test/perf/micro/perf.js +++ b/test/perf/micro/perf.js @@ -248,7 +248,7 @@ } function randmatstat(t) { - var n, P, PTransposed, PMatMul, Q, QTransposed, QMatMul, + var n, P, PTransposed, Pt1P, Pt2P, Q, QTransposed, Pt1Q, Pt2Q, a, b, c, d, aSub, bSub, cSub, dSub, v, w, i, j, k, trP, trQ, v1, v2, w1, w2; n = 5; @@ -259,8 +259,10 @@ PTransposed = new Float64Array( P.length ); QTransposed = new Float64Array( Q.length ); - PMatMul = new Float64Array( n*n ); - QMatMul = new Float64Array( (2*n) * (2*n) ); + Pt1P = new Float64Array( (4*n) * (4*n) ); + Pt2P = new Float64Array( (4*n) * (4*n) ); + Pt1Q = new Float64Array( (2*n) * (2*n) ); + Pt2Q = new Float64Array( (2*n) * (2*n) ); a = new Float64Array( n*n ); b = new Float64Array( n*n ); @@ -307,24 +309,24 @@ } transpose( PTransposed, P, n, 4*n ); - matmulCopy( PMatMul, PTransposed, P, n, 4*n, n ); - matmulCopy( PMatMul, P, P, n, n, n); - matmulCopy( PMatMul, P, P, n, n, n); + matmulCopy( Pt1P, PTransposed, P, 4*n, n, 4*n ); + matmulCopy( Pt2P, Pt1P, Pt1P, 4*n, 4*n, 4*n); + matmulCopy( Pt1P, Pt2P, Pt2P, 4*n, 4*n, 4*n); trP = 0; - for (j = 0; j < n; j++) { - trP += PMatMul[(n+1)*j]; + for (j = 0; j < 4*n; j++) { + trP += Pt1P[(4*n+1)*j]; } v[i] = trP; transpose( QTransposed, Q, 2*n, 2*n ); - matmulCopy( QMatMul, QTransposed, Q, 2*n, 2*n, 2*n ); - matmulCopy( QMatMul, Q, Q, 2*n, 2*n, 2*n); - matmulCopy( QMatMul, Q, Q, 2*n, 2*n, 2*n); + matmulCopy( Pt1Q, QTransposed, Q, 2*n, 2*n, 2*n ); + matmulCopy( Pt2Q, Pt1Q, Pt1Q, 2*n, 2*n, 2*n); + matmulCopy( Pt1Q, Pt2Q, Pt2Q, 2*n, 2*n, 2*n); trQ = 0; for (j = 0; j < 2*n; j++) { - trQ += QMatMul[(2*n+1)*j]; + trQ += Pt1Q[(2*n+1)*j]; } w[i] = trQ; } From 03638e3468abdc63b27b0c025239fb79ef0e71aa Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 8 Nov 2017 22:17:28 -0500 Subject: [PATCH 28/44] fix run-time implementation of ashr_int intrinsic (#24517) (cherry picked from commit 5383a3084d32421f758c8af3782bb8d8fdda2419) --- src/runtime_intrinsics.c | 6 ++---- test/intrinsics.jl | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index e00e13dce82ca..0f5e63a29f656 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -809,10 +809,8 @@ bi_iintrinsic_fast(LLVMXor, xor_op, xor_int, u) bi_iintrinsic_cnvtb_fast(LLVMShl, shl_op, shl_int, u, 1) #define lshr_op(a,b) (b >= 8 * sizeof(a)) ? 0 : a >> b bi_iintrinsic_cnvtb_fast(LLVMLShr, lshr_op, lshr_int, u, 1) -#define ashr_op(a,b) \ - /* if ((signed)a > 0) [in two's complement] ? ... : ...) */ \ - (a >> (host_char_bit * sizeof(a) - 1)) ? ~(b >= 8 * sizeof(a) ? 0 : (~a) >> b) : (b >= 8 * sizeof(a) ? 0 : a >> b) -bi_iintrinsic_cnvtb_fast(LLVMAShr, ashr_op, ashr_int, u, 1) +#define ashr_op(a,b) (b < 0 || b >= 8 * sizeof(a) ? 0 : a >> b) +bi_iintrinsic_cnvtb_fast(LLVMAShr, ashr_op, ashr_int, , 1) //#define bswap_op(a) __builtin_bswap(a) //un_iintrinsic_fast(LLVMByteSwap, bswap_op, bswap_int, u) un_iintrinsic_slow(LLVMByteSwap, bswap_int, u) diff --git a/test/intrinsics.jl b/test/intrinsics.jl index 7e4dbfa46f99f..96ba334a619c9 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -82,3 +82,7 @@ end @test compiled_conv(UInt32, UInt64(0xC000_BA98_8765_4321)) == (0x87654321, 0x0000000087654321, 0xffffffff87654321, 0xc005d4c4, 0xc000ba9880000000) @test_throws ErrorException compiled_conv(Bool, im) + +let f = Core.Intrinsics.ashr_int + @test f(Int8(-17), 1) == -9 +end From b109dfedbcce99898e77ca67596b88a999eee9aa Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 9 Nov 2017 00:13:38 -0500 Subject: [PATCH 29/44] runtime intrinsics: correct several (#24537) flipsign was using the wrong argument several others could trigger an LLVM assertion on divide-by-zero (cherry picked from commit ebd8831cc9bb1ad7c83693e7ff6a5358d9f2bd77) --- src/APInt-C.cpp | 73 +++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp index 9d7092d5338a5..6d9519ffea77e 100644 --- a/src/APInt-C.cpp +++ b/src/APInt-C.cpp @@ -87,34 +87,26 @@ void LLVMMul(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr extern "C" JL_DLLEXPORT void LLVMSDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - CREATE(a) - CREATE(b) - a = a.sdiv(b); - ASSIGN(r, a) + if (LLVMDiv_sov(numbits, pa, pb, pr)) + jl_throw(jl_diverror_exception); } extern "C" JL_DLLEXPORT void LLVMUDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - CREATE(a) - CREATE(b) - a = a.udiv(b); - ASSIGN(r, a) + if (LLVMDiv_uov(numbits, pa, pb, pr)) + jl_throw(jl_diverror_exception); } extern "C" JL_DLLEXPORT void LLVMSRem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - CREATE(a) - CREATE(b) - a = a.srem(b); - ASSIGN(r, a) + if (LLVMRem_sov(numbits, pa, pb, pr)) + jl_throw(jl_diverror_exception); } extern "C" JL_DLLEXPORT void LLVMURem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - CREATE(a) - CREATE(b) - a = a.urem(b); - ASSIGN(r, a) + if (LLVMRem_uov(numbits, pa, pb, pr)) + jl_throw(jl_diverror_exception); } extern "C" JL_DLLEXPORT @@ -277,6 +269,8 @@ extern "C" JL_DLLEXPORT int LLVMDiv_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { CREATE(a) CREATE(b) + if (!b) + return true; bool Overflow; a = a.sdiv_ov(b, Overflow); ASSIGN(r, a) @@ -287,9 +281,10 @@ extern "C" JL_DLLEXPORT int LLVMDiv_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { CREATE(a) CREATE(b) + if (!b) + return true; a = a.udiv(b); ASSIGN(r, a) - // unsigned division cannot overflow return false; } @@ -297,9 +292,10 @@ extern "C" JL_DLLEXPORT int LLVMRem_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { CREATE(a) CREATE(b) + if (!b) + return true; a = a.srem(b); ASSIGN(r, a) - // signed remainder cannot overflow return false; } @@ -307,9 +303,10 @@ extern "C" JL_DLLEXPORT int LLVMRem_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { CREATE(a) CREATE(b) + if (!b) + return true; a = a.urem(b); ASSIGN(r, a) - // unsigned remainder cannot overflow return false; } @@ -394,8 +391,11 @@ int LLVMFPtoUI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integ extern "C" JL_DLLEXPORT void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { - CREATE(a) - double val = a.roundToDouble(true); + double val; + { // end scope before jl_error call + CREATE(a) + val = a.roundToDouble(true); + } if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) @@ -406,8 +406,11 @@ void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPar extern "C" JL_DLLEXPORT void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { - CREATE(a) - double val = a.roundToDouble(false); + double val; + { // end scope before jl_error call + CREATE(a) + val = a.roundToDouble(false); + } if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) @@ -495,24 +498,30 @@ unsigned countTrailingZeros_64(uint64_t Val) { extern "C" JL_DLLEXPORT void jl_LLVMSMod(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - CREATE(a) - CREATE(b) - APInt r = a.srem(b); - if (a.isNegative() != b.isNegative()) { - r = (b + r).srem(b); + { // end scope before jl_error call + CREATE(a) + CREATE(b) + if (!!b) { + APInt r = a.srem(b); + if (a.isNegative() != b.isNegative()) { + r = (b + r).srem(b); + } + ASSIGN(r, r) + return; + } } - ASSIGN(r, r) + jl_throw(jl_diverror_exception); } extern "C" JL_DLLEXPORT void jl_LLVMFlipSign(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { - unsigned numbytes = RoundUpToAlignment(numbits, host_char_bit) / host_char_bit; + unsigned numbytes = (numbits + host_char_bit - 1) / host_char_bit; int signbit = (numbits - 1) % host_char_bit; - int sign = ((unsigned char*)pa)[numbytes - 1] & (1 << signbit) ? -1 : 0; + int sign = ((unsigned char*)pb)[numbytes - 1] & (1 << signbit); if (sign) LLVMNeg(numbits, pa, pr); else - memcpy(pr, pa, numbytes); + memcpy(pr, pa, numbytes); } extern "C" JL_DLLEXPORT From c4d489802f0811dbed1a6de157dfde9026d504c3 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 9 Nov 2017 15:37:09 -0500 Subject: [PATCH 30/44] fix regression in lowering `[x... y]` caused by #24538 (#24552) recursive calls to `expand-forms` should generally be tail calls, matching the usual top-down macro expansion order. (cherry picked from commit 27665e83a757dafdcb838541a40870e3760be2f3) --- src/julia-syntax.scm | 14 +++++++------- test/meta.jl | 5 +++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index bb2cf33723849..0528667b446b2 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2172,7 +2172,7 @@ '=> (lambda (e) (syntax-deprecation #f "Expr(:(=>), ...)" "Expr(:call, :(=>), ...)") - `(call => ,(expand-forms (cadr e)) ,(expand-forms (caddr e)))) + (expand-forms `(call => ,@(cdr e)))) 'cell1d (lambda (e) (error "{ } vector syntax is discontinued")) 'cell2d (lambda (e) (error "{ } matrix syntax is discontinued")) @@ -2268,13 +2268,13 @@ (and (length= e 4) (eq? (cadddr e) ':))) (error "invalid \":\" outside indexing")) - `(call colon ,.(map expand-forms (cdr e)))) + (expand-forms `(call colon ,@(cdr e)))) 'vect (lambda (e) (expand-forms `(call (top vect) ,@(cdr e)))) 'hcat - (lambda (e) (expand-forms `(call hcat ,.(map expand-forms (cdr e))))) + (lambda (e) (expand-forms `(call hcat ,@(cdr e)))) 'vcat (lambda (e) @@ -2297,7 +2297,7 @@ `(call vcat ,@a)))))) 'typed_hcat - (lambda (e) `(call (top typed_hcat) ,(expand-forms (cadr e)) ,.(map expand-forms (cddr e)))) + (lambda (e) (expand-forms `(call (top typed_hcat) ,@(cdr e)))) 'typed_vcat (lambda (e) @@ -2318,8 +2318,8 @@ ,.(apply append rows))) `(call (top typed_vcat) ,t ,@a))))) - '|'| (lambda (e) `(call ctranspose ,(expand-forms (cadr e)))) - '|.'| (lambda (e) `(call transpose ,(expand-forms (cadr e)))) + '|'| (lambda (e) (expand-forms `(call ctranspose ,(cadr e)))) + '|.'| (lambda (e) (expand-forms `(call transpose ,(cadr e)))) 'ccall (lambda (e) @@ -2360,7 +2360,7 @@ ,iter)))) 'flatten - (lambda (e) `(call (top Flatten) ,(expand-forms (cadr e)))) + (lambda (e) (expand-forms `(call (top Flatten) ,(cadr e)))) 'comprehension (lambda (e) diff --git a/test/meta.jl b/test/meta.jl index 6e12549ab10ee..299ed3cec31e3 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -144,3 +144,8 @@ baremodule B end @test B.x == 3 @test B.M.x == 4 + +# caused by #24538. forms that lower to `call` should wrap with `call` before +# recursively calling expand-forms. +@test [(0,0)... 1] == [0 0 1] +@test Float32[(0,0)... 1] == Float32[0 0 1] From ed6b7f9ddbfe0a3d7b6b9b7ddd192e43af11b03b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 11 Nov 2017 23:38:15 +0100 Subject: [PATCH 31/44] print output from info and warnings in one shot, to prevent interleaving when used async (#24530) (cherry picked from commit 1cd62b3b48d494b67eb7a15b031bc2b3379ca97c) --- base/util.jl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/base/util.jl b/base/util.jl index 66972107def5b..e6a014c43f830 100644 --- a/base/util.jl +++ b/base/util.jl @@ -524,9 +524,11 @@ MY INFO: hello world See also [`logging`](@ref). """ function info(io::IO, msg...; prefix="INFO: ") - io = redirect(io, log_info_to, :info) - print_with_color(info_color(), io, prefix; bold = true) - println_with_color(info_color(), io, chomp(string(msg...))) + buf = IOBuffer() + iob = redirect(IOContext(buf, io), log_info_to, :info) + print_with_color(info_color(), iob, prefix; bold = true) + println_with_color(info_color(), iob, chomp(string(msg...))) + print(io, String(take!(buf))) return end info(msg...; prefix="INFO: ") = info(STDERR, msg..., prefix=prefix) @@ -559,16 +561,18 @@ function warn(io::IO, msg...; (key in have_warned) && return push!(have_warned, key) end - io = redirect(io, log_warn_to, :warn) - print_with_color(warn_color(), io, prefix; bold = true) - print_with_color(warn_color(), io, str) + buf = IOBuffer() + iob = redirect(IOContext(buf, io), log_warn_to, :warn) + print_with_color(warn_color(), iob, prefix; bold = true) + print_with_color(warn_color(), iob, str) if bt !== nothing - show_backtrace(io, bt) + show_backtrace(iob, bt) end if filename !== nothing - print(io, "\nwhile loading $filename, in expression starting on line $lineno") + print(iob, "\nwhile loading $filename, in expression starting on line $lineno") end - println(io) + println(iob) + print(io, String(take!(buf))) return end From d6acac2b8449ab8d93f8f420398c39a6f2e31e2e Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Sat, 11 Nov 2017 14:42:14 -0800 Subject: [PATCH 32/44] Update libuv commit (#24190) This includes a fix from upstream libuv that ensures that the thread stack size is set to a multiple of the page size. Fixes #18818 Fixes #24169 (cherry picked from commit 6a23e234e6cc5b4361b5f88614a9ed423dc2c12a) --- .../md5 | 1 + .../sha512 | 1 + deps/libuv.version | 2 +- test/socket.jl | 29 +++++++++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/md5 create mode 100644 deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/sha512 diff --git a/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/md5 b/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/md5 new file mode 100644 index 0000000000000..6a5504c2da819 --- /dev/null +++ b/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/md5 @@ -0,0 +1 @@ +e2d1baa42d69dc5391e2e8bae2596eac diff --git a/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/sha512 b/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/sha512 new file mode 100644 index 0000000000000..d515cb2e45c52 --- /dev/null +++ b/deps/checksums/libuv-d8ab1c6a33e77bf155facb54215dd8798e13825d.tar.gz/sha512 @@ -0,0 +1 @@ +272e3cc7b1290ef19cc941c3b3e6dd39dba7dcb26f0aea8e667c48c56288aa9266020504e0cc90074f02881521b3352079416f564c7eb24ab444326a8f04ca64 diff --git a/deps/libuv.version b/deps/libuv.version index dd4ea6f5e5760..bfc5342bd5f76 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=52d72a52cc7ccd570929990f010ed16e2ec604c8 +LIBUV_SHA1=d8ab1c6a33e77bf155facb54215dd8798e13825d diff --git a/test/socket.jl b/test/socket.jl index 1d215ec237efc..7e9a05b7075a5 100644 --- a/test/socket.jl +++ b/test/socket.jl @@ -351,3 +351,32 @@ let @test test_connect(addr) end +# Issues #18818 and #24169 +mutable struct RLimit + cur::Int64 + max::Int64 +end +function with_ulimit(f::Function, stacksize::Int) + RLIMIT_STACK = 3 # from /usr/include/sys/resource.h + rlim = Ref(RLimit(0, 0)) + # Get the current maximum stack size in bytes + rc = ccall(:getrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim) + @assert rc == 0 + current = rlim[].cur + try + rlim[].cur = stacksize * 1024 + ccall(:setrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim) + f() + finally + rlim[].cur = current + ccall(:setrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim) + end + nothing +end +@static if is_apple() + @testset "Issues #18818 and #24169" begin + with_ulimit(7001) do + @test getaddrinfo("localhost") isa IPAddr + end + end +end From d6fd53c9019502e2cf6240ee97ab3477cd417d31 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Fri, 13 Oct 2017 01:55:54 -0400 Subject: [PATCH 33/44] Upgrade curl to 7.56 (#24112) Fix #23980 (cherry picked from commit a12d6f31805dc4ca7c79d293872943cb42f8440e) --- deps/Versions.make | 2 +- deps/checksums/curl-7.56.0.tar.bz2/md5 | 1 + deps/checksums/curl-7.56.0.tar.bz2/sha512 | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 deps/checksums/curl-7.56.0.tar.bz2/md5 create mode 100644 deps/checksums/curl-7.56.0.tar.bz2/sha512 diff --git a/deps/Versions.make b/deps/Versions.make index 0486758d2bc2b..9be5c36ebd425 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -11,4 +11,4 @@ GMP_VER = 6.1.2 MPFR_VER = 3.1.5 PATCHELF_VER = 0.9 MBEDTLS_VER = 2.3.0 -CURL_VER = 7.53.1 +CURL_VER = 7.56.0 diff --git a/deps/checksums/curl-7.56.0.tar.bz2/md5 b/deps/checksums/curl-7.56.0.tar.bz2/md5 new file mode 100644 index 0000000000000..535621f02b915 --- /dev/null +++ b/deps/checksums/curl-7.56.0.tar.bz2/md5 @@ -0,0 +1 @@ +e0caf257103e0c77cee5be7e9ac66ca4 diff --git a/deps/checksums/curl-7.56.0.tar.bz2/sha512 b/deps/checksums/curl-7.56.0.tar.bz2/sha512 new file mode 100644 index 0000000000000..f6ceb99733939 --- /dev/null +++ b/deps/checksums/curl-7.56.0.tar.bz2/sha512 @@ -0,0 +1 @@ +ba17a9fdc4b540d6053fa542bd875f321d009b9ba0cb56b16fe6c217f3856ab061f2a6c735771a0eadc28338889d071884680b4d4c243b4179872abb29915e3b From 06623d7d27264e487839d9ad4770f36a999fcffd Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 8 Nov 2017 14:37:26 -0500 Subject: [PATCH 34/44] clarifications on display vs show (#24513) From the [mailing list](https://discourse.julialang.org/t/output-like-print/6930/7?u=stevengj) (cherry picked from commit 307773217a79d267910aba9af9a12310d9f704d8) --- base/multimedia.jl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/base/multimedia.jl b/base/multimedia.jl index db765279a44f6..3bfcc5bc63b71 100644 --- a/base/multimedia.jl +++ b/base/multimedia.jl @@ -187,6 +187,30 @@ end xdisplayable(D::Display, args...) = applicable(display, D, args...) +""" + display(x) + display(d::Display, x) + display(mime, x) + display(d::Display, mime, x) + +Display `x` using the topmost applicable display in the display stack, typically using the +richest supported multimedia output for `x`, with plain-text [`STDOUT`](@ref) output as a fallback. +The `display(d, x)` variant attempts to display `x` on the given display `d` only, throwing +a [`MethodError`](@ref) if `d` cannot display objects of this type. + +In general, you cannot assume that `display` output goes to `STDOUT` (unlike [`print(x)`](@ref) or +[`show(x)`](@ref)). For example, `display(x)` may open up a separate window with an image. +`display(x)` means "show `x` in the best way you can for the current output device(s)." +If you want REPL-like text output that is guaranteed to go to `STDOUT`, use +[`show(STDOUT, "text/plain", x)`](@ref) instead. + +There are also two variants with a `mime` argument (a MIME type string, such as +`"image/png"`), which attempt to display `x` using the requested MIME type *only*, throwing +a `MethodError` if this type is not supported by either the display(s) or by `x`. With these +variants, one can also supply the "raw" data in the requested MIME type by passing +`x::AbstractString` (for MIME types with text-based storage, such as text/html or +application/postscript) or `x::Vector{UInt8}` (for binary MIME types). +""" function display(x) for i = length(displays):-1:1 if xdisplayable(displays[i], x) From 0ba49ca89ab528da62b511b5a017225cf9f52e00 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 9 Nov 2017 00:42:10 -0500 Subject: [PATCH 35/44] Skip memmove in del_at_beg when not needed. (#24503) This was likely previously done on linux in the glibc implementation. Fix #24494 (cherry picked from commit d7a171eb2e650192f07e93466d0ccd4747061a0d) --- src/array.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/array.c b/src/array.c index e073727538a56..5ae407d7dbe87 100644 --- a/src/array.c +++ b/src/array.c @@ -831,7 +831,9 @@ STATIC_INLINE void jl_array_del_at_beg(jl_array_t *a, size_t idx, size_t dec, nbtotal++; if (idx > 0) memmove(newdata, olddata, nb1); - memmove(newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1); + // Move the rest of the data if the offset changed + if (newoffs != offset) + memmove(newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1); a->data = newdata; } else { From d2ee1a7b73e3681294911efb691d5796d983586f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Nov 2017 14:09:50 -0600 Subject: [PATCH 36/44] Show Dates.format docstring with code table (#24607) (cherry picked from commit 973e0599dc182ea578b97cd4d97dc9893e18710c) --- doc/src/stdlib/dates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/stdlib/dates.md b/doc/src/stdlib/dates.md index a7ebd9f3d64dd..3c15c41575628 100644 --- a/doc/src/stdlib/dates.md +++ b/doc/src/stdlib/dates.md @@ -27,7 +27,7 @@ Base.Dates.DateTime(::Base.Dates.Period...) Base.Dates.DateTime(::Function, ::Any...) Base.Dates.DateTime(::Base.Dates.TimeType) Base.Dates.DateTime(::AbstractString, ::AbstractString) -Base.Dates.format +Base.Dates.format(::Base.Dates.TimeType, ::AbstractString) Base.Dates.DateFormat Base.Dates.@dateformat_str Base.Dates.DateTime(::AbstractString, ::Base.Dates.DateFormat) From c10fea82431c46ec872cccc42b49862e39b104a4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 14 Nov 2017 15:06:32 +0100 Subject: [PATCH 37/44] Pass codegen params to setup_module from emit_function. Ref #24604 (cherry picked from commit 7c46e3d6e4e74e4e8961e89b2d4478ca9a810058) --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 1847c063595d5..df198b03eca27 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5308,7 +5308,7 @@ static std::unique_ptr emit_function( // allocate Function declarations and wrapper objects Module *M = new Module(ctx.name, jl_LLVMContext); - jl_setup_module(M); + jl_setup_module(M, params); jl_returninfo_t returninfo = {}; Function *f = NULL; Function *fwrap = NULL; From 7809470083a9d047df1a730bba095703c24a085c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 14 Nov 2017 09:06:19 +0100 Subject: [PATCH 38/44] Remove unused compiler-rt patch. Ref #24601 (cherry picked from commit 16ce645609fe1cd5936f773294f54a23221eac55) --- deps/patches/compiler-rt-3.7.1.patch | 73 ---------------------------- 1 file changed, 73 deletions(-) delete mode 100644 deps/patches/compiler-rt-3.7.1.patch diff --git a/deps/patches/compiler-rt-3.7.1.patch b/deps/patches/compiler-rt-3.7.1.patch deleted file mode 100644 index 3c91eaebfd8f4..0000000000000 --- a/deps/patches/compiler-rt-3.7.1.patch +++ /dev/null @@ -1,73 +0,0 @@ -From efecb2c285bd444b6def43ac62e5f0278df387eb Mon Sep 17 00:00:00 2001 -From: Keno Fischer -Date: Mon, 5 Oct 2015 22:24:12 +0000 -Subject: [PATCH] [compiler-rt] Properly detect lack of available system - libraries for arch in clang_darwin.mk - -Summary: This is the Makefile analog of r247833, except that the test also had to be changed such that clang actually attempts to link the program as opposed to just building it. Because of that change, I also switched the order to checking for ld/clang architecture support, because now lack of ld support would make the clang check fail. This fixes PR24776. - -Reviewers: beanz - -Subscribers: llvm-commits - -Differential Revision: http://reviews.llvm.org/D13425 - -git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@249358 91177308-0d34-0410-b5e6-96231b3b80d8 ---- - make/platform/clang_darwin.mk | 16 ++++++++-------- - make/platform/clang_darwin_test_input.c | 9 +++++++++ - 2 files changed, 17 insertions(+), 8 deletions(-) - -diff --git a/make/platform/clang_darwin.mk b/make/platform/clang_darwin.mk -index 3a034b8..dff2694 100644 ---- a/projects/compiler-rt/make/platform/clang_darwin.mk -+++ b/projects/compiler-rt/make/platform/clang_darwin.mk -@@ -17,23 +17,23 @@ CheckArches = \ - result=""; \ - if [ "X$(3)" != X ]; then \ - for arch in $(1); do \ -- if $(CC) -arch $$arch -c \ -+ if $(LD) -v 2>&1 | grep "configured to support" \ -+ | tr ' ' '\n' | grep "^$$arch$$" >/dev/null 2>/dev/null; then \ -+ if $(CC) -arch $$arch \ - -integrated-as \ - $(ProjSrcRoot)/make/platform/clang_darwin_test_input.c \ - -isysroot $(3) \ - -o /dev/null > /dev/null 2> /dev/null; then \ -- if $(LD) -v 2>&1 | grep "configured to support" \ -- | tr ' ' '\n' | grep "^$$arch$$" >/dev/null 2>/dev/null; then \ -- result="$$result$$arch "; \ -+ result="$$result$$arch "; \ - else \ - printf 1>&2 \ -- "warning: clang_darwin.mk: dropping arch '$$arch' from lib '$(2)'";\ -- printf 1>&2 " (ld does not support it)\n"; \ -+ "warning: clang_darwin.mk: dropping arch '$$arch' from lib '$(2)'"; \ -+ printf 1>&2 " (clang or system libraries do not support it)\n"; \ - fi; \ - else \ - printf 1>&2 \ -- "warning: clang_darwin.mk: dropping arch '$$arch' from lib '$(2)'"; \ -- printf 1>&2 " (clang does not support it)\n"; \ -+ "warning: clang_darwin.mk: dropping arch '$$arch' from lib '$(2)'";\ -+ printf 1>&2 " (ld does not support it)\n"; \ - fi; \ - done; \ - fi; \ -diff --git a/make/platform/clang_darwin_test_input.c b/make/platform/clang_darwin_test_input.c -index b7074b8..b406a28 100644 ---- a/projects/compiler-rt/make/platform/clang_darwin_test_input.c -+++ b/projects/compiler-rt/make/platform/clang_darwin_test_input.c -@@ -4,3 +4,12 @@ - #include - #include - #include -+#include -+ -+// Force us to link at least one symbol in a system library -+// to detect systems where we don't have those for a given -+// architecture. -+int main(int argc, const char **argv) { -+ int x; -+ memcpy(&x,&argc,sizeof(int)); -+} From c59edf9db79e12f1e03204e1b4e95dcf85490a7d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 14 Nov 2017 09:23:23 +0100 Subject: [PATCH 39/44] Add patch to build compiler-rt on recent glibc. Ref #24601 (cherry picked from commit fca7881c33066ded863fbc431660c357c1a32657) --- deps/llvm.mk | 3 + .../compiler_rt-3.9-glibc_2.25.90.patch | 118 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 deps/patches/compiler_rt-3.9-glibc_2.25.90.patch diff --git a/deps/llvm.mk b/deps/llvm.mk index f8cb58f36dc82..34d6f47eab4d9 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -497,6 +497,9 @@ $(eval $(call LLVM_PATCH,llvm-PR29010-i386-xmm)) # Remove for 4.0 $(eval $(call LLVM_PATCH,llvm-D32593)) $(eval $(call LLVM_PATCH,llvm-D33179)) $(eval $(call LLVM_PATCH,llvm-3.9.0-D37576-NVPTX-sm_70)) # NVPTX, Remove for 6.0 +ifeq ($(BUILD_LLVM_CLANG),1) +$(eval $(call LLVM_PATCH,compiler_rt-3.9-glibc_2.25.90)) # Remove for 5.0 +endif endif # LLVM_VER ifeq ($(LLVM_VER),3.7.1) diff --git a/deps/patches/compiler_rt-3.9-glibc_2.25.90.patch b/deps/patches/compiler_rt-3.9-glibc_2.25.90.patch new file mode 100644 index 0000000000000..8fe993a637512 --- /dev/null +++ b/deps/patches/compiler_rt-3.9-glibc_2.25.90.patch @@ -0,0 +1,118 @@ +From 8a5e425a68de4d2c80ff00a97bbcb3722a4716da Mon Sep 17 00:00:00 2001 +From: Kostya Serebryany +Date: Thu, 13 Jul 2017 21:59:01 +0000 +Subject: [PATCH] Fix sanitizer build against latest glibc + +Summary: +libsanitizer doesn't build against latest glibc anymore, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81066 for details. +One of the changes is that stack_t changed from typedef struct sigaltstack { ... } stack_t; to typedef struct { ... } stack_t; for conformance reasons. +And the other change is that the glibc internal __need_res_state macro is now ignored, so when doing +``` +#define __need_res_state +#include +``` +the effect is now the same as just +``` +#include +``` +and thus one doesn't get just the +``` +struct __res_state { ... }; +``` +definition, but newly also the +``` +extern struct __res_state *__res_state(void) __attribute__ ((__const__)); +``` +prototype. So __res_state is no longer a type, but a function. + +Reviewers: kcc, ygribov + +Reviewed By: kcc + +Subscribers: kubamracek + +Differential Revision: https://reviews.llvm.org/D35246 + +git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@307969 91177308-0d34-0410-b5e6-96231b3b80d8 +--- + lib/sanitizer_common/sanitizer_linux.cc | 3 +-- + lib/sanitizer_common/sanitizer_linux.h | 4 +--- + lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc | 2 +- + lib/tsan/rtl/tsan_platform_linux.cc | 2 +- + 4 files changed, 4 insertions(+), 7 deletions(-) + +diff --git a/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +index a79a2a155..8c3c1e5d6 100644 +--- a/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc ++++ b/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +@@ -629,8 +629,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { + } + #endif + +-uptr internal_sigaltstack(const struct sigaltstack *ss, +- struct sigaltstack *oss) { ++uptr internal_sigaltstack(const void *ss, void *oss) { + return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); + } + +diff --git a/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +index ee336f7dd..11cad6b80 100644 +--- a/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.h ++++ b/projects/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +@@ -21,7 +21,6 @@ + #include "sanitizer_platform_limits_posix.h" + + struct link_map; // Opaque type returned by dlopen(). +-struct sigaltstack; + + namespace __sanitizer { + // Dirent structure for getdents(). Note that this structure is different from +@@ -30,8 +29,7 @@ struct linux_dirent; + + // Syscall wrappers. + uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count); +-uptr internal_sigaltstack(const struct sigaltstack* ss, +- struct sigaltstack* oss); ++uptr internal_sigaltstack(const void* ss, void* oss); + uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset); + +diff --git a/projects/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/projects/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +index 03f73ae88..d7fa5f645 100644 +--- a/projects/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc ++++ b/projects/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +@@ -287,7 +287,7 @@ static int TracerThread(void* argument) { + + // Alternate stack for signal handling. + InternalScopedBuffer handler_stack_memory(kHandlerStackSize); +- struct sigaltstack handler_stack; ++ stack_t handler_stack; + internal_memset(&handler_stack, 0, sizeof(handler_stack)); + handler_stack.ss_sp = handler_stack_memory.data(); + handler_stack.ss_size = kHandlerStackSize; +diff --git a/projects/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/projects/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +index 0ba01babe..ead1e5704 100644 +--- a/projects/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc ++++ b/projects/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +@@ -286,7 +286,7 @@ void InitializePlatform() { + int ExtractResolvFDs(void *state, int *fds, int nfd) { + #if SANITIZER_LINUX && !SANITIZER_ANDROID + int cnt = 0; +- __res_state *statp = (__res_state*)state; ++ struct __res_state *statp = (struct __res_state*)state; + for (int i = 0; i < MAXNS && cnt < nfd; i++) { + if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1) + fds[cnt++] = statp->_u._ext.nssocks[i]; +diff --git a/projects/compiler-rt/lib/esan/esan_sideline_linux.cpp b/projects/compiler-rt/lib/esan/esan_sideline_linux.cpp +index d04f5909d6a2..bc272dfe49f8 100644 +--- a/projects/compiler-rt/lib/esan/esan_sideline_linux.cpp ++++ b/projects/compiler-rt/lib/esan/esan_sideline_linux.cpp +@@ -70,7 +70,7 @@ int SidelineThread::runSideline(void *Arg) { + + // Set up a signal handler on an alternate stack for safety. + InternalScopedBuffer StackMap(SigAltStackSize); +- struct sigaltstack SigAltStack; ++ stack_t SigAltStack; + SigAltStack.ss_sp = StackMap.data(); + SigAltStack.ss_size = SigAltStackSize; + SigAltStack.ss_flags = 0; From 0f12eacaa1f8510f0aca838d4dbbd98fbae30566 Mon Sep 17 00:00:00 2001 From: Pablo Zubieta Date: Sun, 19 Nov 2017 10:22:40 -0600 Subject: [PATCH 40/44] Allow broadcating with custom types that loose to Tuple --- base/broadcast.jl | 6 +++--- test/broadcast.jl | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 53b7bc0fc7dd9..a07f4193aab1c 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -354,9 +354,9 @@ tuplebroadcast_maxtuple(A, B) = tuplebroadcast_maxtuple(A, tuplebroadcast_maxtuple(Bs...)) tuplebroadcast_maxtuple(A::NTuple{N,Any}, ::NTuple{N,Any}...) where {N} = A # Here we use the containertype trait to easier disambiguate between methods -_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Any}, A, B) = (nothing,) -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Any}, A, B) = A -_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Tuple}, A, B) = B +_tuplebroadcast_maxtuple(::Any, ::Any, A, B) = (nothing,) +_tuplebroadcast_maxtuple(::Type{Tuple}, ::Any, A, B) = A +_tuplebroadcast_maxtuple(::Any, ::Type{Tuple}, A, B) = B _tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B::Tuple{Any}) = A _tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, B) = B _tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, ::Tuple{Any}) = A diff --git a/test/broadcast.jl b/test/broadcast.jl index fed20c55d314f..9fe4b4fdd3eae 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -452,6 +452,20 @@ Base.Broadcast.broadcast_c(f, ::Type{Array19745}, A, Bs...) = @test isa(aa .* aa', Array19745) end +# broadcast with a custom type that looses to tuple +struct DataValue{T} + value::T +end + +Base.Broadcast._containertype(::Type{<:DataValue}) = DataValue +Base.Broadcast.promote_containertype(::Type{Tuple}, ::Type{DataValue}) = Tuple +Base.Broadcast.promote_containertype(::Type{DataValue}, ::Type{Tuple}) = Tuple +Base.Broadcast._broadcast_getindex(::Type{DataValue}, A, I) = A.value + +@testset "Broadcast with tuple and a custom type" begin + @test DataValue(1) .+ (1, 2) == (2, 3) +end + # broadcast should only "peel off" one container layer @test get.([Nullable(1), Nullable(2)]) == [1, 2] let io = IOBuffer() From 7d0840ce4824127b65a41dc9f61a0d4e93f8b48b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 13 Nov 2017 16:21:17 -0500 Subject: [PATCH 41/44] another fix to run-time `ashr_int` (#24575) (cherry picked from commit a282b5fc577fad0d33358cd9b7d8dd0793f3164d) --- src/runtime_intrinsics.c | 2 +- test/intrinsics.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 0f5e63a29f656..1ff6765794474 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -809,7 +809,7 @@ bi_iintrinsic_fast(LLVMXor, xor_op, xor_int, u) bi_iintrinsic_cnvtb_fast(LLVMShl, shl_op, shl_int, u, 1) #define lshr_op(a,b) (b >= 8 * sizeof(a)) ? 0 : a >> b bi_iintrinsic_cnvtb_fast(LLVMLShr, lshr_op, lshr_int, u, 1) -#define ashr_op(a,b) (b < 0 || b >= 8 * sizeof(a) ? 0 : a >> b) +#define ashr_op(a,b) ((b < 0 || b >= 8 * sizeof(a)) ? a >> (8*sizeof(a) - 1) : a >> b) bi_iintrinsic_cnvtb_fast(LLVMAShr, ashr_op, ashr_int, , 1) //#define bswap_op(a) __builtin_bswap(a) //un_iintrinsic_fast(LLVMByteSwap, bswap_op, bswap_int, u) diff --git a/test/intrinsics.jl b/test/intrinsics.jl index 96ba334a619c9..a28968a10458b 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -85,4 +85,8 @@ end let f = Core.Intrinsics.ashr_int @test f(Int8(-17), 1) == -9 + @test f(Int32(-1), 33) == -1 + @test f(Int32(-1), -1) == -1 + @test f(Int32(-1), -10) == -1 + @test f(Int32(2), -1) == 0 end From efba001b9159b617ae2dd260b69496bc2556c8dc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 15 Nov 2017 20:05:51 -0500 Subject: [PATCH 42/44] runtime-intrinsics: fix definition of shifts The compiled version does zext, whereas this was doing sext. (cherry picked from commit bceb764ba3c884bf791ef9a8e2c3144af496a069) --- src/runtime_intrinsics.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 1ff6765794474..609ad83879a3f 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -549,7 +549,7 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name, void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b); unsigned sz = jl_datatype_size(ty); unsigned sz2 = next_power_of_two(sz); - unsigned szb = jl_datatype_size(tyb); + unsigned szb = cvtb ? jl_datatype_size(tyb) : sz; if (sz2 > sz) { /* round type up to the appropriate c-type and set/clear the unused bits */ void *pa2 = alloca(sz2); @@ -558,10 +558,12 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name, pa = pa2; } if (sz2 > szb) { - /* round type up to the appropriate c-type and set/clear/truncate the unused bits */ + /* round type up to the appropriate c-type and set/clear/truncate the unused bits + * (zero-extend if cvtb is set, since in that case b is unsigned while the sign of a comes from the op) + */ void *pb2 = alloca(sz2); memcpy(pb2, pb, szb); - memset((char*)pb2 + szb, getsign(pb, sz), sz2 - szb); + memset((char*)pb2 + szb, cvtb ? 0 : getsign(pb, szb), sz2 - szb); pb = pb2; } jl_value_t *newv = lambda2(ty, pa, pb, sz, sz2, list); From 3522df129cf4ab870eb0fef6f6eb6f975f7e59a5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 20 Nov 2017 18:24:15 -0500 Subject: [PATCH 43/44] revert part of cf2ed662 to fix JuliaDB test failure Ref #24678 (cherry picked from commit 2f19f3f4613787fc2b707e3507abc52745548c84) --- base/reflection.jl | 5 +---- test/inference.jl | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index c9a4230c44fa4..85ddf9cbb71bf 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -515,10 +515,7 @@ function _methods_by_ftype(t::ANY, lim::Int, world::UInt, min::Array{UInt,1}, ma end end if 1 < nu <= 64 - ms = _methods_by_ftype(Any[tp...], t, length(tp), lim, [], world, min, max) - if all(m->isleaftype(m[1]), ms) - return ms - end + return _methods_by_ftype(Any[tp...], t, length(tp), lim, [], world, min, max) end # XXX: the following can return incorrect answers that the above branch would have corrected return ccall(:jl_matching_methods, Any, (Any, Cint, Cint, UInt, Ptr{UInt}, Ptr{UInt}), t, lim, 0, world, min, max) diff --git a/test/inference.jl b/test/inference.jl index fa11804428db8..207a45f378092 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -963,14 +963,14 @@ end # approximate static parameters due to unions let T1 = Array{Float64}, T2 = Array{_1,2} where _1 inference_test_copy(a::T) where {T<:Array} = ccall(:jl_array_copy, Ref{T}, (Any,), a) - rt = Base.return_types(inference_test_copy, (Union{T1,T2},))[1] + rt = Union{Base.return_types(inference_test_copy, (Union{T1,T2},))...} @test rt >: T1 && rt >: T2 el(x::T) where {T} = eltype(T) - rt = Base.return_types(el, (Union{T1,Array{Float32,2}},))[1] + rt = Union{Base.return_types(el, (Union{T1,Array{Float32,2}},))...} @test rt >: Union{Type{Float64}, Type{Float32}} g(x::Ref{T}) where {T} = T - rt = Base.return_types(g, (Union{Ref{Array{Float64}}, Ref{Array{Float32}}},))[1] + rt = Union{Base.return_types(g, (Union{Ref{Array{Float64}}, Ref{Array{Float32}}},))...} @test rt >: Union{Type{Array{Float64}}, Type{Array{Float32}}} end From 6736f45eb1b270642ed12e0f6ec1b71f491b59d5 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Fri, 24 Nov 2017 19:04:00 -0800 Subject: [PATCH 44/44] Listen on the first free port from 9009 Partially reverts the backport of #21818 in 0.6.1. Fixes #24722. Revert client_socket_reuse --- NEWS.md | 3 ++- base/distributed/cluster.jl | 2 +- base/distributed/managers.jl | 6 +++++- doc/src/manual/parallel-computing.md | 9 ++++----- test/distributed_exec.jl | 3 ++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/NEWS.md b/NEWS.md index 62e2983c8e7f5..13d26cdb4dac4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -259,7 +259,8 @@ This section lists changes that do not have deprecation warnings. rather than from environment variables ([#19636]). * Workers now listen on an ephemeral port assigned by the OS. Previously workers would - listen on the first free port available from 9009 ([#21818]). + listen on the first free port available from 9009 ([#21818]). Version 0.6.1 only. + Reverted in 0.6.2 Library improvements diff --git a/base/distributed/cluster.jl b/base/distributed/cluster.jl index c12f8ed847176..1d1f74d7f23c8 100644 --- a/base/distributed/cluster.jl +++ b/base/distributed/cluster.jl @@ -153,7 +153,7 @@ function start_worker(out::IO, cookie::AbstractString) init_worker(cookie) interface = IPv4(LPROC.bind_addr) if LPROC.bind_port == 0 - (port, sock) = listenany(interface, UInt16(0)) + (port, sock) = listenany(interface, UInt16(9009)) LPROC.bind_port = port else sock = listen(interface, LPROC.bind_port) diff --git a/base/distributed/managers.jl b/base/distributed/managers.jl index 169214287cd04..1ea79e25144a9 100644 --- a/base/distributed/managers.jl +++ b/base/distributed/managers.jl @@ -489,7 +489,11 @@ function bind_client_port(s) end function connect_to_worker(host::AbstractString, port::Integer) - s = socket_reuse_port() + # Revert support for now. Client socket port number reuse + # does not play well in a scenario where worker processes are repeatedly + # created and torn down, i.e., when the new workers end up reusing a + # a previous listen port. + s = TCPSocket() connect(s, host, UInt16(port)) # Avoid calling getaddrinfo if possible - involves a DNS lookup diff --git a/doc/src/manual/parallel-computing.md b/doc/src/manual/parallel-computing.md index 8b3c15dc379c3..4d2fb1e97de39 100644 --- a/doc/src/manual/parallel-computing.md +++ b/doc/src/manual/parallel-computing.md @@ -1231,8 +1231,8 @@ as local laptops, departmental clusters, or even the cloud. This section covers requirements for the inbuilt `LocalManager` and `SSHManager`: * The master process does not listen on any port. It only connects out to the workers. - * Each worker binds to only one of the local interfaces and listens on an ephemeral port number - assigned by the OS. + * Each worker binds to only one of the local interfaces and listens on the first free port starting + from `9009`. * `LocalManager`, used by `addprocs(N)`, by default binds only to the loopback interface. This means that workers started later on remote hosts (or by anyone with malicious intentions) are unable to connect to the cluster. An `addprocs(4)` followed by an `addprocs(["remote_host"])` will fail. @@ -1250,9 +1250,8 @@ requirements for the inbuilt `LocalManager` and `SSHManager`: authenticated via public key infrastructure (PKI). Authentication credentials can be supplied via `sshflags`, for example ```sshflags=`-e ` ```. - In an all-to-all topology (the default), all workers connect to each other via plain TCP sockets. - The security policy on the cluster nodes must thus ensure free connectivity between workers for - the ephemeral port range (varies by OS). + Note that worker-worker connections are still plain TCP and the local security policy on the remote + cluster must allow for free connections between worker nodes, at least for ports 9009 and above. Securing and encrypting all worker-worker traffic (via SSH) or encrypting individual messages can be done via a custom ClusterManager. diff --git a/test/distributed_exec.jl b/test/distributed_exec.jl index 821f3c1a31eb0..86293697a5c95 100644 --- a/test/distributed_exec.jl +++ b/test/distributed_exec.jl @@ -53,7 +53,8 @@ if is_unix() s = TCPSocket(delay = false) is_linux() && Base.Distributed.bind_client_port(s) if ccall(:jl_tcp_reuseport, Int32, (Ptr{Void},), s.handle) == 0 - reuseport_tests() + # Client reuse port has been disabled in 0.6.2 +# reuseport_tests() else info("SO_REUSEPORT is unsupported, skipping reuseport tests.") end