Merge pull request #24519 from JuliaLang/aa/backports-0.6.2

Backports for 0.6.2

.travis.yml

@@ -1,5 +1,6 @@
 language: cpp
-sudo: false
+sudo: required
+dist: trusty
 matrix:
   include:
     - os: linux
@@ -71,8 +72,11 @@ before_install:
         ln -s /usr/bin/g++-5 $HOME/bin/x86_64-linux-gnu-g++;
         gcc --version;
         BAR="bar -i 30";
-        BUILDOPTS="-j3 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1";
+        BUILDOPTS="-j5 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1";
         echo "override ARCH=$ARCH" >> Make.user;
+        sudo sh -c "echo 0 > /proc/sys/net/ipv6/conf/lo/disable_ipv6";
+        export JULIA_CPU_CORES=4;
+        export JULIA_TEST_MAXRSS_MB=1200;
         TESTSTORUN="all";
       elif [ `uname` = "Darwin" ]; then
         brew update;
@@ -94,6 +98,8 @@ before_install:
         export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$DYLD_FALLBACK_LIBRARY_PATH\" \$1";
         export BUILDOPTS="$BUILDOPTS spawn=\$(JULIA_MACOS_SPAWN)";
         make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make;
+        export JULIA_CPU_CORES=2;
+        export JULIA_TEST_MAXRSS_MB=600;
         TESTSTORUN="all --skip linalg/triangular subarray"; fi # TODO: re enable these if possible without timing out
     - git clone -q git://git.kitenet.net/moreutils
 script:
@@ -132,9 +138,8 @@ script:
         /tmp/julia/bin/julia-debug --precompiled=no -e 'true'
     - /tmp/julia/bin/julia -e 'versioninfo()'
     - pushd /tmp/julia/share/julia/test
-    - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 &&
-        /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
-        /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg
+    - /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
+      /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg
     - popd
     # test that the embedding code works on our installation
     - mkdir /tmp/embedding-test &&

Make.inc

@@ -709,6 +709,13 @@ endif
 
 JULIA_CPU_TARGET ?= native
 
+ifneq ($(OS),WINNT)
+# Windows headers with this configuration conflicts with LLVM
+# (Symbol renames are done with macros)
+# We mainly need this on linux for cgmemmgr so don't worry about windows for now...
+JCXXFLAGS += -D_FILE_OFFSET_BITS=64
+endif
+
 # Set some ARCH-specific flags
 ifneq ($(USEICC),1)
 ifeq ($(ISX86),1)

Makefile

@@ -55,7 +55,7 @@ julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
 
 # Build the HTML docs (skipped if already exists, notably in tarballs)
-$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl \) -prune -o -type f -print)
+$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print)
 	@$(MAKE) docs
 
 # doc needs to live under $(build_docdir), not under $(build_datarootdir)/julia/
@@ -154,11 +154,12 @@ release-candidate: release testall
 	@echo 5. Create tag, push to github "\(git tag v\`cat VERSION\` && git push --tags\)"		#"` # These comments deal with incompetent syntax highlighting rules
 	@echo 6. Clean out old .tar.gz files living in deps/, "\`git clean -fdx\`" seems to work	#"`
 	@echo 7. Replace github release tarball with tarballs created from make light-source-dist and make full-source-dist
-	@echo 8. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms
-	@echo 9. Upload to AWS, update https://julialang.org/downloads and http://status.julialang.org/stable links
-	@echo 10. Update checksums on AWS for tarball and packaged binaries
-	@echo 11. Announce on mailing lists
-	@echo 12. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20
+	@echo 8. Check that 'make && make install && make test' succeed with unpacked tarballs even without Internet access.
+	@echo 9. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms
+	@echo 10. Upload to AWS, update https://julialang.org/downloads and http://status.julialang.org/stable links
+	@echo 11. Update checksums on AWS for tarball and packaged binaries
+	@echo 12. Announce on mailing lists
+	@echo 13. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20
 	@echo
 
 $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir)

NEWS.md

@@ -259,7 +259,8 @@ This section lists changes that do not have deprecation warnings.
     rather than from environment variables ([#19636]).
 
   * Workers now listen on an ephemeral port assigned by the OS. Previously workers would
-    listen on the first free port available from 9009 ([#21818]).
+    listen on the first free port available from 9009 ([#21818]). Version 0.6.1 only.
+    Reverted in 0.6.2
 
 
 Library improvements

base/Enums.jl

@@ -29,10 +29,10 @@ end
 @noinline enum_argument_error(typename, x) = throw(ArgumentError(string("invalid value for Enum $(typename): $x")))
 
 """
-    @enum EnumName[::BaseType] EnumValue1[=x] EnumValue2[=y]
+    @enum EnumName[::BaseType] value1[=x] value2[=y]
 
 Create an `Enum{BaseType}` subtype with name `EnumName` and enum member values of
-`EnumValue1` and `EnumValue2` with optional assigned values of `x` and `y`, respectively.
+`value1` and `value2` with optional assigned values of `x` and `y`, respectively.
 `EnumName` can be used just like other types and enum member values as regular values, such as
 
 ```jldoctest

base/array.jl

@@ -445,7 +445,7 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
     return a
 end
 
-_collect_indices(::Tuple{}, A) = copy!(Vector{eltype(A)}(), A)
+_collect_indices(::Tuple{}, A) = copy!(Array{eltype(A)}(), A)
 _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) =
     copy!(Array{eltype(A)}(length.(indsA)), A)
 function _collect_indices(indsA, A)

base/asyncmap.jl

@@ -420,5 +420,5 @@ returning a collection.
 """
 function asyncmap!(f, r, c1, c...; ntasks=0, batch_size=nothing)
     foreach(identity, AsyncCollector(f, r, c1, c...; ntasks=ntasks, batch_size=batch_size))
-    c
+    r
 end

base/broadcast.jl

@@ -128,6 +128,8 @@ end
 end
 
 Base.@propagate_inbounds _broadcast_getindex(A, I) = _broadcast_getindex(containertype(A), A, I)
+# `(x,)`, where `x` is a scalar, broadcasts the same way as `[x]` or `x`
+Base.@propagate_inbounds _broadcast_getindex(::Type{Tuple}, A::Tuple{Any}, I) = A[1]
 Base.@propagate_inbounds _broadcast_getindex(::Type{Array}, A::Ref, I) = A[]
 Base.@propagate_inbounds _broadcast_getindex(::ScalarType, A, I) = A
 Base.@propagate_inbounds _broadcast_getindex(::Any, A, I) = A[I]
@@ -334,13 +336,32 @@ end
 end
 @inline broadcast_c(f, ::Type{Any}, a...) = f(a...)
 @inline broadcast_c(f, ::Type{Tuple}, A, Bs...) =
-    tuplebroadcast(f, first_tuple(A, Bs...), A, Bs...)
+    tuplebroadcast(f, tuplebroadcast_maxtuple(A, Bs...), A, Bs...)
 @inline tuplebroadcast(f, ::NTuple{N,Any}, As...) where {N} =
     ntuple(k -> f(tuplebroadcast_getargs(As, k)...), Val{N})
 @inline tuplebroadcast(f, ::NTuple{N,Any}, ::Type{T}, As...) where {N,T} =
     ntuple(k -> f(T, tuplebroadcast_getargs(As, k)...), Val{N})
-first_tuple(A::Tuple, Bs...) = A
-@inline first_tuple(A, Bs...) = first_tuple(Bs...)
+# When the result of broadcast is a tuple it can only come from mixing n-tuples
+# of the same length with scalars and 1-tuples. So, in order to have a
+# type-stable broadcast, we need to find a tuple of maximum length (except when
+# there are only scalars, empty tuples and 1-tuples, in which case the
+# returned value will be an empty tuple).
+# The following methods compare broadcast arguments pairwise to determine the
+# length of the final tuple.
+tuplebroadcast_maxtuple(A, B) =
+    _tuplebroadcast_maxtuple(containertype(A), containertype(B), A, B)
+@inline tuplebroadcast_maxtuple(A, Bs...) =
+    tuplebroadcast_maxtuple(A, tuplebroadcast_maxtuple(Bs...))
+tuplebroadcast_maxtuple(A::NTuple{N,Any}, ::NTuple{N,Any}...) where {N} = A
+# Here we use the containertype trait to easier disambiguate between methods
+_tuplebroadcast_maxtuple(::Any, ::Any, A, B) = (nothing,)
+_tuplebroadcast_maxtuple(::Type{Tuple}, ::Any, A, B) = A
+_tuplebroadcast_maxtuple(::Any, ::Type{Tuple}, A, B) = B
+_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B::Tuple{Any}) = A
+_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, B) = B
+_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, ::Tuple{Any}) = A
+_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B) =
+    throw(DimensionMismatch("tuples could not be broadcast to a common size"))
 tuplebroadcast_getargs(::Tuple{}, k) = ()
 @inline tuplebroadcast_getargs(As, k) =
     (_broadcast_getindex(first(As), k), tuplebroadcast_getargs(tail(As), k)...)

base/distributed/Distributed.jl

@@ -10,7 +10,8 @@ import Base: getindex, wait, put!, take!, fetch, isready, push!, length,
 using Base: Process, Semaphore, JLOptions, AnyDict, buffer_writes, wait_connected,
             VERSION_STRING, sync_begin, sync_add, sync_end, async_run_thunk,
             binding_module, notify_error, atexit, julia_exename, julia_cmd,
-            AsyncGenerator, display_error, acquire, release, invokelatest
+            AsyncGenerator, display_error, acquire, release, invokelatest, warn_once,
+            shell_escape, uv_error
 
 # NOTE: clusterserialize.jl imports additional symbols from Base.Serializer for use
 

base/distributed/cluster.jl

@@ -153,7 +153,7 @@ function start_worker(out::IO, cookie::AbstractString)
     init_worker(cookie)
     interface = IPv4(LPROC.bind_addr)
     if LPROC.bind_port == 0
-        (port, sock) = listenany(interface, UInt16(0))
+        (port, sock) = listenany(interface, UInt16(9009))
         LPROC.bind_port = port
     else
         sock = listen(interface, LPROC.bind_port)

base/distributed/managers.jl

@@ -184,7 +184,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched,
     cmd = `cd $dir '&&' $tval $exename $exeflags`
 
     # shell login (-l) with string command (-c) to launch julia process
-    cmd = `sh -l -c $(Base.shell_escape(cmd))`
+    cmd = `sh -l -c $(shell_escape(cmd))`
 
     # remote launch with ssh with given ssh flags / host / port information
     # -T → disable pseudo-terminal allocation
@@ -194,7 +194,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched,
     #                          forwarded connections are causing collisions
     # -n → Redirects stdin from /dev/null (actually, prevents reading from stdin).
     #      Used when running ssh in the background.
-    cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(Base.shell_escape(cmd))`
+    cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(shell_escape(cmd))`
 
     # launch the remote Julia process
 
@@ -377,7 +377,7 @@ connection to worker with id `pid`, specified by `config` and return a pair of `
 objects. Messages from `pid` to current process will be read off `instrm`, while messages to
 be sent to `pid` will be written to `outstrm`. The custom transport implementation must
 ensure that messages are delivered and received completely and in order.
-`Base.connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between
+`connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between
 workers.
 """
 function connect(manager::ClusterManager, pid::Int, config::WorkerConfig)
@@ -481,15 +481,19 @@ end
 function bind_client_port(s)
     err = ccall(:jl_tcp_bind, Int32, (Ptr{Void}, UInt16, UInt32, Cuint),
                             s.handle, hton(client_port[]), hton(UInt32(0)), 0)
-    Base.uv_error("bind() failed", err)
+    uv_error("bind() failed", err)
 
     _addr, port = Base._sockname(s, true)
     client_port[] = port
     return s
 end
 
 function connect_to_worker(host::AbstractString, port::Integer)
-    s = socket_reuse_port()
+    # Revert support for now. Client socket port number reuse
+    # does not play well in a scenario where worker processes are repeatedly
+    # created and torn down, i.e., when the new workers end up reusing a
+    # a previous listen port.
+    s = TCPSocket()
     connect(s, host, UInt16(port))
 
     # Avoid calling getaddrinfo if possible - involves a DNS lookup
@@ -516,7 +520,7 @@ end
 Implemented by cluster managers.
 It is called on the master process, by [`rmprocs`](@ref).
 It should cause the remote worker specified by `pid` to exit.
-`Base.kill(manager::ClusterManager.....)` executes a remote `exit()`
+`kill(manager::ClusterManager.....)` executes a remote `exit()`
 on `pid`.
 """
 function kill(manager::ClusterManager, pid::Int, config::WorkerConfig)

base/linalg/lq.jl

@@ -69,13 +69,8 @@ function getindex(A::LQ, d::Symbol)
     end
 end
 
-function getindex(A::LQPackedQ, i::Integer, j::Integer)
-    x = zeros(eltype(A), size(A, 1))
-    x[i] = 1
-    y = zeros(eltype(A), size(A, 2))
-    y[j] = 1
-    return dot(x, A*y)
-end
+getindex(A::LQPackedQ, i::Integer, j::Integer) =
+    A_mul_B!(A, setindex!(zeros(eltype(A), size(A, 2)), 1, j))[i]
 
 getq(A::LQ) = LQPackedQ(A.factors, A.τ)
 

base/linalg/qr.jl

@@ -695,11 +695,12 @@ function A_ldiv_B!(A::QRPivoted{T}, B::StridedMatrix{T}, rcond::Real) where T<:B
     nr = min(mA,nA)
     nrhs = size(B, 2)
     if nr == 0
-        return zeros(T, 0, nrhs), 0
+        return B, 0
     end
     ar = abs(A.factors[1])
     if ar == 0
-        return zeros(T, nA, nrhs), 0
+        B[1:nA, :] = 0
+        return B, 0
     end
     rnk = 1
     xmin = ones(T, 1)
@@ -791,23 +792,8 @@ _cut_B(x::AbstractVector, r::UnitRange) = length(x)  > length(r) ? x[r]   : x
 _cut_B(X::AbstractMatrix, r::UnitRange) = size(X, 1) > length(r) ? X[r,:] : X
 
 ## append right hand side with zeros if necessary
-function _append_zeros(b::AbstractVector, T::Type, n)
-    if n > length(b)
-        x = zeros(T, n)
-        return copy!(x, b)
-    else
-        return copy_oftype(b, T)
-    end
-end
-function _append_zeros(B::AbstractMatrix, T::Type, n)
-    if n > size(B, 1)
-        X = zeros(T, (n, size(B, 2)))
-        X[1:size(B,1), :] = B
-        return X
-    else
-        return copy_oftype(B, T)
-    end
-end
+_zeros(::Type{T}, b::AbstractVector, n::Integer) where {T} = zeros(T, max(length(b), n))
+_zeros(::Type{T}, B::AbstractMatrix, n::Integer) where {T} = zeros(T, max(size(B, 1), n), size(B, 2))
 
 function (\)(A::Union{QR{TA},QRCompactWY{TA},QRPivoted{TA}}, B::AbstractVecOrMat{TB}) where {TA,TB}
     S = promote_type(TA,TB)
@@ -816,7 +802,10 @@ function (\)(A::Union{QR{TA},QRCompactWY{TA},QRPivoted{TA}}, B::AbstractVecOrMat
 
     AA = convert(Factorization{S}, A)
 
-    X = A_ldiv_B!(AA, _append_zeros(B, S, n))
+    X = _zeros(S, B, n)
+    X[1:size(B, 1), :] = B
+
+    A_ldiv_B!(AA, X)
 
     return _cut_B(X, 1:n)
 end
@@ -838,7 +827,10 @@ function (\)(A::Union{QR{T},QRCompactWY{T},QRPivoted{T}}, BIn::VecOrMat{Complex{
 #                                                 |x4|y4|
     B = reshape(transpose(reinterpret(T, BIn, (2, length(BIn)))), size(BIn, 1), 2*size(BIn, 2))
 
-    X = A_ldiv_B!(A, _append_zeros(B, T, n))
+    X = _zeros(T, B, n)
+    X[1:size(B, 1), :] = B
+
+    A_ldiv_B!(A, X)
 
 # |z1|z3|  reinterpret  |x1|x2|x3|x4|  transpose  |x1|y1|  reshape  |x1|y1|x3|y3|
 # |z2|z4|      <-       |y1|y2|y3|y4|     <-      |x2|y2|     <-    |x2|y2|x4|y4|

base/multimedia.jl

@@ -187,6 +187,30 @@ end
 
 xdisplayable(D::Display, args...) = applicable(display, D, args...)
 
+"""
+    display(x)
+    display(d::Display, x)
+    display(mime, x)
+    display(d::Display, mime, x)
+
+Display `x` using the topmost applicable display in the display stack, typically using the
+richest supported multimedia output for `x`, with plain-text [`STDOUT`](@ref) output as a fallback.
+The `display(d, x)` variant attempts to display `x` on the given display `d` only, throwing
+a [`MethodError`](@ref) if `d` cannot display objects of this type.
+
+In general, you cannot assume that `display` output goes to `STDOUT` (unlike [`print(x)`](@ref) or
+[`show(x)`](@ref)).  For example, `display(x)` may open up a separate window with an image.
+`display(x)` means "show `x` in the best way you can for the current output device(s)."
+If you want REPL-like text output that is guaranteed to go to `STDOUT`, use
+[`show(STDOUT, "text/plain", x)`](@ref) instead.
+
+There are also two variants with a `mime` argument (a MIME type string, such as
+`"image/png"`), which attempt to display `x` using the requested MIME type *only*, throwing
+a `MethodError` if this type is not supported by either the display(s) or by `x`. With these
+variants, one can also supply the "raw" data in the requested MIME type by passing
+`x::AbstractString` (for MIME types with text-based storage, such as text/html or
+application/postscript) or `x::Vector{UInt8}` (for binary MIME types).
+"""
 function display(x)
     for i = length(displays):-1:1
         if xdisplayable(displays[i], x)

base/reflection.jl

@@ -390,10 +390,10 @@ Return a collection of all instances of the given type, if applicable. Mostly us
 enumerated types (see `@enum`).
 
 ```jldoctest
-julia> @enum Colors Red Blue Green
+julia> @enum Color red blue green
 
-julia> instances(Colors)
-(Red::Colors = 0, Blue::Colors = 1, Green::Colors = 2)
+julia> instances(Color)
+(red::Color = 0, blue::Color = 1, green::Color = 2)
 ```
 """
 function instances end
@@ -515,10 +515,7 @@ function _methods_by_ftype(t::ANY, lim::Int, world::UInt, min::Array{UInt,1}, ma
         end
     end
     if 1 < nu <= 64
-        ms = _methods_by_ftype(Any[tp...], t, length(tp), lim, [], world, min, max)
-        if all(m->isleaftype(m[1]), ms)
-            return ms
-        end
+        return _methods_by_ftype(Any[tp...], t, length(tp), lim, [], world, min, max)
     end
     # XXX: the following can return incorrect answers that the above branch would have corrected
     return ccall(:jl_matching_methods, Any, (Any, Cint, Cint, UInt, Ptr{UInt}, Ptr{UInt}), t, lim, 0, world, min, max)

base/show.jl

@@ -1589,7 +1589,7 @@ function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices)
                 if length(ind) > 10
                     if ii == ind[4] && all(d->idxs[d]==first(tailinds[d]),1:i-1)
                         for j=i+1:nd
-                            szj = size(a,j+2)
+                            szj = length(indices(a, j+2))
                             indj = tailinds[j]
                             if szj>10 && first(indj)+2 < idxs[j] <= last(indj)-3
                                 @goto skip

base/strings/errors.jl

@@ -3,7 +3,7 @@
 ##    Error messages for Unicode / UTF support
 
 const UTF_ERR_SHORT             = "invalid UTF-8 sequence starting at index <<1>> (0x<<2>> missing one or more continuation bytes)"
-const UTF_ERR_INVALID_INDEX     = "invalid character index"
+const UTF_ERR_INVALID_INDEX     = "invalid character index <<1>> (0x<<2>> is a continuation byte)"
 
 mutable struct UnicodeError <: Exception
     errmsg::AbstractString      ##< A UTF_ERR_ message