Merge pull request JuliaLang#15385 from JuliaLang/tk/backports-0.4.4

[release-0.4] WIP: backports for 0.4.4

Makefile

@@ -562,7 +562,14 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686))
 	$(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \
 	7z x -y 7z920.exe 7z.exe 7z.dll && \
 	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \
-		"mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \
+		"mingw32-libexpat1 mingw32-zlib1" && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libstdc++6-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \
+	for i in *.rpm; do 7z x -y $$i; done && \
+	for i in *.cpio; do 7z x -y $$i; done && \
 	cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . && \
 	$(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-32-bit.7z.exe
 else ifeq ($(ARCH),x86_64)
@@ -572,7 +579,14 @@ else ifeq ($(ARCH),x86_64)
 	mv _7z.dll 7z.dll && \
 	mv _7z.exe 7z.exe && \
 	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \
-		"mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \
+		"mingw64-libexpat1 mingw64-zlib1" && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libstdc++6-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \
+	for i in *.rpm; do 7z x -y $$i; done && \
+	for i in *.cpio; do 7z x -y $$i; done && \
 	cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . && \
 	$(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-64-bit.7z.exe
 else

base/Terminals.jl

@@ -201,7 +201,14 @@ start_reading(t::UnixTerminal) = start_reading(t.in_stream)
 stop_reading(t::UnixTerminal) = stop_reading(t.in_stream)
 eof(t::UnixTerminal) = eof(t.in_stream)
 
-@unix_only hascolor(t::TTYTerminal) = (startswith(t.term_type, "xterm") || success(`tput setaf 0`))
+@unix_only function hascolor(t::TTYTerminal)
+    startswith(t.term_type, "xterm") && return true
+    try
+        return success(`tput setaf 0`)
+    catch
+        return false
+    end
+end
 @windows_only hascolor(t::TTYTerminal) = true
 
 end # module

base/dict.jl

@@ -495,15 +495,21 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys))
     vals = Array(V, newsz)
     count0 = h.count
     count = 0
+    maxprobe = max(16, newsz>>6)
 
     for i = 1:sz
         if olds[i] == 0x1
             k = oldk[i]
             v = oldv[i]
-            index = hashindex(k, newsz)
+            index0 = index = hashindex(k, newsz)
             while slots[index] != 0
                 index = (index & (newsz-1)) + 1
             end
+            if index - index0 > maxprobe
+                # rare condition: new table size causes more grouping of keys than before
+                # see issue #15077
+                return rehash!(h, newsz*2)
+            end
             slots[index] = 0x1
             keys[index] = k
             vals[index] = v

base/docs/helpdb.jl

@@ -4011,46 +4011,67 @@ Note that workers do not run a ``.juliarc.jl`` startup script, nor do they synch
 """
 addprocs()
 
-doc"""
-```rst
-..  addprocs(machines; tunnel=false, sshflags=``, max_parallel=10, exeflags=``) -> List of process identifiers
+"""
+```
+addprocs(machines; keyword_args...) -> List of process identifiers
+```
 
-Add processes on remote machines via SSH.
-Requires julia to be installed in the same location on each node, or to be available via a shared file system.
+Add processes on remote machines via SSH. Requires julia to be installed in the same
+location on each node, or to be available via a shared file system.
 
-``machines`` is a vector of machine specifications.  Worker are started for each specification.
+`machines` is a vector of machine specifications.  Worker are started for each specification.
 
-A machine specification is either a string ``machine_spec`` or a tuple - ``(machine_spec, count)``
+A machine specification is either a string `machine_spec` or a tuple - `(machine_spec, count)`.
 
-``machine_spec`` is a string of the form ``[user@]host[:port] [bind_addr[:port]]``. ``user`` defaults
-to current user, ``port`` to the standard ssh port. If ``[bind_addr[:port]]`` is specified, other
-workers will connect to this worker at the specified ``bind_addr`` and ``port``.
+`machine_spec` is a string of the form `[user@]host[:port] [bind_addr[:port]]`. `user` defaults
+to current user, `port` to the standard ssh port. If `[bind_addr[:port]]` is specified, other
+workers will connect to this worker at the specified `bind_addr` and `port`.
 
-``count`` is the number of workers to be launched on the specified host. If specified as ``:auto``
+`count` is the number of workers to be launched on the specified host. If specified as `:auto`
 it will launch as many workers as the number of cores on the specific host.
 
 
 Keyword arguments:
 
-``tunnel`` : if ``true`` then SSH tunneling will be used to connect to the worker from the master process.
+* `tunnel`: if `true` then SSH tunneling will be used to connect to the worker from the
+            master process. Default is `false`.
+
+* `sshflags`: specifies additional ssh options, e.g.
+
+    sshflags=`-i /home/foo/bar.pem`
+
+* `max_parallel`: specifies the maximum number of workers connected to in parallel at a host.
+                  Defaults to 10.
+
+* `dir`: specifies the working directory on the workers. Defaults to the host's current
+         directory (as found by `pwd()`)
 
-``sshflags`` : specifies additional ssh options, e.g. :literal:`sshflags=\`-i /home/foo/bar.pem\`` .
+* `exename`: name of the julia executable. Defaults to `"\$JULIA_HOME/julia"` or
+             `"\$JULIA_HOME/julia-debug"` as the case may be.
 
-``max_parallel`` : specifies the maximum number of workers connected to in parallel at a host. Defaults to 10.
+* `exeflags`: additional flags passed to the worker processes.
 
-``dir`` :  specifies the working directory on the workers. Defaults to the host's current directory (as found by ``pwd()``)
+* `topology`: Specifies how the workers connect to each other. Sending a message
+            between unconnected workers results in an error.
 
-``exename`` :  name of the julia executable. Defaults to "$JULIA_HOME/julia" or "$JULIA_HOME/julia-debug" as the case may be.
+  + `topology=:all_to_all`  :  All processes are connected to each other.
+                      This is the default.
+
+  + `topology=:master_slave`  :  Only the driver process, i.e. pid 1 connects to the
+                        workers. The workers do not connect to each other.
+
+  + `topology=:custom`  :  The `launch` method of the cluster manager specifes the
+                  connection topology via fields `ident` and `connect_idents` in
+                  `WorkerConfig`. A worker with a cluster manager identity `ident`
+                  will connect to all workers specified in `connect_idents`.
 
-``exeflags`` :  additional flags passed to the worker processes.
 
 Environment variables :
 
 If the master process fails to establish a connection with a newly launched worker within 60.0 seconds,
 the worker treats it a fatal situation and terminates. This timeout can be controlled via environment
-variable ``JULIA_WORKER_TIMEOUT``. The value of ``JULIA_WORKER_TIMEOUT`` on the master process, specifies
+variable `JULIA_WORKER_TIMEOUT`. The value of `JULIA_WORKER_TIMEOUT` on the master process, specifies
 the number of seconds a newly launched worker waits for connection establishment.
-```
 """
 addprocs(machines)
 
@@ -10606,15 +10627,6 @@ This is only needed if your module depends on a file that is not used via `inclu
 """
 include_dependency
 
-doc"""
-    __precompile__(isprecompilable::Bool=true)
-
-Specify whether the file calling this function is precompilable. If `isprecompilable` is `true`, then `__precompile__` throws an exception when the file is loaded by `using`/`import`/`require` *unless* the file is being precompiled, and in a module file it causes the module to be automatically precompiled when it is imported. Typically, `__precompile__()` should occur before the `module` declaration in the file, or better yet `VERSION >= v"0.4" && __precompile__()` in order to be backward-compatible with Julia 0.3.
-
-If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in order to throw an error if Julia attempts to precompile it.
-"""
-__precompile__
-
 doc"""
     randn!([rng], A::Array{Float64,N})
 
@@ -11082,7 +11094,7 @@ DivideError
 doc"""
     AssertionError([msg])
 
-The asserted condition did not evalutate to `true`.
+The asserted condition did not evaluate to `true`.
 Optional argument `msg` is a descriptive error string.
 """
 AssertionError

base/linalg/blas.jl

@@ -57,6 +57,7 @@ export
 
 
 const libblas = Base.libblas_name
+const liblapack = Base.liblapack_name
 
 import ..LinAlg: BlasReal, BlasComplex, BlasFloat, BlasInt, DimensionMismatch, chksquare, axpy!
 
@@ -239,10 +240,10 @@ function axpy!{T<:BlasFloat,Ta<:Number,Ti<:Integer}(alpha::Ta, x::Array{T}, rx::
         throw(DimensionMismatch("ranges of differing lengths"))
     end
     if minimum(rx) < 1 || maximum(rx) > length(x)
-        throw(BoundsError("range out of bounds for x, of length $(length(x))"))
+        throw(ArgumentError("range out of bounds for x, of length $(length(x))"))
     end
     if minimum(ry) < 1 || maximum(ry) > length(y)
-        throw(BoundsError("range out of bounds for y, of length $(length(y))"))
+        throw(ArgumentError("range out of bounds for y, of length $(length(y))"))
     end
     axpy!(length(rx), convert(T, alpha), pointer(x)+(first(rx)-1)*sizeof(T), step(rx), pointer(y)+(first(ry)-1)*sizeof(T), step(ry))
     y
@@ -342,10 +343,10 @@ for (fname, elty) in ((:dgbmv_,:Float64),
 end
 
 ### symv
-for (fname, elty) in ((:dsymv_,:Float64),
-                      (:ssymv_,:Float32),
-                      (:zsymv_,:Complex128),
-                      (:csymv_,:Complex64))
+for (fname, elty, lib) in ((:dsymv_,:Float64,libblas),
+                           (:ssymv_,:Float32,libblas),
+                           (:zsymv_,:Complex128,liblapack),
+                           (:csymv_,:Complex64,liblapack))
     # Note that the complex symv are not BLAS but auiliary functions in LAPACK
     @eval begin
              #      SUBROUTINE DSYMV(UPLO,N,ALPHA,A,LDA,X,INCX,BETA,Y,INCY)
@@ -366,7 +367,7 @@ for (fname, elty) in ((:dsymv_,:Float64),
             if m != length(y)
                 throw(DimensionMismatch("A has size $(size(A)), and y has length $(length(y))"))
             end
-            ccall(($(blasfunc(fname)), libblas), Void,
+            ccall(($(blasfunc(fname)), $lib), Void,
                 (Ptr{UInt8}, Ptr{BlasInt}, Ptr{$elty}, Ptr{$elty},
                  Ptr{BlasInt}, Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty},
                  Ptr{$elty}, Ptr{BlasInt}),
@@ -565,17 +566,17 @@ for (fname, elty) in ((:dger_,:Float64),
 end
 
 ### syr
-for (fname, elty) in ((:dsyr_,:Float64),
-                      (:ssyr_,:Float32),
-                      (:zsyr_,:Complex128),
-                      (:csyr_,:Complex64))
+for (fname, elty, lib) in ((:dsyr_,:Float64,libblas),
+                           (:ssyr_,:Float32,libblas),
+                           (:zsyr_,:Complex128,liblapack),
+                           (:csyr_,:Complex64,liblapack))
     @eval begin
         function syr!(uplo::Char, α::$elty, x::StridedVector{$elty}, A::StridedMatrix{$elty})
             n = chksquare(A)
             if length(x) != n
                 throw(DimensionMismatch("A has size ($n,$n), x has length $(length(x))"))
             end
-            ccall(($(blasfunc(fname)), libblas), Void,
+            ccall(($(blasfunc(fname)), $lib), Void,
                 (Ptr{UInt8}, Ptr{BlasInt}, Ptr{$elty}, Ptr{$elty},
                  Ptr{BlasInt}, Ptr{$elty}, Ptr{BlasInt}),
                  &uplo, &n, &α, x,
@@ -943,10 +944,10 @@ end # module
 function copy!{T<:BlasFloat,Ti<:Integer}(dest::Array{T}, rdest::Union{UnitRange{Ti},Range{Ti}},
                                           src::Array{T}, rsrc::Union{UnitRange{Ti},Range{Ti}})
     if minimum(rdest) < 1 || maximum(rdest) > length(dest)
-        throw(BoundsError("range out of bounds for dest, of length $(length(dest))"))
+        throw(ArgumentError("range out of bounds for dest, of length $(length(dest))"))
     end
     if minimum(rsrc) < 1 || maximum(rsrc) > length(src)
-        throw(BoundsError("range out of bounds for src, of length $(length(src))"))
+        throw(ArgumentError("range out of bounds for src, of length $(length(src))"))
     end
     if length(rdest) != length(rsrc)
         throw(DimensionMismatch("ranges must be of the same length"))

base/linalg/cholesky.jl

@@ -185,7 +185,7 @@ function getindex{T<:BlasFloat}(C::CholeskyPivoted{T}, d::Symbol)
 end
 
 show{T,S<:AbstractMatrix}(io::IO, C::Cholesky{T,S}) =
-    (println("$(typeof(C)) with factor:");show(io,C[:UL]))
+    (println(io, "$(typeof(C)) with factor:");show(io,C[:UL]))
 
 A_ldiv_B!{T<:BlasFloat,S<:AbstractMatrix}(C::Cholesky{T,S}, B::StridedVecOrMat{T}) =
     LAPACK.potrs!(C.uplo, C.factors, B)

base/linalg/generic.jl

@@ -447,21 +447,20 @@ function axpy!(α, x::AbstractArray, y::AbstractArray)
 end
 
 function axpy!{Ti<:Integer,Tj<:Integer}(α, x::AbstractArray, rx::AbstractArray{Ti}, y::AbstractArray, ry::AbstractArray{Tj})
-    if length(x) != length(y)
-        throw(DimensionMismatch("x has length $(length(x)), but y has length $(length(y))"))
+    if length(rx) != length(ry)
+        throw(DimensionMismatch("rx has length $(length(rx)), but ry has length $(length(ry))"))
     elseif minimum(rx) < 1 || maximum(rx) > length(x)
         throw(BoundsError(x, rx))
     elseif minimum(ry) < 1 || maximum(ry) > length(y)
         throw(BoundsError(y, ry))
-    elseif length(rx) != length(ry)
-        throw(ArgumentError("rx has length $(length(rx)), but ry has length $(length(ry))"))
     end
     for i = 1:length(rx)
         @inbounds y[ry[i]] += x[rx[i]]*α
     end
     y
 end
 
+
 # Elementary reflection similar to LAPACK. The reflector is not Hermitian but ensures that tridiagonalization of Hermitian matrices become real. See lawn72
 @inline function reflector!(x::AbstractVector)
     n = length(x)

base/linalg/matmul.jl

@@ -256,7 +256,7 @@ function syrk_wrapper!{T<:BlasFloat}(C::StridedMatrix{T}, tA::Char, A::StridedVe
         return matmul3x3!(C,tA,tAt,A,A)
     end
 
-    if stride(A, 1) == 1 && stride(A, 2) >= size(A, 1)
+    if stride(A, 1) == stride(C, 1) == 1 && stride(A, 2) >= size(A, 1) && stride(C, 2) >= size(C, 1)
         return copytri!(BLAS.syrk!('U', tA, one(T), A, zero(T), C), 'U')
     end
     return generic_matmatmul!(C, tA, tAt, A, A)
@@ -287,7 +287,7 @@ function herk_wrapper!{T<:BlasReal}(C::Union{StridedMatrix{T}, StridedMatrix{Com
     # Result array does not need to be initialized as long as beta==0
     #    C = Array(T, mA, mA)
 
-    if stride(A, 1) == 1 && stride(A, 2) >= size(A, 1)
+    if stride(A, 1) == stride(C, 1) == 1 && stride(A, 2) >= size(A, 1) && stride(C, 2) >= size(C, 1)
         return copytri!(BLAS.herk!('U', tA, one(T), A, zero(T), C), 'U', true)
     end
     return generic_matmatmul!(C,tA, tAt, A, A)
@@ -325,7 +325,7 @@ function gemm_wrapper!{T<:BlasFloat}(C::StridedVecOrMat{T}, tA::Char, tB::Char,
         return matmul3x3!(C,tA,tB,A,B)
     end
 
-    if stride(A, 1) == stride(B, 1) == 1 && stride(A, 2) >= size(A, 1) && stride(B, 2) >= size(B, 1)
+    if stride(A, 1) == stride(B, 1) == stride(C, 1) == 1 && stride(A, 2) >= size(A, 1) && stride(B, 2) >= size(B, 1) && stride(C, 2) >= size(C, 1)
         return BLAS.gemm!(tA, tB, one(T), A, B, zero(T), C)
     end
     generic_matmatmul!(C, tA, tB, A, B)
@@ -434,11 +434,12 @@ const Cbuf = Array(UInt8, tilebufsize)
 function generic_matmatmul!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S})
     mA, nA = lapack_size(tA, A)
     mB, nB = lapack_size(tB, B)
+    mC, nC = size(C)
 
-    if mA == nA == nB == 2
+    if mA == nA == mB == nB == mC == nC == 2
         return matmul2x2!(C, tA, tB, A, B)
     end
-    if mA == nA == nB == 3
+    if mA == nA == mB == nB == mC == nC == 3
         return matmul3x3!(C, tA, tB, A, B)
     end
     _generic_matmatmul!(C, tA, tB, A, B)
@@ -450,14 +451,14 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV
     mA, nA = lapack_size(tA, A)
     mB, nB = lapack_size(tB, B)
     if mB != nA
-        throw(DimensionMismatch("matrix A has dimensions ($mA, $nB), matrix B has dimensions ($mB, $nB)"))
+        throw(DimensionMismatch("matrix A has dimensions ($mA,$nA), matrix B has dimensions ($mB,$nB)"))
     end
     if size(C,1) != mA || size(C,2) != nB
-        throw(DimensionMismatch("result C has dimensions $(size(C)), needs ($mA, $nB)"))
+        throw(DimensionMismatch("result C has dimensions $(size(C)), needs ($mA,$nB)"))
     end
 
     tile_size = 0
-    if isbits(R) && isbits(T) && isbits(S)
+    if isbits(R) && isbits(T) && isbits(S) && (tA == 'N' || tB != 'N')
         tile_size = floor(Int,sqrt(tilebufsize/max(sizeof(R),sizeof(S),sizeof(T))))
     end
     @inbounds begin

base/linalg/tridiag.jl

@@ -77,7 +77,7 @@ function diag{T}(M::SymTridiagonal{T}, n::Integer=0)
     elseif absn<size(M,1)
         return zeros(T,size(M,1)-absn)
     else
-        throw(BoundsError("$n-th diagonal of a $(size(M)) matrix doesn't exist!"))
+        throw(ArgumentError("$n-th diagonal of a $(size(M)) matrix doesn't exist!"))
     end
 end
 
@@ -249,7 +249,7 @@ det(A::SymTridiagonal) = det_usmani(A.ev, A.dv, A.ev)
 
 function getindex{T}(A::SymTridiagonal{T}, i::Integer, j::Integer)
     if !(1 <= i <= size(A,2) && 1 <= j <= size(A,2))
-        throw(BoundsError("(i,j) = ($i,$j) not within matrix of size $(size(A))"))
+        throw(BoundsError(A, (i,j)))
     end
     if i == j
         return A.dv[i]
@@ -339,13 +339,13 @@ function diag{T}(M::Tridiagonal{T}, n::Integer=0)
     elseif abs(n) < size(M,1)
         return zeros(T,size(M,1)-abs(n))
     else
-        throw(BoundsError("$n-th diagonal of a $(size(M)) matrix doesn't exist!"))
+        throw(ArgumentError("$n-th diagonal of a $(size(M)) matrix doesn't exist!"))
     end
 end
 
 function getindex{T}(A::Tridiagonal{T}, i::Integer, j::Integer)
     if !(1 <= i <= size(A,2) && 1 <= j <= size(A,2))
-        throw(BoundsError("(i,j) = ($i,$j) not within matrix of size $(size(A))"))
+        throw(BoundsError(A, (i,j)))
     end
     if i == j
         return A.d[i]